DATA1901_G5_Final_report.rmd

---
title: "How Much Does Distance to Train Stations Matter?"
author: "Group 5; # ??"
subtitle: "DATA1901 Project 2 report"
date: "University of Sydney | DATA1901 | April 2022"
output:
  html_document:
    fig_caption: yes
    number_sections: yes
    self_contained: yes
    theme: flatly
    toc: true
    toc_depth: 3
    toc_float: true
    code_folding: hide
---

<br>

# Executive Summary

Based on data analysis using 2036 data points from five geographically close suburbs, we can conclude that distance from a train station has a small but relatively insignificant effect on townhouse prices. Therefore, new home buyers can choose more conveniently located homes without worrying about a significant increase in price.

<br>

# Full Report

## Initial Data Analysis (IDA)

```{r, echo=F, message=F}
library(tidygeocoder)
library(tidyverse)
library(broom)
library(dplyr)
library(rafalib)
library(plotly)
library(rvest)
library(lubridate)
library(geosphere)
library(parallel)
```

```{r, echo=F, message=F}
# House scraping: get_df_suburb
get_df_suburb <- function(location = "2151/Parramatta/"){
  # adapted from https://embracingtherandom.com/r/web-scraping/rent-scraping/
  # determine how many pages to scroll through
  tryCatch({
    location <- gsub("\\s+", "+", location)
    # print(location)
    url <- paste0("https://www.auhouseprices.com/sold/list/NSW/",
                  location,
                  "1/?type=townhouse&ymin=0&ymax=0&bmin=0&bmax=0&pmin=0&pmax=0&sort=date&kw=") # type set to townhouse, no other filtering
    # print(url)
    webpage <- read_html(url)

    # get the number of properties and the number of property displayed on each page
    find_page_number <- webpage  %>%  html_nodes("h2") %>%  html_text()
    find_page_number <- find_page_number[1]
    numbers <- as.numeric(regmatches(find_page_number, gregexpr("[0-9]+", find_page_number))[[1]])
    end_page <- ceiling(numbers[3] / numbers[2]) # number of total properties / number on page  = total number of pages

    df <- NULL

    # print(paste0(location, ": begins 0/4"))
    # print(paste0( "Current suburb: ", location) )
    # print(paste0( "Total pages ", end_page) )

    for (this_page in c(1:end_page)){
    # # print(paste0( "Processing page ", this_page) )
    if (this_page %% 5 == 0){
      # print(paste0("Page processed: ", this_page, "/", end_page))
    }

    # get website text
    url <- paste0("https://www.auhouseprices.com/sold/list/NSW/",
                  location,
                  this_page,
                  "/?type=townhouse&ymin=0&ymax=0&bmin=0&bmax=0&pmin=0&pmax=0&sort=date&kw=") # type set to townhouse, no other filtering
    webpage <- read_html(url)

    result <- webpage  %>%  html_nodes("li") %>%  html_text()
    # end of the relevant content
    result <-  result[ 1: grep("current", result) ]
    # remove the redundant "listed price"
    result <-  result[ !grepl("List", result) ]
    # remove the price listed with rent
    result <-  result[ !grepl("Rent", result) ]

    # filter information on price and number of bedroom/bathroom/carspace
    price_bedroom  <- result[ grep("\\$", result)]
    price_bedroom <- strsplit( price_bedroom , "\\$")
    bedroom <- lapply(price_bedroom, `[`, 1)
    bedroom <- strsplit(unlist( trimws( bedroom) ) , "\\s+")

    price <-  lapply(price_bedroom, `[`, 2)
    price <- trimws(price)
    price <- as.numeric(gsub(",","", price ))

    # filter information on sold month and year
    # note sometimes the price is not listed , therefore only get the ones with the price
    timesold  <- result[ grep("\\$", result)-1]
    timesold <-  trimws( gsub("Sold on","", timesold ))

    # whether to use day month year or just month year
    timesold <- lapply(timesold , function(x){
      check_format <- strsplit(x, "\\s")
      if (length(check_format[[1]]) == 3){
        x <- dmy(x)
      }else if (length(check_format[[1]]) == 2){
        x <- my(x)
      }else{
        x <-  as.Date(paste0(x, "-01-01"))
      }
      x
    })
    timesold <- do.call("c", timesold)

    # get address of these properties
    address <- webpage  %>%  html_nodes("h4") %>%  html_text()
    # end of the relevant content
    address <-  address[ 1: grep("Auction History", address) -1 ]

    #decide which address contain sold price
    sold_info <- grep("Sold on", result) #entry with sold info
    price_info <- grep("\\$", result) #entry with price info
    contain_price <- sold_info  %in% c(price_info-1) #for every sold entry, the immediate next row should be price, if not, then this sold entry does not have price record
    address <- address[contain_price] #only record those property that has price recorded

    temp_df <- data.frame( address = address,
                           bedroom = as.numeric( unlist( lapply( bedroom, `[`, 1) ) ) ,
                           bathroom = as.numeric(  unlist( lapply( bedroom, `[`, 2) )) ,
                           carspace =  as.numeric( unlist( lapply( bedroom, `[`, 3) )),
                           soldprice = price ,
                           yearsold =timesold )

    df <- rbind(df, temp_df)
  }
  # Borrowed from ChatGPT
  # create a new column called "index" with a sequence of numbers
  df <- df %>% mutate(House_ID = 1:nrow(.))
  # move the "index" column to the front of the data frame
  df <- df[, c("House_ID", names(df)[-ncol(df)])]

  # print(paste0("Page processed: ", this_page, "/", end_page))
  # print(paste0(location, ": 1/4: get_df_suburb: creating data frame done!"))
  return(df)
  }, error = function(e) {
    # Error handling code
    # Set the file path and name
    file_path <- "main1_export_Brandon_log/"
    file_name <- "main1_export_Brandon_log.txt"

    # Create the directory if it doesn't exist
    if(!dir.exists(file_path)){
      dir.create(file_path)
    }

    # Write location to the file
    write(location, file.path(file_path, file_name), append = TRUE)
    return(NULL)
  })
}


add_distance_between <- function(lat, lon, fixed_lat, fixed_lon) {
  dist <- distHaversine(c(lon, lat), c(fixed_lon, fixed_lat))
  return(dist)
}


get_l_suburb_dist <- function(df_suburb, suburb_lat, suburb_lon, location) {
  l_suburb <- df_suburb %>% geocode(address, method = 'arcgis', lat=latitude, long=longitude)
  # print(paste0(location, ": 2/4: get_l_suburb: done!"))
  l_suburb_dist <- data.frame(
    l_suburb, distance_to_train_station = apply(
      l_suburb[,c("latitude","longitude")], 1, function(x) add_distance_between(x[1], x[2], suburb_lat, suburb_lon))
  )
  # print(paste0(location, ": 3/4: get_l_suburb_dist: done!"))
  return(l_suburb_dist)
}


export_l_suburb_dist_csv <- function(location, l_suburb_dist) {
  # Writing the `l_granville_houseprice.csv` file in "~/csv_cache/"
  file_name <- paste0("l_", gsub("/", "_", location), "houseprice.csv")
  # print(file_name)
  file_path <- file.path("csv_cache", file_name)  # specify file path
  write.csv(l_suburb_dist, file_path, row.names = FALSE)  # export as CSV file
  # print(paste0(location, ": 4/4: export_l_suburb_dist_csv: done!"))
  return("Result: csv export finished")
}


export_a_suburb <- function(location, suburb_lat, suburb_lon) {
  df_suburb <- get_df_suburb(location)

  # Check if df_suburb is NULL (meaning an error occurred in get_df_suburb)
  if (is.null(df_suburb)) {
    return(NULL)
  }

  l_suburb_dist <- get_l_suburb_dist(df_suburb, suburb_lat, suburb_lon, location)
  export_l_suburb_dist_csv(location, l_suburb_dist)
  # print(paste0(location, ": Finish csv export"))
}


clear_log <- function() {
  # Set the file path and name
  file_path <- "main1_export_Brandon_log"
  file_name <- "main1_export_Brandon_log.txt"

  # Check if file exists before removing it
  if (file.exists(file.path(file_path, file_name))) {
    file.remove(file.path(file_path, file_name))
  }
}


# Modified export_all_suburbs function with progress bar
export_all_suburbs <- function(file_name) {
  cat("Exporting into csv_cache/ begins:\n")
  # Clear the log
  a <- clear_log()
  # create directory if it doesn't exist
  if (!dir.exists("~/csv_cache")) {
  dir.create("csv_cache")
  }

  # Read the input file
  suburbs_input <- read.table(file_name, header = FALSE, sep = ",", col.names = c("location", "latitude", "longitude"), strip.white = TRUE, comment.char = "", quote = "")

  # Filter out rows starting with a '#' character
  suburbs_input <- suburbs_input[!grepl("^#", suburbs_input$location), ]

  # Randomize the order of rows
  random_order <- sample(nrow(suburbs_input))
  suburbs_input <- suburbs_input[random_order, ]

  # Loop through each row in the input file and call export_a_suburb function
  for (i in 1:nrow(suburbs_input)) {
    location <- as.character(suburbs_input[i, "location"])
    latitude <- as.numeric(suburbs_input[i, "latitude"])
    longitude <- as.numeric(suburbs_input[i, "longitude"])

    export_a_suburb(location, latitude, longitude)

    # Print progress bar
    progress <- i / nrow(suburbs_input)
    num_hashes <- floor(progress * 100 / 2) # Assuming each '#' represents 2% of the progress
    num_spaces <- 50 - num_hashes # Assuming the progress bar has 50 characters in total
    cat("\n")
    cat(sprintf("#%s%s (%.0f%%)\n", paste(rep("#", num_hashes), collapse = ""), paste(rep(" ", num_spaces), collapse = ""), progress * 100))
  }

  return(NULL)
}

```

```{r, echo=F, message=F, eval=FALSE}
export_all_suburbs("main1_INPUT.txt")
```

### Reading in files

We first generate a list of full list of names and the longitude and latitude of the train stations of these respective suburbs. The list is stored in `main1_INPUT.txt` (Appendix 1 ??). Then we have the R code (Appendix 2 ??) reading this `.txt` file to output the `.csv` files for each suburbs (Appendix 3 ?? Or github link). Finally we read in the data through the code below.

```{r}
# Get the list of CSV files in the 'csv_cache' directory
csv_files <- list.files(path = "csv_cache", pattern = "*.csv", full.names = TRUE)

# Initialize an empty data frame to store the combined data
combined_df <- data.frame()

# Loop through each file in the csv_files list
for (file in csv_files) {
  # Read the CSV file
  location_data <- read.csv(file)
  
  # Categorize distance
  location_data$"distance_to_train_station(km)" <- location_data$distance_to_train_station / 1000
  
  # Classing distance
  location_data$distance_class <- cut(location_data$"distance_to_train_station(km)",
                                      breaks = c(0, 0.250, 0.500, 0.750, 1.000, 1.250, 1.500, 1.750, 2.000, 2.250, 2.500, 3.000, 3.250, 3.500, 3.750, 4.000))
  
  # Combine the processed data frame with the combined_df data frame
  combined_df <- rbind(combined_df, location_data)
}

# Inspect the combined number of suburbs
print(paste0("Total number of suburbs: ", length(csv_files)))

# Inspect the combined data frame
tail(combined_df)
```

Data used for the report was scraped from the internet using the following link: <https://www.auhouseprices.com/sold/list/NSW/>. 

In total, we analysed 136 suburbs across Sydney, containing a total of 29786 data entries. Each data entry contains a  complete buy/sell history.

We used these variables and cleaned the data in the following ways:

- Distance from train station (km) [QUANTITATIVE]
Address was operationalised into longitude and latitude. These coordinates were used to calculate straight line distance to train station and classed into 250m intervals
- Selling price [QUANTITATIVE]

### Limitations

A function was created to calculate straight line distance from townhouses to train stations, which inaccurately represents travel distance between the two. Some townhouses are likely closer to stations from neighbouring suburbs instead. The relevance of trains as a mode of transport may differ between different suburbs. Additionally, train stations often coincide with commercial centres which may affect selling price.

### Assumptions

A significant assumption was that no amenities close to train stations would increase the price of townhouses (e.g. shops, schools), which may be confounding variables. Another assumption was that all stations, regardless of how major, had an equal effect on selling prices. 

<br>

## Research Question

**What is the effect of distance to train stations on Sydney's housing prices?**

<br>

## Research Theme

Distances from stations were classed into 250 metre intervals to increase the readability of graphical summaries, as the data points produced cluttered scatterplots. A side-by-side boxplot was used to compare whether distance correlated to a change in price. The boxplot suggests there is no correlation between proximity to train stations and selling price. The residual plot illustrates clustering of data points on the bottom-left. Without random scatter, the data is not homoscedastic, hence a linear model is not appropriate.

The numerical summary suggested no correlation. The median selling price for houses between 0 and 250 metres was $506000, it increased to $560000 between 1.75 and 2 kilometres, then decreased to $360000 between 3.75 and 4 kilometres. The fluctuation in median selling price over distance discounts the possibility of a linear correlation. Properties in Sydney within 400 metres of train stations have higher price growth (4.5%) compared to properties between 800 and 1600 metres (0.3%)(Forbes, 2021). Other research suggests the train stations have an insignificant correlation with property prices (r=0.091) (p=0.380)(Berawi et al., 2020). Research suggests that number of rooms and building size was the most significant contributor to property pricing close to stations(Berawi et al., 2020). From our graphs, we see that an increase in car spaces and bathrooms was also linked to an increase in price, and so this could potentially be a confounding variable.

The number of confounding variables alongside a more complex trend could account for the lack of correlation observed. Prices seemed to increase with the number of bedrooms, car-spaces and bathrooms. Yet after controlling for them, there was still no correlation. This suggests there are further confounding variables unaccounted for.To account for inflation, a boxplot of selling price between 2000 and 2023 in Western Sydney suburbs was plotted. There was a general increase in townhouse price over the years. Inflation is also a significant confounding variable that has had a substantial effect on selling price. The complex interaction of variables which affect property price could explain the absence of a correlation. 


<br>

## Related Articles

Proximity to schools and hospitals had a significant effect on house pricing(Berawi et al., 2020). Positive effects of stations on property pricing include retail activity, reduced travel cost or convenience whilst increased crime or noise have negative effects(Bowes & Ihlanfeldt, 2001). These confounding variables could explain the null result and must be controlled in further investigation.


<br>

## References

Berawi, M. A., Miraj, P., Saroji, G., & Sari, M. (2020). Impact of rail transit station proximity to commercial property prices: Utilizing big data in Urban Real Estate. Journal of Big Data, 7(1), 1–17. https://doi.org/10.1186/s40537-020-00348-z 

Bowes, D. R., & Ihlanfeldt, K. R. (2001). Identifying the impacts of rail transit stations on residential property values. Journal of Urban Economics, 50(1), 1–25. https://doi.org/10.1006/juec.2001.2214 

Forbes, K. (2021, August 12). Does a train station increase the value of a property? Metropole Property Strategists. Retrieved April 10, 2023, from https://metropole.com.au/how-have-train-stations-affected-property-prices-in-sydney/#:~:text=It%20found%20that%20properties%20within,a%20growth%20rate%20of%200.3%25. 

<br>

## Acknowledgments
When did you team meet (date and time), and what did each team member contribute?

??

<br>

## Appendix (Optional)









## Filtering Data
```{r}
combined_df_1bed <-filter(combined_df, bedroom ==1)
combined_df_2bed <-filter(combined_df, bedroom ==2)
combined_df_3bed <-filter(combined_df, bedroom ==3)
combined_df_4bed <-filter(combined_df, bedroom ==4)
combined_df_5bed <-filter(combined_df, bedroom ==5)
```


```{r}
par(mfrow=c(1,2))
ggplot(combined_df_1bed, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs Distance from Train Station for 1 Bedroom", x="Distance from Train Station(km)", y="Selling Price (x$100000)", fill = "Number of Carspaces")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))+
  theme(plot.title = element_text(hjust=0.25))

ggplot(combined_df_1bed, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5, aes(fill=factor(carspace))) +
  labs(title = "Sold Price vs Distance from Train Station for 1 Bedroom", x="Distance from Train Station(km)", y="Selling Price (x$100000)", fill = "Number of Carspaces")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))+
  theme(plot.title = element_text(hjust=0.25))
```

```{r}
ggplot(combined_df_2bed, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs Distance from Train Station for 2 Bedrooms", x="Distance from Train Station(km)", y="Selling Price (x$100000)", fill = "Number of Carspaces")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))+
  theme(plot.title = element_text(hjust=0.25))
summary(combined_df_2bed$soldprice)

ggplot(combined_df_2bed, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5, aes(fill=factor(carspace))) +
  labs(title = "Sold Price vs Distance from Train Station for 2 Bedrooms", x="Distance from Train Station(km)", y="Selling Price (x$100000)", fill = "Number of Carspaces")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))+
  theme(plot.title = element_text(hjust=0.25))
summary(combined_df_2bed$soldprice)
```

```{r}
ggplot(combined_df_3bed, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs Distance from Train Station for 3 Bedrooms", x="Distance from Train Station(km)", y="Selling Price (x$100000)", fill = "Number of Carspaces")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))+
  theme(plot.title = element_text(hjust=0.25))
summary(combined_df_3bed$soldprice)

ggplot(combined_df_3bed, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5, aes(fill=factor(carspace))) +
  labs(title = "Sold Price vs Distance from Train Station for 3 Bedrooms", x="Distance from Train Station(km)", y="Selling Price (x$100000)", fill = "Number of Carspaces")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))+
  theme(plot.title = element_text(hjust=0.25))
summary(combined_df_3bed$soldprice)
```

```{r}
ggplot(combined_df_4bed, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs Distance from Train Station for 4 Bedrooms", x="Distance from Train Station(km)", y="Selling Price (x$100000)", fill = "Number of Carspaces")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))+
  theme(plot.title = element_text(hjust=0.25))
summary(combined_df_4bed$soldprice)

ggplot(combined_df_4bed, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5, aes(fill=factor(carspace))) +
  labs(title = "Sold Price vs Distance from Train Station for 4 Bedrooms", x="Distance from Train Station(km)", y="Selling Price (x$100000)", fill = "Number of Carspaces")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))+
  theme(plot.title = element_text(hjust=0.25))
summary(combined_df_4bed$soldprice)
```

```{r}
ggplot(combined_df_5bed, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs Distance from Train Station for 5 Bedrooms", x="Distance from Train Station(km)", y="Selling Price (x$100000)", fill = "Number of Carspaces")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))+
  theme(plot.title = element_text(hjust=0.25))
summary(combined_df_5bed$soldprice)

ggplot(combined_df_5bed, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5, aes(fill=factor(carspace))) +
  labs(title = "Sold Price vs Distance from Train Station for 5 Bedrooms", x="Distance from Train Station(km)", y="Selling Price (x$100000)", fill = "Number of Carspaces")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))+
  theme(plot.title = element_text(hjust=0.25))
summary(combined_df_5bed$soldprice)
```

# Filtering Data by Carspaces and Bedrooms
```{r}
combined_df_1bed_1car <-filter(combined_df, bedroom ==1, carspace == 1)

combined_df_2bed_1car <-filter(combined_df, bedroom ==2, carspace == 1)

combined_df_2bed_2car <-filter(combined_df, bedroom ==2, carspace == 2)

combined_df_3bed_1car <-filter(combined_df, bedroom ==3, carspace == 1)

combined_df_3bed_2car <-filter(combined_df, bedroom ==3, carspace == 2)

combined_df_3bed_3car <-filter(combined_df, bedroom ==3, carspace == 3)

combined_df_3bed_4car <-filter(combined_df, bedroom ==3, carspace == 4)

combined_df_4bed_1car <-filter(combined_df, bedroom ==4, carspace == 1)

combined_df_4bed_2car <-filter(combined_df, bedroom ==4, carspace == 2)

combined_df_4bed_3car <-filter(combined_df, bedroom ==4, carspace == 3)

combined_df_4bed_4car <-filter(combined_df, bedroom ==4, carspace == 4)

combined_df_5bed_1car <-filter(combined_df, bedroom ==5, carspace == 1)

combined_df_5bed_2car <-filter(combined_df, bedroom ==5, carspace == 2)

combined_df_5bed_3car <-filter(combined_df, bedroom ==5, carspace == 3)
```


#### 1 bedroom
```{r}
ggplot(combined_df_1bed_1car, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs Distance from Train Station for 1 Bedroom and 1 Carspace", x="Distance from Train Station(km)", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_1bed_1car$soldprice)
```

#### 2 bedrooms 
```{r}
ggplot(combined_df_2bed_1car, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs Distance from Train Station for 2 Bedrooms and 1 Carspace", x="Distance from Train Station(km)", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_2bed_1car$soldprice)

ggplot(combined_df_2bed_2car, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs Distance from Train Station for 2 Bedrooms and 2 Carspaces", x="Distance from Train Station(km)", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_2bed_2car$soldprice)
```

#### 3 bedrooms
```{r}
ggplot(combined_df_3bed_1car, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs Distance from Train Station for 3 Bedrooms and 1 Carspace", x="Distance from Train Station(km)", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_3bed_1car$soldprice)

ggplot(combined_df_3bed_2car, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs Distance from Train Station for 3 Bedrooms and 2 Carspaces", x="Distance from Train Station(km)", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_3bed_2car$soldprice)

ggplot(combined_df_3bed_3car, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs Distance from Train Station for 3 Bedrooms and 3 Carspaces", x="Distance from Train Station(km)", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_3bed_2car$soldprice)

ggplot(combined_df_3bed_4car, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs Distance from Train Station for 3 Bedrooms and 4 Carspaces", x="Distance from Train Station(km)", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_3bed_4car$soldprice)
```

#### 4 bedrooms
```{r}
ggplot(combined_df_4bed_1car, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs Distance from Train Station for 4 Bedrooms and 1 Carspace", x="Distance from Train Station(km)", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_4bed_1car$soldprice)

ggplot(combined_df_4bed_2car, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs Distance from Train Station for 4 Bedrooms and 2 Carspaces", x="Distance from Train Station(km)", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_4bed_2car$soldprice)

ggplot(combined_df_4bed_3car, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs Distance from Train Station for 4 Bedrooms and 3 Carspaces", x="Distance from Train Station(km)", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_4bed_3car$soldprice)

ggplot(combined_df_4bed_4car, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs Distance from Train Station for 4 Bedrooms and 4 Carspaces", x="Distance from Train Station(km)", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_4bed_4car$soldprice)
```

#### 5 bedrooms
```{r}
ggplot(combined_df_5bed_1car, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs Distance from Train Station for 5 Bedrooms and 1 Carspace", x="Distance from Train Station(km)", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_5bed_1car$soldprice)

ggplot(combined_df_5bed_2car, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs Distance from Train Station for 5 Bedrooms and 2 Carspaces", x="Distance from Train Station(km)", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_5bed_2car$soldprice)

ggplot(combined_df_5bed_3car, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs Distance from Train Station for 5 Bedrooms and 3 Carspaces", x="Distance from Train Station(km)", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_5bed_3car$soldprice)
```

# Creating a column for Year
```{r}
combined_df$Year <- as.factor(format(as.Date(combined_df$yearsold), "%Y"))
```

```{r}
# Filtering by year
combined_df_0.00 <-filter(combined_df, distance_class == "(0,0.25]")
combined_df_0.25 <-filter(combined_df, distance_class == "(0.25,0.5]")
combined_df_0.50 <-filter(combined_df, distance_class == "(0.5,0.75]")
combined_df_0.75 <-filter(combined_df, distance_class == "(0.75,1]")
combined_df_1.00 <-filter(combined_df, distance_class == "(1,1.25]")
combined_df_1.25 <-filter(combined_df, distance_class == "(1.25,1.5]")
combined_df_1.50 <-filter(combined_df, distance_class == "(1.5,1.75]")
combined_df_1.75 <-filter(combined_df, distance_class == "(1.75,2]")
combined_df_2.00 <-filter(combined_df, distance_class == "(2,2.25]")
combined_df_2.25 <-filter(combined_df, distance_class == "(2.25,2.5]")
combined_df_2.50 <-filter(combined_df, distance_class == "(2.5,2.75]")
combined_df_2.75 <-filter(combined_df, distance_class == "(2.75,3]")
combined_df_3.00 <-filter(combined_df, distance_class == "(3,3.25]")
combined_df_3.25 <-filter(combined_df, distance_class == "(3.25,3.5]")
combined_df_3.50 <-filter(combined_df, distance_class == "(3.5,3.75]")
combined_df_3.75 <-filter(combined_df, distance_class == "(3.75,4]")
```

```{r}
ggplot(combined_df_0.00, aes(x = Year, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs year for townhouses 0 to 0.25km from train station", x="Year", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_0.00$soldprice)

ggplot(combined_df_0.25, aes(x = Year, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs year for townhouses 0.25 to 0.50km from train station", x="Year", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_0.25$soldprice)

ggplot(combined_df_0.50, aes(x = Year, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs year for townhouses 0.50 to 0.75km from train station", x="Year", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_0.50$soldprice)

ggplot(combined_df_0.75, aes(x = Year, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs year for townhouses 0.75 to 1.00km from train station", x="Year", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_0.75$soldprice)

ggplot(combined_df_1.00, aes(x = Year, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs year for townhouses 1.00 to 1.25km from train station", x="Year", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_1.00$soldprice)

ggplot(combined_df_1.25, aes(x = Year, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs year for townhouses 1.25 to 1.50km from train station", x="Year", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_1.25$soldprice)

ggplot(combined_df_1.50, aes(x = Year, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs year for townhouses 1.50 to 1.75km from train station", x="Year", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_1.50$soldprice)

ggplot(combined_df_1.75, aes(x = Year, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs year for townhouses 1.75 to 2.00km from train station", x="Year", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_1.75$soldprice)

ggplot(combined_df_2.00, aes(x = Year, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs year for townhouses 2.00 to 2.25km from train station", x="Year", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_2.00$soldprice)

ggplot(combined_df_2.25, aes(x = Year, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs year for townhouses 2.25 to 2.50km from train station", x="Year", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_2.25$soldprice)

ggplot(combined_df_2.50, aes(x = Year, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs year for townhouses 2.50 to 2.75km from train station", x="Year", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_2.50$soldprice)

ggplot(combined_df_2.75, aes(x = Year, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs year for townhouses 2.75 to 3.00km from train station", x="Year", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_2.75$soldprice)

ggplot(combined_df_3.00, aes(x = Year, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs year for townhouses 3.00 to 3.25km from train station", x="Year", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_3.00$soldprice)

ggplot(combined_df_3.25, aes(x = Year, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs year for townhouses 3.25 to 3.75km from train station", x="Year", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_3.25$soldprice)

ggplot(combined_df_3.50, aes(x = Year, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs year for townhouses 3.50 to 3.75km from train station", x="Year", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_3.50$soldprice)

ggplot(combined_df_3.75, aes(x = Year, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs year for townhouses 3.75 to 4.00km from train station", x="Year", y="Selling Price (x$100000)")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))
summary(combined_df_3.75$soldprice)
```

```{r}
ggplot(combined_df, aes(x = Year, y = soldprice/100000))+
    geom_point(aes(color=distance_class)) +
    labs(title = "Sold Price over Years", x="Year", y="Selling Price (x$100000)", fill = "Number of Carspaces")+
    theme_bw()+
    theme(axis.text.x = element_text(angle=45,hjust=1))+
    theme(plot.title = element_text(hjust=0.25))

ggplot(combined_df, aes(x = Year, y = soldprice/100000))+
    geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
    labs(title = "Sold Price over Years", x="Year", y="Selling Price (x$100000)", fill = "Number of Carspaces")+
    theme_bw()+
    theme(axis.text.x = element_text(angle=45,hjust=1))+
    theme(plot.title = element_text(hjust=0.25))

ggplot(combined_df, aes(x = factor(bedroom), y = soldprice/100000))+
    geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
    labs(title = "Sold Price for Different Numbers of Bedrooms", x="Number of Bedrooms", y="Selling Price (x$100000)")+
    theme_bw()+
    theme(axis.text.x = element_text(angle=45,hjust=1))+
    theme(plot.title = element_text(hjust=0.25))

ggplot(combined_df, aes(x = factor(bathroom), y = soldprice/100000))+
    geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
    labs(title = "Sold Price for Different Numbers of Bathrooms", x="Number of Bathrooms", y="Selling Price (x$100000)")+
    theme_bw()+
    theme(axis.text.x = element_text(angle=45,hjust=1))+
    theme(plot.title = element_text(hjust=0.25))

ggplot(combined_df, aes(x = factor(carspace), y = soldprice/100000))+
    geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
    labs(title = "Sold Price for Different Numbers of Carspaces", x="Number of Carspaces", y="Selling Price (x$100000)")+
    theme_bw()+
    theme(axis.text.x = element_text(angle=45,hjust=1))+
    theme(plot.title = element_text(hjust=0.25))
```

# Two added graphs from Jasmine Mon Apr 17, 2023 7 pm
```{r}
q1 <- quantile(combined_df$soldprice, 0.25)
q3 <- quantile(combined_df$soldprice, 0.75)
iqr <- q3 - q1
combined <- subset(combined_df, soldprice >= q1 - 1.5*iqr & soldprice <= q3 + 1.5*iqr)

# I changed the `na.rm` to be TRUE to remove all invalid N/A data points
Q1 <- quantile(combined_df$`distance_to_train_station(km)`, 0.25, na.rm = TRUE)
Q3 <- quantile(combined_df$`distance_to_train_station(km)`, 0.75, na.rm = TRUE)
IQR <- Q3 - Q1

# What I've changed here at 7:05 AM, Apr 17, 2023, Monday
# `subset(combined_df ...` <- `subset(combined, ...`
combined <- subset(combined_df, `distance_to_train_station(km)` >= Q1 - 1.5*IQR & `distance_to_train_station(km)` <= Q3 + 1.5*IQR)

ggplot(combined, aes(x = distance_class, y = soldprice/100000))+
geom_boxplot(outlier.colour = "blue", outlier.size=1.5) +
  labs(title = "Sold Price vs Distance from Train Station", x="Distance from Train Station(km)", y="Selling Price (x$100000)", fill = "Number of Carspaces")+
  theme_bw()+
  theme(axis.text.x = element_text(angle=45,hjust=1))+
  theme(plot.title = element_text(hjust=0.25))

model <- lm(soldprice ~ `distance_to_train_station(km)`, data = combined)
plot(combined$"distance_to_train_station(km)", resid(model), main = "Residual Plot", xlab = "Distance to train station (km)", ylab = "Residuals", cex=0.15)
abline(h=0)
```