Skip to content

Commit

Permalink
Merge pull request #70 from oven-2023/feature/crawling
Browse files Browse the repository at this point in the history 
fix: 크롤링 에러 수정
  • Loading branch information
@haen-su
haen-su authored Nov 22, 2023
2 parents 34b12f6 + bc518d9 commit f8547e3
Show file tree
Hide file tree
Showing 5 changed files with 116 additions and 26 deletions.
19 changes: 15 additions & 4 deletions src/main/java/com/oven/server/api/work/domain/Work.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@
@Getter
@ToString
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class Work extends BaseEntity {

@Id
Expand All @@ -37,11 +35,24 @@ public class Work extends BaseEntity {

private String poster;

@Builder.Default
@OneToMany(mappedBy = "work")
private List<WorkProvider> providerList = new ArrayList<>();
private List<WorkProvider> providerList;

@Embedded
private Genre genre;

@Builder
public Work(String titleKr, String titleEng, int year, String ageRating, String director, String actor, String summary, String poster, Genre genre) {
this.titleKr = titleKr;
this.titleEng = titleEng;
this.year = year;
this.ageRating = ageRating;
this.director = director;
this.actor = actor;
this.summary = summary;
this.poster = poster;
this.genre = genre;
this.providerList = new ArrayList<>();
}

}
11 changes: 9 additions & 2 deletions src/main/java/com/oven/server/api/work/domain/WorkProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,15 @@

import com.oven.server.common.BaseEntity;
import jakarta.persistence.*;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;

@Entity
@Getter
@Setter
@Slf4j
@NoArgsConstructor
public class WorkProvider extends BaseEntity {

@Id
Expand All @@ -25,4 +26,10 @@ public class WorkProvider extends BaseEntity {
@JoinColumn(name = "provider_id")
private Provider provider;

@Builder
public WorkProvider(Work work, Provider provider) {
this.work = work;
this.provider = provider;
}

}
46 changes: 26 additions & 20 deletions src/main/java/com/oven/server/api/work/service/CrawlingService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import org.openqa.selenium.*;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

Expand All @@ -36,10 +37,12 @@ public class CrawlingService {
private final WorkRepository workRepository;
private final ProviderRepository providerRepository;
private final WorkProviderRepository workProviderRepository;
@Value("${chromedriver.path}")
private String chromeDriverPath;

public void saveWork() {

String chromeDriverPath = "/Users/siyoung/졸업프로젝트/oven-server/chromedriver";
System.out.println(chromeDriverPath);

//세션 시작
ChromeOptions options = new ChromeOptions();
Expand Down Expand Up @@ -69,7 +72,7 @@ public void saveWork() {
WebElement item = homeDriver.findElement(By.cssSelector(".MovieItem.grid"));

var stTime = new Date().getTime(); //현재시간
while (new Date().getTime() < stTime + 10000) { //10초 동안 무한스크롤 지속
while (new Date().getTime() < stTime + 40000) { //40초 동안 무한스크롤 지속
Thread.sleep(500); //리소스 초과 방지
//executeScript: 해당 페이지에 JavaScript 명령을 보내는 거
((JavascriptExecutor) homeDriver).executeScript("window.scrollTo(0, document.body.scrollHeight)", item);
Expand Down Expand Up @@ -290,7 +293,8 @@ public void saveWork() {
.genre(genre)
.build();

Work savedWork = workRepository.saveAndFlush(work);
workRepository.saveAndFlush(work);
log.info("-------savedWorkId: {}", work.getId());

// provider
List<WebElement> providers = detailDriver.findElements(By.className("price-item-provider"));
Expand All @@ -299,43 +303,43 @@ public void saveWork() {
log.info("----provider: " + provider.findElement(By.tagName("span")).getText() + "--------");
switch (provider.findElement(By.tagName("span")).getText()) {
case "넷플릭스":
saveWorkProvider("NETFLIX", savedWork);
saveWorkProvider("NETFLIX", work);
break;
case "티빙":
saveWorkProvider("TVING", savedWork);
saveWorkProvider("TVING", work);
break;
case "웨이브":
saveWorkProvider("WAVVE", savedWork);
saveWorkProvider("WAVVE", work);
break;
case "왓챠":
saveWorkProvider("WATCHA", savedWork);
saveWorkProvider("WATCHA", work);
break;
case "디즈니+":
saveWorkProvider("DISNEY_PLUS", savedWork);
saveWorkProvider("DISNEY_PLUS", work);
break;
case "쿠팡플레이":
saveWorkProvider("COUPANG_PLAY", savedWork);
saveWorkProvider("COUPANG_PLAY", work);
break;
case "Apple TV":
saveWorkProvider("APPLE_TV", savedWork);
saveWorkProvider("APPLE_TV", work);
break;
case "네이버 시리즈온":
saveWorkProvider("NAVER_SERIESON", savedWork);
saveWorkProvider("NAVER_SERIESON", work);
break;
case "Google Play 무비":
saveWorkProvider("GOOGLE_PLAY", savedWork);
saveWorkProvider("GOOGLE_PLAY", work);
break;
case "U+모바일tv":
saveWorkProvider("UPLUS_MOBILE_TV", savedWork);
saveWorkProvider("UPLUS_MOBILE_TV", work);
break;
case "아마존 프라임 비디오":
saveWorkProvider("AMAZON_PRIME_VIDEO", savedWork);
saveWorkProvider("AMAZON_PRIME_VIDEO", work);
break;
case "라프텔":
saveWorkProvider("LAFTEL", savedWork);
saveWorkProvider("LAFTEL", work);
break;
case "씨네폭스":
saveWorkProvider("CINEFOX", savedWork);
saveWorkProvider("CINEFOX", work);
break;
}
}
Expand Down Expand Up @@ -363,10 +367,12 @@ private void saveWorkProvider(String providerName, Work work) {
log.info("--------work: " + work + "--------");
log.info("--------findProvider: " + findProvider + "--------");

WorkProvider workProvider = new WorkProvider();
workProvider.setProvider(findProvider);
workProvider.setWork(work);
workProviderRepository.save(workProvider);
WorkProvider workProvider = WorkProvider.builder()
.provider(findProvider)
.work(work)
.build();

workProviderRepository.saveAndFlush(workProvider);
}

public void saveRatingFile() {
Expand Down
2 changes: 2 additions & 0 deletions src/main/java/com/oven/server/config/SecurityConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,8 @@ public SecurityFilterChain configure(HttpSecurity httpSecurity) throws Exception
.requestMatchers(swagger).permitAll()
.requestMatchers("/auth/**").permitAll()
.requestMatchers("/ws/endpoint").permitAll()
.requestMatchers("/crawling").permitAll()
.requestMatchers("/writing").permitAll()
.anyRequest().authenticated()

.and()
Expand Down
64 changes: 64 additions & 0 deletions src/test/java/com/oven/server/SaveWorkTest.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package com.oven.server;

import com.oven.server.api.work.domain.Genre;
import com.oven.server.api.work.domain.Provider;
import com.oven.server.api.work.domain.Work;
import com.oven.server.api.work.domain.WorkProvider;
import com.oven.server.api.work.repository.ProviderRepository;
import com.oven.server.api.work.repository.WorkProviderRepository;
import com.oven.server.api.work.repository.WorkRepository;
import com.oven.server.common.exception.BaseException;
import com.oven.server.common.response.ResponseCode;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.annotation.Rollback;
import org.springframework.transaction.annotation.Transactional;

@SpringBootTest
public class SaveWorkTest {

@Autowired
private WorkRepository workRepository;
@Autowired
private ProviderRepository providerRepository;

@Autowired
private WorkProviderRepository workProviderRepository;


@Test
@Transactional
public void saveWork() {

Genre genre = new Genre();
genre.setAction(true);
genre.setComedy(true);

Provider provider = providerRepository.findById(1L).orElseThrow(
() -> new BaseException(ResponseCode.PROVIDER_NOT_FOUND)
);

Work work = Work.builder()
.titleKr("하이")
.titleEng("Hi")
.year(2023)
.ageRating("12세이상 관람가")
.poster("hi.html")
.summary("hihihihihibye")
.director("김현수, 임채리, 박시영")
.actor("김현수, 임채리, 박시영")
.genre(genre)
.build();

workRepository.saveAndFlush(work);

WorkProvider workProvider = WorkProvider.builder()
.provider(provider)
.work(work)
.build();

workProviderRepository.saveAndFlush(workProvider);
}

}

0 comments on commit f8547e3

Please sign in to comment.