You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
u := launcher.New().
Set("--no-sandbox").
MustLaunch()
browser := rod.New().ControlURL(u).MustConnect()
defer browser.MustClose()
// We create a pool that will hold at most 3 pages which means the max concurrency is 3
pool := rod.NewPagePool(2)
// Create a page if needed
create := func() *rod.Page {
// We use MustIncognito to isolate pages with each other
return browser.MustIncognito().MustPage()
}
imgList = make(map[string][]string)
var lock sync.Mutex
// Run jobs concurrently
wg := sync.WaitGroup{}
// 遍历列表
for _, item := range list {
wg.Add(1)
go func(item model.SpiderList) {
defer wg.Done()
page := pool.Get(create)
defer pool.Put(page)
var resultsImg []string
var err error
fmt.Println("采集图片:", item.Id)
resultsImg, err = Job(item.Url, page)
if err != nil {
fmt.Println(item.Url)
fmt.Println(err) // context deadline exceeded
return
}
lock.Lock() // 加锁
imgList[strconv.FormatInt(item.Id, 10)] = resultsImg
lock.Unlock() // 解锁
//time.Sleep(time.Second * time.Duration(utils.RandInt(1, 3)))
}(item)
}
wg.Wait()
pool.Cleanup(func(p *rod.Page) { defer p.MustClose() })
//rodUtils.Pause() // pause goroutine
return imgList
func Img(list []model.SpiderList) (imgList map[string][]string) {
}
func Job(url string, page *rod.Page) ([]string, error) {
}
出现context deadline exceeded错误,程序异常挂起,排查下来,page没有释放。
Current version is v0.114.5
The text was updated successfully, but these errors were encountered: