^q^

src/extractor/afreeca_downloader.py

@@ -1,5 +1,5 @@
 import downloader
-from utils import Soup, Downloader, get_outdir, Session, LazyUrl, try_n, format_filename, clean_title, get_print
+from utils import Soup, Downloader, get_outdir, Session, LazyUrl, try_n, format_filename, get_print
 import ree as re
 from timee import sleep, time
 import os

src/extractor/artstation_downloader.py

@@ -5,6 +5,7 @@
 from translator import tr_
 from timee import sleep
 from utils import Downloader, Soup, get_print, lazy, Session, try_n, LazyUrl, clean_title
+import clf2
 
 
 class Image(object):
@@ -34,8 +35,10 @@ def init(self):
             pass#raise NotImplementedError('Single post')
         else:
             self.url = self.url_main
-
-        self.session = Session()
+
+        # 3849
+        self.session = Session('chrome')
+        #clf2.solve(self.url, session=self.session, cw=self.cw)
 
     @lazy
     def id(self):

src/extractor/avgle_downloader.py

@@ -22,7 +22,7 @@ def init(self):
         if not self.cw.data_:
             link = 'https://github.com/KurtBestor/Hitomi-Downloader/wiki/Chrome-Extension'
             webbrowser.open(link)
-            return self.Invalid('No data; See: {}'.format(link))
+            raise errors.Invalid('No data; See: {}'.format(link))
 
     def read(self):
         video = get_video(self.url, cw=self.cw)

src/extractor/bdsmlr_downloader.py

@@ -8,6 +8,7 @@
 from timee import sleep
 from error_printer import print_error
 import clf2
+import errors
 
 
 @Downloader.register
@@ -18,7 +19,7 @@ class Downloader_bdsmlr(Downloader):
 
     def init(self):
         if u'bdsmlr.com/post/' in self.url:
-            return self.Invalid(tr_(u'개별 다운로드는 지원하지 않습니다: {}').format(self.url), fail=False)
+            raise errors.Invalid(tr_(u'개별 다운로드는 지원하지 않습니다: {}').format(self.url))
 
         self.url = 'https://{}.bdsmlr.com'.format(self.id_)
         self.session = Session()

src/extractor/bili_downloader.py

@@ -35,7 +35,7 @@ class Video(object):
 
     def __init__(self, url, referer, id, p):
         ext = os.path.splitext(url.split('?')[0])[1]
-        self.filename = (u'{}.part{}{}').format(id, p, ext)
+        self.filename = '{}.part{}{}'.format(id, p, ext)
         self.url = LazyUrl(referer, lambda _: url, self, detect_local=False)
 
 
@@ -53,11 +53,11 @@ def fix_url(url, cw=None):
     if meta:
         url_new = meta.attrs['content']
         if tail:
-            url_new = u'{}?{}'.format(url_new, tail)
-        print_(u'redirect: {} -> {}'.format(url, url_new))
+            url_new = '{}?{}'.format(url_new, tail)
+        print_('redirect: {} -> {}'.format(url, url_new))
     else:
         url_new = url
-        print_(u'no redirect')
+        print_('no redirect')
     return url_new
 
 
@@ -97,10 +97,10 @@ def read(self):
         self.setIcon(thumb)
         title = info['title']
         if page is not None:
-            title += (u'_p{}').format(page)
+            title += '_p{}'.format(page)
         title = format_filename(title, self.id_, '.mp4')[:-4]
         n = int(math.ceil(8.0 / len(videos)))
-        self.print_(('n_threads: {}').format(n))
+        self.print_('n_threads: {}'.format(n))
         self.enableSegment(n_threads=n, overwrite=True)
         self.title = title
 
@@ -201,7 +201,7 @@ def get_videos(url, cw=None, depth=0):
                 raise Exception(msg)
         quality = video_info['quality']
         resolution = get_resolution_(quality)
-        s = (u'resolution: {}').format(resolution)
+        s = 'resolution: {}'.format(resolution)
         print_(s)
 
         # 2184

src/extractor/discord_emoji_downloader.py

@@ -27,6 +27,7 @@
 """
 from utils import Downloader, clean_title
 import requests
+import errors
 
 
 @Downloader.register
@@ -53,23 +54,23 @@ def read(self):
             account_info = response.json()
             if response.status_code == 400:
                 if account_info.get("captcha_key"):
-                    return self.Invalid(
+                    raise errors.Invalid(
                         "먼저 웹 또는 디스코드 앱에서 로그인하신후 캡차를 인증해주세요."
                     )  # 메세지 박스 return하니까 멈춰서 raise로 해놨어요
                 else:
-                    return self.Invalid("이메일 또는 비밀번호가 잘못되었습니다. 확인후 다시 시도해주세요.")
+                    raise errors.Invalid("이메일 또는 비밀번호가 잘못되었습니다. 확인후 다시 시도해주세요.")
             else:
                 if not account_info["token"]:
-                    return self.Invalid("토큰을 받아오지 못했어요. 2단계인증을 사용중이신경우 토큰을 이용해 요청해주세요.")
+                    raise errors.Invalid("토큰을 받아오지 못했어요. 2단계인증을 사용중이신경우 토큰을 이용해 요청해주세요.")
                 else:
                     token = account_info["token"]
         else:
-            return self.Invalid("인자값이 더 많이왔어요.")
+            raise errors.Invalid("인자값이 더 많이왔어요.")
 
         guild_info_response = self.get_emoji_list(token, int(guild_id))  # 토큰과 함께 get요청함
 
         if guild_info_response.status_code != 200:
-            return self.Invalid("정상적인 토큰이 아니거나 서버를 찾을수없어요. 맞는 토큰인지, 해당 서버에 접속해있는지 확인해주세요.")
+            raise errors.Invalid("정상적인 토큰이 아니거나 서버를 찾을수없어요. 맞는 토큰인지, 해당 서버에 접속해있는지 확인해주세요.")
         else:
             guild_info = guild_info_response.json()
 
@@ -86,7 +87,7 @@ def read(self):
                 )
                 self.urls.append(base_url + param + "?v=1")  # 인자 합치기
         else:
-            return self.Invalid("해당 서버에는 이모지가 없어요")
+            raise errors.Invalid("해당 서버에는 이모지가 없어요")
 
     def get_emoji_list(self, token: str, guild_id: int) -> dict:
         response = requests.get(

src/extractor/etc_downloader.py

@@ -201,13 +201,6 @@ def __init__(self, f, f_audio, info, session, referer, cw=None):
         self.filename = format_filename(title, self.id, ext, header=self.header)
 
     def pp(self, filename):
-        if self.cw:
-            with self.cw.convert(self):
-                return self._pp(filename)
-        else:
-            return self._pp(filename)
-
-    def _pp(self, filename):
         if self.f_audio:
             f = BytesIO()
             downloader.download(self.f_audio['url'], buffer=f, referer=self.referer, session=self.session)

src/extractor/fc2_downloader.py

@@ -1,6 +1,6 @@
 import downloader
 import ree as re
-from utils import urljoin, Downloader, format_filename, Soup, LazyUrl, get_print
+from utils import urljoin, Downloader, format_filename, Soup, LazyUrl, get_print, Session
 from m3u8_tools import M3u8_stream
 from io import BytesIO
 PATTERN_ID = r'/content/([^/]+)'
@@ -23,7 +23,9 @@ def key_id(cls, url):
         return re.find(PATTERN_ID, url) or url
 
     def read(self):
-        info = get_info(self.url, self.cw)
+        self.session = Session()
+        self.session.cookies.set('_ac', '1', domain='.video.fc2.com')
+        info = get_info(self.url, self.session, self.cw)
 
         video = info['videos'][0]
 
@@ -38,25 +40,26 @@ def read(self):
 
 class Video(object):
 
-    def __init__(self, url, url_thumb, referer, title, id_):
+    def __init__(self, url, url_thumb, referer, title, id_, session):
         self._url = url
         self.url = LazyUrl(referer, self.get, self)
         self.filename = format_filename(title, id_, '.mp4')
         self.url_thumb = url_thumb
+        self.session = session
 
     def get(self, referer):
-        ext = downloader.get_ext(self._url, referer=referer)
+        ext = downloader.get_ext(self._url, session=self.session, referer=referer)
         if ext == '.m3u8':
-            video = M3u8_stream(self._url, n_thread=4)
+            video = M3u8_stream(self._url, referer=referer, session=self.session, n_thread=4)
         else:
             video = self._url
         return video
 
 
-def get_info(url, cw=None):
+def get_info(url, session, cw=None):
     print_ = get_print(cw)
     info = {'videos': []}
-    html = downloader.read_html(url)
+    html = downloader.read_html(url, session=session)
     soup = Soup(html)
     info['title'] = soup.find('h2', class_='videoCnt_title').text.strip()
 
@@ -69,11 +72,12 @@ def get_info(url, cw=None):
     hdr = {
         'X-FC2-Video-Access-Token': token,
         }
-    data = downloader.read_json(url_api, url, headers=hdr)
+    data = downloader.read_json(url_api, url, session=session, headers=hdr)
 
-    url_video = urljoin(url, data['playlist'].get('nq') or data['playlist']['sample'])
+    pl = data['playlist']
+    url_video = urljoin(url, pl.get('hq') or pl.get('nq') or pl['sample']) #3784
     url_thumb = soup.find('meta', {'property':'og:image'})['content']
-    video = Video(url_video, url_thumb, url, info['title'], id_)
+    video = Video(url_video, url_thumb, url, info['title'], id_, session)
     info['videos'].append(video)
 
     return info

src/extractor/file_downloader.py

@@ -24,16 +24,18 @@ def read(self):
                 name = qs[key][(-1)]
                 break
         else:
-            name = os.path.basename(self.url)
+            name = self.url
             for esc in ['?', '#']:
                 name = name.split(esc)[0]
+            name = os.path.basename(name.strip('/'))
 
-        ext = get_ext(name)
+        try:
+            ext = downloader.get_ext(self.url)
+        except:
+            ext = ''
         if not ext:
-            try:
-                ext = downloader.get_ext(self.url)
-            except:
-                ext = ''
+            ext = get_ext(name)
+
         name = os.path.splitext(name)[0]
 
         self.urls.append(self.url)

src/extractor/hanime_downloader.py

@@ -1,5 +1,5 @@
 import downloader
-from utils import Session, Downloader, get_outdir, try_n, Soup, format_filename, clean_title
+from utils import Session, Downloader, get_outdir, try_n, Soup, format_filename, clean_title, get_print, get_resolution
 import ree as re, json
 from io import BytesIO
 import os
@@ -42,7 +42,7 @@ class Downloader_hanime(Downloader):
     display_name = 'hanime.tv'
 
     def read(self):
-        video, session = get_video(self.url)
+        video, session = get_video(self.url, cw=self.cw)
         self.video = video
 
         self.urls.append(video.url)
@@ -53,7 +53,8 @@ def read(self):
 
 
 @try_n(8)
-def get_video(url, session=None):
+def get_video(url, session=None, cw=None):
+    print_ = get_print(cw)
     if session is None:
         session = Session()
         session.headers['User-Agent'] = downloader.hdr['User-Agent']
@@ -94,15 +95,30 @@ def get_video(url, session=None):
         url_video = stream['url']
         if not url_video or 'deprecated.' in url_video:
             continue
+        stream['height'] = int(stream['height'])
         streams_good.append(stream)
 
     if not streams_good:
         raise Exception('No video available')
     print('len(streams_good):', len(streams_good))
+    res = get_resolution()
+
+    def print_stream(stream):
+        print_([stream['extension'], stream['height'], stream['filesize_mbs'], stream['url']])
+
+    steams_filtered = []
     for stream in streams_good:
-        print(stream['extension'], stream['width'], stream['filesize_mbs'], stream['url'])
+        print_stream(stream)
+        if stream['height'] <= res: #3712
+            steams_filtered.append(stream)
 
-    stream = streams_good[0]
+    if steams_filtered:
+        stream = sorted(steams_filtered, key=lambda _: _['height'])[-1]
+    else:
+        stream = sorted(streams_good, key=lambda _: _['height'])[0]
+
+    print_('Final stream:')
+    print_stream(stream)
     return Video(info, stream), session
 
 

src/extractor/hentaicosplay_downloader.py

@@ -0,0 +1,83 @@
+#coding: utf8
+import downloader
+from utils import Downloader, Session, Soup, LazyUrl, urljoin, get_ext, clean_title
+import ree as re
+from translator import tr_
+import clf2
+from ratelimit import limits, sleep_and_retry
+
+
+
+class Image:
+
+    def __init__(self, url, referer, p, session):
+        self._url = url
+        self._p = p
+        self.url = LazyUrl(referer, self.get, self)
+        self.session = session
+
+    @sleep_and_retry
+    @limits(2, 1)
+    def get(self, referer):
+        soup = downloader.read_soup(self._url, referer, session=self.session)
+        div = soup.find('div', id='display_image_detail')
+        url = urljoin(self._url, div.find('img').parent['href'])
+        ext = get_ext(url)
+        self.filename = '{:04}{}'.format(self._p, ext)
+        return url, self._url
+
+
+@Downloader.register
+class Downloader_hentaicosplay(Downloader):
+    type = 'hentaicosplay'
+    URLS = ['hentai-cosplays.com']
+    icon = None
+    display_name = 'Hentai Cosplay'
+    MAX_CORE = 4
+
+    @classmethod
+    def fix_url(cls, url):
+        url = re.sub(r'/page/[0-9]+', '', url)
+        url = re.sub(r'/attachment/[0-9]+', '', url)
+        url = re.sub(r'([a-zA-Z]+\.)hentai-cosplays\.com', 'hentai-cosplays.com', url)
+        return url
+
+    def init(self):
+        self.session = Session()
+
+    def read(self):
+        if '/image/' not in self.url:
+            raise NotImplementedError('Not a post')
+
+        res = clf2.solve(self.url, session=self.session, cw=self.cw)
+        soup = Soup(res['html'])
+        title = soup.find('h2').text
+        paginator = soup.find('div', id='paginator')
+        pages = [self.url]
+        for a in paginator.findAll('a'):
+            href = a.get('href')
+            if not href:
+                continue
+            href = urljoin(self.url, href)
+            if href not in pages:
+                pages.append(href)
+
+        imgs = []
+        for i, page in enumerate(pages):
+            if page == self.url:
+                soup_page =  soup
+            else:
+                soup_page = downloader.read_soup(page, session=self.session)
+            view = soup_page.find('div', id='post')
+            for img in view.findAll('img'):
+                href = img.parent['href']
+                href = urljoin(page, href)
+                img = Image(href, page, len(imgs), self.session)
+                imgs.append(img)
+            self.cw.setTitle('{} {} ({} / {})'.format(tr_('읽는 중...'), title, i+1, len(pages)))
+
+        for img in imgs:
+            self.urls.append(img.url)
+
+        self.title = clean_title(title)
+