add http/https proxy support and version 1.0.6

README.md

@@ -60,12 +60,14 @@ pip3 install hget -U
 
 ```shell
 $ hget -h 
-usage: hget [-h] [-o <file>] [--dir <dir>] [-n <int>] [-c <int>] [-t <int>] [-s <str>] [-d] [-q] [-v] [--access-key <str>] [--secrets-key <str>] [--noreload] <url>
+usage: hget [-h] [-o <file>] [--dir <dir>] [-n <int>] [-c <int>] [-t <int>] [-s <str>] [-d] [-q] [-v] [-p <str>] [--proxy-user <str>] [--proxy-password <str>] [--use-proxy-env] [--noreload]
+            [--access-key <str>] [--secrets-key <str>]
+            <url>
 
-An interruptable and resumable download accelerator.
+An interruptable and resumable download accelerator supplementary of wget/axel.
 
 positional arguments:
-  <url>                 download url, http/https/s3/ftp support
+  <url>                 download url, http/https/ftp/s3 support
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -85,6 +87,14 @@ optional arguments:
   -v, --version         show program's version number and exit
   --noreload            tells hget to NOT use the auto-reloader
 
+proxy arguments:
+  -p <str>, --proxy <str>
+                        proxy url, statswith http/https
+  --proxy-user <str>    set USER as proxy username
+  --proxy-password <str>
+                        set PASS as proxy password
+  --use-proxy-env       use HTTP_PROXY or HTTPS_PROXY environment variables for proxy url
+
 aws arguments:
   --access-key <str>    access key if necessary
   --secrets-key <str>   secrets key if necessary
@@ -103,6 +113,10 @@ aws arguments:
 | -d/--debug       | `debug`模式，更多的`logging`输出                             |
 | -q/--quiet       | 禁止除错误外的全部屏幕输出                                   |
 | -v/--version     | 打印软件版本并退出                                           |
+| -p/--proxy       | 使用的代理url                                                |
+| --proxy-user     | 使用的代理用户名                                             |
+| --proxy-password | 使用的代理密码                                               |
+| --use-proxy-env  | 使用系统环境变量HTTP_PROXY或HTTPS_PROXY environment的值作为代理url |
 | --access-key     | 亚马逊云对象存储访问key，`s3`地址生效，没有可以不提供        |
 | --secrets-key    | 亚马逊云对象存储私有key，`s3`地址生效，没有可以不提供        |
 | --noreload       | 禁止自动重载，当网络异常或程序异常中断情况下，不进行重置并继续下载 |

src/_version.py

@@ -1 +1 @@
-__version__ = "1.0.5"
+__version__ = "1.0.6"

src/main.py

@@ -19,6 +19,7 @@ def _main(args):
     hget(args.url, outfile, args.num, quiet=args.quiet,
          tcp_conn=args.connections, timeout=args.timeout,
          access_key=args.access_key, secrets_key=args.secrets_key,
+         proxy=args.proxy, proxy_user=args.proxy_user, proxy_pass=args.proxy_password, proxy_env=args.use_proxy_env,
          max_speed=args.max_speed)
 
 

src/src.py

@@ -109,7 +109,10 @@ async def download(self):
         self.timeout = ClientTimeout(total=60*60*24, sock_read=2400)
         self.connector = TCPConnector(
             limit=self.tcp_conn, ssl=False)
-        async with ClientSession(connector=self.connector, timeout=self.timeout) as session:
+        trust_env = False
+        if self.extra.get("proxy_env"):
+            trust_env = self.extra.pop("proxy_env")
+        async with ClientSession(connector=self.connector, timeout=self.timeout, trust_env=trust_env) as session:
             await self.get_range(session)
             _thread.start_new_thread(self.check_offset, (self.datatimeout,))
             _thread.start_new_thread(self._auto_write_offset, ())
@@ -191,7 +194,11 @@ async def fetch(self, session, pbar=None, headers=None):
                 headers["Range"] = 'bytes={0}-{1}'.format(s, e)
                 self.loger.debug(
                     "Start %s %s", asyncio.current_task().get_name(), headers["Range"])
-                async with session.get(self.url, headers=headers, timeout=self.timeout, params=self.extra) as req:
+                proxy, proxy_auth = self.extra.get("proxy"), None
+                if self.extra.get("proxy_user") and self.extra.get("proxy_pass") and self.extra.get("proxy"):
+                    proxy_auth = BasicAuth(self.extra.get(
+                        "proxy_user"), self.extra.get("proxy_pass"))
+                async with session.get(self.url, headers=headers, timeout=self.timeout, params=self.extra, proxy=proxy, proxy_auth=proxy_auth) as req:
                     with open(self.outfile, 'r+b') as f:
                         f.seek(s, os.SEEK_SET)
                         async for chunk in req.content.iter_chunked(self.chunk_size):

src/utils.py

@@ -25,7 +25,7 @@
 from botocore import UNSIGNED
 from botocore.config import Config
 from botocore.exceptions import ReadTimeoutError
-from aiohttp import ClientSession, TCPConnector, ClientTimeout, hdrs
+from aiohttp import ClientSession, TCPConnector, ClientTimeout, BasicAuth, hdrs
 from aiohttp.client_reqrep import ClientRequest
 from aiohttp.client_exceptions import *
 from concurrent.futures import ThreadPoolExecutor
@@ -238,15 +238,24 @@ def parseArg():
                         help='suppress all output except error or download success', default=False)
     parser.add_argument('-v', '--version',
                         action='version', version="v" + __version__)
+    parser.add_argument('--noreload', dest='use_reloader', action='store_false',
+                        help='tells hget to NOT use the auto-reloader')
+    parser.add_argument("url", type=str,
+                        help="download url, http/https/ftp/s3 support", metavar="<url>")
+    proxy = parser.add_argument_group("proxy arguments")
+    proxy.add_argument('-p', '--proxy', type=str,
+                       help='proxy url, statswith http/https', metavar="<str>")
+    proxy.add_argument('--proxy-user', type=str,
+                       help='set USER as proxy username', metavar="<str>")
+    proxy.add_argument('--proxy-password', type=str,
+                       help='set PASS as proxy password', metavar="<str>")
+    proxy.add_argument('--use-proxy-env', default=False, action='store_true',
+                       help='use HTTP_PROXY or HTTPS_PROXY environment variables for proxy url')
     aws = parser.add_argument_group("aws arguments")
     aws.add_argument('--access-key', dest='access_key', type=str,
                      help='access key if necessary', metavar="<str>")
     aws.add_argument('--secrets-key', dest='secrets_key', type=str,
                      help='secrets key if necessary', metavar="<str>")
-    parser.add_argument('--noreload', dest='use_reloader', action='store_false',
-                        help='tells hget to NOT use the auto-reloader')
-    parser.add_argument("url", type=str,
-                        help="download url, http/https/ftp/s3 support", metavar="<url>")
     return parser.parse_args()