Merge branch 'release-1.2'

AUTHORS.rst

@@ -13,3 +13,4 @@ Contributors
 * @kevinlekiller - Compressed Header Support
 * @bettse - Various bugfixes
 * @guillp - Group management scripts and bugfixes
+* @ukharley - Logging modifications

README.md

@@ -347,7 +347,7 @@ To build the webui from source, first modify the config to include your indexer
 
     > cd webui/app/scripts
     > vim config.js
-    > [add host url]
+    > [add host url and port]
 
 Then initiate the build:
 

api.py

@@ -1,9 +1,10 @@
 import json
-
+import argparse
 import regex
 import bottle
 from bottle import request, response
 import xmltodict
+import traceback
 
 from pynab import log
 import pynab.api
@@ -33,6 +34,11 @@ def serve_static(path):
     return bottle.static_file(path, root='./webui/dist/fonts/')
 
 
+@app.get('/bower_components/:path#.+#')
+def serve_static(path):
+    return bottle.static_file(path, root='./webui/dist/bower_components/')
+
+
 @app.get('/api')
 def api():
     log.debug('Handling request for {0}.'.format(request.fullpath))
@@ -57,13 +63,13 @@ def api():
 @app.get('/')
 @app.get('/index.html')
 def index():
-    if config.site['webui']:
+    if config.api.get('webui'): # disabled by default ? not really useful for a single user install
         raise bottle.static_file('index.html', root='./webui/dist')
 
 
 @app.get('/favicon.ico')
 def index():
-    if config.site['webui']:
+    if config.api.get('webui'):
         raise bottle.static_file('favicon.ico', root='./webui/dist')
 
 
@@ -108,5 +114,34 @@ def get_link(route=''):
     return url
 
 
+def daemonize(pidfile):
+    try:
+        import traceback
+        from daemonize import Daemonize
+        daemon = Daemonize(app='pynab', pid=pidfile, action=main)
+        daemon.start()
+    except SystemExit:
+        raise
+    except:
+        log.critical(traceback.format_exc())
+
+
+def main():
+    bottle.run(app=app, host=config.api.get('api_host', '0.0.0.0'), port=config.api.get('api_port', 8080))
+
+
 if __name__ == '__main__':
-    bottle.run(app=app, host=config.site.get('api_host', '0.0.0.0'), port=config.site.get('api_port', 8080))
+    argparser = argparse.ArgumentParser(description="Pynab main indexer script")
+    argparser.add_argument('-d', '--daemonize', action='store_true', help='run as a daemon')
+    argparser.add_argument('-p', '--pid-file', help='pid file (when -d)')
+
+    args = argparser.parse_args()
+
+    if args.daemonize:
+        pidfile = args.pid_file or config.api.get('api_pid_file')
+        if not pidfile:
+            log.error("A pid file is required to run as a daemon, please supply one either in the config file '{}' or as argument".format(config.__file__))
+        else:
+            daemonize(pidfile)
+    else:
+        main()

config.sample.py

@@ -1,7 +1,7 @@
 import logging
 
-site = {
-    # general site settings
+api = {
+    # api settings
     # ---------------------
 
     # title: shows on the rss feed, can be whatever
@@ -19,11 +19,8 @@
     # your administrator email (shows on rss feed)
     'email': '',
 
-	# enable web interface
-	'webui': True,
-
-    # api settings
-    # ------------
+    # enable web interface
+    'webui': True,
 
     # result_limit: maximum search results for rss feeds
     # make sure there's no quotes around it
@@ -41,6 +38,12 @@
     # usually 8080
     'api_port': 8080,
 
+    # pid_file: process file for the api, if daemonized
+    # make sure it's writable, leave blank for nginx
+    'pid_file': ''
+}
+
+scan = {
     # scanning settings
     # -----------------
 
@@ -70,6 +73,13 @@
     # set this to 3 days or so, don't set it to 0
     'dead_binary_age': 3,
 
+    # pid_file: process file for the scanner, if daemonized
+    # make sure it's writable, leave blank for nginx
+    'pid_file': ''
+
+}
+
+postprocess = {
     # release processing settings
     # ---------------------------
 
@@ -86,9 +96,6 @@
     # 100% completion resulted in about 11,000 unmatched releases after 4 weeks over 6 groups
     # lowering that to 99% built an extra 3,500 releases
 
-    # postprocessing settings
-    # -----------------------
-
     # postprocess_wait: time to sleep between postprocess.py loops
     # setting this to 0 may be horrible to online APIs, but if you've got a good
     # local db it should be fine
@@ -131,20 +138,7 @@
     # so if we can't find a match for some movie, wait 7 days before trying that movie again
     # there's really no benefit to setting this low - anywhere from a week to several months is fine
     'fetch_blacklist_duration': 7,
-
-    # logging settings
-    # ----------------
-    # logging_file: a filepath or None to go to stdout
-    'logging_file': None,
-
-    # logging.x where DEBUG, INFO, WARNING, ERROR, etc
-    # generally, debug if something goes wrong, info for normal usage
-    'logging_level': logging.DEBUG,
-
-    # max_log_size: maximum size of logfiles before they get rotated
-    # number, in bytes (this is 50mb)
-    'max_log_size': 50*1024*1024,
-
+
     # regex update settings
     # ---------------------
 
@@ -158,7 +152,22 @@
     # generally leave alone
     'blacklist_url': 'https://raw.github.com/kevinlekiller/Newznab-Blacklist/master/New/blacklists.txt',
 
+}
 
+log = {
+    # logging settings
+    # ----------------
+    # logging_file: a filepath or None to go to stdout
+    'logging_file': None,
+
+    # logging.x where DEBUG, INFO, WARNING, ERROR, etc
+    # generally, debug if something goes wrong, info for normal usage
+    'logging_level': logging.DEBUG,
+
+    # max_log_size: maximum size of logfiles before they get rotated
+    # number, in bytes (this is 50mb)
+    'max_log_size': 50*1024*1024,
+
 }
 
 # mongodb config

install.py

@@ -83,15 +83,15 @@
             print('Problem inserting data into MongoDB.')
             sys.exit(0)
 
-    if config.site['regex_url']:
+    if config.postprocess.get('regex_url'):
         print('Updating regex...')
         pynab.util.update_regex()
     else:
         print('Could not update regex - no update url/key in config.py.')
         print('If you don\'t have one, buy a Newznab+ license or find your own regexes.')
         print('You won\'t be able to build releases without appropriate regexes.')
 
-    if config.site['blacklist_url']:
+    if config.postprocess.get('blacklist_url'):
         print('Updating binary blacklist...')
         pynab.util.update_blacklist()
     else:

postprocess.py

@@ -44,35 +44,47 @@ def process_imdb():
     # print MP log as well
     multiprocessing.log_to_stderr().setLevel(logging.DEBUG)
 
-    # take care of REQ releases first
-    for release in db.releases.find({'search_name': {'$regex': 'req', '$options': '-i'}}):
-        pynab.releases.strip_req(release)
-
     # start with a quick post-process
-    log.info('Starting with a quick post-process to clear out the cruft that\'s available locally...')
+    log.info('starting with a quick post-process to clear out the cruft that\'s available locally...')
     scripts.quick_postprocess.local_postprocess()
 
     while True:
+        # take care of REQ releases first
+        for release in db.releases.find({'search_name': {'$regex': 'req', '$options': '-i'}}):
+            pynab.releases.strip_req(release)
+
+        # delete passworded releases first so we don't bother processing them
+        if config.postprocess.get('delete_passworded', True):
+            if config.postprocess.get('delete_potentially_passworded', True):
+                query = {'passworded': {'$in': [True, 'potentially']}}
+            else:
+                query = {'passworded': True}
+            db.releases.remove(query)
+
+        # delete any nzbs that don't have an associated release
+        # and delete any releases that don't have an nzb
+
+
         # grab and append tvrage data to tv releases
         tvrage_p = None
-        if config.site['process_tvrage']:
+        if config.postprocess.get('process_tvrage', True):
             tvrage_p = multiprocessing.Process(target=process_tvrage)
             tvrage_p.start()
 
         imdb_p = None
-        if config.site['process_imdb']:
+        if config.postprocess.get('process_imdb', True):
             imdb_p = multiprocessing.Process(target=process_imdb)
             imdb_p.start()
 
         # grab and append nfo data to all releases
         nfo_p = None
-        if config.site['process_nfos']:
+        if config.postprocess.get('process_nfos', True):
             nfo_p = multiprocessing.Process(target=process_nfos)
             nfo_p.start()
 
         # check for passwords, file count and size
         rar_p = None
-        if config.site['process_rars']:
+        if config.postprocess.get('process_rars', True):
             rar_p = multiprocessing.Process(target=process_rars)
             rar_p.start()
 
@@ -92,11 +104,12 @@ def process_imdb():
         scripts.rename_bad_releases.rename_bad_releases(8010)
         scripts.rename_bad_releases.rename_bad_releases(7020)
 
-        if config.site['delete_bad_releases']:
+        if config.postprocess.get('delete_bad_releases', False):
             pass
             #log.info('Deleting bad releases...')
             # not confident in this yet
 
         # wait for the configured amount of time between cycles
-        log.info('Sleeping for {:d} seconds...'.format(config.site['postprocess_wait']))
-        time.sleep(config.site['postprocess_wait'])
+        postprocess_wait = config.postprocess.get('postprocess_wait', 1)
+        log.info('sleeping for {:d} seconds...'.format(postprocess_wait))
+        time.sleep(postprocess_wait)

pynab/__init__.py

@@ -3,18 +3,58 @@
 
 __author__ = 'James Meneghello'
 __email__ = 'murodese@gmail.com'
-__version__ = '1.1.0'
+__version__ = '1.2.0'
 
 import logging
 import config
 import logging.handlers
+import os
+import colorlog
+import inspect
+import sys
 
 log = logging.getLogger(__name__)
-log.setLevel(config.site['logging_level'])
+log.setLevel(config.log.get('logging_level', logging.DEBUG))
 
-if config.site['logging_file']:
-    handler = logging.handlers.RotatingFileHandler(config.site['logging_file'], maxBytes=config.site['max_log_size'], backupCount=5, encoding='utf-8')
-    handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
+logging_file = config.log.get('logging_file')
+log_descriptor = None
+
+formatter = colorlog.ColoredFormatter(
+        "%(log_color)s%(asctime)s - %(levelname)s - %(reset)s %(blue)s%(message)s",
+        datefmt=None,
+        reset=True,
+        log_colors={
+                'DEBUG':    'cyan',
+                'INFO':     'green',
+                'WARNING':  'yellow',
+                'ERROR':    'red',
+                'CRITICAL': 'red',
+        }
+)
+
+if logging_file:
+    frame = inspect.currentframe()
+    info=inspect.getouterframes(frame)
+    c=0
+    for n in info:
+        if n[4] and c > 1: # c > 1 skips this module itself
+            if n[3] == '<module>': # from my testing (on Windows), the first module found is the calling module
+                break
+        c += 1
+    if c >= len(info):
+        sys.exit(1)
+    name, _ = os.path.splitext(os.path.basename(inspect.stack()[c][1].rstrip(os.sep)))
+    file, ext = os.path.splitext(config.log.get('logging_file'))
+    logging_file = ''.join([file, '_', name, ext])
+
+    handler = logging.handlers.RotatingFileHandler(logging_file, maxBytes=config.log.get('max_log_size', 50*1024*1024), backupCount=5, encoding='utf-8')
+    handler.setFormatter(formatter)
     log.addHandler(handler)
+    log_descriptor = handler.stream.fileno()
 else:
-    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s')
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+    log.addHandler(handler)
+
+# set up root_dir for use with templates etc
+root_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')