Merge branch 'release-1.1'

README.md

@@ -20,14 +20,11 @@ limits or the like. If you'd like to use this software and want to add such
 functionality, please feel free to fork it! I won't have time to work on it
 beyond my own needs, but this is what open source is for.
 
-Note that because this is purely for API access, THERE IS NO WEB FRONTEND. You
-cannot add users through a web interface, manage releases, etc. There isn't a
-frontend. Something like 99.9% of the usage of my old Newznab server was API-only
+Note that because this is purely for API access, the WebUI is very simple. You
+cannot add users through a web interface, manage releases, etc. 
+Something like 99.9% of the usage of my old Newznab server was API-only
 for Sickbeard, Couchpotato, Headphones etc - so it's low-priority.
 
-@DanielSchaffer is working on a WebUI for Pynab here: https://github.com/DanielSchaffer/pynab/
-
-
 Features
 --------
 
@@ -67,9 +64,9 @@ Technical Differences to Newznab
   - Big IO savings
   - Generally quicker than off the HDD
   - You don't run into filesystem problems from having 2.5 million files in a few directories
-- No web interface
+- Very simple query interface
   - The vast majority of access to my indexer was API-based (1-5 web hits per month vs 50,000+ api hits)
-  - I'll probably make a very simple query interface
+  - It's not a replacement for Newznab if you have a lot of direct user interaction
 - Simplified authentication
   - No more usernames, passwords, or anything, really.
   - API key access required for everything - sure, it can be sniffed very easily, but it always could be. Worst that can happen: someone uses the API.
@@ -328,7 +325,7 @@ To update indexes (generally only run if a commit message tells you to):
 
 Update regex (run it every now and then, but it doesn't update that often):
 
-	> python3 scripts/update_regex.py
+    > python3 scripts/update_regex.py
 
 Quickly match releases to local post-processing databases (run this pretty often, 
 it'll probably be incorporated into start.py at some point):
@@ -340,6 +337,29 @@ Categorise all uncategorised releases - this runs automatically after import.
 	> python3 scripts/process_uncategorised.py
 
 
+### Building the WebUI ###
+
+Requires NPM and probably a few other things (post an issue if you're missing stuff):
+
+    > sudo apt-get install npm
+
+To build the webui from source, first modify the config to include your indexer host:
+
+    > cd webui/app/scripts
+    > vim config.js
+    > [add host url]
+
+Then initiate the build:
+
+    > cd webui
+    > npm install
+    > bower install
+    > grunt build
+
+This will build a working and optimised version of the UI into the dist/ directory, which
+will then be hosted by your webserver as part of api.py. Note that you can disable the web
+interface in the main configuration.
+
 F.A.Q.
 ======
 
@@ -397,4 +417,4 @@ Acknowledgements
 - The Newznab team, for creating a great piece of software
 - Everyone who contributed to the NN+ regex collection
 - Kevinlekiller, for his blacklist regex
-- Everyone who's sent in issues and tested the software
+- Everyone who's sent in issues and tested the software

api.py

@@ -3,12 +3,35 @@
 import bottle
 from bottle import request, response
 import xmltodict
+import json
 
 from pynab import log
 import pynab.api
+import config
 
 app = application = bottle.Bottle()
 
+#bottle.debug(True)
+
+@app.get('/scripts/:path#.+#')
+def serve_static(path):
+    return bottle.static_file(path, root='./webui/dist/scripts/')
+
+
+@app.get('/styles/:path#.+#')
+def serve_static(path):
+    return bottle.static_file(path, root='./webui/dist/styles/')
+
+
+@app.get('/views/:path#.+#')
+def serve_static(path):
+	return bottle.static_file(path, root='./webui/dist/views/')
+
+
+@app.get('/fonts/:path#.+#')
+def serve_static(path):
+	return bottle.static_file(path, root='./webui/dist/fonts/')
+
 
 @app.get('/api')
 def api():
@@ -31,15 +54,28 @@ def api():
     return pynab.api.api_error(202)
 
 
+@app.get('/')
+@app.get('/index.html')
+def index():
+	if config.site['webui']:
+	    raise bottle.static_file('index.html', root='./webui/dist')
+
+
 def switch_output(data):
     output_format = request.query.o or 'xml'
+    output_callback = request.query.callback or None
+
     if output_format == 'xml':
         # return as xml
         response.set_header('Content-type', 'application/rss+xml')
         return data
     elif output_format == 'json':
-        # bottle auto-converts into json
-        return xmltodict.parse(data)
+        if output_callback:
+            response.content_type = 'application/javascript'
+            return '{}({})'.format(output_callback, json.dumps(xmltodict.parse(data, attr_prefix='')))
+        else:
+            # bottle auto-converts a python dict into json
+            return xmltodict.parse(data, attr_prefix='')
     else:
         return pynab.api.api_error(201)
 
@@ -67,4 +103,4 @@ def get_link(route=''):
 
 
 if __name__ == '__main__':
-    bottle.run(app=app, host='0.0.0.0', port=8080)
+    bottle.run(app=app, host='0.0.0.0', port=8080)

config.sample.py

@@ -19,6 +19,9 @@
     # your administrator email (shows on rss feed)
     'email': '',
 
+	# enable web interface
+	'webui': True,
+
     # api settings
     # ------------
 
@@ -197,4 +200,4 @@
     'user': '',
     'passwd': '',
     'db': 'newznab',
-}
+}

postprocess.py

@@ -23,19 +23,19 @@ def mp_error(msg, *args):
 
 
 def process_tvrage():
-    pynab.tvrage.process(0)
+    pynab.tvrage.process(500)
 
 
 def process_nfos():
-    pynab.nfos.process(0)
+    pynab.nfos.process(500)
 
 
 def process_rars():
-    pynab.rars.process(0)
+    pynab.rars.process(500)
 
 
 def process_imdb():
-    pynab.imdb.process(0)
+    pynab.imdb.process(500)
 
 
 if __name__ == '__main__':
@@ -93,7 +93,8 @@ def process_imdb():
         scripts.rename_bad_releases.rename_bad_releases(7020)
 
         if config.site['delete_bad_releases']:
-            log.info('Deleting bad releases...')
+            pass
+            #log.info('Deleting bad releases...')
             # not confident in this yet
 
         # wait for the configured amount of time between cycles

pynab/__init__.py

@@ -3,7 +3,7 @@
 
 __author__ = 'James Meneghello'
 __email__ = 'murodese@gmail.com'
-__version__ = '1.0.0'
+__version__ = '1.1.0'
 
 import logging
 import config

pynab/groups.py

@@ -51,6 +51,7 @@ def backfill(group_name, date=None):
         if target_article > start:
             start = target_article
 
+        retries = 0
         while True:
             messages = server.scan(group_name, start, end)
 
@@ -64,12 +65,18 @@ def backfill(group_name, date=None):
                                              'first': start
                                          }
                                      })
-                    pass
+                    retries = 0
                 else:
                     log.error('{}: Failed while saving parts.'.format(group_name))
                     if server.connection:
                         server.connection.quit()
                     return False
+            else:
+                    log.error('Problem updating group - trying again...')
+                    retries += 1
+                    # keep trying the same block 3 times, then skip
+                    if retries <= 3:
+                        continue
 
             if start == target_article:
                 if server.connection:
@@ -139,15 +146,22 @@ def update(group_name):
 
         start_date = server.post_date(group_name, start)
         end_date = server.post_date(group_name, end)
-        total_date = end_date - start_date
 
-        log.debug('{}: Start: {:d} ({}) End: {:d} ({}) Total: {:d} ({} days, {} hours, {} minutes)'
-        .format(
-            group_name, start, start_date,
-            end, end_date,
-            total, total_date.days, total_date.seconds // 3600, (total_date.seconds // 60) % 60
-        )
-        )
+        if start_date and end_date:
+            total_date = end_date - start_date
+
+            log.debug('{}: Start: {:d} ({}) End: {:d} ({}) Total: {:d} ({} days, {} hours, {} minutes)'
+                .format(
+                    group_name, start, start_date,
+                    end, end_date,
+                    total, total_date.days, total_date.seconds // 3600, (total_date.seconds // 60) % 60
+                )
+            )
+        else:
+            log.debug('{}: Group is semi-broken - not all debug output is available. Start: {}, End: {}, Total: {}'
+                .format(group_name, start, end, total)
+            )
+
         if total > 0:
             if not group['last']:
                 log.info('{}: Starting new group with {:d} days and {:d} new parts.'
@@ -176,6 +190,7 @@ def update(group_name):
                                                  'last': end
                                              }
                                          })
+                        retries = 0
                     else:
                         log.error('{}: Failed while saving parts.'.format(group_name))
                         if server.connection:
@@ -184,12 +199,6 @@ def update(group_name):
                             except:
                                 pass
                         return False
-                else:
-                    log.error('Problem updating group - trying again...')
-                    retries += 1
-                    # keep trying the same block 3 times, then skip
-                    if retries <= 3:
-                        continue
 
                 if end == last:
                     if server.connection:

pynab/imdb.py

@@ -2,7 +2,7 @@
 import unicodedata
 import difflib
 import datetime
-
+import pymongo
 import requests
 import pytz
 
@@ -85,7 +85,7 @@ def process(limit=100, online=True):
                 {'imdb.attempted': {'$lte': expiry}}
             ]
         })
-    for release in db.releases.find(query).limit(limit):
+    for release in db.releases.find(query).limit(limit).sort('posted', pymongo.DESCENDING).batch_size(50):
         process_release(release, online)
 
 

pynab/nfos.py

@@ -44,7 +44,7 @@ def process(limit=5, category=0):
         if category:
             query['category._id'] = int(category)
 
-        for release in db.releases.find(query).limit(limit).sort('posted', pymongo.ASCENDING):
+        for release in db.releases.find(query).limit(limit).sort('posted', pymongo.DESCENDING).batch_size(50):
             log.debug('Checking for NFO in {}...'.format(release['search_name']))
             nzb = pynab.nzbs.get_nzb_dict(release['nzb'])
 

pynab/rars.py

@@ -3,6 +3,7 @@
 import regex
 import shutil
 import subprocess
+import pymongo
 
 import lib.rar
 from pynab import log
@@ -13,6 +14,7 @@
 from pynab.server import Server
 import config
 
+
 MAYBE_PASSWORDED_REGEX = regex.compile('\.(ace|cab|tar|gz|url)$', regex.I)
 PASSWORDED_REGEX = regex.compile('password\.url', regex.I)
 
@@ -210,7 +212,7 @@ def process(limit=20, category=0):
         query = {'passworded': None}
         if category:
             query['category._id'] = int(category)
-        for release in db.releases.find(query).limit(limit):
+        for release in db.releases.find(query).limit(limit).sort('posted', pymongo.DESCENDING).batch_size(50):
             log.debug('Processing rar part for {}...'.format(release['name']))
             nzb = pynab.nzbs.get_nzb_dict(release['nzb'])