Merge branch 'master' into new-login-page

README.md

@@ -97,7 +97,7 @@ use for development:
 
 The first time you set up the application, the database needs to be initialized:
 
-    $ ./develop.sh manage init_db --create-db
+    $ ./develop.sh manage init-db --create-db
 
 The web server should now be accessible at **http://localhost:80/**.
 

admin/schema_updates/17.sql

@@ -0,0 +1,31 @@
+BEGIN;
+
+CREATE TYPE dataset_project_type AS ENUM ('musicbrainz', 'listenbrainz');
+
+CREATE TABLE dataset (
+    id          INTEGER GENERATED ALWAYS AS IDENTITY,
+    name        TEXT NOT NULL,
+    description TEXT,
+    project     dataset_project_type NOT NULL
+);
+
+CREATE TABLE dataset_user (
+    id          INTEGER GENERATED ALWAYS AS IDENTITY,
+    user_id     INTEGER NOT NULL,
+    dataset_id  INTEGER NOT NULL
+);
+
+ALTER TABLE dataset ADD CONSTRAINT dataset_pkey PRIMARY KEY (id);
+ALTER TABLE dataset_user ADD CONSTRAINT dataset_user_pkey PRIMARY KEY (id);
+
+ALTER TABLE dataset_user
+    ADD CONSTRAINT dataset_user_user_id_fkey FOREIGN KEY (user_id)
+    REFERENCES "user" (id) MATCH SIMPLE
+    ON UPDATE NO ACTION ON DELETE NO ACTION;
+
+ALTER TABLE dataset_user
+    ADD CONSTRAINT dataset_user_dataset_id_fkey FOREIGN KEY (dataset_id)
+    REFERENCES "dataset" (id) MATCH SIMPLE
+    ON UPDATE NO ACTION ON DELETE NO ACTION;
+
+COMMIT;

admin/schema_updates/18.sql

@@ -0,0 +1 @@
+ALTER TYPE dataset_project_type ADD VALUE 'critiquebrainz';

admin/sql/create_foreign_keys.sql

@@ -10,6 +10,16 @@ ALTER TABLE "user"
   REFERENCES tier (id) MATCH SIMPLE
   ON UPDATE CASCADE ON DELETE SET NULL;
 
+ALTER TABLE dataset_user
+    ADD CONSTRAINT dataset_user_user_id_fkey FOREIGN KEY (user_id)
+    REFERENCES "user" (id) MATCH SIMPLE
+    ON UPDATE NO ACTION ON DELETE NO ACTION;
+
+ALTER TABLE dataset_user
+    ADD CONSTRAINT dataset_user_dataset_id_fkey FOREIGN KEY (dataset_id)
+    REFERENCES "dataset" (id) MATCH SIMPLE
+    ON UPDATE NO ACTION ON DELETE NO ACTION;
+
 ALTER TABLE token_log
   ADD CONSTRAINT token_log_token_value_fkey FOREIGN KEY (token_value)
   REFERENCES token (value) MATCH SIMPLE

admin/sql/create_primary_keys.sql

@@ -9,5 +9,7 @@ ALTER TABLE payment ADD CONSTRAINT payment_pkey PRIMARY KEY (id);
 ALTER TABLE oauth_client ADD CONSTRAINT oauth_client_pkey PRIMARY KEY (client_id);
 ALTER TABLE oauth_grant ADD CONSTRAINT oauth_grant_pkey PRIMARY KEY (id);
 ALTER TABLE oauth_token ADD CONSTRAINT oauth_token_pkey PRIMARY KEY (id);
+ALTER TABLE dataset ADD CONSTRAINT dataset_pkey PRIMARY KEY (id);
+ALTER TABLE dataset_user ADD CONSTRAINT dataset_user_pkey PRIMARY KEY (id);
 
 COMMIT;

admin/sql/create_tables.sql

@@ -10,6 +10,13 @@ CREATE TABLE tier (
   "primary"  BOOLEAN           NOT NULL
 );
 
+CREATE TABLE dataset (
+    id          INTEGER GENERATED ALWAYS AS IDENTITY,
+    name        TEXT NOT NULL,
+    description TEXT,
+    project     dataset_project_type NOT NULL
+);
+
 CREATE TABLE "user" (
   id               SERIAL            NOT NULL, -- PK
   is_commercial    BOOLEAN           NOT NULL,
@@ -37,6 +44,12 @@ CREATE TABLE "user" (
   featured         BOOLEAN           NOT NULL
 );
 
+CREATE TABLE dataset_user (
+    id          INTEGER GENERATED ALWAYS AS IDENTITY,
+    user_id     INTEGER NOT NULL,
+    dataset_id  INTEGER NOT NULL
+);
+
 CREATE TABLE token (
   value     CHARACTER VARYING NOT NULL, -- PK
   is_active BOOLEAN           NOT NULL,

admin/sql/create_types.sql

@@ -23,3 +23,9 @@ CREATE TYPE payment_currency AS ENUM (
   'usd',
   'eur'
 );
+
+CREATE TYPE dataset_project_type AS ENUM (
+  'musicbrainz',
+  'listenbrainz',
+  'critiquebrainz'
+);

admin/sql/drop_tables.sql

@@ -9,5 +9,7 @@ DROP TABLE IF EXISTS token_log    CASCADE;
 DROP TABLE IF EXISTS token        CASCADE;
 DROP TABLE IF EXISTS "user"       CASCADE;
 DROP TABLE IF EXISTS tier         CASCADE;
+DROP TABLE IF EXISTS dataset_user CASCADE;
+DROP TABLE IF EXISTS dataset      CASCADE;
 
 COMMIT;

manage.py

@@ -8,6 +8,8 @@
 import os
 import click
 
+import logging
+
 ADMIN_SQL_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'admin', 'sql')
 
 cli = click.Group()
@@ -105,6 +107,16 @@ def send_invoices():
         qb.send_invoices()
 
 
+@cli.command()
+def send_invoice_reminders():
+    """ Send invoices reminders about invoices that remain unpaid."""
+
+    logging.getLogger().setLevel(logging.DEBUG)
+    with create_app().app_context():
+        qb = QuickBooksInvoiceSender()
+        qb.send_invoice_reminders()
+
+
 def _run_psql(script, uri, database=None):
     hostname, port, db_name, username, password = _explode_db_uri(uri)
     script = os.path.join(ADMIN_SQL_DIR, script)

metabrainz/__init__.py

@@ -1,10 +1,10 @@
 import os
 import pprint
-import subprocess
 import sys
 
 import stripe
 from brainzutils.flask import CustomFlask
+from brainzutils import sentry
 from flask import send_from_directory, request
 from metabrainz.admin.quickbooks.views import QuickBooksView
 from time import sleep
@@ -50,10 +50,10 @@ def create_app(debug=None, config_path = None):
                 '..', 'config.py'
             ))
 
-    # Load configuration files: If we're running under a docker deployment, wait until 
+    # Load configuration files: If we're running under a docker deployment, wait until
     # the consul configuration is available.
     if deploy_env:
-        consul_config = os.path.join( os.path.dirname(os.path.realpath(__file__)), 
+        consul_config = os.path.join( os.path.dirname(os.path.realpath(__file__)),
             '..', 'consul_config.py')
 
         print("loading consul %s" % consul_config)
@@ -84,10 +84,9 @@ def create_app(debug=None, config_path = None):
     print('Configuration values are as follows: ')
     print(pprint.pformat(app.config, indent=4))
 
-    app.init_loggers(
-        file_config=app.config.get('LOG_FILE'),
-        sentry_config=app.config.get('LOG_SENTRY'),
-    )
+    sentry_config = app.config.get('LOG_SENTRY')
+    if sentry_config:
+        sentry.init_sentry(**sentry_config)
 
     # Database
     from metabrainz import db
@@ -143,6 +142,40 @@ def create_tables():
     # Blueprints
     _register_blueprints(app)
 
+    # ADMIN SECTION
+
+    from flask_admin import Admin
+    from metabrainz.admin.views import HomeView
+    admin = Admin(app, index_view=HomeView(name='Pending users'), template_mode='bootstrap3')
+
+    # Models
+    from metabrainz.model.user import UserAdminView
+    from metabrainz.model.payment import PaymentAdminView
+    from metabrainz.model.tier import TierAdminView
+    from metabrainz.model.dataset import DatasetAdminView
+    admin.add_view(UserAdminView(model.db.session, category='Users', endpoint="user_model"))
+    admin.add_view(PaymentAdminView(model.db.session, category='Payments', endpoint="payment_model"))
+    admin.add_view(TierAdminView(model.db.session, endpoint="tier_model"))
+    admin.add_view(DatasetAdminView(model.db.session, endpoint="dataset_model"))
+
+    # Custom stuff
+    from metabrainz.admin.views import CommercialUsersView
+    from metabrainz.admin.views import UsersView
+    from metabrainz.admin.views import PaymentsView
+    from metabrainz.admin.views import TokensView
+    from metabrainz.admin.views import StatsView
+    admin.add_view(CommercialUsersView(name='Commercial users', category='Users'))
+    admin.add_view(UsersView(name='Search', category='Users'))
+    admin.add_view(PaymentsView(name='All', category='Payments'))
+    admin.add_view(TokensView(name='Access tokens', category='Users'))
+    admin.add_view(StatsView(name='Statistics', category='Statistics'))
+    admin.add_view(StatsView(name='Top IPs', endpoint="statsview/top-ips", category='Statistics'))
+    admin.add_view(StatsView(name='Top Tokens', endpoint="statsview/top-tokens", category='Statistics'))
+    admin.add_view(StatsView(name='Supporters', endpoint="statsview/supporters", category='Statistics'))
+
+    if app.config["QUICKBOOKS_CLIENT_ID"]:
+        admin.add_view(QuickBooksView(name='Invoices', endpoint="quickbooks/", category='Quickbooks'))
+
     stripe.api_key = app.config["STRIPE_KEYS"]["SECRET"]
 
     return app

metabrainz/admin/forms.py

@@ -1,4 +1,4 @@
-from flask_wtf import Form
+from flask_wtf import FlaskForm
 from flask_wtf.file import FileField, FileAllowed
 from wtforms import StringField, BooleanField, SelectField, TextAreaField
 from wtforms.fields.html5 import EmailField, URLField, DecimalField
@@ -20,7 +20,7 @@
 )
 
 
-class UserEditForm(Form):
+class UserEditForm(FlaskForm):
     # General info
     musicbrainz_id = StringField("MusicBrainz Username")
     contact_name = StringField("Name")
@@ -64,6 +64,6 @@ class UserEditForm(Form):
     def __init__(self, defaults=None, **kwargs):
         for key, val in defaults.items():
             kwargs.setdefault(key, val)
-        Form.__init__(self, **kwargs)
+        FlaskForm.__init__(self, **kwargs)
         self.tier.choices = [(str(t["id"]), t["name"]) for t in db_tier.get_all()]
         self.tier.choices.insert(0, ("None", "None"))

metabrainz/admin/quickbooks/views.py

@@ -11,7 +11,7 @@
 from flask_admin import expose, BaseView
 from quickbooks import QuickBooks
 from quickbooks.objects.customer import Customer
-from quickbooks.objects.invoice import Invoice
+from quickbooks.objects.invoice import Invoice, DeliveryInfo
 from quickbooks.objects.detailline import SalesItemLineDetail
 from intuitlib.exceptions import AuthClientError
 from werkzeug.exceptions import BadRequest, InternalServerError
@@ -82,10 +82,14 @@ def create_invoices(client, invoices):
             new_invoice.Line[0].Amount = "%d" % round(float(invoice['price']) * float(invoice['qty']))
             new_invoice.CustomField[1].StringValue = invoice['begin']
             new_invoice.CustomField[2].StringValue = invoice['end']
+            new_invoice.DeliveryInfo = DeliveryInfo()
+            new_invoice.DeliveryInfo.DeliveryType = "Email"
+            new_invoice.DeliveryInfo.DeliveryTime = datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%dT%H:%M:%S")
+
             try:
                 new_invoice.save(qb=client)
             except quickbooks.exceptions.QuickbooksException as err:
-                flash("failed to create invoice for %s (%s)" % (new_invoice.CustomerRef.name, str(err)))
+                flash("failed to create invoice for %s (%s -- %s)" % (new_invoice.CustomerRef.name, str(err), err.detail))
 
 
     @expose('/', methods=['GET'])

metabrainz/api/views/musicbrainz.py

@@ -1,5 +1,5 @@
 from flask import Blueprint, jsonify, send_from_directory, current_app
-from flask.helpers import safe_join
+from werkzeug.utils import safe_join
 from werkzeug.wrappers import Response
 from werkzeug.urls import iri_to_uri
 from metabrainz.api.decorators import token_required, tracked