From 052a3c33506bc386f3b766e39c451a4fc5b57cce Mon Sep 17 00:00:00 2001 From: Jonas Drotleff Date: Fri, 17 Jun 2022 19:20:29 +0200 Subject: [PATCH] Implement full text search on all models --- alembic/versions/b6f593818541_search.py | 32 ++++++++++++++ .../c7d3e7e86fe0_all_models_search.py | 43 +++++++++++++++++++ doku/__init__.py | 2 +- doku/blueprints/base.py | 16 ++++--- doku/blueprints/resources.py | 7 ++- doku/blueprints/snippet.py | 12 +++++- doku/blueprints/stylesheets.py | 8 +++- doku/blueprints/template.py | 16 +++---- doku/models/__init__.py | 6 +++ doku/models/document.py | 15 ++++++- doku/models/resource.py | 15 ++++++- doku/models/snippet.py | 16 ++++++- doku/models/template.py | 27 +++++++++++- doku/static/src/sites/templates.vue | 15 +++++++ doku/templates/components/document_list.html | 6 +-- doku/templates/components/pagination.html | 8 ++-- doku/templates/sites/index.html | 11 +++++ doku/templates/sites/resources.html | 9 ++++ doku/templates/sites/snippets.html | 11 ++++- doku/templates/sites/stylesheets.html | 9 ++++ doku/templates/sites/templates.html | 2 + 21 files changed, 253 insertions(+), 33 deletions(-) create mode 100644 alembic/versions/b6f593818541_search.py create mode 100644 alembic/versions/c7d3e7e86fe0_all_models_search.py diff --git a/alembic/versions/b6f593818541_search.py b/alembic/versions/b6f593818541_search.py new file mode 100644 index 0000000..cc254a0 --- /dev/null +++ b/alembic/versions/b6f593818541_search.py @@ -0,0 +1,32 @@ +"""empty message + +Revision ID: b6f593818541 +Revises: c2d7aa36d0e7 +Create Date: 2022-06-17 18:35:58.043728 + +""" +from alembic import op +import sqlalchemy as sa +from doku.models import TSVector + + +# revision identifiers, used by Alembic. + +revision = 'b6f593818541' +down_revision = 'c2d7aa36d0e7' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('doku_document', sa.Column('__ts_vector__', TSVector(), sa.Computed("to_tsvector('english', name)", persisted=True), nullable=True)) + op.create_index('doku_document___ts_vector__', 'doku_document', ['__ts_vector__'], unique=False, postgresql_using='gin') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index('doku_document___ts_vector__', table_name='doku_document', postgresql_using='gin') + op.drop_column('doku_document', '__ts_vector__') + # ### end Alembic commands ### diff --git a/alembic/versions/c7d3e7e86fe0_all_models_search.py b/alembic/versions/c7d3e7e86fe0_all_models_search.py new file mode 100644 index 0000000..a151aa9 --- /dev/null +++ b/alembic/versions/c7d3e7e86fe0_all_models_search.py @@ -0,0 +1,43 @@ +"""empty message + +Revision ID: c7d3e7e86fe0 +Revises: b6f593818541 +Create Date: 2022-06-17 18:58:45.565407 + +""" +from alembic import op +import sqlalchemy as sa +from doku.models import TSVector + + +# revision identifiers, used by Alembic. +revision = 'c7d3e7e86fe0' +down_revision = 'b6f593818541' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('doku_resource', sa.Column('__ts_vector__', TSVector(), sa.Computed("to_tsvector('english', name)", persisted=True), nullable=True)) + op.create_index('doku_resource___ts_vector__', 'doku_resource', ['__ts_vector__'], unique=False, postgresql_using='gin') + op.add_column('doku_snippet', sa.Column('__ts_vector__', TSVector(), sa.Computed("to_tsvector('english', name)", persisted=True), nullable=True)) + op.create_index('doku_snippet___ts_vector__', 'doku_snippet', ['__ts_vector__'], unique=False, postgresql_using='gin') + op.add_column('doku_stylesheet', sa.Column('__ts_vector__', TSVector(), sa.Computed("to_tsvector('english', name)", persisted=True), nullable=True)) + op.create_index('doku_stylesheet___ts_vector__', 'doku_stylesheet', ['__ts_vector__'], unique=False, postgresql_using='gin') + op.add_column('doku_template', sa.Column('__ts_vector__', TSVector(), sa.Computed("to_tsvector('english', name)", persisted=True), nullable=True)) + op.create_index('doku_template___ts_vector__', 'doku_template', ['__ts_vector__'], unique=False, postgresql_using='gin') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index('doku_template___ts_vector__', table_name='doku_template', postgresql_using='gin') + op.drop_column('doku_template', '__ts_vector__') + op.drop_index('doku_stylesheet___ts_vector__', table_name='doku_stylesheet', postgresql_using='gin') + op.drop_column('doku_stylesheet', '__ts_vector__') + op.drop_index('doku_snippet___ts_vector__', table_name='doku_snippet', postgresql_using='gin') + op.drop_column('doku_snippet', '__ts_vector__') + op.drop_index('doku_resource___ts_vector__', table_name='doku_resource', postgresql_using='gin') + op.drop_column('doku_resource', '__ts_vector__') + # ### end Alembic commands ### diff --git a/doku/__init__.py b/doku/__init__.py index a203d87..cdd2171 100644 --- a/doku/__init__.py +++ b/doku/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.5.0" +__version__ = "0.5.1" __author__ = "Jonas Drotleff " __all__ = ["create_app", "cli"] diff --git a/doku/blueprints/base.py b/doku/blueprints/base.py index 22f1cd3..f162274 100644 --- a/doku/blueprints/base.py +++ b/doku/blueprints/base.py @@ -1,4 +1,4 @@ -from flask import Blueprint, render_template +from flask import Blueprint, render_template, request from doku.models import db from doku.models.document import Document @@ -15,9 +15,15 @@ def index(): ordering, order, direction = get_ordering( Document, default_order="last_updated", default_dir="desc" ) - documents = ( - db.session.query(Document).order_by(ordering).paginate(page=page, per_page=10) - ) + query: str = request.args.get("query", "", type=str) + documents = db.session.query(Document) + if query != "": + documents = documents.filter(Document.__ts_vector__.match(query)) + documents = documents.order_by(ordering).paginate(page=page, per_page=10) return render_template( - "sites/index.html", documents=documents, order=order, direction=direction + "sites/index.html", + documents=documents, + order=order, + direction=direction, + query=query, ) diff --git a/doku/blueprints/resources.py b/doku/blueprints/resources.py index e87b088..d0e29a6 100644 --- a/doku/blueprints/resources.py +++ b/doku/blueprints/resources.py @@ -43,13 +43,18 @@ def index(): db.session.add(resource) db.session.commit() page = get_pagination_page() - resources = db.session.query(Resource).paginate(page=page, per_page=10) + query: str = request.args.get("query", "", type=str) + resources = db.session.query(Resource) + if query != "": + resources = resources.filter(Resource.__ts_vector__.match(query)) + resources = resources.paginate(page=page, per_page=10) resource_schemas = ResourceSchema(session=db.session, many=True) return render_template( "sites/resources.html", resources_json=resource_schemas.dumps(resources.items), resources=resources, + query=query, ) diff --git a/doku/blueprints/snippet.py b/doku/blueprints/snippet.py index 3f8792f..c843903 100644 --- a/doku/blueprints/snippet.py +++ b/doku/blueprints/snippet.py @@ -15,8 +15,16 @@ @login_required def index(): page = get_pagination_page() - snippets = db.session.query(Snippet).paginate(page=page, per_page=10) - return render_template("sites/snippets.html", snippets=snippets) + query: str = request.args.get("query", "", type=str) + snippets = db.session.query(Snippet) + if query != "": + snippets = snippets.filter(Snippet.__ts_vector__.match(query)) + snippets = snippets.paginate(page=page, per_page=10) + return render_template( + "sites/snippets.html", + snippets=snippets, + query=query, + ) @bp.route("/", methods=["GET", "POST"]) diff --git a/doku/blueprints/stylesheets.py b/doku/blueprints/stylesheets.py index dc04281..74d811b 100644 --- a/doku/blueprints/stylesheets.py +++ b/doku/blueprints/stylesheets.py @@ -46,7 +46,11 @@ def index(): db.session.commit() page = get_pagination_page() - stylesheets = db.session.query(Stylesheet).paginate(page=page, per_page=10) + query: str = request.args.get("query", "", type=str) + stylesheets = db.session.query(Stylesheet) + if query != "": + stylesheets = stylesheets.filter(Stylesheet.__ts_vector__.match(query)) + stylesheets = stylesheets.paginate(page=page, per_page=10) stylesheet_schemas = StylesheetSchema( session=db.session, many=True, include=("source",) @@ -54,5 +58,5 @@ def index(): return render_template( "sites/stylesheets.html", stylesheets_json=stylesheet_schemas.dumps(stylesheets.items), - stylesheets=stylesheets, + stylesheets=stylesheets, query=query ) diff --git a/doku/blueprints/template.py b/doku/blueprints/template.py index 2f9ba80..0fff655 100644 --- a/doku/blueprints/template.py +++ b/doku/blueprints/template.py @@ -1,4 +1,4 @@ -from flask import Blueprint, render_template +from flask import Blueprint, render_template, request from doku import db from doku.models.schemas import TemplateSchema, StylesheetSchema @@ -13,19 +13,17 @@ @login_required def index_all(): page = get_pagination_page() - ordering, order, direction = get_ordering( - Template, default_order="name", default_dir="asc" - ) - templates = ( - db.session.query(Template).order_by(ordering).paginate(page=page, per_page=10) - ) + query: str = request.args.get("query", "", type=str) + templates = db.session.query(Template) + if query != "": + templates = templates.filter(Template.__ts_vector__.match(query)) + templates = templates.paginate(page=page, per_page=10) template_schema = TemplateSchema(session=db.session, many=True) return render_template( "sites/templates.html", templates_json=template_schema.dumps(templates.items), templates=templates, - order=order, - direction=direction, + query=query, ) diff --git a/doku/models/__init__.py b/doku/models/__init__.py index efc24dc..2d1b658 100644 --- a/doku/models/__init__.py +++ b/doku/models/__init__.py @@ -3,10 +3,16 @@ from flask_babel import format_timedelta, format_datetime from flask_sqlalchemy import SQLAlchemy from marshmallow_sqlalchemy import auto_field +from sqlalchemy import TypeDecorator +from sqlalchemy.dialects.postgresql import TSVECTOR db = SQLAlchemy(session_options={"autoflush": False}) +class TSVector(TypeDecorator): + impl = TSVECTOR + + class DateMixin: created_on = db.Column(db.DateTime(timezone=True), default=datetime.now) last_updated = db.Column( diff --git a/doku/models/document.py b/doku/models/document.py index 00bb625..e8984c0 100644 --- a/doku/models/document.py +++ b/doku/models/document.py @@ -1,6 +1,7 @@ from flask import session +from sqlalchemy import Index -from doku.models import db, DateMixin +from doku.models import db, DateMixin, TSVector class Document(db.Model, DateMixin): @@ -34,6 +35,18 @@ class Document(db.Model, DateMixin): overlaps="variables, document", ) + __ts_vector__ = db.Column( + TSVector(), + db.Computed( + "to_tsvector('english', name)", + persisted=True + ) + ) + + __table_args__ = ( + Index('doku_document___ts_vector__', __ts_vector__, postgresql_using='gin'), + ) + def __init__(self, *args, author_id=None, author=None, **kwargs): if author_id is None and author is None: author_id = session.get("id", None) diff --git a/doku/models/resource.py b/doku/models/resource.py index 2d82a5f..b271914 100644 --- a/doku/models/resource.py +++ b/doku/models/resource.py @@ -2,9 +2,10 @@ import string from flask import url_for +from sqlalchemy import Index from werkzeug.utils import secure_filename -from doku.models import db, DateMixin +from doku.models import db, DateMixin, TSVector class Resource(db.Model, DateMixin): @@ -16,6 +17,18 @@ class Resource(db.Model, DateMixin): name = db.Column(db.String(255), unique=False, nullable=False) filename = db.Column(db.String(255), unique=True, nullable=False) + __ts_vector__ = db.Column( + TSVector(), + db.Computed( + "to_tsvector('english', name)", + persisted=True + ) + ) + + __table_args__ = ( + Index('doku_resource___ts_vector__', __ts_vector__, postgresql_using='gin'), + ) + @property def url(self): return url_for("resources.view", resource_id=self.id) diff --git a/doku/models/snippet.py b/doku/models/snippet.py index ae7bd34..f1359c0 100644 --- a/doku/models/snippet.py +++ b/doku/models/snippet.py @@ -1,6 +1,6 @@ -from sqlalchemy import event +from sqlalchemy import event, Index -from doku.models import db, DateMixin +from doku.models import db, DateMixin, TSVector from doku.utils.markdown import compile_content @@ -23,6 +23,18 @@ class Snippet(db.Model, DateMixin): used_by = db.relationship("Variable") + __ts_vector__ = db.Column( + TSVector(), + db.Computed( + "to_tsvector('english', name)", + persisted=True + ) + ) + + __table_args__ = ( + Index('doku_snippet___ts_vector__', __ts_vector__, postgresql_using='gin'), + ) + def __str__(self): return f"Snippet {self.name}" diff --git a/doku/models/template.py b/doku/models/template.py index 6f61e47..1376edc 100644 --- a/doku/models/template.py +++ b/doku/models/template.py @@ -4,9 +4,10 @@ from jinja2 import Environment, meta from jinja2 import Template as Jinja2Template from pygments.formatters.html import HtmlFormatter +from sqlalchemy import Index from weasyprint import HTML, CSS -from doku.models import db, DateMixin +from doku.models import db, DateMixin, TSVector from doku.utils.weasyfetch import url_fetcher DEFAULT_TEMPLATE = """ @@ -49,6 +50,18 @@ class Template(db.Model, DateMixin): "Stylesheet", secondary=template_stylesheet_relation, back_populates="templates" ) + __ts_vector__ = db.Column( + TSVector(), + db.Computed( + "to_tsvector('english', name)", + persisted=True + ) + ) + + __table_args__ = ( + Index('doku_template___ts_vector__', __ts_vector__, postgresql_using='gin'), + ) + def __str__(self): return self.name @@ -110,6 +123,18 @@ class Stylesheet(db.Model, DateMixin): MAX_CONTENT_LENGTH = 125000 + __ts_vector__ = db.Column( + TSVector(), + db.Computed( + "to_tsvector('english', name)", + persisted=True + ) + ) + + __table_args__ = ( + Index('doku_stylesheet___ts_vector__', __ts_vector__, postgresql_using='gin'), + ) + def __str__(self): return self.name diff --git a/doku/static/src/sites/templates.vue b/doku/static/src/sites/templates.vue index 58ef8c0..3476690 100644 --- a/doku/static/src/sites/templates.vue +++ b/doku/static/src/sites/templates.vue @@ -9,6 +9,15 @@ Create new template +
+ +
+ +
+ +
state.template.templates, }), diff --git a/doku/templates/components/document_list.html b/doku/templates/components/document_list.html index a5560ed..c838d2c 100644 --- a/doku/templates/components/document_list.html +++ b/doku/templates/components/document_list.html @@ -1,11 +1,11 @@ {% extends "components/list.html" %} {% block head %} - NAME + NAME TEMPLATE - LAST UPDATED + LAST UPDATED {% endblock head %} {% block body %} @@ -28,4 +28,4 @@ {% endfor %} -{% endblock body %} \ No newline at end of file +{% endblock body %} diff --git a/doku/templates/components/pagination.html b/doku/templates/components/pagination.html index 7726596..dd55e20 100644 --- a/doku/templates/components/pagination.html +++ b/doku/templates/components/pagination.html @@ -1,13 +1,13 @@ \ No newline at end of file + diff --git a/doku/templates/sites/index.html b/doku/templates/sites/index.html index f9bec20..b6e9a03 100644 --- a/doku/templates/sites/index.html +++ b/doku/templates/sites/index.html @@ -2,6 +2,17 @@ {% block content %} {{ super() }}
+
+ + + +
+ +
+ +
{% with items=documents.items %} {% include "components/document_list.html" %} diff --git a/doku/templates/sites/resources.html b/doku/templates/sites/resources.html index 45a8dc8..b0e4535 100644 --- a/doku/templates/sites/resources.html +++ b/doku/templates/sites/resources.html @@ -5,6 +5,15 @@

Resources

+
+ +
+ +
+ +
diff --git a/doku/templates/sites/snippets.html b/doku/templates/sites/snippets.html index 2830898..6344e47 100644 --- a/doku/templates/sites/snippets.html +++ b/doku/templates/sites/snippets.html @@ -13,6 +13,15 @@

Snippets

+
+ +
+ +
+ +
{% for item in snippets.items %}
@@ -23,4 +32,4 @@

Snippets

{% with pagination=snippets, page_url='snippet.index' %} {% include "components/pagination.html" %} {% endwith %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/doku/templates/sites/stylesheets.html b/doku/templates/sites/stylesheets.html index e1bef3f..b8af6ff 100644 --- a/doku/templates/sites/stylesheets.html +++ b/doku/templates/sites/stylesheets.html @@ -5,6 +5,15 @@

Stylesheets

+
+ +
+ +
+ +
diff --git a/doku/templates/sites/templates.html b/doku/templates/sites/templates.html index 8b8066c..cbca48a 100644 --- a/doku/templates/sites/templates.html +++ b/doku/templates/sites/templates.html @@ -15,6 +15,8 @@ window.templates = {{ templates_json|tojson }}; window.templateApi = '{{ url_for('api.v1.template.api') }}'; {{ create_csrf_token(output=False) }} + window._formPage = "{{ templates.page }}"; + window._formQuery = "{{ query }}"; {% endblock %}