improve index and api for localized title

catalog/book/models.py

@@ -23,6 +23,7 @@
 from django.db import models
 from django.utils.translation import gettext_lazy as _
 from loguru import logger
+from ninja import Field
 
 from catalog.common import (
     BaseSchema,
@@ -36,18 +37,23 @@
     PrimaryLookupIdDescriptor,
     jsondata,
 )
-from catalog.common.models import SCRIPT_CHOICES
+from catalog.common.models import (
+    LOCALE_CHOICES_JSONFORM,
+    SCRIPT_CHOICES,
+    LanguageListField,
+)
+from common.models.lang import get_current_locales
 from common.models.misc import uniq
 
 from .utils import *
 
 
 class EditionInSchema(ItemInSchema):
-    subtitle: str | None = None
+    subtitle: str | None = Field(default=None, alias="display_subtitle")
     orig_title: str | None = None
     author: list[str]
     translator: list[str]
-    language: str | None = None
+    language: list[str]
     pub_house: str | None = None
     pub_year: int | None = None
     pub_month: int | None = None
@@ -63,6 +69,45 @@ class EditionSchema(EditionInSchema, BaseSchema):
     pass
 
 
+EDITION_LOCALIZED_TITLE_SCHEMA = {
+    "type": "list",
+    "items": {
+        "type": "dict",
+        "keys": {
+            "lang": {
+                "type": "string",
+                "title": _("locale"),
+                "choices": LOCALE_CHOICES_JSONFORM,
+            },
+            "text": {"type": "string", "title": _("text content")},
+        },
+        "required": ["lang", "s"],
+    },
+    "minItems": 1,
+    "maxItems": 1,
+    # "uniqueItems": True,
+}
+
+EDITION_LOCALIZED_SUBTITLE_SCHEMA = {
+    "type": "list",
+    "items": {
+        "type": "dict",
+        "keys": {
+            "lang": {
+                "type": "string",
+                "title": _("locale"),
+                "choices": LOCALE_CHOICES_JSONFORM,
+            },
+            "text": {"type": "string", "title": _("text content")},
+        },
+        "required": ["lang", "s"],
+    },
+    "minItems": 0,
+    "maxItems": 1,
+    # "uniqueItems": True,
+}
+
+
 class Edition(Item):
     if TYPE_CHECKING:
         works: "models.ManyToManyField[Work, Edition]"
@@ -76,8 +121,10 @@ class Edition(Item):
     # goodreads = LookupIdDescriptor(IdType.Goodreads)
 
     METADATA_COPY_LIST = [
-        "title",
-        "subtitle",
+        "localized_title",
+        "localized_subtitle",
+        # "title",
+        # "subtitle",
         "author",
         "pub_house",
         "pub_year",
@@ -94,9 +141,18 @@ class Edition(Item):
         "localized_description",
         "contents",
     ]
-    subtitle = jsondata.CharField(
-        _("subtitle"), null=True, blank=True, default=None, max_length=500
+    # force Edition to have only one title
+    localized_title_schema = EDITION_LOCALIZED_TITLE_SCHEMA
+    localized_subtitle = jsondata.JSONField(
+        verbose_name=_("subtitle"),
+        null=False,
+        blank=True,
+        default=list,
+        schema=EDITION_LOCALIZED_SUBTITLE_SCHEMA,
     )
+    # subtitle = jsondata.CharField(
+    #     _("subtitle"), null=True, blank=True, default=None, max_length=500
+    # )
     orig_title = jsondata.CharField(
         _("original title"), null=True, blank=True, default=None, max_length=500
     )
@@ -114,14 +170,7 @@ class Edition(Item):
         blank=True,
         default=list,
     )
-    language = jsondata.CharField(
-        _("language"),
-        null=False,
-        blank=True,
-        default=None,
-        max_length=500,
-        choices=SCRIPT_CHOICES,
-    )
+    language = LanguageListField()
     pub_house = jsondata.CharField(
         _("publishing house"), null=True, blank=False, default=None, max_length=500
     )
@@ -148,6 +197,13 @@ class Edition(Item):
     price = jsondata.CharField(_("price"), null=True, blank=True, max_length=500)
     imprint = jsondata.CharField(_("imprint"), null=True, blank=True, max_length=500)
 
+    def get_localized_subtitle(self) -> str | None:
+        return self.localized_title[0]["text"] if self.localized_subtitle else None
+
+    @property
+    def display_subtitle(self) -> str | None:
+        return self.get_localized_subtitle()
+
     @property
     def isbn10(self):
         return isbn_13_to_10(self.isbn)
@@ -265,25 +321,26 @@ class Work(Item):
     douban_work = PrimaryLookupIdDescriptor(IdType.DoubanBook_Work)
     goodreads_work = PrimaryLookupIdDescriptor(IdType.Goodreads_Work)
     editions = models.ManyToManyField(Edition, related_name="works")
+    language = LanguageListField()
     author = jsondata.ArrayField(
         verbose_name=_("author"),
         base_field=models.CharField(max_length=500),
         null=True,
         blank=True,
         default=list,
     )
-    other_title = jsondata.ArrayField(
-        verbose_name=_("other title"),
-        base_field=models.CharField(blank=True, default="", max_length=200),
-        null=True,
-        blank=True,
-        default=list,
-    )
+    # other_title = jsondata.ArrayField(
+    #     verbose_name=_("other title"),
+    #     base_field=models.CharField(blank=True, default="", max_length=200),
+    #     null=True,
+    #     blank=True,
+    #     default=list,
+    # )
     METADATA_COPY_LIST = [
-        "title",
-        "other_title",
+        "localized_title",
         "author",
-        "brief",
+        "language",
+        "localized_description",
     ]
     # TODO: we have many duplicates due to 302
     # a lazy fix is to remove smaller DoubanBook_Work ids

catalog/common/jsondata.py

@@ -15,6 +15,7 @@
 from django.utils import dateparse, timezone
 from django.utils.encoding import force_bytes
 from django.utils.translation import gettext_lazy as _
+from django_jsonform.forms.fields import JSONFormField as DJANGO_JSONFormField
 
 # from django.db.models import JSONField as DJANGO_JSONField
 # from jsoneditor.fields.django3_jsonfield import JSONField as DJANGO_JSONField
@@ -24,6 +25,20 @@
 from loguru import logger
 
 
+class Patched_DJANGO_JSONField(DJANGO_JSONField):
+    def formfield(self, **kwargs):
+        schema = getattr(self.model, self.attname + "_schema", self.schema)
+        return super().formfield(
+            **{
+                "form_class": DJANGO_JSONFormField,
+                "schema": schema,
+                "model_name": self.model.__name__,
+                "file_handler": self.file_handler,
+                **kwargs,
+            }
+        )
+
+
 def _get_crypter():
     configured_keys = [settings.SECRET_KEY] + settings.SECRET_KEY_FALLBACKS
     keys = [Fernet(b64encode(sha256(force_bytes(k)).digest())) for k in configured_keys]
@@ -299,7 +314,7 @@ def from_json(self, value):  # backward compatible with dirty legacy data
         return []
 
 
-class JSONField(JSONFieldMixin, DJANGO_JSONField):
+class JSONField(JSONFieldMixin, Patched_DJANGO_JSONField):
     pass
 
 

catalog/common/models.py

@@ -268,13 +268,15 @@ class LocalizedTitleSchema(Schema):
 
 
 class ItemInSchema(Schema):
-    title: str
-    brief: str
+    title: str = Field(alias="display_title")
+    description: str = Field(default=None, alias="display_description")
     localized_title: list[LocalizedTitleSchema] = []
     localized_description: list[LocalizedTitleSchema] = []
     cover_image_url: str | None
     rating: float | None
     rating_count: int | None
+    # brief is deprecated
+    brief: str = Field(deprecated=True, alias="display_description")
 
 
 class ItemSchema(BaseSchema, ItemInSchema):
@@ -296,13 +298,14 @@ def get_locale_choices_for_jsonform(choices):
         "keys": {
             "lang": {
                 "type": "string",
-                "title": _("language"),
+                "title": _("locale"),
                 "choices": LOCALE_CHOICES_JSONFORM,
             },
-            "text": {"type": "string", "title": _("content")},
+            "text": {"type": "string", "title": _("text content")},
         },
         "required": ["lang", "s"],
     },
+    "minItems": 1,
     "uniqueItems": True,
 }
 
@@ -313,10 +316,14 @@ def get_locale_choices_for_jsonform(choices):
         "keys": {
             "lang": {
                 "type": "string",
-                "title": _("language"),
+                "title": _("locale"),
                 "choices": LOCALE_CHOICES_JSONFORM,
             },
-            "text": {"type": "string", "title": _("content"), "widget": "textarea"},
+            "text": {
+                "type": "string",
+                "title": _("text content"),
+                "widget": "textarea",
+            },
         },
         "required": ["lang", "s"],
     },
@@ -669,8 +676,8 @@ def update_lookup_ids(self, lookup_ids: list[tuple[str, str]]):
                 self.primary_lookup_id_value = v
 
     METADATA_COPY_LIST = [
-        "title",
-        "brief",
+        # "title",
+        # "brief",
         "localized_title",
         "localized_description",
     ]  # list of metadata keys to copy from resource to item

catalog/game/models.py

@@ -43,7 +43,6 @@ class Game(Item):
         # "brief",
         # "other_title",
         "localized_title",
-        "localized_description",
         "designer",
         "artist",
         "developer",
@@ -53,6 +52,7 @@ class Game(Item):
         "genre",
         "platform",
         "official_site",
+        "localized_description",
     ]
 
     other_title = jsondata.ArrayField(

catalog/management/commands/catalog.py

@@ -1,4 +1,5 @@
 import pprint
+import re
 
 from django.contrib.contenttypes.models import ContentType
 from django.core.management.base import BaseCommand
@@ -51,20 +52,36 @@ def handle(self, *args, **options):
         self.stdout.write(self.style.SUCCESS(f"Done."))
 
     def localize(self):
-        for i in tqdm(Item.objects.all()):
+        for i in tqdm(
+            Item.objects.filter(is_deleted=False, merged_to_item__isnull=True)
+        ):
             localized_title = [{"lang": detect_language(i.title), "text": i.title}]
-            if hasattr(i, "orig_title") and i.orig_title:  # type:ignore
-                localized_title += [
-                    {
-                        "lang": detect_language(i.orig_title),  # type:ignore
-                        "text": i.orig_title,  # type:ignore
-                    }
-                ]
-            if hasattr(i, "other_title") and i.other_title:  # type:ignore
-                for title in i.other_title:  # type:ignore
-                    localized_title += [{"lang": detect_language(title), "text": title}]
-            localized_desc = [{"lang": detect_language(i.brief), "text": i.brief}]
+            if i.__class__ != Edition:
+                if hasattr(i, "orig_title") and i.orig_title:  # type:ignore
+                    localized_title += [
+                        {
+                            "lang": detect_language(i.orig_title),  # type:ignore
+                            "text": i.orig_title,  # type:ignore
+                        }
+                    ]
+                if hasattr(i, "other_title") and i.other_title:  # type:ignore
+                    for title in i.other_title:  # type:ignore
+                        localized_title += [
+                            {"lang": detect_language(title), "text": title}
+                        ]
+            else:
+                # Edition has no other_title
+                subtitle = i.metadata.get("subtitle")
+                i.metadata["localized_subtitle"] = (
+                    [{"lang": detect_language(subtitle), "text": subtitle}]
+                    if subtitle
+                    else []
+                )
+                lang = i.metadata.get("language")
+                if isinstance(lang, str):
+                    i.metadata["language"] = [lang]
             i.localized_title = uniq(localized_title)
+            localized_desc = [{"lang": detect_language(i.brief), "text": i.brief}]
             i.localized_description = localized_desc
             i.save(update_fields=["metadata"])
 

catalog/movie/models.py

@@ -44,10 +44,10 @@ class Movie(Item):
     douban_movie = PrimaryLookupIdDescriptor(IdType.DoubanMovie)
 
     METADATA_COPY_LIST = [
-        "title",
+        # "title",
+        # "other_title",
         "localized_title",
         "orig_title",
-        # "other_title",
         "director",
         "playwright",
         "actor",