Merge branch 'galaxyproject:dev' into FastAPI_library_contents

client/src/api/schema/schema.ts

@@ -339,7 +339,7 @@ export interface paths {
         };
         /**
          * Displays information about and/or content of a dataset.
-         * @description **Note**: Due to the multipurpose nature of this endpoint, which can receive a wild variety of parameters
+         * @description **Note**: Due to the multipurpose nature of this endpoint, which can receive a wide variety of parameters
          *     and return different kinds of responses, the documentation here will be limited.
          *     To get more information please check the source code.
          */
@@ -18853,6 +18853,10 @@ export interface operations {
                 hda_ldda?: components["schemas"]["DatasetSourceType"];
                 /** @description The type of information about the dataset to be requested. Each of these values may require additional parameters in the request and may return different responses. */
                 data_type?: components["schemas"]["RequestDataType"] | null;
+                /** @description Maximum number of items to return. Currently only applies to `data_type=raw_data` requests */
+                limit?: number | null;
+                /** @description Starts at the beginning skip the first ( offset - 1 ) items and begin returning at the Nth item. Currently only applies to `data_type=raw_data` requests */
+                offset?: number | null;
                 /** @description View to be passed to the serializer */
                 view?: string | null;
                 /** @description Comma-separated list of keys to be passed to the serializer */

lib/galaxy/app.py

@@ -655,7 +655,7 @@ def __init__(self, configure_logging=True, use_converters=True, use_display_appl
         # Load security policy.
         self.security_agent = self.model.security_agent
         self.host_security_agent = galaxy.model.security.HostAgent(
-            model=self.security_agent.model, permitted_actions=self.security_agent.permitted_actions
+            self.security_agent.sa_session, permitted_actions=self.security_agent.permitted_actions
         )
 
         # We need the datatype registry for running certain tasks that modify HDAs, and to build the registry we need

lib/galaxy/datatypes/dataproviders/base.py

@@ -36,6 +36,7 @@ def stop( self ): self.endpoint = source.tell(); raise StopIteration()
         Building a giant list by sweeping all possible dprov classes doesn't make sense
     For now - I'm burying them in the class __init__s - but I don't like that
 """
+MAX_LIMIT = 10000
 
 
 # ----------------------------------------------------------------------------- base classes
@@ -233,21 +234,20 @@ class LimitedOffsetDataProvider(FilteredDataProvider):
     settings = {"limit": "int", "offset": "int"}
 
     # TODO: may want to squash this into DataProvider
-    def __init__(self, source, offset=0, limit=None, **kwargs):
+    def __init__(self, source, offset=0, limit=MAX_LIMIT, **kwargs):
         """
         :param offset:  the number of data to skip before providing.
         :param limit:   the final number of data to provide.
         """
         super().__init__(source, **kwargs)
 
-        # how many valid data to skip before we start outputing data - must be positive
-        #   (diff to support neg. indeces - must be pos.)
-        self.offset = max(offset, 0)
+        # how many valid data to skip before we start outputting data - must be positive
+        self.offset = offset
 
-        # how many valid data to return - must be positive (None indicates no limit)
+        # how many valid data to return - must be positive
+        if limit is None:
+            limit = MAX_LIMIT
         self.limit = limit
-        if self.limit is not None:
-            self.limit = max(self.limit, 0)
 
     def __iter__(self):
         """

lib/galaxy/managers/groups.py

@@ -13,8 +13,6 @@
 from galaxy.managers.context import ProvidesAppContext
 from galaxy.model import Group
 from galaxy.model.base import transaction
-from galaxy.model.db.role import get_roles_by_ids
-from galaxy.model.db.user import get_users_by_ids
 from galaxy.model.scoped_session import galaxy_scoped_session
 from galaxy.schema.fields import Security
 from galaxy.schema.groups import (
@@ -54,13 +52,11 @@ def create(self, trans: ProvidesAppContext, payload: GroupCreatePayload):
 
         group = model.Group(name=name)
         sa_session.add(group)
-        user_ids = payload.user_ids
-        users = get_users_by_ids(sa_session, user_ids)
-        role_ids = payload.role_ids
-        roles = get_roles_by_ids(sa_session, role_ids)
-        trans.app.security_agent.set_entity_group_associations(groups=[group], roles=roles, users=users)
-        with transaction(sa_session):
-            sa_session.commit()
+
+        trans.app.security_agent.set_group_user_and_role_associations(
+            group, user_ids=payload.user_ids, role_ids=payload.role_ids
+        )
+        sa_session.commit()
 
         encoded_id = Security.security.encode_id(group.id)
         item = group.to_dict(view="element")
@@ -88,23 +84,12 @@ def update(self, trans: ProvidesAppContext, group_id: int, payload: GroupUpdateP
         if name := payload.name:
             self._check_duplicated_group_name(sa_session, name)
             group.name = name
-            sa_session.add(group)
-
-        users = None
-        if payload.user_ids is not None:
-            users = get_users_by_ids(sa_session, payload.user_ids)
-
-        roles = None
-        if payload.role_ids is not None:
-            roles = get_roles_by_ids(sa_session, payload.role_ids)
+            sa_session.commit()
 
-        self._app.security_agent.set_entity_group_associations(
-            groups=[group], roles=roles, users=users, delete_existing_assocs=False
+        self._app.security_agent.set_group_user_and_role_associations(
+            group, user_ids=payload.user_ids, role_ids=payload.role_ids
         )
 
-        with transaction(sa_session):
-            sa_session.commit()
-
         encoded_id = Security.security.encode_id(group.id)
         item = group.to_dict(view="element")
         item["url"] = self._url_for(trans, "show_group", group_id=encoded_id)

lib/galaxy/managers/workflows.py

@@ -1660,7 +1660,7 @@ def _workflow_to_dict_instance(self, trans, stored, workflow, legacy=True):
         inputs = {}
         for step in workflow.input_steps:
             step_type = step.type
-            step_label = step.label or step.tool_inputs.get("name")
+            step_label = step.label or step.tool_inputs and step.tool_inputs.get("name")
             if step_label:
                 label = step_label
             elif step_type == "data_input":
@@ -1954,7 +1954,7 @@ def __set_default_label(self, step, module, state):
         to the actual `label` attribute which is available for all module types, unique, and mapped to its own database column.
         """
         if not module.label and module.type in ["data_input", "data_collection_input"]:
-            new_state = safe_loads(state)
+            new_state = safe_loads(state) or {}
             default_label = new_state.get("name")
             if default_label and util.unicodify(default_label).lower() not in [
                 "input dataset",

lib/galaxy/model/__init__.py

@@ -780,7 +780,7 @@ class User(Base, Dictifiable, RepresentById):
     id: Mapped[int] = mapped_column(primary_key=True)
     create_time: Mapped[datetime] = mapped_column(default=now, nullable=True)
     update_time: Mapped[datetime] = mapped_column(default=now, onupdate=now, nullable=True)
-    email: Mapped[str] = mapped_column(TrimmedString(255), index=True)
+    email: Mapped[str] = mapped_column(TrimmedString(255), index=True, unique=True)
     username: Mapped[Optional[str]] = mapped_column(TrimmedString(255), index=True, unique=True)
     password: Mapped[str] = mapped_column(TrimmedString(255))
     last_password_change: Mapped[Optional[datetime]] = mapped_column(default=now)
@@ -849,14 +849,6 @@ class User(Base, Dictifiable, RepresentById):
     all_notifications: Mapped[List["UserNotificationAssociation"]] = relationship(
         back_populates="user", cascade_backrefs=False
     )
-    non_private_roles: Mapped[List["UserRoleAssociation"]] = relationship(
-        viewonly=True,
-        primaryjoin=(
-            lambda: (User.id == UserRoleAssociation.user_id)
-            & (UserRoleAssociation.role_id == Role.id)
-            & not_(Role.name == User.email)
-        ),
-    )
 
     preferences: AssociationProxy[Any]
 
@@ -2967,10 +2959,11 @@ def __init__(self, name=None):
 
 class UserGroupAssociation(Base, RepresentById):
     __tablename__ = "user_group_association"
+    __table_args__ = (UniqueConstraint("user_id", "group_id"),)
 
     id: Mapped[int] = mapped_column(primary_key=True)
-    user_id: Mapped[int] = mapped_column(ForeignKey("galaxy_user.id"), index=True, nullable=True)
-    group_id: Mapped[int] = mapped_column(ForeignKey("galaxy_group.id"), index=True, nullable=True)
+    user_id: Mapped[int] = mapped_column(ForeignKey("galaxy_user.id"), index=True)
+    group_id: Mapped[int] = mapped_column(ForeignKey("galaxy_group.id"), index=True)
     create_time: Mapped[datetime] = mapped_column(default=now, nullable=True)
     update_time: Mapped[datetime] = mapped_column(default=now, onupdate=now, nullable=True)
     user: Mapped["User"] = relationship(back_populates="groups")
@@ -3685,10 +3678,11 @@ class HistoryUserShareAssociation(Base, UserShareAssociation):
 
 class UserRoleAssociation(Base, RepresentById):
     __tablename__ = "user_role_association"
+    __table_args__ = (UniqueConstraint("user_id", "role_id"),)
 
     id: Mapped[int] = mapped_column(primary_key=True)
-    user_id: Mapped[int] = mapped_column(ForeignKey("galaxy_user.id"), index=True, nullable=True)
-    role_id: Mapped[int] = mapped_column(ForeignKey("role.id"), index=True, nullable=True)
+    user_id: Mapped[int] = mapped_column(ForeignKey("galaxy_user.id"), index=True)
+    role_id: Mapped[int] = mapped_column(ForeignKey("role.id"), index=True)
     create_time: Mapped[datetime] = mapped_column(default=now, nullable=True)
     update_time: Mapped[datetime] = mapped_column(default=now, onupdate=now, nullable=True)
 
@@ -3703,10 +3697,11 @@ def __init__(self, user, role):
 
 class GroupRoleAssociation(Base, RepresentById):
     __tablename__ = "group_role_association"
+    __table_args__ = (UniqueConstraint("group_id", "role_id"),)
 
     id: Mapped[int] = mapped_column(primary_key=True)
-    group_id: Mapped[int] = mapped_column(ForeignKey("galaxy_group.id"), index=True, nullable=True)
-    role_id: Mapped[int] = mapped_column(ForeignKey("role.id"), index=True, nullable=True)
+    group_id: Mapped[int] = mapped_column(ForeignKey("galaxy_group.id"), index=True)
+    role_id: Mapped[int] = mapped_column(ForeignKey("role.id"), index=True)
     create_time: Mapped[datetime] = mapped_column(default=now, nullable=True)
     update_time: Mapped[datetime] = mapped_column(default=now, onupdate=now, nullable=True)
     group: Mapped["Group"] = relationship(back_populates="roles")
@@ -4549,7 +4544,9 @@ class DatasetInstance(RepresentById, UsesCreateAndUpdateTime, _HasTable):
     creating_job_associations: List[Union[JobToOutputDatasetCollectionAssociation, JobToOutputDatasetAssociation]]
     copied_from_history_dataset_association: Optional["HistoryDatasetAssociation"]
     copied_from_library_dataset_dataset_association: Optional["LibraryDatasetDatasetAssociation"]
+    dependent_jobs: List[JobToInputLibraryDatasetAssociation]
     implicitly_converted_datasets: List["ImplicitlyConvertedDatasetAssociation"]
+    implicitly_converted_parent_datasets: List["ImplicitlyConvertedDatasetAssociation"]
 
     validated_states = DatasetValidatedState
 

lib/galaxy/model/mapping.py

@@ -97,7 +97,7 @@ def _build_model_mapping(engine, map_install_models, thread_local_log) -> Galaxy
         model_modules.append(tool_shed_install)
 
     model_mapping = GalaxyModelMapping(model_modules, engine)
-    model_mapping.security_agent = GalaxyRBACAgent(model_mapping)
+    model_mapping.security_agent = GalaxyRBACAgent(model_mapping.session)
     model_mapping.thread_local_log = thread_local_log
     return model_mapping
 

lib/galaxy/model/migrations/alembic/versions_gxy/13fe10b8e35b_add_not_null_constraints_to_user_group_.py

@@ -0,0 +1,42 @@
+"""Add not-null constraints to user_group_association
+
+Revision ID: 13fe10b8e35b
+Revises: 56ddf316dbd0
+Create Date: 2024-09-09 21:26:26.032842
+
+"""
+
+from alembic import op
+
+from galaxy.model.migrations.data_fixes.association_table_fixer import UserGroupAssociationNullFix
+from galaxy.model.migrations.util import (
+    alter_column,
+    transaction,
+)
+
+# revision identifiers, used by Alembic.
+revision = "13fe10b8e35b"
+down_revision = "56ddf316dbd0"
+branch_labels = None
+depends_on = None
+
+table_name = "user_group_association"
+
+
+def upgrade():
+    with transaction():
+        _remove_records_with_nulls()
+        alter_column(table_name, "user_id", nullable=False)
+        alter_column(table_name, "group_id", nullable=False)
+
+
+def downgrade():
+    with transaction():
+        alter_column(table_name, "user_id", nullable=True)
+        alter_column(table_name, "group_id", nullable=True)
+
+
+def _remove_records_with_nulls():
+    """Remove associations having null as user_id or group_id"""
+    connection = op.get_bind()
+    UserGroupAssociationNullFix(connection).run()

lib/galaxy/model/migrations/alembic/versions_gxy/1cf595475b58_email_column_unique_constraint.py

@@ -0,0 +1,52 @@
+"""Email column unique constraint
+
+Revision ID: 1cf595475b58
+Revises: d619fdfa6168
+Create Date: 2024-07-03 19:53:22.443016
+"""
+
+from alembic import op
+
+from galaxy.model.database_object_names import build_index_name
+from galaxy.model.migrations.data_fixes.user_table_fixer import EmailDeduplicator
+from galaxy.model.migrations.util import (
+    create_index,
+    drop_index,
+    index_exists,
+    transaction,
+)
+
+# revision identifiers, used by Alembic.
+revision = "1cf595475b58"
+down_revision = "d619fdfa6168"
+branch_labels = None
+depends_on = None
+
+
+table_name = "galaxy_user"
+column_name = "email"
+index_name = build_index_name(table_name, [column_name])
+
+
+def upgrade():
+    with transaction():
+        _fix_duplicate_emails()
+        # Existing databases may have an existing index we no longer need
+        # New databases will not have that index, so we must check.
+        if index_exists(index_name, table_name, False):
+            drop_index(index_name, table_name)
+        # Create a UNIQUE index
+        create_index(index_name, table_name, [column_name], unique=True)
+
+
+def downgrade():
+    with transaction():
+        drop_index(index_name, table_name)
+        # Restore a non-unique index
+        create_index(index_name, table_name, [column_name])
+
+
+def _fix_duplicate_emails():
+    """Fix records with duplicate usernames"""
+    connection = op.get_bind()
+    EmailDeduplicator(connection).run()

lib/galaxy/model/migrations/alembic/versions_gxy/1fdd615f2cdb_add_not_null_constraints_to_user_role_.py

@@ -0,0 +1,42 @@
+"""Add not-null constraints to user_role_association
+
+Revision ID: 1fdd615f2cdb
+Revises: 349dd9d9aac9
+Create Date: 2024-09-09 21:28:11.987054
+
+"""
+
+from alembic import op
+
+from galaxy.model.migrations.data_fixes.association_table_fixer import UserRoleAssociationNullFix
+from galaxy.model.migrations.util import (
+    alter_column,
+    transaction,
+)
+
+# revision identifiers, used by Alembic.
+revision = "1fdd615f2cdb"
+down_revision = "349dd9d9aac9"
+branch_labels = None
+depends_on = None
+
+table_name = "user_role_association"
+
+
+def upgrade():
+    with transaction():
+        _remove_records_with_nulls()
+        alter_column(table_name, "user_id", nullable=False)
+        alter_column(table_name, "role_id", nullable=False)
+
+
+def downgrade():
+    with transaction():
+        alter_column(table_name, "user_id", nullable=True)
+        alter_column(table_name, "role_id", nullable=True)
+
+
+def _remove_records_with_nulls():
+    """Remove associations having null as user_id or role_id"""
+    connection = op.get_bind()
+    UserRoleAssociationNullFix(connection).run()