Load all data on record boards (twentyhq#5070)

## Context

For users with many records, only the first n*60 records were loaded on
board views (n being the number of visible columns). This was because of
the following behavior:
- watch for end of column visibility changes. If an end of column is
visible, try to fetch more. However, watching for visbility changes is
not reliable enough.

## What we want

If an end of column is visible, try to fetch more. If no more records is
availble in pagination, do not fetch more

packages/twenty-front/src/modules/object-record/record-board/hooks/internal/useRecordBoardStates.ts

@@ -4,16 +4,17 @@ import { isLastRecordBoardColumnComponentFamilyState } from '@/object-record/rec
 import { isRecordBoardCardSelectedComponentFamilyState } from '@/object-record/record-board/states/isRecordBoardCardSelectedComponentFamilyState';
 import { isRecordBoardCompactModeActiveComponentState } from '@/object-record/record-board/states/isRecordBoardCompactModeActiveComponentState';
 import { isRecordBoardFetchingRecordsComponentState } from '@/object-record/record-board/states/isRecordBoardFetchingRecordsComponentState';
-import { onRecordBoardFetchMoreVisibilityChangeComponentState } from '@/object-record/record-board/states/onRecordBoardFetchMoreVisibilityChangeComponentState';
 import { recordBoardColumnIdsComponentState } from '@/object-record/record-board/states/recordBoardColumnIdsComponentState';
 import { recordBoardFieldDefinitionsComponentState } from '@/object-record/record-board/states/recordBoardFieldDefinitionsComponentState';
 import { recordBoardFiltersComponentState } from '@/object-record/record-board/states/recordBoardFiltersComponentState';
 import { recordBoardKanbanFieldMetadataNameComponentState } from '@/object-record/record-board/states/recordBoardKanbanFieldMetadataNameComponentState';
 import { recordBoardObjectSingularNameComponentState } from '@/object-record/record-board/states/recordBoardObjectSingularNameComponentState';
 import { recordBoardRecordIdsByColumnIdComponentFamilyState } from '@/object-record/record-board/states/recordBoardRecordIdsByColumnIdComponentFamilyState';
+import { recordBoardShouldFetchMoreInColumnComponentFamilyState } from '@/object-record/record-board/states/recordBoardShouldFetchMoreInColumnComponentFamilyState';
 import { recordBoardSortsComponentState } from '@/object-record/record-board/states/recordBoardSortsComponentState';
 import { recordBoardColumnsComponentFamilySelector } from '@/object-record/record-board/states/selectors/recordBoardColumnsComponentFamilySelector';
 import { recordBoardSelectedRecordIdsComponentSelector } from '@/object-record/record-board/states/selectors/recordBoardSelectedRecordIdsComponentSelector';
+import { recordBoardShouldFetchMoreComponentSelector } from '@/object-record/record-board/states/selectors/recordBoardShouldFetchMoreComponentFamilySelector';
 import { recordBoardVisibleFieldDefinitionsComponentSelector } from '@/object-record/record-board/states/selectors/recordBoardVisibleFieldDefinitionsComponentSelector';
 import { useAvailableScopeIdOrThrow } from '@/ui/utilities/recoil-scope/scopes-internal/hooks/useAvailableScopeId';
 import { getScopeIdOrUndefinedFromComponentId } from '@/ui/utilities/recoil-scope/utils/getScopeIdOrUndefinedFromComponentId';
@@ -90,8 +91,12 @@ export const useRecordBoardStates = (recordBoardId?: string) => {
       scopeId,
     ),
 
-    onFetchMoreVisibilityChangeState: extractComponentState(
-      onRecordBoardFetchMoreVisibilityChangeComponentState,
+    shouldFetchMoreInColumnFamilyState: extractComponentFamilyState(
+      recordBoardShouldFetchMoreInColumnComponentFamilyState,
+      scopeId,
+    ),
+    shouldFetchMoreSelector: extractComponentReadOnlySelector(
+      recordBoardShouldFetchMoreComponentSelector,
       scopeId,
     ),
   };

packages/twenty-front/src/modules/object-record/record-board/hooks/useRecordBoard.ts

@@ -11,8 +11,8 @@ export const useRecordBoard = (recordBoardId?: string) => {
     objectSingularNameState,
     selectedRecordIdsSelector,
     isCompactModeActiveState,
-    onFetchMoreVisibilityChangeState,
     kanbanFieldMetadataNameState,
+    shouldFetchMoreSelector,
   } = useRecordBoardStates(recordBoardId);
 
   const { setColumns } = useSetRecordBoardColumns(recordBoardId);
@@ -32,6 +32,6 @@ export const useRecordBoard = (recordBoardId?: string) => {
     setKanbanFieldMetadataName,
     selectedRecordIdsSelector,
     isCompactModeActiveState,
-    onFetchMoreVisibilityChangeState,
+    shouldFetchMoreSelector,
   };
 };

packages/twenty-front/src/modules/object-record/record-board/record-board-column/components/RecordBoardColumnFetchMoreLoader.tsx

@@ -1,8 +1,10 @@
+import { useContext, useEffect } from 'react';
 import { useInView } from 'react-intersection-observer';
 import styled from '@emotion/styled';
-import { useRecoilValue } from 'recoil';
+import { useRecoilValue, useSetRecoilState } from 'recoil';
 
 import { useRecordBoardStates } from '@/object-record/record-board/hooks/internal/useRecordBoardStates';
+import { RecordBoardColumnContext } from '@/object-record/record-board/record-board-column/contexts/RecordBoardColumnContext';
 import { GRAY_SCALE } from '@/ui/theme/constants/GrayScale';
 
 const StyledText = styled.div`
@@ -16,17 +18,20 @@ const StyledText = styled.div`
 `;
 
 export const RecordBoardColumnFetchMoreLoader = () => {
-  const { isFetchingRecordState, onFetchMoreVisibilityChangeState } =
+  const { columnDefinition } = useContext(RecordBoardColumnContext);
+  const { isFetchingRecordState, shouldFetchMoreInColumnFamilyState } =
     useRecordBoardStates();
   const isFetchingRecord = useRecoilValue(isFetchingRecordState);
 
-  const onFetchMoreVisibilityChange = useRecoilValue(
-    onFetchMoreVisibilityChangeState,
+  const shouldFetchMore = useSetRecoilState(
+    shouldFetchMoreInColumnFamilyState(columnDefinition.id),
   );
 
-  const { ref } = useInView({
-    onChange: onFetchMoreVisibilityChange,
-  });
+  const { ref, inView } = useInView();
+
+  useEffect(() => {
+    shouldFetchMore(inView);
+  }, [shouldFetchMore, inView]);
 
   return (
     <div ref={ref}>

packages/twenty-front/src/modules/object-record/record-board/states/recordBoardShouldFetchMoreInColumnComponentFamilyState.ts

@@ -0,0 +1,7 @@
+import { createComponentFamilyState } from '@/ui/utilities/state/component-state/utils/createComponentFamilyState';
+
+export const recordBoardShouldFetchMoreInColumnComponentFamilyState =
+  createComponentFamilyState<boolean, string>({
+    key: 'onRecordBoardFetchMoreIrecordBoardShouldFetchMoreInColumnComponentFamilyStatesVisibleComponentFamilyState',
+    defaultValue: false,
+  });

packages/twenty-front/src/modules/object-record/record-board/states/selectors/recordBoardShouldFetchMoreComponentFamilySelector.ts

@@ -0,0 +1,28 @@
+import { recordBoardColumnIdsComponentState } from '@/object-record/record-board/states/recordBoardColumnIdsComponentState';
+import { recordBoardShouldFetchMoreInColumnComponentFamilyState } from '@/object-record/record-board/states/recordBoardShouldFetchMoreInColumnComponentFamilyState';
+import { createComponentReadOnlySelector } from '@/ui/utilities/state/component-state/utils/createComponentReadOnlySelector';
+
+export const recordBoardShouldFetchMoreComponentSelector =
+  createComponentReadOnlySelector<boolean>({
+    key: 'recordBoardShouldFetchMoreComponentSelector',
+    get:
+      ({ scopeId }: { scopeId: string }) =>
+      ({ get }) => {
+        const columnIds = get(
+          recordBoardColumnIdsComponentState({
+            scopeId,
+          }),
+        );
+
+        const shouldFetchMoreInColumns = columnIds.map((columnId) => {
+          return get(
+            recordBoardShouldFetchMoreInColumnComponentFamilyState({
+              scopeId,
+              familyKey: columnId,
+            }),
+          );
+        });
+
+        return shouldFetchMoreInColumns.some(Boolean);
+      },
+  });

packages/twenty-front/src/modules/object-record/record-index/components/RecordIndexBoardContainerEffect.tsx

@@ -1,6 +1,6 @@
 import { useCallback, useEffect } from 'react';
 import { useNavigate } from 'react-router-dom';
-import { useRecoilValue, useSetRecoilState } from 'recoil';
+import { useRecoilValue } from 'recoil';
 
 import { useObjectMetadataItem } from '@/object-metadata/hooks/useObjectMetadataItem';
 import { useRecordActionBar } from '@/object-record/record-action-bar/hooks/useRecordActionBar';
@@ -33,7 +33,7 @@ export const RecordIndexBoardContainerEffect = ({
     setObjectSingularName,
     selectedRecordIdsSelector,
     setFieldDefinitions,
-    onFetchMoreVisibilityChangeState,
+    shouldFetchMoreSelector,
     setKanbanFieldMetadataName,
   } = useRecordBoard(recordBoardId);
 
@@ -43,28 +43,31 @@ export const RecordIndexBoardContainerEffect = ({
     viewBarId,
   });
 
-  const setOnFetchMoreVisibilityChange = useSetRecoilState(
-    onFetchMoreVisibilityChangeState,
-  );
-
   const recordIndexKanbanFieldMetadataId = useRecoilValue(
     recordIndexKanbanFieldMetadataIdState,
   );
 
-  useEffect(() => {
-    setOnFetchMoreVisibilityChange(() => () => {
-      if (!loading) {
-        fetchMoreRecords?.();
-      }
-    });
-  }, [fetchMoreRecords, loading, setOnFetchMoreVisibilityChange]);
-
   const navigate = useNavigate();
 
   const navigateToSelectSettings = useCallback(() => {
     navigate(`/settings/objects/${objectMetadataItem.namePlural}`);
   }, [navigate, objectMetadataItem.namePlural]);
 
+  const columnDefinitions =
+    computeRecordBoardColumnDefinitionsFromObjectMetadata(
+      objectMetadataItem,
+      recordIndexKanbanFieldMetadataId ?? '',
+      navigateToSelectSettings,
+    );
+
+  const shouldFetchMore = useRecoilValue(shouldFetchMoreSelector());
+
+  useEffect(() => {
+    if (!loading && shouldFetchMore) {
+      fetchMoreRecords?.();
+    }
+  }, [columnDefinitions, fetchMoreRecords, loading, shouldFetchMore]);
+
   const { resetRecordSelection } = useRecordBoardSelection(recordBoardId);
 
   useEffect(() => {

packages/twenty-front/src/modules/views/view-picker/components/ViewPickerCreateOrEditContent.tsx

@@ -166,7 +166,10 @@ export const ViewPickerCreateOrEditContent = () => {
                 label="Stages"
                 fullWidth
                 value={viewPickerKanbanFieldMetadataId}
-                onChange={(value) => setViewPickerKanbanFieldMetadataId(value)}
+                onChange={(value) => {
+                  setViewPickerIsDirty(true);
+                  setViewPickerKanbanFieldMetadataId(value);
+                }}
                 options={
                   availableFieldsForKanban.length > 0
                     ? availableFieldsForKanban.map((field) => ({

packages/twenty-front/src/modules/views/view-picker/components/ViewPickerCreateOrEditContentEffect.tsx

@@ -64,10 +64,14 @@ export const ViewPickerCreateOrEditContentEffect = () => {
   ]);
 
   useEffect(() => {
-    if (availableFieldsForKanban.length > 0) {
+    if (availableFieldsForKanban.length > 0 && !viewPickerIsDirty) {
       setViewPickerKanbanFieldMetadataId(availableFieldsForKanban[0].id);
     }
-  }, [availableFieldsForKanban, setViewPickerKanbanFieldMetadataId]);
+  }, [
+    availableFieldsForKanban,
+    setViewPickerKanbanFieldMetadataId,
+    viewPickerIsDirty,
+  ]);
 
   return <></>;
 };