feat: Optimize large dataset rendering perf

- lazy loading using intersection observer for on-demand graph renders
- data sampling to reduce point density for large datasets
- chunk loading to prevent browser freezing
- memoization and React.memo to prevent unnecessary re-renders
- debounced resize handling
- feature toggles for fine-tuning optimization parameters

frontend/src/components/widgets/DataVisualizationWidget.jsx

@@ -1,22 +1,82 @@
-import React, { useEffect, useRef, useState } from "react";
+import React, { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import { debounce, processDataInChunks, sampleData } from "../../utils/dataProcessing";
 
+import { FEATURES } from "../../config/features";
 import Plot from "react-plotly.js";
 
 const DataVisualizationWidget = ({ id, data, content, error }) => {
   const [isLoading, setIsLoading] = useState(true);
   const [plotError, setPlotError] = useState(null);
+  const [isVisible, setIsVisible] = useState(false);
+  const [processedData, setProcessedData] = useState(null);
   const plotContainerRef = useRef(null);
+  const observerRef = useRef(null);
 
+  // Memoize the plot data processing
+  const processPlotData = useCallback((plotData) => {
+    if (!plotData?.data || !Array.isArray(plotData.data)) return plotData;
+
+    return {
+      ...plotData,
+      data: plotData.data.map(trace => ({
+        ...trace,
+        x: FEATURES.OPTIMIZED_VISUALIZATION 
+          ? sampleData(trace.x, FEATURES.DATA_SAMPLING_THRESHOLD)
+          : trace.x,
+        y: FEATURES.OPTIMIZED_VISUALIZATION 
+          ? sampleData(trace.y, FEATURES.DATA_SAMPLING_THRESHOLD)
+          : trace.y,
+      }))
+    };
+  }, []);
+
+  // Memoize the processed data
+  const memoizedData = useMemo(() => {
+    if (!data) return null;
+    return processPlotData(data);
+  }, [data, processPlotData]);
+
+  // Setup Intersection Observer for lazy loading
   useEffect(() => {
-    setIsLoading(false);
+    if (!FEATURES.OPTIMIZED_VISUALIZATION) {
+      setIsVisible(true);
+      return;
+    }
+
+    const options = {
+      root: null,
+      rootMargin: '50px',
+      threshold: 0.1
+    };
+
+    const observer = new IntersectionObserver((entries) => {
+      entries.forEach(entry => {
+        if (entry.isIntersecting) {
+          setIsVisible(true);
+          observer.disconnect();
+        }
+      });
+    }, options);
+
+    if (plotContainerRef.current) {
+      observer.observe(plotContainerRef.current);
+    }
+
+    observerRef.current = observer;
+    return () => observer.disconnect();
+  }, []);
+
+  // Handle data processing and loading
+  useEffect(() => {
+    if (!isVisible) return;
+
+    setIsLoading(true);
     setPlotError(null);
 
-    // If we receive HTML content (from backend's fig.to_html())
     if (typeof content === 'string' && content.includes('<div id="plotly">')) {
       try {
         if (plotContainerRef.current) {
           plotContainerRef.current.innerHTML = content;
-          // Execute any scripts that came with the plot
           const scripts = plotContainerRef.current.getElementsByTagName('script');
           Array.from(scripts).forEach(script => {
             const newScript = document.createElement('script');
@@ -31,8 +91,42 @@ const DataVisualizationWidget = ({ id, data, content, error }) => {
         setPlotError("Failed to render Plotly visualization");
         console.error("Plot rendering error:", err);
       }
+    } else if (memoizedData?.data) {
+      if (FEATURES.OPTIMIZED_VISUALIZATION) {
+        processDataInChunks(
+          memoizedData.data,
+          FEATURES.PROGRESSIVE_LOADING_CHUNK_SIZE,
+          (chunk) => {
+            setProcessedData(prevData => ({
+              ...memoizedData,
+              data: [...(prevData?.data || []), ...chunk]
+            }));
+          }
+        );
+      } else {
+        setProcessedData(memoizedData);
+      }
     }
-  }, [content]);
+
+    setIsLoading(false);
+  }, [isVisible, content, memoizedData]);
+
+  // Handle resize events with debouncing
+  const debouncedResize = useMemo(() => 
+    debounce(() => {
+      if (plotContainerRef.current) {
+        window.Plotly.Plots.resize(plotContainerRef.current);
+      }
+    }, FEATURES.RESIZE_DEBOUNCE_MS),
+    []
+  );
+
+  useEffect(() => {
+    if (FEATURES.OPTIMIZED_VISUALIZATION) {
+      window.addEventListener('resize', debouncedResize);
+      return () => window.removeEventListener('resize', debouncedResize);
+    }
+  }, [debouncedResize]);
 
   if (error || plotError) {
     return (
@@ -42,19 +136,17 @@ const DataVisualizationWidget = ({ id, data, content, error }) => {
     );
   }
 
-  if (isLoading) {
+  if (!isVisible || isLoading) {
     return (
-      <div className="flex items-center justify-center h-64">
+      <div className="flex items-center justify-center h-64" ref={plotContainerRef}>
         <div className="animate-spin rounded-full h-8 w-8 border-b-2 border-blue-600"></div>
       </div>
     );
   }
 
-  // If we have direct Plotly data
-  if (data?.data && Array.isArray(data.data)) {
-    const { data: plotData, layout = {}, config = {} } = data;
+  if (processedData?.data) {
+    const { layout = {}, config = {} } = processedData;
 
-    // Default config for better UX
     const defaultConfig = {
       responsive: true,
       displayModeBar: true,
@@ -63,7 +155,6 @@ const DataVisualizationWidget = ({ id, data, content, error }) => {
       ...config
     };
 
-    // Enhanced layout for better visuals
     const enhancedLayout = {
       font: { family: 'Inter, system-ui, sans-serif' },
       paper_bgcolor: 'transparent',
@@ -74,11 +165,11 @@ const DataVisualizationWidget = ({ id, data, content, error }) => {
 
     try {
       return (
-        <div className="w-full h-full">
+        <div className="w-full h-full" ref={plotContainerRef}>
           <div className="relative w-full" style={{ minHeight: '400px' }}>
             <Plot
-              key={id} // Add key to force re-render on data change
-              data={plotData}
+              key={id}
+              data={processedData.data}
               layout={enhancedLayout}
               config={defaultConfig}
               className="w-full h-full"
@@ -102,7 +193,6 @@ const DataVisualizationWidget = ({ id, data, content, error }) => {
     }
   }
 
-  // For HTML content from backend
   return (
     <div className="w-full h-full">
       <div 
@@ -113,4 +203,6 @@ const DataVisualizationWidget = ({ id, data, content, error }) => {
   );
 };
 
-export default DataVisualizationWidget;
+export default React.memo(DataVisualizationWidget);
+
+

frontend/src/config/features.js

@@ -0,0 +1,6 @@
+export const FEATURES = {
+    OPTIMIZED_VISUALIZATION: true, // Toggle for visualization optimizations
+    DATA_SAMPLING_THRESHOLD: 100, // Number of points above which to apply sampling
+    PROGRESSIVE_LOADING_CHUNK_SIZE: 100, // Number of points to load in each chunk
+    RESIZE_DEBOUNCE_MS: 10, // Debounce time for resize events in milliseconds
+}; 

frontend/src/utils/dataProcessing.js

@@ -0,0 +1,37 @@
+// Utility function to sample data points
+export const sampleData = (data, threshold) => {
+    if (!Array.isArray(data) || data.length <= threshold) return data;
+
+    const samplingRate = Math.ceil(data.length / threshold);
+    return data.filter((_, index) => index % samplingRate === 0);
+};
+
+// Process data in chunks
+export const processDataInChunks = (data, chunkSize, callback) => {
+    let currentIndex = 0;
+
+    const processNextChunk = () => {
+        const chunk = data.slice(currentIndex, currentIndex + chunkSize);
+        callback(chunk, currentIndex);
+        currentIndex += chunkSize;
+
+        if (currentIndex < data.length) {
+            requestAnimationFrame(processNextChunk);
+        }
+    };
+
+    processNextChunk();
+};
+
+// Debounce function
+export const debounce = (func, wait) => {
+    let timeout;
+    return function executedFunction(...args) {
+        const later = () => {
+            clearTimeout(timeout);
+            func(...args);
+        };
+        clearTimeout(timeout);
+        timeout = setTimeout(later, wait);
+    };
+};