From cce4eebe0ed867d52d0065989e1ff8b6a0f26a5d Mon Sep 17 00:00:00 2001 From: mdshamoon Date: Thu, 12 Sep 2024 14:53:35 +0530 Subject: [PATCH 01/14] moving data statistics out --- src/components/SideDrawer/SideDrawer.tsx | 12 ++-- .../LowerSectionTabs/StatisticsPane.tsx | 42 ++---------- src/config/menu.tsx | 6 ++ src/pages/data-statistics/index.tsx | 65 +++++++++++++++++++ 4 files changed, 86 insertions(+), 39 deletions(-) create mode 100644 src/pages/data-statistics/index.tsx diff --git a/src/components/SideDrawer/SideDrawer.tsx b/src/components/SideDrawer/SideDrawer.tsx index 4c2ef22c8..9bfd7ad09 100644 --- a/src/components/SideDrawer/SideDrawer.tsx +++ b/src/components/SideDrawer/SideDrawer.tsx @@ -93,7 +93,7 @@ const closedMixin = (theme: Theme): CSSObject => ({ width: `calc(${theme.spacing(8)} + 1px)`, }, }); - const Drawer = styled(MuiDrawer, { +const Drawer = styled(MuiDrawer, { shouldForwardProp: (prop) => prop !== 'open', })(({ theme, open }) => ({ width: drawerWidth, @@ -166,7 +166,7 @@ export const SideDrawer = ({ openMenu, setOpenMenu }: any) => { !item.parent && ( @@ -272,7 +272,9 @@ export const SideDrawer = ({ openMenu, setOpenMenu }: any) => { justifyContent: 'center', }} > - Documentation + + Documentation + { marginTop: 1, }} > - Privacy Policy + + Privacy Policy + { - const [debouncedValue, setDebouncedValue] = useState(value); - - useEffect(() => { - const handler = setTimeout(() => { - setDebouncedValue(value); - }, delay); - - return () => { - clearTimeout(handler); - }; - }, [value, delay]); - - return debouncedValue; -}; - interface StatisticsPaneProps { - height: number; + modelToPreview: DbtSourceModel | null; } export interface DateTimeFilter { @@ -148,15 +131,15 @@ export const pollTaskStatus = async ( return new Promise(poll); }; -export const StatisticsPane: React.FC = ({ height }) => { - const [modelToPreview, setModelToPreview] = useState(); - - const debouncedHeight = useDebounce(height, 500); +export const StatisticsPane: React.FC = ({ + modelToPreview, +}) => { + // const [modelToPreview, setModelToPreview] = useState(); const [rowCount, setRowCount] = useState(-1); const { data: session } = useSession(); const toastContext = useContext(GlobalContext); - const { previewAction } = usePreviewAction(); + // Row Data: The data to be displayed. const [data, setData] = useState([]); @@ -410,14 +393,6 @@ export const StatisticsPane: React.FC = ({ height }) => { }; }, []); - useEffect(() => { - if (previewAction.type === 'preview') { - setModelToPreview(previewAction.data); - } else if (previewAction.type === 'clear-preview') { - setModelToPreview(null); - } - }, [previewAction]); - useEffect(() => { if (modelToPreview) { setData([]); @@ -500,7 +475,7 @@ export const StatisticsPane: React.FC = ({ height }) => { - + {getHeaderGroups().map((headerGroup) => ( @@ -595,7 +570,6 @@ export const StatisticsPane: React.FC = ({ height }) => { display: 'flex', alignItems: 'center', justifyContent: 'center', - height: debouncedHeight, }} > @@ -608,7 +582,6 @@ export const StatisticsPane: React.FC = ({ height }) => { display: 'flex', alignItems: 'center', justifyContent: 'center', - height: debouncedHeight, }} > No data (0 rows) available to generate insights @@ -620,7 +593,6 @@ export const StatisticsPane: React.FC = ({ height }) => { display: 'flex', alignItems: 'center', justifyContent: 'center', - height: debouncedHeight, }} > Select a table to view diff --git a/src/config/menu.tsx b/src/config/menu.tsx index daf3a4b53..0e747c683 100644 --- a/src/config/menu.tsx +++ b/src/config/menu.tsx @@ -88,6 +88,12 @@ export const sideMenu: MenuOption[] = [ }, { index: 4, + title: 'Data Statistics', + path: '/data-statistics', + icon: (selected: boolean) => , + }, + { + index: 5, title: 'User management', path: '/user-management', icon: () => , diff --git a/src/pages/data-statistics/index.tsx b/src/pages/data-statistics/index.tsx new file mode 100644 index 000000000..dbf89de3e --- /dev/null +++ b/src/pages/data-statistics/index.tsx @@ -0,0 +1,65 @@ +import { Elementary } from '@/components/DBT/Elementary'; +import { PageHead } from '@/components/PageHead'; +import { DbtSourceModel } from '@/components/TransformWorkflow/FlowEditor/Components/Canvas'; +import { StatisticsPane } from '@/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/StatisticsPane'; +import { Autocomplete } from '@/components/UI/Autocomplete/Autocomplete'; +import { httpGet } from '@/helpers/http'; +import { Box, Typography } from '@mui/material'; +import { useSession } from 'next-auth/react'; +import { useEffect, useState } from 'react'; +import styles from '@/styles/Home.module.css'; + +export default function DataStatisticsPage() { + const { data: session } = useSession(); + const [sourceModels, setSourcesModels] = useState([]); + const [selectedSourceModel, setSelectedSourceModel] = + useState(null); + console.log(selectedSourceModel); + + const fetchSourcesModels = () => { + httpGet(session, 'transform/dbt_project/sources_models/') + .then((response: DbtSourceModel[]) => { + setSourcesModels(response); + }) + .catch((error) => { + console.log(error); + }); + }; + + useEffect(() => { + fetchSourcesModels(); + }, []); + return ( + <> + +
+ + Data statistics + + + + Select table + + option.input_name} + groupBy={(option) => option.schema} + onChange={(model) => setSelectedSourceModel(model)} + /> + + +
+ + ); +} From c5d27674f91fab8307923005c94b8bc867e4c48c Mon Sep 17 00:00:00 2001 From: mdshamoon Date: Mon, 16 Sep 2024 01:01:04 +0530 Subject: [PATCH 02/14] moving data statistics out --- .../LowerSectionTabs/StatisticsPane.tsx | 12 +----- src/pages/data-statistics/index.tsx | 42 ++++++++++--------- 2 files changed, 24 insertions(+), 30 deletions(-) diff --git a/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/StatisticsPane.tsx b/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/StatisticsPane.tsx index 5738e57a0..b8d38eaef 100644 --- a/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/StatisticsPane.tsx +++ b/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/StatisticsPane.tsx @@ -587,15 +587,5 @@ export const StatisticsPane: React.FC = ({ No data (0 rows) available to generate insights ) - ) : ( - - Select a table to view - - ); + ) : null; }; diff --git a/src/pages/data-statistics/index.tsx b/src/pages/data-statistics/index.tsx index dbf89de3e..d21b5e4d4 100644 --- a/src/pages/data-statistics/index.tsx +++ b/src/pages/data-statistics/index.tsx @@ -32,31 +32,35 @@ export default function DataStatisticsPage() { return ( <> -
- - Data statistics - - +
+ - Select table + Data statistics - option.input_name} - groupBy={(option) => option.schema} - onChange={(model) => setSelectedSourceModel(model)} - /> + + + Select table + + option.input_name} + groupBy={(option) => option.schema} + onChange={(model) => setSelectedSourceModel(model)} + /> + + +
From 20563f74577d033903ac49731dbcf93077148688 Mon Sep 17 00:00:00 2001 From: mdshamoon Date: Tue, 17 Sep 2024 14:54:32 +0530 Subject: [PATCH 03/14] added new explore page --- src/assets/icons/explore.tsx | 31 ++++ src/components/Explore/Explore.tsx | 143 ++++++++++++++++++ .../LowerSectionTabs/StatisticsPane.tsx | 54 ++++++- .../FlowEditor/Components/ProjectTree.tsx | 19 +-- .../FlowEditor/FlowEditor.tsx | 18 ++- src/config/menu.tsx | 14 +- src/pages/data-statistics/index.tsx | 69 --------- src/pages/explore/index.tsx | 10 ++ 8 files changed, 261 insertions(+), 97 deletions(-) create mode 100644 src/assets/icons/explore.tsx create mode 100644 src/components/Explore/Explore.tsx delete mode 100644 src/pages/data-statistics/index.tsx create mode 100644 src/pages/explore/index.tsx diff --git a/src/assets/icons/explore.tsx b/src/assets/icons/explore.tsx new file mode 100644 index 000000000..f3b5c45ff --- /dev/null +++ b/src/assets/icons/explore.tsx @@ -0,0 +1,31 @@ +import * as React from 'react'; +const SvgComponent = (props: any) => ( + + + + + + + + +); +export default SvgComponent; diff --git a/src/components/Explore/Explore.tsx b/src/components/Explore/Explore.tsx new file mode 100644 index 000000000..7cb9ee187 --- /dev/null +++ b/src/components/Explore/Explore.tsx @@ -0,0 +1,143 @@ +import { PageHead } from '@/components/PageHead'; +import { DbtSourceModel } from '@/components/TransformWorkflow/FlowEditor/Components/Canvas'; +import { StatisticsPane } from '@/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/StatisticsPane'; + +import { httpGet } from '@/helpers/http'; +import { Box, Dialog, Divider, IconButton, Tab, Tabs } from '@mui/material'; +import { useSession } from 'next-auth/react'; +import { useRouter } from 'next/navigation'; +import { useEffect, useState } from 'react'; +import { Transition } from '@/components/DBT/DBTTransformType'; +import { ResizableBox } from 'react-resizable'; +import ProjectTree from '@/components/TransformWorkflow/FlowEditor/Components/ProjectTree'; +import PreviewPane from '@/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/PreviewPane'; +import { NodeApi } from 'react-arborist'; +import Close from '@mui/icons-material/Close'; +import { usePreviewAction } from '@/contexts/FlowEditorPreviewContext'; + +export const Explore = () => { + const { data: session } = useSession(); + const [selectedTab, setSelectedTab] = useState<'preview' | 'statistics'>( + 'preview' + ); + const router = useRouter(); + + const [dialogueOpen, setDialogueOpen] = useState(true); + const [width, setWidth] = useState(260); + + const [height, setheight] = useState(500); + const [sourceModels, setSourcesModels] = useState([]); + + const { setPreviewAction } = usePreviewAction(); + + const fetchSourcesModels = () => { + httpGet(session, 'transform/dbt_project/sources_models/') + .then((response: DbtSourceModel[]) => { + setSourcesModels(response); + }) + .catch((error) => { + console.log(error); + }); + }; + + useEffect(() => { + fetchSourcesModels(); + }, []); + + useEffect(() => { + const dialogBox = document.querySelector('.MuiDialog-container'); + + if (dialogBox) { + const fullHeight = dialogBox?.clientHeight - 50; + setheight(fullHeight); + } + }, [sourceModels]); + + const onResize = (event: any, { size }: any) => { + setWidth(size.width); + }; + + const handleTabChange = ( + event: React.SyntheticEvent, + newValue: 'preview' | 'statistics' + ) => { + setSelectedTab(newValue); + }; + + const handleNodeClick = (nodes: NodeApi[]) => { + if (nodes.length > 0 && nodes[0].isLeaf) { + setPreviewAction({ type: 'preview', data: nodes[0].data }); + } + }; + return ( + <> + + + + + + + + + + + + + + + + + { + setDialogueOpen(false); + router.push('/pipeline/ingest'); + }} + > + + + + + {selectedTab === 'preview' && } + + {selectedTab === 'statistics' && ( + + )} + + + + + + + ); +}; diff --git a/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/StatisticsPane.tsx b/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/StatisticsPane.tsx index b8d38eaef..922b60159 100644 --- a/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/StatisticsPane.tsx +++ b/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/StatisticsPane.tsx @@ -32,9 +32,26 @@ import { Session } from 'next-auth'; import { DateTimeInsights } from '@/components/Charts/DateTimeInsights'; import { StringInsights } from '@/components/Charts/StringInsights'; import { NumberInsights } from '@/components/Charts/NumberInsights'; +import { usePreviewAction } from '@/contexts/FlowEditorPreviewContext'; + +const useDebounce = (value: number, delay: number) => { + const [debouncedValue, setDebouncedValue] = useState(value); + + useEffect(() => { + const handler = setTimeout(() => { + setDebouncedValue(value); + }, delay); + + return () => { + clearTimeout(handler); + }; + }, [value, delay]); + + return debouncedValue; +}; interface StatisticsPaneProps { - modelToPreview: DbtSourceModel | null; + height: number; } export interface DateTimeFilter { @@ -131,10 +148,10 @@ export const pollTaskStatus = async ( return new Promise(poll); }; -export const StatisticsPane: React.FC = ({ - modelToPreview, -}) => { - // const [modelToPreview, setModelToPreview] = useState(); +export const StatisticsPane: React.FC = ({ height }) => { + const [modelToPreview, setModelToPreview] = useState(); + + const debouncedHeight = useDebounce(height, 500); const [rowCount, setRowCount] = useState(-1); const { data: session } = useSession(); @@ -143,6 +160,8 @@ export const StatisticsPane: React.FC = ({ // Row Data: The data to be displayed. const [data, setData] = useState([]); + const { previewAction } = usePreviewAction(); + const columns: ColumnDef[] = useMemo( () => [ { @@ -393,6 +412,14 @@ export const StatisticsPane: React.FC = ({ }; }, []); + useEffect(() => { + if (previewAction.type === 'preview') { + setModelToPreview(previewAction.data); + } else if (previewAction.type === 'clear-preview') { + setModelToPreview(null); + } + }, [previewAction]); + useEffect(() => { if (modelToPreview) { setData([]); @@ -475,7 +502,7 @@ export const StatisticsPane: React.FC = ({ - +
{getHeaderGroups().map((headerGroup) => ( @@ -567,6 +594,7 @@ export const StatisticsPane: React.FC = ({ ) : ( = ({ = ({ No data (0 rows) available to generate insights ) - ) : null; + ) : ( + + Select a table to view + + ); }; diff --git a/src/components/TransformWorkflow/FlowEditor/Components/ProjectTree.tsx b/src/components/TransformWorkflow/FlowEditor/Components/ProjectTree.tsx index 938dd77fd..7379173be 100644 --- a/src/components/TransformWorkflow/FlowEditor/Components/ProjectTree.tsx +++ b/src/components/TransformWorkflow/FlowEditor/Components/ProjectTree.tsx @@ -1,6 +1,6 @@ import { Box, Tooltip, Typography } from '@mui/material'; import React, { useContext, useEffect, useState } from 'react'; -import { Tree, NodeApi } from 'react-arborist'; +import { Tree } from 'react-arborist'; import FolderIcon from '@mui/icons-material/Folder'; import FolderOpenIcon from '@mui/icons-material/FolderOpen'; import TocIcon from '@/assets/icons/datatable.svg'; @@ -72,12 +72,15 @@ const Node = ({ node, style, dragHandle }: any) => { interface ProjectTreeProps { dbtSourceModels: DbtSourceModel[]; + handleNodeClick: any; } // type TreeData = Partial & { children: TreeData[] }; -const ProjectTree = ({ dbtSourceModels }: ProjectTreeProps) => { - const { setCanvasAction } = useCanvasAction(); +const ProjectTree = ({ + dbtSourceModels, + handleNodeClick, +}: ProjectTreeProps) => { const { ref, width, height } = useResizeObserver(); const [projectTreeData, setProjectTreeData] = useState([]); const globalContext = useContext(GlobalContext); @@ -118,16 +121,6 @@ const ProjectTree = ({ dbtSourceModels }: ProjectTreeProps) => { } }, [dbtSourceModels]); - const handleNodeClick = (nodes: NodeApi[]) => { - if (nodes.length > 0 && nodes[0].isLeaf) { - console.log( - 'adding a node to canvas from project tree component', - nodes[0].data - ); - setCanvasAction({ type: 'add-srcmodel-node', data: nodes[0].data }); - } - }; - return ( { const [width, setWidth] = useState(260); - + const { setCanvasAction } = useCanvasAction(); const onResize = (event: any, { size }: any) => { setWidth(size.width); }; + + const handleNodeClick = (nodes: NodeApi[]) => { + if (nodes.length > 0 && nodes[0].isLeaf) { + console.log( + 'adding a node to canvas from project tree component', + nodes[0].data + ); + setCanvasAction({ type: 'add-srcmodel-node', data: nodes[0].data }); + } + }; return ( - + diff --git a/src/config/menu.tsx b/src/config/menu.tsx index 0e747c683..3681bf2de 100644 --- a/src/config/menu.tsx +++ b/src/config/menu.tsx @@ -6,6 +6,7 @@ import PipelineIcon from '@/assets/icons/pipeline'; import OrchestrateIcon from '@/assets/icons/orchestrate'; import DataQualityIcon from '@/assets/icons/dataQuality'; import SupervisorAccountIcon from '@mui/icons-material/SupervisorAccount'; +import ExploreIcon from '@/assets/icons/explore'; import { primaryColor } from './theme'; @@ -79,6 +80,12 @@ export const sideMenu: MenuOption[] = [ }, { index: 3, + title: 'Explore', + path: '/explore', + icon: (selected: boolean) => , + }, + { + index: 4, title: 'Data Quality', path: '/data-quality', icon: (selected: boolean) => , @@ -86,12 +93,7 @@ export const sideMenu: MenuOption[] = [ // hide: !showElementaryMenu, minimize: true, }, - { - index: 4, - title: 'Data Statistics', - path: '/data-statistics', - icon: (selected: boolean) => , - }, + { index: 5, title: 'User management', diff --git a/src/pages/data-statistics/index.tsx b/src/pages/data-statistics/index.tsx deleted file mode 100644 index d21b5e4d4..000000000 --- a/src/pages/data-statistics/index.tsx +++ /dev/null @@ -1,69 +0,0 @@ -import { Elementary } from '@/components/DBT/Elementary'; -import { PageHead } from '@/components/PageHead'; -import { DbtSourceModel } from '@/components/TransformWorkflow/FlowEditor/Components/Canvas'; -import { StatisticsPane } from '@/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/StatisticsPane'; -import { Autocomplete } from '@/components/UI/Autocomplete/Autocomplete'; -import { httpGet } from '@/helpers/http'; -import { Box, Typography } from '@mui/material'; -import { useSession } from 'next-auth/react'; -import { useEffect, useState } from 'react'; -import styles from '@/styles/Home.module.css'; - -export default function DataStatisticsPage() { - const { data: session } = useSession(); - const [sourceModels, setSourcesModels] = useState([]); - const [selectedSourceModel, setSelectedSourceModel] = - useState(null); - console.log(selectedSourceModel); - - const fetchSourcesModels = () => { - httpGet(session, 'transform/dbt_project/sources_models/') - .then((response: DbtSourceModel[]) => { - setSourcesModels(response); - }) - .catch((error) => { - console.log(error); - }); - }; - - useEffect(() => { - fetchSourcesModels(); - }, []); - return ( - <> - -
- - - Data statistics - - - - Select table - - option.input_name} - groupBy={(option) => option.schema} - onChange={(model) => setSelectedSourceModel(model)} - /> - - - - - -
- - ); -} diff --git a/src/pages/explore/index.tsx b/src/pages/explore/index.tsx new file mode 100644 index 000000000..32730e25e --- /dev/null +++ b/src/pages/explore/index.tsx @@ -0,0 +1,10 @@ +import { Explore } from '@/components/Explore/Explore'; +import { PreviewActionProvider } from '@/contexts/FlowEditorPreviewContext'; + +export default function ExplorePage() { + return ( + + + + ); +} From 236128909aaf208113a67b5f3045fb614cbc8b79 Mon Sep 17 00:00:00 2001 From: mdshamoon Date: Mon, 23 Sep 2024 00:05:18 +0530 Subject: [PATCH 04/14] added basic test case --- src/components/Explore/Explore.tsx | 2 +- .../Explore/__tests__/Explore.test.tsx | 48 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 src/components/Explore/__tests__/Explore.test.tsx diff --git a/src/components/Explore/Explore.tsx b/src/components/Explore/Explore.tsx index 7cb9ee187..18e229f23 100644 --- a/src/components/Explore/Explore.tsx +++ b/src/components/Explore/Explore.tsx @@ -5,7 +5,7 @@ import { StatisticsPane } from '@/components/TransformWorkflow/FlowEditor/Compon import { httpGet } from '@/helpers/http'; import { Box, Dialog, Divider, IconButton, Tab, Tabs } from '@mui/material'; import { useSession } from 'next-auth/react'; -import { useRouter } from 'next/navigation'; +import { useRouter } from 'next/router'; import { useEffect, useState } from 'react'; import { Transition } from '@/components/DBT/DBTTransformType'; import { ResizableBox } from 'react-resizable'; diff --git a/src/components/Explore/__tests__/Explore.test.tsx b/src/components/Explore/__tests__/Explore.test.tsx new file mode 100644 index 000000000..d7a0a5978 --- /dev/null +++ b/src/components/Explore/__tests__/Explore.test.tsx @@ -0,0 +1,48 @@ +import { getByText, render, screen, waitFor } from '@testing-library/react'; +import { SessionProvider } from 'next-auth/react'; +import { Session } from 'next-auth'; +import { Explore } from '../Explore'; +import userEvent from '@testing-library/user-event'; +import '@testing-library/jest-dom'; + +const user = userEvent.setup(); + +jest.mock('next/router', () => ({ + useRouter() { + return { + push: jest.fn(), + }; + }, +})); + +window.ResizeObserver = + window.ResizeObserver || + jest.fn().mockImplementation(() => ({ + disconnect: jest.fn(), + observe: jest.fn(), + unobserve: jest.fn(), + })); + +const mockSession: Session = { + expires: 'false', + user: { email: 'a' }, +}; + +const mockedFetch = jest.fn().mockResolvedValueOnce({ + ok: true, + json: jest.fn().mockResolvedValueOnce([]), +}); +(global as any).fetch = mockedFetch; + +it('renders the explore page with preview and data statistics tab', async () => { + render( + + + + ); + + await waitFor(() => { + expect(screen.getByText('Preview')).toBeInTheDocument(); + expect(screen.getByText('Data statistics')).toBeInTheDocument(); + }); +}); From bce67a338ca777857b2215df4a68c3d5a5f97fc5 Mon Sep 17 00:00:00 2001 From: mdshamoon Date: Mon, 23 Sep 2024 00:06:36 +0530 Subject: [PATCH 05/14] added basic test case --- src/components/Explore/__tests__/Explore.test.tsx | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/components/Explore/__tests__/Explore.test.tsx b/src/components/Explore/__tests__/Explore.test.tsx index d7a0a5978..5aa23ffe0 100644 --- a/src/components/Explore/__tests__/Explore.test.tsx +++ b/src/components/Explore/__tests__/Explore.test.tsx @@ -1,12 +1,9 @@ -import { getByText, render, screen, waitFor } from '@testing-library/react'; +import { render, screen, waitFor } from '@testing-library/react'; import { SessionProvider } from 'next-auth/react'; import { Session } from 'next-auth'; import { Explore } from '../Explore'; -import userEvent from '@testing-library/user-event'; import '@testing-library/jest-dom'; -const user = userEvent.setup(); - jest.mock('next/router', () => ({ useRouter() { return { From d078f3bcb6af600ad36efb61ddbb7c4e83e215ce Mon Sep 17 00:00:00 2001 From: Ishankoradia Date: Wed, 9 Oct 2024 13:33:58 +0530 Subject: [PATCH 06/14] merge conflict --- .../FlowEditor/Components/ProjectTree.tsx | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/components/TransformWorkflow/FlowEditor/Components/ProjectTree.tsx b/src/components/TransformWorkflow/FlowEditor/Components/ProjectTree.tsx index 730b28f11..21a9c98a2 100644 --- a/src/components/TransformWorkflow/FlowEditor/Components/ProjectTree.tsx +++ b/src/components/TransformWorkflow/FlowEditor/Components/ProjectTree.tsx @@ -115,13 +115,6 @@ const ProjectTree = ({ dbtSourceModels, handleNodeClick }: ProjectTreeProps) => } }, [dbtSourceModels]); - const handleNodeClick = (nodes: NodeApi[]) => { - if (nodes.length > 0 && nodes[0].isLeaf) { - console.log('adding a node to canvas from project tree component', nodes[0].data); - setCanvasAction({ type: 'add-srcmodel-node', data: nodes[0].data }); - } - }; - return ( Date: Thu, 10 Oct 2024 22:07:21 +0530 Subject: [PATCH 07/14] make the project tree's sync click controllable --- src/components/Explore/Explore.tsx | 23 +++++++++++----- .../FlowEditor/Components/Canvas.tsx | 22 +++++++++++---- .../FlowEditor/Components/ProjectTree.tsx | 27 +++++++++---------- .../FlowEditor/FlowEditor.tsx | 21 ++++++++++----- 4 files changed, 62 insertions(+), 31 deletions(-) diff --git a/src/components/Explore/Explore.tsx b/src/components/Explore/Explore.tsx index 04b601d55..d9147d208 100644 --- a/src/components/Explore/Explore.tsx +++ b/src/components/Explore/Explore.tsx @@ -1,12 +1,15 @@ import { PageHead } from '@/components/PageHead'; -import { DbtSourceModel } from '@/components/TransformWorkflow/FlowEditor/Components/Canvas'; +import { + DbtSourceModel, + WarehouseTable, +} from '@/components/TransformWorkflow/FlowEditor/Components/Canvas'; import { StatisticsPane } from '@/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/StatisticsPane'; import { httpGet } from '@/helpers/http'; import { Box, Dialog, Divider, IconButton, Tab, Tabs } from '@mui/material'; import { useSession } from 'next-auth/react'; import { useRouter } from 'next/router'; -import { useEffect, useState } from 'react'; +import { useContext, useEffect, useState } from 'react'; import { Transition } from '@/components/DBT/DBTTransformType'; import { ResizableBox } from 'react-resizable'; import ProjectTree from '@/components/TransformWorkflow/FlowEditor/Components/ProjectTree'; @@ -14,24 +17,28 @@ import PreviewPane from '@/components/TransformWorkflow/FlowEditor/Components/Lo import { NodeApi } from 'react-arborist'; import Close from '@mui/icons-material/Close'; import { usePreviewAction } from '@/contexts/FlowEditorPreviewContext'; +import { successToast } from '../ToastMessage/ToastHelper'; +import { GlobalContext } from '@/contexts/ContextProvider'; export const Explore = () => { const { data: session } = useSession(); const [selectedTab, setSelectedTab] = useState<'preview' | 'statistics'>('preview'); const router = useRouter(); + const globalContext = useContext(GlobalContext); const [dialogueOpen, setDialogueOpen] = useState(true); const [width, setWidth] = useState(260); const [height, setheight] = useState(500); - const [sourceModels, setSourcesModels] = useState([]); + const [sourceModels, setSourcesModels] = useState([]); const { setPreviewAction } = usePreviewAction(); const fetchSourcesModels = () => { - httpGet(session, 'transform/dbt_project/sources_models/') - .then((response: DbtSourceModel[]) => { + httpGet(session, 'warehouse/sync_tables') + .then((response: WarehouseTable[]) => { setSourcesModels(response); + successToast('Tables synced with warehouse', [], globalContext); }) .catch((error) => { console.log(error); @@ -85,7 +92,11 @@ export const Explore = () => { maxConstraints={[550, Infinity]} resizeHandles={['e']} > - + diff --git a/src/components/TransformWorkflow/FlowEditor/Components/Canvas.tsx b/src/components/TransformWorkflow/FlowEditor/Components/Canvas.tsx index e29e5b568..e9710c4cd 100644 --- a/src/components/TransformWorkflow/FlowEditor/Components/Canvas.tsx +++ b/src/components/TransformWorkflow/FlowEditor/Components/Canvas.tsx @@ -67,15 +67,27 @@ export interface OperationNodeData { seq?: number; } -export type DbtSourceModel = { - source_name: string; +// export type DbtSourceModel = { +// source_name: string; +// input_name: string; +// input_type: 'model' | 'source'; +// schema: string; +// id: string; +// type: typeof SRC_MODEL_NODE; +// isDummy?: boolean; +// }; + +export interface WarehouseTable { + id: string; input_name: string; - input_type: 'model' | 'source'; schema: string; - id: string; type: typeof SRC_MODEL_NODE; +} +export interface DbtSourceModel extends WarehouseTable { + source_name: string; + input_type: 'model' | 'source'; isDummy?: boolean; -}; +} // export interface OperationNodeType extends NodeProps { // data: OperationNodeData; diff --git a/src/components/TransformWorkflow/FlowEditor/Components/ProjectTree.tsx b/src/components/TransformWorkflow/FlowEditor/Components/ProjectTree.tsx index 21a9c98a2..2123fd87d 100644 --- a/src/components/TransformWorkflow/FlowEditor/Components/ProjectTree.tsx +++ b/src/components/TransformWorkflow/FlowEditor/Components/ProjectTree.tsx @@ -4,7 +4,7 @@ import { Tree } from 'react-arborist'; import FolderIcon from '@mui/icons-material/Folder'; import FolderOpenIcon from '@mui/icons-material/FolderOpen'; import TocIcon from '@/assets/icons/datatable.svg'; -import { DbtSourceModel } from './Canvas'; +import { DbtSourceModel, WarehouseTable } from './Canvas'; import AddIcon from '@mui/icons-material/Add'; import { useCanvasAction } from '@/contexts/FlowEditorCanvasContext'; import useResizeObserver from 'use-resize-observer'; @@ -13,7 +13,8 @@ import Image from 'next/image'; import ReplayIcon from '@mui/icons-material/Replay'; import { GlobalContext } from '@/contexts/ContextProvider'; -const Node = ({ node, style, dragHandle }: any) => { +const Node = ({ node, style, dragHandle, tree, handleSyncClick }: any) => { + console.log(tree.props); const globalContext = useContext(GlobalContext); const permissions = globalContext?.Permissions.state || []; const width = node.tree.props.width; @@ -23,7 +24,7 @@ const Node = ({ node, style, dragHandle }: any) => { const data: DbtSourceModel = node.data; let name: string | JSX.Element = !node.isLeaf ? data.schema : data.input_name; name = trimString(name, stringLengthWithWidth); - const { setCanvasAction } = useCanvasAction(); + return ( { }} onClick={(event) => { event.stopPropagation(); - if (permissions.includes('can_sync_sources')) - setCanvasAction({ type: 'sync-sources', data: null }); + console.log('here clicking the sync button'); + handleSyncClick(); }} /> @@ -69,19 +70,18 @@ const Node = ({ node, style, dragHandle }: any) => { }; interface ProjectTreeProps { - dbtSourceModels: DbtSourceModel[]; - handleNodeClick: any; + dbtSourceModels: WarehouseTable[]; + handleNodeClick: (...args: any) => void; + handleSyncClick: (...args: any) => void; } -// type TreeData = Partial & { children: TreeData[] }; - -const ProjectTree = ({ dbtSourceModels, handleNodeClick }: ProjectTreeProps) => { +const ProjectTree = ({ dbtSourceModels, handleNodeClick, handleSyncClick }: ProjectTreeProps) => { const { ref, width, height } = useResizeObserver(); const [projectTreeData, setProjectTreeData] = useState([]); const globalContext = useContext(GlobalContext); const permissions = globalContext?.Permissions.state || []; - const constructAndSetProjectTreeData = (dbtSourceModels: DbtSourceModel[]) => { + const constructAndSetProjectTreeData = (dbtSourceModels: WarehouseTable[]) => { // group by schema and push dbtSourceModels under the children key const leafNodesBySchema = dbtSourceModels.reduce( (acc, dbtSourceModel) => { @@ -93,7 +93,7 @@ const ProjectTree = ({ dbtSourceModels, handleNodeClick }: ProjectTreeProps) => } return acc; }, - {} as { [key: string]: DbtSourceModel[] } + {} as { [key: string]: WarehouseTable[] } ); // construct the tree data @@ -110,7 +110,6 @@ const ProjectTree = ({ dbtSourceModels, handleNodeClick }: ProjectTreeProps) => useEffect(() => { if (dbtSourceModels) { - console.log('rerendeirng project tree'); constructAndSetProjectTreeData(dbtSourceModels); } }, [dbtSourceModels]); @@ -149,7 +148,7 @@ const ProjectTree = ({ dbtSourceModels, handleNodeClick }: ProjectTreeProps) => rowHeight={30} onSelect={permissions.includes('can_create_dbt_model') ? handleNodeClick : undefined} > - {Node} + {(props) => } diff --git a/src/components/TransformWorkflow/FlowEditor/FlowEditor.tsx b/src/components/TransformWorkflow/FlowEditor/FlowEditor.tsx index d1104c539..a8c941ee0 100644 --- a/src/components/TransformWorkflow/FlowEditor/FlowEditor.tsx +++ b/src/components/TransformWorkflow/FlowEditor/FlowEditor.tsx @@ -38,6 +38,7 @@ const UpperSection = ({ setTempLockCanvas, }: UpperSectionProps) => { const [width, setWidth] = useState(260); + const globalContext = useContext(GlobalContext); const { setCanvasAction } = useCanvasAction(); const onResize = (event: any, { size }: any) => { setWidth(size.width); @@ -49,6 +50,13 @@ const UpperSection = ({ setCanvasAction({ type: 'add-srcmodel-node', data: nodes[0].data }); } }; + + const initiateSyncSources = () => { + const permissions = globalContext?.Permissions.state || []; + if (permissions.includes('can_sync_sources')) { + setCanvasAction({ type: 'sync-sources', data: null }); + } + }; return ( - + @@ -262,13 +274,10 @@ const FlowEditor = ({}) => { // Clear previous logs setDbtRunLogs([]); - const orgslug = globalContext?.CurrentOrg.state.slug; - const syncSourcesHashKey = `syncsources-${orgslug}`; - const response: any = await httpPost(session, `transform/dbt_project/sync_sources/`, {}); - if (response?.task_id && orgslug) { - await pollForSyncSourcesTask(response.task_id, syncSourcesHashKey); + if (response?.task_id && response?.hashkey) { + await pollForSyncSourcesTask(response.task_id, response.hashkey); } } catch (error) { console.error(error); From a372576567c2888983b72b91956f44150f5509c6 Mon Sep 17 00:00:00 2001 From: Ishankoradia Date: Fri, 11 Oct 2024 11:28:50 +0530 Subject: [PATCH 08/14] add a loader for the first time, might take some time --- src/components/Explore/Explore.tsx | 6 +++ .../FlowEditor/Components/ProjectTree.tsx | 51 ++++++++++++------- 2 files changed, 39 insertions(+), 18 deletions(-) diff --git a/src/components/Explore/Explore.tsx b/src/components/Explore/Explore.tsx index d9147d208..396c5000d 100644 --- a/src/components/Explore/Explore.tsx +++ b/src/components/Explore/Explore.tsx @@ -25,6 +25,7 @@ export const Explore = () => { const [selectedTab, setSelectedTab] = useState<'preview' | 'statistics'>('preview'); const router = useRouter(); const globalContext = useContext(GlobalContext); + const [loading, setLoading] = useState(false); const [dialogueOpen, setDialogueOpen] = useState(true); const [width, setWidth] = useState(260); @@ -35,6 +36,7 @@ export const Explore = () => { const { setPreviewAction } = usePreviewAction(); const fetchSourcesModels = () => { + setLoading(true); httpGet(session, 'warehouse/sync_tables') .then((response: WarehouseTable[]) => { setSourcesModels(response); @@ -42,6 +44,9 @@ export const Explore = () => { }) .catch((error) => { console.log(error); + }) + .finally(() => { + setLoading(false); }); }; @@ -96,6 +101,7 @@ export const Explore = () => { dbtSourceModels={sourceModels} handleNodeClick={handleNodeClick} handleSyncClick={fetchSourcesModels} + isSyncing={loading} /> diff --git a/src/components/TransformWorkflow/FlowEditor/Components/ProjectTree.tsx b/src/components/TransformWorkflow/FlowEditor/Components/ProjectTree.tsx index 2123fd87d..3315f41e2 100644 --- a/src/components/TransformWorkflow/FlowEditor/Components/ProjectTree.tsx +++ b/src/components/TransformWorkflow/FlowEditor/Components/ProjectTree.tsx @@ -1,4 +1,4 @@ -import { Box, Tooltip, Typography } from '@mui/material'; +import { Box, CircularProgress, Tooltip, Typography } from '@mui/material'; import React, { useContext, useEffect, useState } from 'react'; import { Tree } from 'react-arborist'; import FolderIcon from '@mui/icons-material/Folder'; @@ -6,15 +6,15 @@ import FolderOpenIcon from '@mui/icons-material/FolderOpen'; import TocIcon from '@/assets/icons/datatable.svg'; import { DbtSourceModel, WarehouseTable } from './Canvas'; import AddIcon from '@mui/icons-material/Add'; -import { useCanvasAction } from '@/contexts/FlowEditorCanvasContext'; import useResizeObserver from 'use-resize-observer'; import { trimString } from '@/utils/common'; import Image from 'next/image'; import ReplayIcon from '@mui/icons-material/Replay'; import { GlobalContext } from '@/contexts/ContextProvider'; +import SyncIcon from '@/assets/icons/sync.svg'; +import styles from '@/styles/Common.module.css'; -const Node = ({ node, style, dragHandle, tree, handleSyncClick }: any) => { - console.log(tree.props); +const Node = ({ node, style, dragHandle, handleSyncClick, isSyncing }: any) => { const globalContext = useContext(GlobalContext); const permissions = globalContext?.Permissions.state || []; const width = node.tree.props.width; @@ -48,22 +48,31 @@ const Node = ({ node, style, dragHandle, tree, handleSyncClick }: any) => { {name} {node.isLeaf && } - {!node.isLeaf && node.level === 0 && ( - - + { + event.stopPropagation(); + console.log('here clicking the sync button'); + handleSyncClick(); + }} + /> + + ) : ( + { - event.stopPropagation(); - console.log('here clicking the sync button'); - handleSyncClick(); }} + size={24} /> - - )} + ))} ); @@ -73,9 +82,15 @@ interface ProjectTreeProps { dbtSourceModels: WarehouseTable[]; handleNodeClick: (...args: any) => void; handleSyncClick: (...args: any) => void; + isSyncing?: boolean; } -const ProjectTree = ({ dbtSourceModels, handleNodeClick, handleSyncClick }: ProjectTreeProps) => { +const ProjectTree = ({ + dbtSourceModels, + handleNodeClick, + handleSyncClick, + isSyncing = false, +}: ProjectTreeProps) => { const { ref, width, height } = useResizeObserver(); const [projectTreeData, setProjectTreeData] = useState([]); const globalContext = useContext(GlobalContext); @@ -148,7 +163,7 @@ const ProjectTree = ({ dbtSourceModels, handleNodeClick, handleSyncClick }: Proj rowHeight={30} onSelect={permissions.includes('can_create_dbt_model') ? handleNodeClick : undefined} > - {(props) => } + {(props) => } From 799836ca432c4c04e09e06e03a95b65cf8f456d6 Mon Sep 17 00:00:00 2001 From: himanshudube97 Date: Wed, 16 Oct 2024 11:53:18 +0530 Subject: [PATCH 09/14] issue-Let the text in data insights say 'Select a table from the left pane to view' #1275 resolved --- .../FlowEditor/Components/LowerSectionTabs/PreviewPane.tsx | 2 +- .../FlowEditor/Components/LowerSectionTabs/StatisticsPane.tsx | 2 +- .../LowerSectionTabs/__tests__/PreviewPane.test.tsx | 4 ++-- .../LowerSectionTabs/__tests__/StatisticsPane.test.tsx | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/PreviewPane.tsx b/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/PreviewPane.tsx index e3722925f..df7c71075 100644 --- a/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/PreviewPane.tsx +++ b/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/PreviewPane.tsx @@ -295,7 +295,7 @@ const PreviewPane = ({ height }: { height: number }) => { height: height, }} > - Select a table to view + Select a table from the left pane to view ); }; diff --git a/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/StatisticsPane.tsx b/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/StatisticsPane.tsx index d77fc0559..66b72e95c 100644 --- a/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/StatisticsPane.tsx +++ b/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/StatisticsPane.tsx @@ -578,7 +578,7 @@ export const StatisticsPane: React.FC = ({ height }) => { height: debouncedHeight, }} > - Select a table to view + Select a table from the left pane to view ); }; diff --git a/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/__tests__/PreviewPane.test.tsx b/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/__tests__/PreviewPane.test.tsx index 0a5a89169..8f8d360bf 100644 --- a/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/__tests__/PreviewPane.test.tsx +++ b/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/__tests__/PreviewPane.test.tsx @@ -109,11 +109,11 @@ describe('PreviewPane Component', () => { }); }); - test('renders "Select a table to view" message when no table is selected', () => { + test('renders "Select a table from the left pane to view " message when no table is selected', () => { usePreviewAction.mockReturnValue({ previewAction: { type: 'clear-preview' } }); render(); - expect(screen.getByText('Select a table to view')).toBeInTheDocument(); + expect(screen.getByText('Select a table from the left pane to view ')).toBeInTheDocument(); }); }); diff --git a/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/__tests__/StatisticsPane.test.tsx b/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/__tests__/StatisticsPane.test.tsx index f3277456f..d5f79db15 100644 --- a/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/__tests__/StatisticsPane.test.tsx +++ b/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/__tests__/StatisticsPane.test.tsx @@ -122,7 +122,7 @@ describe('StatisticsPane', () => { ); // Check for the message when no model is selected - expect(screen.getByText(/Select a table to view/i)).toBeInTheDocument(); + expect(screen.getByText(/Select a table from the left pane to view /i)).toBeInTheDocument(); }); it('should resolve when status is completed', async () => { From 81d6370b832e54288b07e1fe4a20b2ad70d8c1af Mon Sep 17 00:00:00 2001 From: himanshudube97 Date: Wed, 16 Oct 2024 13:38:49 +0530 Subject: [PATCH 10/14] Data Insights: Remove the store folder #1273 --- .../FlowEditor/Components/ProjectTree.tsx | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/components/TransformWorkflow/FlowEditor/Components/ProjectTree.tsx b/src/components/TransformWorkflow/FlowEditor/Components/ProjectTree.tsx index 3315f41e2..9c683ccf6 100644 --- a/src/components/TransformWorkflow/FlowEditor/Components/ProjectTree.tsx +++ b/src/components/TransformWorkflow/FlowEditor/Components/ProjectTree.tsx @@ -11,8 +11,6 @@ import { trimString } from '@/utils/common'; import Image from 'next/image'; import ReplayIcon from '@mui/icons-material/Replay'; import { GlobalContext } from '@/contexts/ContextProvider'; -import SyncIcon from '@/assets/icons/sync.svg'; -import styles from '@/styles/Common.module.css'; const Node = ({ node, style, dragHandle, handleSyncClick, isSyncing }: any) => { const globalContext = useContext(GlobalContext); @@ -24,6 +22,11 @@ const Node = ({ node, style, dragHandle, handleSyncClick, isSyncing }: any) => { const data: DbtSourceModel = node.data; let name: string | JSX.Element = !node.isLeaf ? data.schema : data.input_name; name = trimString(name, stringLengthWithWidth); + useEffect(() => { + if (!node.isLeaf && node.level === 0 && !node.isOpen) { + node.toggle(); + } + }, [node]); return ( { width: (250 * width) / 270 + 'px', opacity: permissions.includes('can_create_dbt_model') ? 1 : 0.5, }} - onClick={() => (node.isLeaf ? undefined : node.toggle())} + onClick={() => (node.isLeaf || node.level === 0 ? undefined : node.toggle())} > {node.isLeaf ? ( @@ -120,7 +123,7 @@ const ProjectTree = ({ }; }); - setProjectTreeData([{ id: '0', schema: 'Store', children: treeData }]); + setProjectTreeData([{ id: '0', schema: 'Schemas', children: treeData }]); }; useEffect(() => { @@ -128,7 +131,6 @@ const ProjectTree = ({ constructAndSetProjectTreeData(dbtSourceModels); } }, [dbtSourceModels]); - return ( Date: Wed, 16 Oct 2024 17:25:39 +0530 Subject: [PATCH 11/14] fixed the margis ui etc --- src/components/Explore/Explore.tsx | 7 ++++++- .../Components/LowerSectionTabs/PreviewPane.tsx | 14 +++++++------- .../Components/LowerSectionTabs/StatisticsPane.tsx | 14 ++++++++------ .../TransformWorkflow/FlowEditor/FlowEditor.tsx | 2 +- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/components/Explore/Explore.tsx b/src/components/Explore/Explore.tsx index 396c5000d..db866e0c2 100644 --- a/src/components/Explore/Explore.tsx +++ b/src/components/Explore/Explore.tsx @@ -120,7 +120,12 @@ export const Explore = () => { diff --git a/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/PreviewPane.tsx b/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/PreviewPane.tsx index df7c71075..bafea67e6 100644 --- a/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/PreviewPane.tsx +++ b/src/components/TransformWorkflow/FlowEditor/Components/LowerSectionTabs/PreviewPane.tsx @@ -184,16 +184,16 @@ const PreviewPane = ({ height }: { height: number }) => { alignItems: 'center', display: 'flex', justifyContent: 'space-between', - padding: '5px', + padding: '8px 8px 8px 44px', }} > - + {modelToPreview?.input_name}