feat(CE): refresh catalog

Multiwoven · Jul 15, 2024 · fba4c85 · fba4c85
1 parent 7c1e24f
commit fba4c85
Show file tree

Hide file tree

Showing 6 changed files with 86 additions and 35 deletions.
diff --git a/ui/src/services/syncs.ts b/ui/src/services/syncs.ts
@@ -9,10 +9,13 @@ import {
 } from '@/views/Activate/Syncs/types';
 import { multiwovenFetch, ApiResponse } from './common';
 
-export const getCatalog = (connectorId: string): Promise<DiscoverResponse> =>
+export const getCatalog = (
+  connectorId: string,
+  refresh: boolean = false,
+): Promise<DiscoverResponse> =>
   multiwovenFetch<null, DiscoverResponse>({
     method: 'get',
-    url: `/connectors/${connectorId}/discover`,
+    url: `/connectors/${connectorId}/discover?refresh=${refresh}`,
   });
 
 export const createSync = (payload: CreateSyncPayload): Promise<ApiResponse<CreateSyncResponse>> =>

diff --git a/ui/src/views/Activate/Syncs/EditSync/EditSync.tsx b/ui/src/views/Activate/Syncs/EditSync/EditSync.tsx
@@ -33,6 +33,7 @@ const EditSync = (): JSX.Element | null => {
   const [selectedSyncMode, setSelectedSyncMode] = useState('');
   const [cursorField, setCursorField] = useState('');
   const activeWorkspaceId = useStore((state) => state.workspaceId);
+  const [refresh, setRefresh] = useState(false);
 
   const { syncId } = useParams();
   const showToast = useCustomToast();
@@ -61,9 +62,9 @@ const EditSync = (): JSX.Element | null => {
     enabled: !!syncData?.destination.id && activeWorkspaceId > 0,
   });
 
-  const { data: catalogData } = useQuery({
+  const { data: catalogData, refetch } = useQuery({
     queryKey: ['syncs', 'catalog', syncData?.destination.id, activeWorkspaceId],
-    queryFn: () => getCatalog(syncData?.destination?.id as string),
+    queryFn: () => getCatalog(syncData?.destination?.id as string, refresh),
     enabled: !!syncData?.destination.id && activeWorkspaceId > 0,
     refetchOnMount: false,
     refetchOnWindowFocus: false,
@@ -145,6 +146,17 @@ const EditSync = (): JSX.Element | null => {
     },
   });
 
+  const handleRefreshCatalog = () => {
+    setRefresh(true);
+  };
+
+  useEffect(() => {
+    if (refresh) {
+      refetch();
+      setRefresh(false);
+    }
+  }, [refresh]);
+
   useEffect(() => {
     if (isError) {
       showToast({
@@ -193,10 +205,19 @@ const EditSync = (): JSX.Element | null => {
       setSelectedStream(selectedStream);
     }
   };
+
   const handleOnConfigChange = (config: FieldMapType[]) => {
     setConfiguration(config);
   };
 
+  useEffect(() => {
+    if (catalogData) {
+      handleOnStreamsLoad(catalogData);
+    }
+  }, [catalogData]);
+
+  const streams = catalogData?.data?.attributes?.catalog?.streams || [];
+
   return (
     <form onSubmit={formik.handleSubmit} style={{ backgroundColor: 'gray.200' }}>
       <Box width='100%' pt='20px'>
@@ -208,13 +229,13 @@ const EditSync = (): JSX.Element | null => {
               <SelectStreams
                 model={syncData?.model}
                 destination={destinationFetchResponse?.data}
-                onStreamsLoad={handleOnStreamsLoad}
                 isEdit
                 setSelectedSyncMode={setSelectedSyncMode}
                 selectedSyncMode={selectedSyncMode}
                 selectedStreamName={syncData?.stream_name}
                 selectedCursorField={cursorField}
                 setCursorField={setCursorField}
+                streams={streams}
               />
               {catalogData?.data.attributes.catalog.schema_mode === SchemaMode.schemaless ? (
                 <MapCustomFields
@@ -235,6 +256,7 @@ const EditSync = (): JSX.Element | null => {
                   data={configuration}
                   isEdit
                   configuration={configuration}
+                  handleRefreshCatalog={handleRefreshCatalog}
                 />
               )}
             </>

diff --git a/ui/src/views/Activate/Syncs/SyncForm/ConfigureSyncs/ConfigureSyncs.tsx b/ui/src/views/Activate/Syncs/SyncForm/ConfigureSyncs/ConfigureSyncs.tsx
@@ -2,7 +2,7 @@ import ContentContainer from '@/components/ContentContainer';
 import { SteppedFormContext } from '@/components/SteppedForm/SteppedForm';
 import { ModelEntity } from '@/views/Models/types';
 import { Box } from '@chakra-ui/react';
-import { FormEvent, useContext, Dispatch, SetStateAction } from 'react';
+import { FormEvent, useContext, Dispatch, SetStateAction, useState, useEffect } from 'react';
 import SelectStreams from './SelectStreams';
 import { Stream, FieldMap as FieldMapType } from '@/views/Activate/Syncs/types';
 import MapFields from './MapFields';
@@ -40,6 +40,7 @@ const ConfigureSyncs = ({
   setCursorField,
 }: ConfigureSyncsProps): JSX.Element | null => {
   const { state, stepInfo, handleMoveForward } = useContext(SteppedFormContext);
+  const [refresh, setRefresh] = useState(false);
 
   const { forms } = state;
 
@@ -74,14 +75,25 @@ const ConfigureSyncs = ({
     handleMoveForward(stepInfo?.formKey as string, payload);
   };
 
-  const { data: catalogData } = useQuery({
+  const { data: catalogData, refetch } = useQuery({
     queryKey: ['syncs', 'catalog', selectedDestination?.id, activeWorkspaceId],
-    queryFn: () => getCatalog(selectedDestination?.id),
+    queryFn: () => getCatalog(selectedDestination?.id, refresh),
     enabled: !!selectedDestination?.id && activeWorkspaceId > 0,
     refetchOnMount: false,
     refetchOnWindowFocus: false,
   });
 
+  const handleRefreshCatalog = () => {
+    setRefresh(true);
+  };
+
+  useEffect(() => {
+    if (refresh) {
+      refetch();
+      setRefresh(false);
+    }
+  }, [refresh]);
+
   if (!catalogData?.data?.attributes?.catalog?.schema_mode) {
     return <Loader />;
   }
@@ -90,6 +102,8 @@ const ConfigureSyncs = ({
     setSchemaMode(SchemaMode.schemaless);
   }
 
+  const streams = catalogData?.data?.attributes?.catalog?.streams || [];
+
   return (
     <Box width='100%' display='flex' justifyContent='center'>
       <ContentContainer>
@@ -103,6 +117,7 @@ const ConfigureSyncs = ({
             selectedSyncMode={selectedSyncMode}
             selectedCursorField={cursorField}
             setCursorField={setCursorField}
+            streams={streams}
           />
           {catalogData?.data?.attributes?.catalog?.schema_mode === SchemaMode.schemaless ? (
             <MapCustomFields
@@ -119,6 +134,7 @@ const ConfigureSyncs = ({
               stream={selectedStream}
               handleOnConfigChange={handleOnConfigChange}
               configuration={configuration}
+              handleRefreshCatalog={handleRefreshCatalog}
             />
           )}
 

diff --git a/ui/src/views/Activate/Syncs/SyncForm/ConfigureSyncs/FieldMap.tsx b/ui/src/views/Activate/Syncs/SyncForm/ConfigureSyncs/FieldMap.tsx
@@ -1,14 +1,16 @@
 import EntityItem from '@/components/EntityItem';
-import { Box, Input } from '@chakra-ui/react';
+import { Box, Input, Button } from '@chakra-ui/react';
 import TemplateMapping from './TemplateMapping/TemplateMapping';
 import { FieldMap as FieldMapType } from '@/views/Activate/Syncs/types';
 import { OPTION_TYPE } from './TemplateMapping/TemplateMapping';
+import { FiRefreshCcw } from 'react-icons/fi';
 
 type FieldMapProps = {
   id: number;
   fieldType: 'model' | 'destination' | 'custom';
   icon: string;
   entityName: string;
+  handleRefreshCatalog?: () => void;
   options?: string[];
   value?: string;
   disabledOptions?: string[];
@@ -17,6 +19,26 @@ type FieldMapProps = {
   selectedConfigOptions?: FieldMapType[] | null;
 };
 
+const RenderRefreshButton = ({ handleRefreshCatalog }: { handleRefreshCatalog: () => void }) => (
+  <Button
+    color='black.500'
+    borderRadius='6px'
+    onClick={handleRefreshCatalog}
+    leftIcon={<FiRefreshCcw color='gray.100' />}
+    backgroundColor='gray.200'
+    variant='shell'
+    height='32px'
+    minWidth={0}
+    width='auto'
+    fontSize='12px'
+    fontWeight={700}
+    lineHeight='18px'
+    letterSpacing='-0.12px'
+  >
+    Refresh
+  </Button>
+);
+
 const FieldMap = ({
   id,
   fieldType,
@@ -27,11 +49,15 @@ const FieldMap = ({
   onChange,
   isDisabled,
   selectedConfigOptions,
+  handleRefreshCatalog,
 }: FieldMapProps): JSX.Element => {
   return (
     <Box width='100%'>
-      <Box marginBottom='10px'>
+      <Box marginBottom='10px' display='flex' justifyContent='space-between'>
         <EntityItem icon={icon} name={entityName} />
+        {fieldType === 'destination' && id === 0 && (
+          <RenderRefreshButton handleRefreshCatalog={handleRefreshCatalog as () => void} />
+        )}
       </Box>
       <Box position='relative'>
         {fieldType === 'custom' ? (

diff --git a/ui/src/views/Activate/Syncs/SyncForm/ConfigureSyncs/MapFields.tsx b/ui/src/views/Activate/Syncs/SyncForm/ConfigureSyncs/MapFields.tsx
@@ -15,6 +15,7 @@ type MapFieldsProps = {
   model: ModelEntity;
   destination: ConnectorItem;
   stream: Stream | null;
+  handleRefreshCatalog: () => void;
   data?: FieldMapType[] | null;
   isEdit?: boolean;
   handleOnConfigChange: (args: FieldMapType[]) => void;
@@ -29,6 +30,7 @@ const MapFields = ({
   isEdit,
   handleOnConfigChange,
   configuration,
+  handleRefreshCatalog,
 }: MapFieldsProps): JSX.Element | null => {
   const [fields, setFields] = useState<FieldMapType[]>([{ from: '', to: '', mapping_type: '' }]);
 
@@ -177,6 +179,7 @@ const MapFields = ({
             onChange={handleOnChange}
             isDisabled={!stream || isRequired}
             selectedConfigOptions={configuration}
+            handleRefreshCatalog={handleRefreshCatalog}
           />
           <Box
             py='20px'

diff --git a/ui/src/views/Activate/Syncs/SyncForm/ConfigureSyncs/SelectStreams.tsx b/ui/src/views/Activate/Syncs/SyncForm/ConfigureSyncs/SelectStreams.tsx
@@ -1,23 +1,23 @@
 import { Box, Text, Select, Tooltip, Input } from '@chakra-ui/react';
-import { DiscoverResponse, Stream } from '@/views/Activate/Syncs/types';
+import { Stream } from '@/views/Activate/Syncs/types';
 import { ModelEntity } from '@/views/Models/types';
 import { useQuery } from '@tanstack/react-query';
 import { getCatalog } from '@/services/syncs';
 import { ConnectorItem } from '@/views/Connectors/types';
 import { getModelPreviewById } from '@/services/models';
-import { useEffect, SetStateAction, Dispatch } from 'react';
+import { SetStateAction, Dispatch } from 'react';
 import { FiInfo } from 'react-icons/fi';
 import { useStore } from '@/stores';
 
 type SelectStreamsProps = {
   model: ModelEntity;
   destination: ConnectorItem;
+  streams: Stream[];
   selectedStreamName?: string;
   selectedSyncMode?: string;
   isEdit?: boolean;
   placeholder?: string;
   onChange?: (stream: Stream) => void;
-  onStreamsLoad?: (catalog: DiscoverResponse) => void;
   selectedStream?: Stream | null;
   setSelectedSyncMode?: Dispatch<SetStateAction<string>>;
   setCursorField?: Dispatch<SetStateAction<string>>;
@@ -27,27 +27,19 @@ type SelectStreamsProps = {
 const SelectStreams = ({
   model,
   destination,
+  streams,
   selectedSyncMode,
   selectedStreamName,
   isEdit,
   placeholder,
   onChange,
-  onStreamsLoad,
   selectedStream,
   setSelectedSyncMode,
   selectedCursorField,
   setCursorField,
 }: SelectStreamsProps): JSX.Element | null => {
   const activeWorkspaceId = useStore((state) => state.workspaceId);
 
-  const { data: catalogData } = useQuery({
-    queryKey: ['syncs', 'catalog', destination.id, activeWorkspaceId],
-    queryFn: () => getCatalog(destination.id),
-    enabled: !!destination.id && activeWorkspaceId > 0,
-    refetchOnMount: false,
-    refetchOnWindowFocus: false,
-  });
-
   const { data: modelDiscoverData } = useQuery({
     queryKey: ['syncs', 'catalog', model.id, activeWorkspaceId],
     queryFn: () => getCatalog(model.id),
@@ -68,13 +60,6 @@ const SelectStreams = ({
 
   const modelColumns = Object.keys(firstRow ?? {});
 
-  useEffect(() => {
-    if (catalogData) {
-      onStreamsLoad?.(catalogData);
-    }
-  }, [catalogData]);
-
-  const streams = catalogData?.data?.attributes?.catalog?.streams;
   let selectedStreamIndex = -1;
 
   const handleOnStreamChange = (streamNumber: string) => {
@@ -87,13 +72,9 @@ const SelectStreams = ({
   };
 
   if (isEdit) {
-    selectedStreamIndex = streams?.findIndex(
-      (stream) => stream.name === selectedStreamName,
-    ) as number;
+    selectedStreamIndex = streams.findIndex((stream) => stream.name === selectedStreamName);
   } else {
-    selectedStreamIndex = streams?.findIndex(
-      (stream) => stream.name === selectedStream?.name,
-    ) as number;
+    selectedStreamIndex = streams.findIndex((stream) => stream.name === selectedStream?.name);
   }
 
   const refreshOptions =