feat: support firecrawl frontend code (#5226)

web/app/components/datasets/documents/detail/settings/index.tsx

@@ -73,6 +73,16 @@ const DocumentSettings = ({ datasetId, documentId }: DocumentSettingsProps) => {
             datasetId={datasetId}
             dataSourceType={documentDetail.data_source_type}
             notionPages={[currentPage]}
+            websitePages={[
+              {
+                title: documentDetail.name,
+                source_url: documentDetail.data_source_info?.url,
+                markdown: '',
+                description: '',
+              },
+            ]}
+            fireCrawlJobId={documentDetail.data_source_info?.job_id}
+            crawlOptions={documentDetail.data_source_info}
             indexingType={indexingTechnique || ''}
             isSetting
             documentDetail={documentDetail}

web/app/components/datasets/documents/index.tsx

@@ -83,6 +83,8 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
   const [notionPageSelectorModalVisible, setNotionPageSelectorModalVisible] = useState(false)
   const [timerCanRun, setTimerCanRun] = useState(true)
   const isDataSourceNotion = dataset?.data_source_type === DataSourceType.NOTION
+  const isDataSourceWeb = dataset?.data_source_type === DataSourceType.WEB
+  const isDataSourceFile = dataset?.data_source_type === DataSourceType.FILE
   const embeddingAvailable = !!dataset?.embedding_available
 
   const query = useMemo(() => {
@@ -211,7 +213,8 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
               <Button type='primary' onClick={routeToDocCreate} className='!h-8 !text-[13px] !shrink-0'>
                 <PlusIcon className='h-4 w-4 mr-2 stroke-current' />
                 {isDataSourceNotion && t('datasetDocuments.list.addPages')}
-                {!isDataSourceNotion && t('datasetDocuments.list.addFile')}
+                {isDataSourceWeb && t('datasetDocuments.list.addUrl')}
+                {isDataSourceFile && t('datasetDocuments.list.addFile')}
               </Button>
             )}
           </div>

web/app/components/datasets/documents/list.tsx

@@ -13,6 +13,7 @@ import cn from 'classnames'
 import dayjs from 'dayjs'
 import { Edit03 } from '../../base/icons/src/vender/solid/general'
 import TooltipPlus from '../../base/tooltip-plus'
+import { Globe01 } from '../../base/icons/src/vender/line/mapsAndTravel'
 import s from './style.module.css'
 import RenameModal from './rename-modal'
 import Switch from '@/app/components/base/switch'
@@ -26,7 +27,7 @@ import type { IndicatorProps } from '@/app/components/header/indicator'
 import Indicator from '@/app/components/header/indicator'
 import { asyncRunSafe } from '@/utils'
 import { formatNumber } from '@/utils/format'
-import { archiveDocument, deleteDocument, disableDocument, enableDocument, syncDocument, unArchiveDocument } from '@/service/datasets'
+import { archiveDocument, deleteDocument, disableDocument, enableDocument, syncDocument, syncWebsite, unArchiveDocument } from '@/service/datasets'
 import NotionIcon from '@/app/components/base/notion-icon'
 import ProgressBar from '@/app/components/base/progress-bar'
 import { DataSourceType, type DocumentDisplayStatus, type SimpleDocumentDetail } from '@/models/datasets'
@@ -146,7 +147,12 @@ export const OperationAction: FC<{
         opApi = disableDocument
         break
       case 'sync':
-        opApi = syncDocument
+        if (data_source_type === 'notion_import')
+          opApi = syncDocument
+
+        else
+          opApi = syncWebsite
+
         break
       default:
         opApi = deleteDocument
@@ -249,7 +255,7 @@ export const OperationAction: FC<{
                   <SettingsIcon />
                   <span className={s.actionName}>{t('datasetDocuments.list.action.settings')}</span>
                 </div>
-                {data_source_type === 'notion_import' && (
+                {['notion_import', DataSourceType.WEB].includes(data_source_type) && (
                   <div className={s.actionItem} onClick={() => onOperate('sync')}>
                     <SyncIcon />
                     <span className={s.actionName}>{t('datasetDocuments.list.action.sync')}</span>
@@ -282,7 +288,7 @@ export const OperationAction: FC<{
           </div>
         }
         btnClassName={open => cn(isListScene ? s.actionIconWrapperList : s.actionIconWrapperDetail, open ? '!bg-gray-100 !shadow-none' : '!bg-transparent')}
-        className={`!w-[200px] h-fit !z-20 ${className}`}
+        className={`flex justify-end !w-[200px] h-fit !z-20 ${className}`}
       />
     )}
     {showModal && <Modal isShow={showModal} onClose={() => setShowModal(false)} className={s.delModal} closable>
@@ -418,10 +424,10 @@ const DocumentList: FC<IDocumentListProps> = ({ embeddingAvailable, documents =
               <td>
                 <div className='group flex items-center justify-between'>
                   <span className={s.tdValue}>
-                    {
-                      doc?.data_source_type === DataSourceType.NOTION
-                        ? <NotionIcon className='inline-flex -mt-[3px] mr-1.5 align-middle' type='page' src={doc.data_source_info.notion_page_icon} />
-                        : <div className={cn(s[`${doc?.data_source_info?.upload_file?.extension ?? fileType}Icon`], s.commonIcon, 'mr-1.5')}></div>
+                    {doc?.data_source_type === DataSourceType.NOTION && <NotionIcon className='inline-flex -mt-[3px] mr-1.5 align-middle' type='page' src={doc.data_source_info.notion_page_icon} />
+                    }
+                    {doc?.data_source_type === DataSourceType.FILE && <div className={cn(s[`${doc?.data_source_info?.upload_file?.extension ?? fileType}Icon`], s.commonIcon, 'mr-1.5')}></div>}
+                    {doc?.data_source_type === DataSourceType.WEB && <Globe01 className='inline-flex -mt-[3px] mr-1.5 align-middle' />
                     }
                     {
                       doc.name