feat: support firecrawl frontend code (#5226)

This commit is contained in:
Joel
2024-06-14 22:02:41 +08:00
committed by GitHub
parent 8d1386df0f
commit 28554350de
51 changed files with 1979 additions and 145 deletions

View File

@@ -1,23 +1,34 @@
import { useEffect, useState } from 'react'
'use client'
import type { FC } from 'react'
import React, { useEffect, useState } from 'react'
import useSWR from 'swr'
import { useTranslation } from 'react-i18next'
import { PlusIcon } from '@heroicons/react/24/solid'
import cn from 'classnames'
import Indicator from '../../../indicator'
import Operate from './operate'
import s from './style.module.css'
import NotionIcon from '@/app/components/base/notion-icon'
import Panel from '../panel'
import { DataSourceType } from '../panel/types'
import type { DataSourceNotion as TDataSourceNotion } from '@/models/common'
import { useAppContext } from '@/context/app-context'
import { fetchNotionConnection } from '@/service/common'
import NotionIcon from '@/app/components/base/notion-icon'
type DataSourceNotionProps = {
const Icon: FC<{
src: string
name: string
className: string
}> = ({ src, name, className }) => {
return (
<NotionIcon
src={src}
name={name}
className={className}
/>
)
}
type Props = {
workspaces: TDataSourceNotion[]
}
const DataSourceNotion = ({
const DataSourceNotion: FC<Props> = ({
workspaces,
}: DataSourceNotionProps) => {
const { t } = useTranslation()
}) => {
const { isCurrentWorkspaceManager } = useAppContext()
const [canConnectNotion, setCanConnectNotion] = useState(false)
const { data } = useSWR(canConnectNotion ? '/oauth/data-source/notion' : null, fetchNotionConnection)
@@ -42,95 +53,32 @@ const DataSourceNotion = ({
if (data?.data)
window.location.href = data.data
}, [data])
return (
<div className='mb-2 border-[0.5px] border-gray-200 bg-gray-50 rounded-xl'>
<div className='flex items-center px-3 py-[9px]'>
<div className={cn(s['notion-icon'], 'w-8 h-8 mr-3 border border-gray-100 rounded-lg')} />
<div className='grow'>
<div className='leading-5 text-sm font-medium text-gray-800'>
{t('common.dataSource.notion.title')}
</div>
{
!connected && (
<div className='leading-5 text-xs text-gray-500'>
{t('common.dataSource.notion.description')}
</div>
)
}
</div>
{
connected
? (
<div
className={
`flex items-center ml-3 px-3 h-7 bg-white border border-gray-200
rounded-md text-xs font-medium text-gray-700
${isCurrentWorkspaceManager ? 'cursor-pointer' : 'grayscale opacity-50 cursor-default'}`
}
onClick={handleConnectNotion}
>
{t('common.dataSource.connect')}
</div>
)
: (
<div
className={
`flex items-center px-3 py-1 min-h-7 bg-white border-[0.5px] border-gray-200 text-xs font-medium text-primary-600 rounded-md
${isCurrentWorkspaceManager ? 'cursor-pointer' : 'grayscale opacity-50 cursor-default'}`
}
onClick={handleConnectNotion}
>
<PlusIcon className='w-[14px] h-[14px] mr-[5px]' />
{t('common.dataSource.notion.addWorkspace')}
</div>
)
}
</div>
{
connected && (
<div className='flex items-center px-3 h-[18px]'>
<div className='text-xs font-medium text-gray-500'>
{t('common.dataSource.notion.connectedWorkspace')}
</div>
<div className='grow ml-3 border-t border-t-gray-100' />
</div>
)
}
{
connected && (
<div className='px-3 pt-2 pb-3'>
{
workspaces.map(workspace => (
<div className={cn(s['workspace-item'], 'flex items-center mb-1 py-1 pr-1 bg-white rounded-lg')} key={workspace.id}>
<NotionIcon
className='ml-3 mr-[6px]'
src={workspace.source_info.workspace_icon}
name={workspace.source_info.workspace_name}
/>
<div className='grow py-[7px] leading-[18px] text-[13px] font-medium text-gray-700 truncate' title={workspace.source_info.workspace_name}>{workspace.source_info.workspace_name}</div>
{
workspace.is_bound
? <Indicator className='shrink-0 mr-[6px]' />
: <Indicator className='shrink-0 mr-[6px]' color='yellow' />
}
<div className='shrink-0 mr-3 text-xs font-medium'>
{
workspace.is_bound
? t('common.dataSource.notion.connected')
: t('common.dataSource.notion.disconnected')
}
</div>
<div className='mr-2 w-[1px] h-3 bg-gray-100' />
<Operate workspace={workspace} onAuthAgain={handleAuthAgain} />
</div>
))
}
</div>
)
}
</div>
<Panel
type={DataSourceType.notion}
isConfigured={connected}
onConfigure={handleConnectNotion}
readonly={!isCurrentWorkspaceManager}
isSupportList
configuredList={workspaces.map(workspace => ({
id: workspace.id,
logo: ({ className }: { className: string }) => (
<Icon
src={workspace.source_info.workspace_icon!}
name={workspace.source_info.workspace_name}
className={className}
/>),
name: workspace.source_info.workspace_name,
isActive: workspace.is_bound,
notionConfig: {
total: workspace.source_info.total || 0,
},
}))}
onRemove={() => { }} // handled in operation/index.tsx
notionActions={{
onChangeAuthorizedPage: handleAuthAgain,
}}
/>
)
}
export default DataSourceNotion
export default React.memo(DataSourceNotion)

View File

@@ -6,17 +6,19 @@ import { EllipsisHorizontalIcon } from '@heroicons/react/24/solid'
import { Menu, Transition } from '@headlessui/react'
import { syncDataSourceNotion, updateDataSourceNotionAction } from '@/service/common'
import Toast from '@/app/components/base/toast'
import type { DataSourceNotion } from '@/models/common'
import { FilePlus02 } from '@/app/components/base/icons/src/vender/line/files'
import { RefreshCw05 } from '@/app/components/base/icons/src/vender/line/arrows'
import { Trash03 } from '@/app/components/base/icons/src/vender/line/general'
type OperateProps = {
workspace: DataSourceNotion
payload: {
id: string
total: number
}
onAuthAgain: () => void
}
export default function Operate({
workspace,
payload,
onAuthAgain,
}: OperateProps) {
const itemClassName = `
@@ -37,11 +39,11 @@ export default function Operate({
mutate({ url: 'data-source/integrates' })
}
const handleSync = async () => {
await syncDataSourceNotion({ url: `/oauth/data-source/notion/${workspace.id}/sync` })
await syncDataSourceNotion({ url: `/oauth/data-source/notion/${payload.id}/sync` })
updateIntegrates()
}
const handleRemove = async () => {
await updateDataSourceNotionAction({ url: `/data-source/integrates/${workspace.id}/disable` })
await updateDataSourceNotionAction({ url: `/data-source/integrates/${payload.id}/disable` })
updateIntegrates()
}
@@ -79,7 +81,7 @@ export default function Operate({
<div>
<div className='leading-5'>{t('common.dataSource.notion.changeAuthorizedPages')}</div>
<div className='leading-5 text-xs text-gray-500'>
{workspace.source_info.total} {t('common.dataSource.notion.pagesAuthorized')}
{payload.total} {t('common.dataSource.notion.pagesAuthorized')}
</div>
</div>
</div>

View File

@@ -0,0 +1,163 @@
'use client'
import type { FC } from 'react'
import React, { useCallback, useState } from 'react'
import { useTranslation } from 'react-i18next'
import {
PortalToFollowElem,
PortalToFollowElemContent,
} from '@/app/components/base/portal-to-follow-elem'
import { Lock01 } from '@/app/components/base/icons/src/vender/solid/security'
import Button from '@/app/components/base/button'
import type { FirecrawlConfig } from '@/models/common'
import Field from '@/app/components/datasets/create/website/firecrawl/base/field'
import Toast from '@/app/components/base/toast'
import { createFirecrawlApiKey } from '@/service/datasets'
import { LinkExternal02 } from '@/app/components/base/icons/src/vender/line/general'
type Props = {
onCancel: () => void
onSaved: () => void
}
const I18N_PREFIX = 'datasetCreation.firecrawl'
const DEFAULT_BASE_URL = 'https://api.firecrawl.dev'
const ConfigFirecrawlModal: FC<Props> = ({
onCancel,
onSaved,
}) => {
const { t } = useTranslation()
const [isSaving, setIsSaving] = useState(false)
const [config, setConfig] = useState<FirecrawlConfig>({
api_key: '',
base_url: '',
})
const handleConfigChange = useCallback((key: string) => {
return (value: string | number) => {
setConfig(prev => ({ ...prev, [key]: value as string }))
}
}, [])
const handleSave = useCallback(async () => {
if (isSaving)
return
let errorMsg = ''
if (config.base_url && !((config.base_url.startsWith('http://') || config.base_url.startsWith('https://'))))
errorMsg = t('common.errorMsg.urlError')
if (!errorMsg) {
if (!config.api_key) {
errorMsg = t('common.errorMsg.fieldRequired', {
field: 'API Key',
})
}
else if (!config.api_key.startsWith('fc-')) {
errorMsg = t(`${I18N_PREFIX}.apiKeyFormatError`)
}
}
if (errorMsg) {
Toast.notify({
type: 'error',
message: errorMsg,
})
return
}
const postData = {
category: 'website',
provider: 'firecrawl',
credentials: {
auth_type: 'bearer',
config: {
api_key: config.api_key,
base_url: config.base_url || DEFAULT_BASE_URL,
},
},
}
try {
setIsSaving(true)
await createFirecrawlApiKey(postData)
Toast.notify({
type: 'success',
message: t('common.api.success'),
})
}
finally {
setIsSaving(false)
}
onSaved()
}, [config.api_key, config.base_url, onSaved, t, isSaving])
return (
<PortalToFollowElem open>
<PortalToFollowElemContent className='w-full h-full z-[60]'>
<div className='fixed inset-0 flex items-center justify-center bg-black/[.25]'>
<div className='mx-2 w-[640px] max-h-[calc(100vh-120px)] bg-white shadow-xl rounded-2xl overflow-y-auto'>
<div className='px-8 pt-8'>
<div className='flex justify-between items-center mb-4'>
<div className='text-xl font-semibold text-gray-900'>{t(`${I18N_PREFIX}.configFirecrawl`)}</div>
</div>
<div className='space-y-4'>
<Field
label='API Key'
labelClassName='!text-sm'
isRequired
value={config.api_key}
onChange={handleConfigChange('api_key')}
placeholder={t(`${I18N_PREFIX}.apiKeyPlaceholder`)!}
/>
<Field
label='Base URL'
labelClassName='!text-sm'
value={config.base_url}
onChange={handleConfigChange('base_url')}
placeholder={DEFAULT_BASE_URL}
/>
</div>
<div className='my-8 flex justify-between items-center h-8'>
<a className='flex items-center space-x-1 leading-[18px] text-xs font-normal text-[#155EEF]' target='_blank' href='https://www.firecrawl.dev/account'>
<span>{t(`${I18N_PREFIX}.getApiKeyLinkText`)}</span>
<LinkExternal02 className='w-3 h-3' />
</a>
<div className='flex'>
<Button
className='mr-2 h-9 text-sm font-medium text-gray-700'
onClick={onCancel}
>
{t('common.operation.cancel')}
</Button>
<Button
className='h-9 text-sm font-medium'
type='primary'
onClick={handleSave}
loading={isSaving}
>
{t('common.operation.save')}
</Button>
</div>
</div>
</div>
<div className='border-t-[0.5px] border-t-black/5'>
<div className='flex justify-center items-center py-3 bg-gray-50 text-xs text-gray-500'>
<Lock01 className='mr-1 w-3 h-3 text-gray-500' />
{t('common.modelProvider.encrypted.front')}
<a
className='text-primary-600 mx-1'
target='_blank' rel='noopener noreferrer'
href='https://pycryptodome.readthedocs.io/en/latest/src/cipher/oaep.html'
>
PKCS1_OAEP
</a>
{t('common.modelProvider.encrypted.back')}
</div>
</div>
</div>
</div>
</PortalToFollowElemContent>
</PortalToFollowElem>
)
}
export default React.memo(ConfigFirecrawlModal)

View File

@@ -0,0 +1,82 @@
'use client'
import type { FC } from 'react'
import React, { useCallback, useEffect, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { useBoolean } from 'ahooks'
import cn from 'classnames'
import Panel from '../panel'
import { DataSourceType } from '../panel/types'
import ConfigFirecrawlModal from './config-firecrawl-modal'
import { fetchFirecrawlApiKey, removeFirecrawlApiKey } from '@/service/datasets'
import type {
DataSourceWebsiteItem,
} from '@/models/common'
import { useAppContext } from '@/context/app-context'
import {
WebsiteProvider,
} from '@/models/common'
import Toast from '@/app/components/base/toast'
type Props = {}
const DataSourceWebsite: FC<Props> = () => {
const { t } = useTranslation()
const { isCurrentWorkspaceManager } = useAppContext()
const [list, setList] = useState<DataSourceWebsiteItem[]>([])
const checkSetApiKey = useCallback(async () => {
const res = await fetchFirecrawlApiKey() as any
const list = res.settings.filter((item: DataSourceWebsiteItem) => item.provider === WebsiteProvider.fireCrawl && !item.disabled)
setList(list)
}, [])
useEffect(() => {
checkSetApiKey()
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [])
const [isShowConfig, {
setTrue: showConfig,
setFalse: hideConfig,
}] = useBoolean(false)
const handleAdded = useCallback(() => {
checkSetApiKey()
hideConfig()
}, [checkSetApiKey, hideConfig])
const handleRemove = useCallback(async () => {
await removeFirecrawlApiKey(list[0].id)
setList([])
Toast.notify({
type: 'success',
message: t('common.api.remove'),
})
}, [list, t])
return (
<>
<Panel
type={DataSourceType.website}
isConfigured={list.length > 0}
onConfigure={showConfig}
readonly={!isCurrentWorkspaceManager}
configuredList={list.map(item => ({
id: item.id,
logo: ({ className }: { className: string }) => (
<div className={cn(className, 'flex items-center justify-center w-5 h-5 bg-white border border-gray-100 text-xs font-medium text-gray-500 rounded ml-3')}>🔥</div>
),
name: 'FireCrawl',
isActive: true,
}))}
onRemove={handleRemove}
/>
{isShowConfig && (
<ConfigFirecrawlModal onSaved={handleAdded} onCancel={hideConfig} />
)}
</>
)
}
export default React.memo(DataSourceWebsite)

View File

@@ -1,6 +1,7 @@
import useSWR from 'swr'
import { useTranslation } from 'react-i18next'
import DataSourceNotion from './data-source-notion'
import DataSourceWebsite from './data-source-website'
import { fetchDataSource } from '@/service/common'
export default function DataSourcePage() {
@@ -12,6 +13,7 @@ export default function DataSourcePage() {
<div className='mb-8'>
<div className='mb-2 text-sm font-medium text-gray-900'>{t('common.dataSource.add')}</div>
<DataSourceNotion workspaces={notionWorkspaces} />
<DataSourceWebsite />
</div>
)
}

View File

@@ -0,0 +1,78 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import { useTranslation } from 'react-i18next'
import cn from 'classnames'
import Indicator from '../../../indicator'
import Operate from '../data-source-notion/operate'
import { DataSourceType } from './types'
import s from './style.module.css'
import { Trash03 } from '@/app/components/base/icons/src/vender/line/general'
export type ConfigItemType = {
id: string
logo: any
name: string
isActive: boolean
notionConfig?: {
total: number
}
}
type Props = {
type: DataSourceType
payload: ConfigItemType
onRemove: () => void
notionActions?: {
onChangeAuthorizedPage: () => void
}
}
const ConfigItem: FC<Props> = ({
type,
payload,
onRemove,
notionActions,
}) => {
const { t } = useTranslation()
const isNotion = type === DataSourceType.notion
const isWebsite = type === DataSourceType.website
const onChangeAuthorizedPage = notionActions?.onChangeAuthorizedPage || function () { }
return (
<div className={cn(s['workspace-item'], 'flex items-center mb-1 py-1 pr-1 bg-white rounded-lg')} key={payload.id}>
<payload.logo className='ml-3 mr-1.5' />
<div className='grow py-[7px] leading-[18px] text-[13px] font-medium text-gray-700 truncate' title={payload.name}>{payload.name}</div>
{
payload.isActive
? <Indicator className='shrink-0 mr-[6px]' />
: <Indicator className='shrink-0 mr-[6px]' color='yellow' />
}
<div className='shrink-0 mr-3 text-xs font-medium uppercase'>
{
payload.isActive
? t(isNotion ? 'common.dataSource.notion.connected' : 'common.dataSource.website.active')
: t(isNotion ? 'common.dataSource.notion.disconnected' : 'common.dataSource.website.inactive')
}
</div>
<div className='mr-2 w-[1px] h-3 bg-gray-100' />
{isNotion && (
<Operate payload={{
id: payload.id,
total: payload.notionConfig?.total || 0,
}} onAuthAgain={onChangeAuthorizedPage}
/>
)}
{
isWebsite && (
<div className='p-2 text-gray-500 cursor-pointer rounded-md hover:bg-black/5' onClick={onRemove} >
<Trash03 className='w-4 h-4 ' />
</div>
)
}
</div>
)
}
export default React.memo(ConfigItem)

View File

@@ -0,0 +1,138 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import { useTranslation } from 'react-i18next'
import { PlusIcon } from '@heroicons/react/24/solid'
import cn from 'classnames'
import type { ConfigItemType } from './config-item'
import ConfigItem from './config-item'
import s from './style.module.css'
import { DataSourceType } from './types'
type Props = {
type: DataSourceType
isConfigured: boolean
onConfigure: () => void
readonly: boolean
isSupportList?: boolean
configuredList: ConfigItemType[]
onRemove: () => void
notionActions?: {
onChangeAuthorizedPage: () => void
}
}
const Panel: FC<Props> = ({
type,
isConfigured,
onConfigure,
readonly,
configuredList,
isSupportList,
onRemove,
notionActions,
}) => {
const { t } = useTranslation()
const isNotion = type === DataSourceType.notion
const isWebsite = type === DataSourceType.website
return (
<div className='mb-2 border-[0.5px] border-gray-200 bg-gray-50 rounded-xl'>
<div className='flex items-center px-3 py-[9px]'>
<div className={cn(s[`${type}-icon`], 'w-8 h-8 mr-3 border border-gray-100 rounded-lg')} />
<div className='grow'>
<div className='flex items-center h-5'>
<div className='text-sm font-medium text-gray-800'>{t(`common.dataSource.${type}.title`)}</div>
{isWebsite && (
<div className='ml-1 leading-[18px] px-1.5 rounded-md bg-white border border-gray-100 text-xs font-medium text-gray-700'>
<span className='text-gray-500'>{t('common.dataSource.website.with')}</span> 🔥 FireCrawl
</div>
)}
</div>
{
!isConfigured && (
<div className='leading-5 text-xs text-gray-500'>
{t(`common.dataSource.${type}.description`)}
</div>
)
}
</div>
{isNotion && (
<>
{
isConfigured
? (
<div
className={
`flex items-center ml-3 px-3 h-7 bg-white border border-gray-200
rounded-md text-xs font-medium text-gray-700
${!readonly ? 'cursor-pointer' : 'grayscale opacity-50 cursor-default'}`
}
onClick={onConfigure}
>
{t('common.dataSource.configure')}
</div>
)
: (
<>
{isSupportList && <div
className={
`flex items-center px-3 py-1 min-h-7 bg-white border-[0.5px] border-gray-200 text-xs font-medium text-primary-600 rounded-md
${!readonly ? 'cursor-pointer' : 'grayscale opacity-50 cursor-default'}`
}
onClick={onConfigure}
>
<PlusIcon className='w-[14px] h-[14px] mr-[5px]' />
{t('common.dataSource.notion.addWorkspace')}
</div>}
</>
)
}
</>
)}
{isWebsite && !isConfigured && (
<div
className={
`flex items-center ml-3 px-3 h-7 bg-white border border-gray-200
rounded-md text-xs font-medium text-gray-700
${!readonly ? 'cursor-pointer' : 'grayscale opacity-50 cursor-default'}`
}
onClick={onConfigure}
>
{t('common.dataSource.configure')}
</div>
)}
</div>
{
isConfigured && (
<div className='flex items-center px-3 h-[18px]'>
<div className='text-xs font-medium text-gray-500'>
{isNotion ? t('common.dataSource.notion.connectedWorkspace') : t('common.dataSource.website.configuredCrawlers')}
</div>
<div className='grow ml-3 border-t border-t-gray-100' />
</div>
)
}
{
isConfigured && (
<div className='px-3 pt-2 pb-3'>
{
configuredList.map(item => (
<ConfigItem
key={item.id}
type={type}
payload={item}
onRemove={onRemove}
notionActions={notionActions} />
))
}
</div>
)
}
</div>
)
}
export default React.memo(Panel)

View File

@@ -3,6 +3,11 @@
background-size: 20px 20px;
}
.website-icon {
background: #ffffff url(../../../../datasets/create/assets/web.svg) center center no-repeat;
background-size: 20px 20px;
}
.workspace-item {
box-shadow: 0px 1px 2px rgba(16, 24, 40, 0.05);
}

View File

@@ -0,0 +1,4 @@
export enum DataSourceType {
notion = 'notion',
website = 'website',
}