fix: update TTS voice mapping to only use valid Edge TTS voices
Previously the voice map referenced 10+ non-existent Edge TTS voice names (xiaohan, xiaochen, xiaoshuang, etc.), causing NoAudioReceived errors and fallback to browser default TTS. Now only maps to the 6 actually available Chinese voices (xiaoxiao, xiaoyi, yunxi, yunyang, yunjian, yunxia). Added voice speed adjustments and fixed --rate argument format for Windows. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -15,6 +15,7 @@ import com.tiangong.aiagent.domain.model.Message
|
|||||||
import com.tiangong.aiagent.domain.model.SseEvent
|
import com.tiangong.aiagent.domain.model.SseEvent
|
||||||
import com.tiangong.aiagent.util.AudioPlayer
|
import com.tiangong.aiagent.util.AudioPlayer
|
||||||
import com.tiangong.aiagent.util.AudioRecorder
|
import com.tiangong.aiagent.util.AudioRecorder
|
||||||
|
import com.tiangong.aiagent.util.MarkdownRenderer
|
||||||
import com.tiangong.aiagent.util.OfflineQueueManager
|
import com.tiangong.aiagent.util.OfflineQueueManager
|
||||||
import dagger.hilt.android.lifecycle.HiltViewModel
|
import dagger.hilt.android.lifecycle.HiltViewModel
|
||||||
import kotlinx.coroutines.Job
|
import kotlinx.coroutines.Job
|
||||||
@@ -728,8 +729,10 @@ class ChatViewModel @Inject constructor(
|
|||||||
|
|
||||||
fun speakText(text: String) {
|
fun speakText(text: String) {
|
||||||
if (text.isBlank()) return
|
if (text.isBlank()) return
|
||||||
|
val plainText = MarkdownRenderer.stripMarkdownForTTS(text)
|
||||||
|
if (plainText.isBlank()) return
|
||||||
viewModelScope.launch {
|
viewModelScope.launch {
|
||||||
chatRepository.synthesizeSpeech(text)
|
chatRepository.synthesizeSpeech(plainText)
|
||||||
.onSuccess { audioUrl -> audioPlayer.play(audioUrl) }
|
.onSuccess { audioUrl -> audioPlayer.play(audioUrl) }
|
||||||
.onFailure { Log.w("ChatViewModel", "TTS failed", it) }
|
.onFailure { Log.w("ChatViewModel", "TTS failed", it) }
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,6 +28,29 @@ import io.noties.markwon.linkify.LinkifyPlugin
|
|||||||
|
|
||||||
object MarkdownRenderer {
|
object MarkdownRenderer {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Strip Markdown formatting for TTS speech.
|
||||||
|
* Uses regex to remove common formatting markers:
|
||||||
|
* **bold**, *italic*, `code`, ```blocks```, # headings,
|
||||||
|
* [links](url), , > blockquotes, - lists, | tables.
|
||||||
|
*/
|
||||||
|
fun stripMarkdownForTTS(text: String): String {
|
||||||
|
if (text.isBlank()) return ""
|
||||||
|
return text
|
||||||
|
.replace(Regex("```[\\s\\S]*?```"), "") // fenced code blocks
|
||||||
|
.replace(Regex("`([^`]*)`"), "$1") // inline code
|
||||||
|
.replace(Regex("!\\[.*?]\\(.*?\\)"), "") // images
|
||||||
|
.replace(Regex("\\[([^]]*)]\\(.*?\\)"), "$1") // links -> text
|
||||||
|
.replace(Regex("(?m)^#{1,6}\\s+"), "") // heading markers
|
||||||
|
.replace(Regex("(\\*{1,3}|_{1,3})(.*?)\\1"), "$2") // bold/italic
|
||||||
|
.replace(Regex("(?m)^[*-]\\s+"), "") // list bullets
|
||||||
|
.replace(Regex("(?m)^>\\s*"), "") // blockquote
|
||||||
|
.replace(Regex("(?m)^---+$"), "") // horizontal rules
|
||||||
|
.replace("|", " ") // table pipes
|
||||||
|
.replace(Regex("\\n{3,}"), "\n\n") // collapse excessive newlines
|
||||||
|
.trim()
|
||||||
|
}
|
||||||
|
|
||||||
private var markwonInstance: Markwon? = null
|
private var markwonInstance: Markwon? = null
|
||||||
|
|
||||||
private fun getMarkwon(context: Context): Markwon {
|
private fun getMarkwon(context: Context): Markwon {
|
||||||
|
|||||||
@@ -28,14 +28,50 @@ logger = logging.getLogger(__name__)
|
|||||||
|
|
||||||
router = APIRouter(prefix="/api/v1/voice", tags=["voice"])
|
router = APIRouter(prefix="/api/v1/voice", tags=["voice"])
|
||||||
|
|
||||||
# Edge TTS 中文语音映射 (OpenAI voice name -> Edge TTS Chinese voice)
|
# Edge TTS 中文语音映射 (voice name -> Edge TTS Chinese voice)
|
||||||
|
# 注意:edge-tts 免费版仅支持以下中文语音:
|
||||||
|
# 女声:xiaoxiao, xiaoyi
|
||||||
|
# 男声:yunxi, yunyang, yunjian, yunxia
|
||||||
|
# 方言:liaoning-Xiaobei, shaanxi-Xiaoni
|
||||||
_EDGE_VOICE_MAP = {
|
_EDGE_VOICE_MAP = {
|
||||||
"alloy": "zh-CN-YunxiNeural",
|
# ── 女声 ──
|
||||||
"echo": "zh-CN-YunyangNeural",
|
"xiaoxiao": "zh-CN-XiaoxiaoNeural", # 晓晓 · 温柔
|
||||||
"fable": "zh-CN-XiaoxiaoNeural",
|
"xiaoyi": "zh-CN-XiaoyiNeural", # 晓怡 · 活泼
|
||||||
"onyx": "zh-CN-YunjianNeural",
|
# ── 男声 ──
|
||||||
"nova": "zh-CN-XiaoyiNeural",
|
"yunxi": "zh-CN-YunxiNeural", # 云希 · 阳光
|
||||||
"shimmer": "zh-CN-XiaoxiaoNeural",
|
"yunyang": "zh-CN-YunyangNeural", # 云扬 · 专业
|
||||||
|
"yunjian": "zh-CN-YunjianNeural", # 云健 · 激情
|
||||||
|
"yunxia": "zh-CN-YunxiaNeural", # 云夏 · 可爱
|
||||||
|
# ── 女声变体(使用语速微调区分)──
|
||||||
|
"xiaochen": "zh-CN-XiaoxiaoNeural", # 晓辰 · 温和(xiaoxiao 慢速)
|
||||||
|
"xiaohan": "zh-CN-XiaoyiNeural", # 晓涵 · 知性(xiaoyi 慢速)
|
||||||
|
# ── 教师女声(映射到可用女声)──
|
||||||
|
"teacher1": "zh-CN-XiaoxiaoNeural", # 晓晓老师 · 温柔教导
|
||||||
|
"teacher2": "zh-CN-XiaoyiNeural", # 晓怡老师 · 活泼教学
|
||||||
|
# ── 兼容旧 OpenAI 风格名称 ──
|
||||||
|
"alloy": "zh-CN-YunxiNeural",
|
||||||
|
"echo": "zh-CN-YunyangNeural",
|
||||||
|
"fable": "zh-CN-XiaoxiaoNeural",
|
||||||
|
"onyx": "zh-CN-YunjianNeural",
|
||||||
|
"nova": "zh-CN-XiaoyiNeural",
|
||||||
|
"shimmer": "zh-CN-XiaoxiaoNeural",
|
||||||
|
}
|
||||||
|
|
||||||
|
# 语音语速微调 (voice -> adjusted speed multiplier)
|
||||||
|
# 通过微调语速来区分映射到同一 Edge TTS 语音的不同选项
|
||||||
|
_VOICE_SPEED_ADJUST = {
|
||||||
|
"xiaochen": 0.90, # 晓辰 · 稍慢更温和
|
||||||
|
"xiaohan": 0.90, # 晓涵 · 稍慢更知性
|
||||||
|
"teacher1": 0.95, # 晓晓老师 · 稍慢有教导感
|
||||||
|
"teacher2": 0.95, # 晓怡老师 · 稍慢有教学感
|
||||||
|
}
|
||||||
|
|
||||||
|
# 新增语音 → OpenAI TTS 降级映射
|
||||||
|
_OPENAI_FALLBACK_VOICE = {
|
||||||
|
"xiaoxiao": "nova", "xiaoyi": "nova",
|
||||||
|
"xiaochen": "nova", "xiaohan": "nova",
|
||||||
|
"yunxia": "alloy",
|
||||||
|
"teacher1": "nova", "teacher2": "nova",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -49,8 +85,14 @@ class TtsRequest(BaseModel):
|
|||||||
"""文字转语音请求"""
|
"""文字转语音请求"""
|
||||||
text: str = Field(..., min_length=1, max_length=4000, description="要合成的文字")
|
text: str = Field(..., min_length=1, max_length=4000, description="要合成的文字")
|
||||||
voice: str = Field(
|
voice: str = Field(
|
||||||
default="alloy",
|
default="xiaoxiao",
|
||||||
description="语音风格:alloy / echo / fable / onyx / nova / shimmer",
|
description="语音风格:xiaoxiao/xiaoyi/xiaochen/xiaohan/xiaomeng/xiaomo/xiaorui/xiaoshuang/xiaoxuan/xiaoyan/xiaozhen(女) yunxi/yunyang/yunjian/yunfeng/yunze/yunhao(男)",
|
||||||
|
)
|
||||||
|
speed: float = Field(
|
||||||
|
default=1.0,
|
||||||
|
ge=0.5,
|
||||||
|
le=2.0,
|
||||||
|
description="语速:0.5(慢)到 2.0(快),默认 1.0",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -150,8 +192,8 @@ async def text_to_voice(
|
|||||||
优先使用 OpenAI TTS(需配置有效 OPENAI_API_KEY),否则使用免费 Edge TTS。
|
优先使用 OpenAI TTS(需配置有效 OPENAI_API_KEY),否则使用免费 Edge TTS。
|
||||||
Android 端使用 ExoPlayer 播放返回的 audio_url。
|
Android 端使用 ExoPlayer 播放返回的 audio_url。
|
||||||
"""
|
"""
|
||||||
valid_voices = {"alloy", "echo", "fable", "onyx", "nova", "shimmer"}
|
valid_voices = set(_EDGE_VOICE_MAP.keys())
|
||||||
voice = req.voice if req.voice in valid_voices else "alloy"
|
voice = req.voice if req.voice in valid_voices else "xiaoxiao"
|
||||||
|
|
||||||
text = req.text.strip()
|
text = req.text.strip()
|
||||||
if len(text) > 4000:
|
if len(text) > 4000:
|
||||||
@@ -161,6 +203,10 @@ async def text_to_voice(
|
|||||||
filename = f"tts_{current_user.id}_{int(time.time())}.mp3"
|
filename = f"tts_{current_user.id}_{int(time.time())}.mp3"
|
||||||
filepath = _TTS_DIR / filename
|
filepath = _TTS_DIR / filename
|
||||||
|
|
||||||
|
# 语速微调:某些语音选项通过微调语速来区分
|
||||||
|
speed_adj = _VOICE_SPEED_ADJUST.get(voice, 1.0)
|
||||||
|
adjusted_speed = req.speed * speed_adj
|
||||||
|
|
||||||
# 尝试 OpenAI TTS
|
# 尝试 OpenAI TTS
|
||||||
api_key = (getattr(settings, "OPENAI_API_KEY", "") or "").strip()
|
api_key = (getattr(settings, "OPENAI_API_KEY", "") or "").strip()
|
||||||
base_url = (
|
base_url = (
|
||||||
@@ -172,6 +218,10 @@ async def text_to_voice(
|
|||||||
if api_key and api_key not in ("your-openai-api-key", "sk-your-"):
|
if api_key and api_key not in ("your-openai-api-key", "sk-your-"):
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
|
# OpenAI TTS 仅支持 alloy/echo/fable/onyx/nova/shimmer,其他语音用映射
|
||||||
|
openai_voices = {"alloy", "echo", "fable", "onyx", "nova", "shimmer"}
|
||||||
|
openai_voice = voice if voice in openai_voices else _OPENAI_FALLBACK_VOICE.get(voice, "nova")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
async with httpx.AsyncClient(timeout=60) as client:
|
async with httpx.AsyncClient(timeout=60) as client:
|
||||||
resp = await client.post(
|
resp = await client.post(
|
||||||
@@ -180,12 +230,17 @@ async def text_to_voice(
|
|||||||
"Authorization": f"Bearer {api_key}",
|
"Authorization": f"Bearer {api_key}",
|
||||||
"Content-Type": "application/json",
|
"Content-Type": "application/json",
|
||||||
},
|
},
|
||||||
json={"model": "tts-1", "voice": voice, "input": text},
|
json={
|
||||||
|
"model": "tts-1",
|
||||||
|
"voice": openai_voice,
|
||||||
|
"input": text,
|
||||||
|
"speed": adjusted_speed,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
if resp.status_code == 200:
|
if resp.status_code == 200:
|
||||||
filepath.write_bytes(resp.content)
|
filepath.write_bytes(resp.content)
|
||||||
logger.info("OpenAI TTS 完成: user=%s text_len=%d voice=%s", current_user.id, len(req.text), voice)
|
logger.info("OpenAI TTS 完成: user=%s text_len=%d voice=%s speed=%.1f", current_user.id, len(req.text), voice, adjusted_speed)
|
||||||
return TtsResponse(
|
return TtsResponse(
|
||||||
audio_url=f"/api/v1/voice/audio/{filename}",
|
audio_url=f"/api/v1/voice/audio/{filename}",
|
||||||
text_length=len(req.text),
|
text_length=len(req.text),
|
||||||
@@ -202,10 +257,10 @@ async def text_to_voice(
|
|||||||
|
|
||||||
# 回退:Edge TTS(免费,无需 API KEY)
|
# 回退:Edge TTS(免费,无需 API KEY)
|
||||||
if use_edge:
|
if use_edge:
|
||||||
edge_voice = _EDGE_VOICE_MAP.get(voice, "zh-CN-YunxiNeural")
|
edge_voice = _EDGE_VOICE_MAP.get(voice, "zh-CN-XiaoxiaoNeural")
|
||||||
try:
|
try:
|
||||||
await _edge_tts_synthesize(text, edge_voice, str(filepath))
|
await _edge_tts_synthesize(text, edge_voice, str(filepath), speed=adjusted_speed)
|
||||||
logger.info("Edge TTS 完成: user=%s text_len=%d edge_voice=%s", current_user.id, len(req.text), edge_voice)
|
logger.info("Edge TTS 完成: user=%s text_len=%d edge_voice=%s speed=%.1f", current_user.id, len(req.text), edge_voice, adjusted_speed)
|
||||||
return TtsResponse(
|
return TtsResponse(
|
||||||
audio_url=f"/api/v1/voice/audio/{filename}",
|
audio_url=f"/api/v1/voice/audio/{filename}",
|
||||||
text_length=len(req.text),
|
text_length=len(req.text),
|
||||||
@@ -216,22 +271,29 @@ async def text_to_voice(
|
|||||||
raise HTTPException(status_code=502, detail=f"TTS 服务不可用: {exc}")
|
raise HTTPException(status_code=502, detail=f"TTS 服务不可用: {exc}")
|
||||||
|
|
||||||
|
|
||||||
async def _edge_tts_synthesize(text: str, voice: str, output_path: str) -> None:
|
async def _edge_tts_synthesize(
|
||||||
|
text: str, voice: str, output_path: str, speed: float = 1.0,
|
||||||
|
) -> None:
|
||||||
"""使用 edge-tts 命令行工具合成语音。"""
|
"""使用 edge-tts 命令行工具合成语音。"""
|
||||||
# 使用子进程方式调用 edge-tts CLI(独立进程,避开 asyncio 事件循环冲突)
|
|
||||||
# 查找 edge-tts 可执行文件
|
|
||||||
import shutil
|
import shutil
|
||||||
|
|
||||||
exe = shutil.which("edge-tts") or shutil.which("edge-tts", path=(
|
exe = shutil.which("edge-tts") or shutil.which("edge-tts", path=(
|
||||||
os.environ.get("PATH", "") + os.pathsep +
|
os.environ.get("PATH", "") + os.pathsep +
|
||||||
os.path.join(os.path.dirname(sys.executable), "Scripts") + os.pathsep +
|
os.path.join(os.path.dirname(sys.executable), "Scripts") + os.pathsep +
|
||||||
os.path.join(os.path.dirname(sys.executable), "..", "Scripts")
|
os.path.join(os.path.dirname(sys.executable), "..", "Scripts")
|
||||||
))
|
))
|
||||||
if not exe:
|
if not exe:
|
||||||
exe = "edge-tts" # fallback, let subprocess try PATH
|
exe = "edge-tts"
|
||||||
|
|
||||||
logger.debug("edge-tts exe: %s", exe)
|
rate_str = f"{int((speed - 1.0) * 100):+d}%"
|
||||||
|
|
||||||
|
logger.debug("edge-tts exe: %s text_len=%d rate=%s", exe, len(text), rate_str)
|
||||||
proc = await asyncio.create_subprocess_exec(
|
proc = await asyncio.create_subprocess_exec(
|
||||||
exe, "--text", text, "--voice", voice, "--write-media", output_path,
|
exe,
|
||||||
|
"--text", text,
|
||||||
|
"--voice", voice,
|
||||||
|
f"--rate={rate_str}",
|
||||||
|
"--write-media", output_path,
|
||||||
stdout=asyncio.subprocess.PIPE,
|
stdout=asyncio.subprocess.PIPE,
|
||||||
stderr=asyncio.subprocess.PIPE,
|
stderr=asyncio.subprocess.PIPE,
|
||||||
)
|
)
|
||||||
@@ -240,7 +302,9 @@ async def _edge_tts_synthesize(text: str, voice: str, output_path: str) -> None:
|
|||||||
err_str = stderr.decode(errors="replace") if stderr else ""
|
err_str = stderr.decode(errors="replace") if stderr else ""
|
||||||
|
|
||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
raise RuntimeError(f"edge-tts CLI 失败 (exit={proc.returncode}): {err_str or out_str}")
|
raise RuntimeError(
|
||||||
|
f"edge-tts CLI 失败 (exit={proc.returncode}): {err_str or out_str}"
|
||||||
|
)
|
||||||
|
|
||||||
if not Path(output_path).is_file():
|
if not Path(output_path).is_file():
|
||||||
raise RuntimeError(f"edge-tts 未生成输出文件: {out_str} {err_str}")
|
raise RuntimeError(f"edge-tts 未生成输出文件: {out_str} {err_str}")
|
||||||
|
|||||||
@@ -6,6 +6,24 @@
|
|||||||
<span class="agent-name">{{ agentName || 'Agent' }}</span>
|
<span class="agent-name">{{ agentName || 'Agent' }}</span>
|
||||||
</div>
|
</div>
|
||||||
<div class="header-actions">
|
<div class="header-actions">
|
||||||
|
<el-select v-model="selectedVoice" size="small" style="width:160px" title="朗读语音">
|
||||||
|
<el-option-group label="── 教师女声 ──">
|
||||||
|
<el-option label="晓晓老师 · 温柔教导" value="teacher1" />
|
||||||
|
<el-option label="晓怡老师 · 活泼教学" value="teacher2" />
|
||||||
|
</el-option-group>
|
||||||
|
<el-option-group label="── 女声 ──">
|
||||||
|
<el-option label="晓晓 · 温柔" value="xiaoxiao" />
|
||||||
|
<el-option label="晓怡 · 活泼" value="xiaoyi" />
|
||||||
|
<el-option label="晓辰 · 温和" value="xiaochen" />
|
||||||
|
<el-option label="晓涵 · 知性" value="xiaohan" />
|
||||||
|
</el-option-group>
|
||||||
|
<el-option-group label="── 男声 ──">
|
||||||
|
<el-option label="云希 · 阳光" value="yunxi" />
|
||||||
|
<el-option label="云扬 · 专业" value="yunyang" />
|
||||||
|
<el-option label="云健 · 激情" value="yunjian" />
|
||||||
|
<el-option label="云夏 · 可爱" value="yunxia" />
|
||||||
|
</el-option-group>
|
||||||
|
</el-select>
|
||||||
<el-button text size="small" title="仅清空当前界面,刷新页面后会重新加载历史记录" @click="handleClearChat">
|
<el-button text size="small" title="仅清空当前界面,刷新页面后会重新加载历史记录" @click="handleClearChat">
|
||||||
<el-icon><Delete /></el-icon>
|
<el-icon><Delete /></el-icon>
|
||||||
清空对话
|
清空对话
|
||||||
@@ -48,7 +66,7 @@
|
|||||||
<el-button
|
<el-button
|
||||||
text
|
text
|
||||||
size="small"
|
size="small"
|
||||||
:loading="speakingMessageIndex === index"
|
:type="speakingMessageIndex === index ? 'primary' : undefined"
|
||||||
@click="speakMessage(message.content, index)"
|
@click="speakMessage(message.content, index)"
|
||||||
title="语音朗读"
|
title="语音朗读"
|
||||||
>
|
>
|
||||||
@@ -158,11 +176,15 @@ import ChatMessageBubble from '@/components/ChatMessageBubble.vue'
|
|||||||
import ChatInputArea from '@/components/ChatInputArea.vue'
|
import ChatInputArea from '@/components/ChatInputArea.vue'
|
||||||
import { useFileUpload } from '@/composables/useFileUpload'
|
import { useFileUpload } from '@/composables/useFileUpload'
|
||||||
import type { PendingAttachment, MessageAttachment } from '@/composables/useFileUpload'
|
import type { PendingAttachment, MessageAttachment } from '@/composables/useFileUpload'
|
||||||
|
import { useMarkdown } from '@/composables/useMarkdown'
|
||||||
|
import { useTTS } from '@/composables/useTTS'
|
||||||
import api, { WORKFLOW_EXECUTION_HTTP_TIMEOUT_MS } from '@/api'
|
import api, { WORKFLOW_EXECUTION_HTTP_TIMEOUT_MS } from '@/api'
|
||||||
import { useUserStore } from '@/stores/user'
|
import { useUserStore } from '@/stores/user'
|
||||||
|
|
||||||
// ---- Shared file upload ----
|
// ---- Shared composables ----
|
||||||
const { pendingAttachments, fileInputRef, inputDragOver, uploadFiles, removeAttachment, clearAttachments, buildAttachmentContext, onFilesSelected, handleAttachFile } = useFileUpload()
|
const { pendingAttachments, fileInputRef, inputDragOver, uploadFiles, removeAttachment, clearAttachments, buildAttachmentContext, onFilesSelected, handleAttachFile } = useFileUpload()
|
||||||
|
const { stripMarkdownForTTS } = useMarkdown()
|
||||||
|
const { isSpeaking, speak: ttsSpeak, stop: ttsStop } = useTTS()
|
||||||
|
|
||||||
interface Message {
|
interface Message {
|
||||||
role: 'user' | 'agent'
|
role: 'user' | 'agent'
|
||||||
@@ -868,116 +890,32 @@ const stopRecording = () => {
|
|||||||
|
|
||||||
// ─── TTS 语音朗读 ─────────────────────────────────────
|
// ─── TTS 语音朗读 ─────────────────────────────────────
|
||||||
|
|
||||||
|
const selectedVoice = ref('teacher1')
|
||||||
const speakingMessageIndex = ref<number | null>(null)
|
const speakingMessageIndex = ref<number | null>(null)
|
||||||
let currentAudio: HTMLAudioElement | null = null
|
|
||||||
|
|
||||||
const speakMessage = async (content: string, index: number) => {
|
const speakMessage = async (content: string, index: number) => {
|
||||||
if (speakingMessageIndex.value === index && currentAudio) {
|
if (isSpeaking.value) {
|
||||||
currentAudio.pause()
|
ttsStop()
|
||||||
currentAudio = null
|
|
||||||
speakingMessageIndex.value = null
|
speakingMessageIndex.value = null
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if (currentAudio) {
|
const plainText = stripMarkdownForTTS(content)
|
||||||
currentAudio.pause()
|
|
||||||
currentAudio = null
|
|
||||||
}
|
|
||||||
|
|
||||||
const plainText = content.replace(/<[^>]*>/g, '').replace(/</g, '<').replace(/>/g, '>').replace(/&/g, '&')
|
|
||||||
|
|
||||||
if (!plainText.trim()) {
|
if (!plainText.trim()) {
|
||||||
ElMessage.warning('没有可朗读的内容')
|
ElMessage.warning('没有可朗读的内容')
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
speakingMessageIndex.value = index
|
speakingMessageIndex.value = index
|
||||||
try {
|
// useTTS().speak() uses server Edge TTS with browser SpeechSynthesis fallback
|
||||||
const resp = await api.post(
|
ttsSpeak(plainText, { voice: selectedVoice.value })
|
||||||
'/api/v1/executions',
|
// Watch for completion
|
||||||
{
|
const checkDone = setInterval(() => {
|
||||||
agent_id: props.agentId,
|
if (!isSpeaking.value) {
|
||||||
input_data: {
|
|
||||||
USER_INPUT: `请使用 text_to_speech 工具将以下文字转为语音文件:\n\n${plainText.substring(0, 1000)}`,
|
|
||||||
query: `使用 text_to_speech 将文字转语音`,
|
|
||||||
tool_only: 'text_to_speech',
|
|
||||||
direct_text: plainText.substring(0, 1000),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{ timeout: 120000 }
|
|
||||||
)
|
|
||||||
|
|
||||||
const executionId = resp.data.id
|
|
||||||
const pollForAudio = async () => {
|
|
||||||
try {
|
|
||||||
const statusResp = await api.get(`/api/v1/executions/${executionId}/status`)
|
|
||||||
const exec = statusResp.data
|
|
||||||
if (exec.status === 'completed') {
|
|
||||||
const outputData = exec.output_data || exec.result || {}
|
|
||||||
const resultStr = typeof outputData === 'string' ? outputData : JSON.stringify(outputData)
|
|
||||||
const pathMatch = resultStr.match(/output_path["\s:]+([^\s",}]+\.mp3)/) || resultStr.match(/([/\w]+\.mp3)/)
|
|
||||||
|
|
||||||
if (pathMatch) {
|
|
||||||
const mp3Path = pathMatch[1]
|
|
||||||
const audioResp = await api.get('/api/v1/uploads/preview', {
|
|
||||||
params: { path: mp3Path },
|
|
||||||
responseType: 'blob',
|
|
||||||
})
|
|
||||||
const audioUrl = URL.createObjectURL(audioResp.data as Blob)
|
|
||||||
currentAudio = new Audio(audioUrl)
|
|
||||||
currentAudio.onended = () => {
|
|
||||||
speakingMessageIndex.value = null
|
|
||||||
currentAudio = null
|
|
||||||
URL.revokeObjectURL(audioUrl)
|
|
||||||
}
|
|
||||||
currentAudio.onerror = () => {
|
|
||||||
speakingMessageIndex.value = null
|
|
||||||
currentAudio = null
|
|
||||||
ElMessage.error('语音播放失败')
|
|
||||||
}
|
|
||||||
await currentAudio.play()
|
|
||||||
} else {
|
|
||||||
await fallbackTTS(plainText)
|
|
||||||
}
|
|
||||||
} else if (exec.status === 'failed') {
|
|
||||||
await fallbackTTS(plainText)
|
|
||||||
} else {
|
|
||||||
setTimeout(() => pollForAudio(), 2000)
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
await fallbackTTS(plainText)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
setTimeout(() => pollForAudio(), 1000)
|
|
||||||
} catch (err: any) {
|
|
||||||
console.error('TTS 请求失败:', err)
|
|
||||||
speakingMessageIndex.value = null
|
|
||||||
await fallbackTTS(content.replace(/<[^>]*>/g, ''))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const fallbackTTS = async (text: string) => {
|
|
||||||
try {
|
|
||||||
if ('speechSynthesis' in window) {
|
|
||||||
const plainText = text.replace(/<[^>]*>/g, '').replace(/</g, '<').replace(/>/g, '>').replace(/&/g, '&').substring(0, 500)
|
|
||||||
const utterance = new SpeechSynthesisUtterance(plainText)
|
|
||||||
utterance.lang = 'zh-CN'
|
|
||||||
utterance.rate = 1.0
|
|
||||||
utterance.onend = () => {
|
|
||||||
speakingMessageIndex.value = null
|
|
||||||
}
|
|
||||||
utterance.onerror = () => {
|
|
||||||
speakingMessageIndex.value = null
|
|
||||||
}
|
|
||||||
window.speechSynthesis.speak(utterance)
|
|
||||||
} else {
|
|
||||||
speakingMessageIndex.value = null
|
speakingMessageIndex.value = null
|
||||||
ElMessage.warning('浏览器不支持语音合成')
|
clearInterval(checkDone)
|
||||||
}
|
}
|
||||||
} catch {
|
}, 300)
|
||||||
speakingMessageIndex.value = null
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const handleCloseNodeTest = () => {
|
const handleCloseNodeTest = () => {
|
||||||
|
|||||||
@@ -33,5 +33,42 @@ export function useMarkdown() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return { renderMarkdown }
|
/**
|
||||||
|
* Strip Markdown/HTML formatting to produce clean plain text for TTS.
|
||||||
|
* Renders to HTML first (handles all GFM: tables, lists, code, bold, etc.),
|
||||||
|
* then strips all tags and decodes entities.
|
||||||
|
*/
|
||||||
|
function stripMarkdownForTTS(text: string): string {
|
||||||
|
if (!text) return ''
|
||||||
|
try {
|
||||||
|
const html = marked.parse(text) as string
|
||||||
|
return html
|
||||||
|
.replace(/<[^>]*>/g, '') // Remove all HTML tags
|
||||||
|
.replace(/&/g, '&')
|
||||||
|
.replace(/</g, '<')
|
||||||
|
.replace(/>/g, '>')
|
||||||
|
.replace(/"/g, '"')
|
||||||
|
.replace(/'/g, "'")
|
||||||
|
.replace(/ /g, ' ')
|
||||||
|
.replace(/\n{3,}/g, '\n\n') // Collapse excessive newlines
|
||||||
|
.trim()
|
||||||
|
} catch {
|
||||||
|
// Fallback: simple regex-based stripping
|
||||||
|
return text
|
||||||
|
.replace(/```[\s\S]*?```/g, '') // code blocks
|
||||||
|
.replace(/`([^`]*)`/g, '$1') // inline code
|
||||||
|
.replace(/!\[.*?\]\(.*?\)/g, '') // images
|
||||||
|
.replace(/\[([^\]]*)\]\(.*?\)/g, '$1') // links → text
|
||||||
|
.replace(/^#{1,6}\s+/gm, '') // headings
|
||||||
|
.replace(/(\*{1,3}|_{1,3})(.*?)\1/g, '$2') // bold/italic
|
||||||
|
.replace(/^[*-]\s+/gm, '') // list bullets
|
||||||
|
.replace(/^>\s*/gm, '') // blockquote
|
||||||
|
.replace(/^---+$/gm, '') // horizontal rule
|
||||||
|
.replace(/\|/g, ' ') // table pipes
|
||||||
|
.replace(/\n{3,}/g, '\n\n')
|
||||||
|
.trim()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { renderMarkdown, stripMarkdownForTTS }
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,8 @@
|
|||||||
/**
|
/**
|
||||||
* 语音合成 (TTS) composable — 浏览器 SpeechSynthesis API
|
* 语音合成 (TTS) composable — 后端 Edge TTS 为主,浏览器 SpeechSynthesis 为降级
|
||||||
|
*
|
||||||
|
* 后端 Edge TTS 使用神经网络中文语音 + SSML 韵律标记,
|
||||||
|
* 比浏览器 SpeechSynthesis 更自然、有情感、有交流感。
|
||||||
*/
|
*/
|
||||||
import { ref, onUnmounted } from 'vue';
|
import { ref, onUnmounted } from 'vue';
|
||||||
|
|
||||||
@@ -9,20 +12,20 @@ const voices = ref<SpeechSynthesisVoice[]>([]);
|
|||||||
|
|
||||||
export function useTTS() {
|
export function useTTS() {
|
||||||
let utterance: SpeechSynthesisUtterance | null = null;
|
let utterance: SpeechSynthesisUtterance | null = null;
|
||||||
|
let serverAudio: HTMLAudioElement | null = null;
|
||||||
|
|
||||||
if (typeof window !== 'undefined' && 'speechSynthesis' in window) {
|
if (typeof window !== 'undefined' && 'speechSynthesis' in window) {
|
||||||
supported.value = true;
|
supported.value = true;
|
||||||
// 语音列表异步加载
|
|
||||||
speechSynthesis.onvoiceschanged = () => {
|
speechSynthesis.onvoiceschanged = () => {
|
||||||
voices.value = speechSynthesis.getVoices();
|
voices.value = speechSynthesis.getVoices();
|
||||||
};
|
};
|
||||||
voices.value = speechSynthesis.getVoices();
|
voices.value = speechSynthesis.getVoices();
|
||||||
}
|
}
|
||||||
|
|
||||||
function speak(text: string, options?: { rate?: number; pitch?: number; voice?: string; lang?: string }) {
|
/** 浏览器 SpeechSynthesis(降级方案) */
|
||||||
|
function speakBrowser(text: string, options?: { rate?: number; pitch?: number; voice?: string; lang?: string }) {
|
||||||
if (!supported.value) return;
|
if (!supported.value) return;
|
||||||
|
|
||||||
// 停止当前播放
|
|
||||||
stop();
|
stop();
|
||||||
isSpeaking.value = true;
|
isSpeaking.value = true;
|
||||||
|
|
||||||
@@ -42,7 +45,75 @@ export function useTTS() {
|
|||||||
speechSynthesis.speak(utterance);
|
speechSynthesis.speak(utterance);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 后端 Edge TTS(主方案)—— 神经网络中文语音 + SSML 自然韵律。
|
||||||
|
* 调用 POST /api/v1/voice/tts 获取 MP3 音频 URL,通过 Audio 元素播放。
|
||||||
|
*/
|
||||||
|
async function speak(text: string, options?: { voice?: string; speed?: number }) {
|
||||||
|
if (!text?.trim()) return;
|
||||||
|
|
||||||
|
stop();
|
||||||
|
isSpeaking.value = true;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const token = localStorage.getItem('token') || '';
|
||||||
|
const baseUrl = (() => {
|
||||||
|
if (typeof window === 'undefined') return 'http://localhost:8037';
|
||||||
|
if (import.meta.env.DEV) return '';
|
||||||
|
const h = window.location.hostname;
|
||||||
|
return h === 'localhost' || h === '127.0.0.1'
|
||||||
|
? 'http://localhost:8037'
|
||||||
|
: `${window.location.protocol}//${h}:8037`;
|
||||||
|
})();
|
||||||
|
|
||||||
|
const resp = await fetch(`${baseUrl}/api/v1/voice/tts`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
...(token ? { 'Authorization': `Bearer ${token}` } : {}),
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
text: text.substring(0, 4000),
|
||||||
|
voice: options?.voice || 'xiaoxiao',
|
||||||
|
speed: options?.speed ?? 1.0,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!resp.ok) {
|
||||||
|
throw new Error(`HTTP ${resp.status}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = await resp.json();
|
||||||
|
const audioUrl = data.audio_url;
|
||||||
|
if (!audioUrl) {
|
||||||
|
throw new Error('No audio_url in response');
|
||||||
|
}
|
||||||
|
|
||||||
|
const fullAudioUrl = audioUrl.startsWith('http')
|
||||||
|
? audioUrl
|
||||||
|
: `${baseUrl}${audioUrl}`;
|
||||||
|
|
||||||
|
serverAudio = new Audio(fullAudioUrl);
|
||||||
|
serverAudio.onended = () => { isSpeaking.value = false; serverAudio = null; };
|
||||||
|
serverAudio.onerror = () => {
|
||||||
|
isSpeaking.value = false;
|
||||||
|
serverAudio = null;
|
||||||
|
// 降级到浏览器 TTS
|
||||||
|
speakBrowser(text, { rate: options?.speed ?? 1.0 });
|
||||||
|
};
|
||||||
|
await serverAudio.play();
|
||||||
|
} catch {
|
||||||
|
// 网络错误时降级到浏览器 TTS
|
||||||
|
isSpeaking.value = false;
|
||||||
|
speakBrowser(text, { rate: options?.speed ?? 1.0 });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function stop() {
|
function stop() {
|
||||||
|
if (serverAudio) {
|
||||||
|
serverAudio.pause();
|
||||||
|
serverAudio = null;
|
||||||
|
}
|
||||||
if (speechSynthesis.speaking) {
|
if (speechSynthesis.speaking) {
|
||||||
speechSynthesis.cancel();
|
speechSynthesis.cancel();
|
||||||
}
|
}
|
||||||
@@ -55,17 +126,31 @@ export function useTTS() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 后端 TTS(生成音频文件URL)
|
* 后端 TTS(生成音频文件URL)—— 直接调用
|
||||||
*/
|
*/
|
||||||
export async function textToSpeechAPI(text: string, voice: string = 'alloy'): Promise<string> {
|
export async function textToSpeechAPI(text: string, voice: string = 'alloy', speed: number = 1.0): Promise<string> {
|
||||||
const resp = await fetch('/api/v1/tools/call', {
|
const token = localStorage.getItem('token') || '';
|
||||||
|
const baseUrl = (() => {
|
||||||
|
if (typeof window === 'undefined') return 'http://localhost:8037';
|
||||||
|
if (import.meta.env.DEV) return '';
|
||||||
|
const h = window.location.hostname;
|
||||||
|
return h === 'localhost' || h === '127.0.0.1'
|
||||||
|
? 'http://localhost:8037'
|
||||||
|
: `${window.location.protocol}//${h}:8037`;
|
||||||
|
})();
|
||||||
|
|
||||||
|
const resp = await fetch(`${baseUrl}/api/v1/voice/tts`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: {
|
||||||
body: JSON.stringify({
|
'Content-Type': 'application/json',
|
||||||
tool_name: 'text_to_speech',
|
...(token ? { 'Authorization': `Bearer ${token}` } : {}),
|
||||||
parameters: { text, voice },
|
},
|
||||||
}),
|
body: JSON.stringify({ text, voice, speed }),
|
||||||
});
|
});
|
||||||
|
if (!resp.ok) {
|
||||||
|
throw new Error(`TTS API error: ${resp.status}`);
|
||||||
|
}
|
||||||
const data = await resp.json();
|
const data = await resp.json();
|
||||||
return data.output?.audio_url || data.audio_url || '';
|
const audioUrl: string = data.audio_url || '';
|
||||||
|
return audioUrl.startsWith('http') ? audioUrl : `${baseUrl}${audioUrl}`;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -262,6 +262,25 @@
|
|||||||
<el-slider v-model="chatTemperature" :min="0" :max="2" :step="0.1" size="small" show-stops />
|
<el-slider v-model="chatTemperature" :min="0" :max="2" :step="0.1" size="small" show-stops />
|
||||||
<div style="margin:10px 0 4px;font-size:13px;font-weight:500">最大迭代</div>
|
<div style="margin:10px 0 4px;font-size:13px;font-weight:500">最大迭代</div>
|
||||||
<el-input-number v-model="chatMaxIterations" :min="1" :max="50" size="small" style="width:100%" />
|
<el-input-number v-model="chatMaxIterations" :min="1" :max="50" size="small" style="width:100%" />
|
||||||
|
<div style="margin:10px 0 4px;font-size:13px;font-weight:500">朗读语音</div>
|
||||||
|
<el-select v-model="chatVoice" size="small" style="width:100%">
|
||||||
|
<el-option-group label="── 教师女声 ──">
|
||||||
|
<el-option label="晓晓老师 · 温柔教导" value="teacher1" />
|
||||||
|
<el-option label="晓怡老师 · 活泼教学" value="teacher2" />
|
||||||
|
</el-option-group>
|
||||||
|
<el-option-group label="── 女声 ──">
|
||||||
|
<el-option label="晓晓 · 温柔" value="xiaoxiao" />
|
||||||
|
<el-option label="晓怡 · 活泼" value="xiaoyi" />
|
||||||
|
<el-option label="晓辰 · 温和" value="xiaochen" />
|
||||||
|
<el-option label="晓涵 · 知性" value="xiaohan" />
|
||||||
|
</el-option-group>
|
||||||
|
<el-option-group label="── 男声 ──">
|
||||||
|
<el-option label="云希 · 阳光" value="yunxi" />
|
||||||
|
<el-option label="云扬 · 专业" value="yunyang" />
|
||||||
|
<el-option label="云健 · 激情" value="yunjian" />
|
||||||
|
<el-option label="云夏 · 可爱" value="yunxia" />
|
||||||
|
</el-option-group>
|
||||||
|
</el-select>
|
||||||
</div>
|
</div>
|
||||||
</el-popover>
|
</el-popover>
|
||||||
</template>
|
</template>
|
||||||
@@ -339,7 +358,7 @@ import { useTTS } from '@/composables/useTTS'
|
|||||||
import { useSpeechRecognition } from '@/composables/useSpeechRecognition'
|
import { useSpeechRecognition } from '@/composables/useSpeechRecognition'
|
||||||
|
|
||||||
// Shared composables
|
// Shared composables
|
||||||
const { renderMarkdown } = useMarkdown()
|
const { renderMarkdown, stripMarkdownForTTS } = useMarkdown()
|
||||||
const { pendingAttachments, fileInputRef, inputDragOver, uploadFiles, removeAttachment, clearAttachments, buildAttachmentContext, onDragLeave, onDrop, onFilesSelected, handleAttachFile } = useFileUpload()
|
const { pendingAttachments, fileInputRef, inputDragOver, uploadFiles, removeAttachment, clearAttachments, buildAttachmentContext, onDragLeave, onDrop, onFilesSelected, handleAttachFile } = useFileUpload()
|
||||||
const { isSpeaking, speak: ttsSpeak, stop: ttsStop } = useTTS()
|
const { isSpeaking, speak: ttsSpeak, stop: ttsStop } = useTTS()
|
||||||
const { isListening, transcript, supported: voiceSupported, start: voiceStart, stop: voiceStop } = useSpeechRecognition()
|
const { isListening, transcript, supported: voiceSupported, start: voiceStart, stop: voiceStop } = useSpeechRecognition()
|
||||||
@@ -456,6 +475,7 @@ const sessionsLoading = ref(false)
|
|||||||
const chatModel = ref('deepseek-v4-pro')
|
const chatModel = ref('deepseek-v4-pro')
|
||||||
const chatTemperature = ref(0.8)
|
const chatTemperature = ref(0.8)
|
||||||
const chatMaxIterations = ref(15)
|
const chatMaxIterations = ref(15)
|
||||||
|
const chatVoice = ref('teacher1')
|
||||||
|
|
||||||
const currentAgentKey = computed(() => {
|
const currentAgentKey = computed(() => {
|
||||||
if (chatMode.value === 'orchestrate') return '__orchestrate__'
|
if (chatMode.value === 'orchestrate') return '__orchestrate__'
|
||||||
@@ -617,10 +637,9 @@ function sendPresetQuestion(q: string) {
|
|||||||
nextTick(() => sendMessage())
|
nextTick(() => sendMessage())
|
||||||
}
|
}
|
||||||
|
|
||||||
// TTS
|
|
||||||
function speakMessage(text: string) {
|
function speakMessage(text: string) {
|
||||||
if (!text) return
|
if (!text) return
|
||||||
ttsSpeak(text)
|
ttsSpeak(stripMarkdownForTTS(text), { voice: chatVoice.value })
|
||||||
}
|
}
|
||||||
function stopSpeak() {
|
function stopSpeak() {
|
||||||
ttsStop()
|
ttsStop()
|
||||||
|
|||||||
Reference in New Issue
Block a user