feat: Doubao-style video call with scenario selection, screen share & fullscreen camera
Browse filesRedesign the companion (视频通话) page to match Doubao's video calling experience:
- Scenario Selection: 6 scenarios (模拟面试, 英语陪练, 唱歌, 同声传译, 成语接龙, 心情树洞) in a floating grid
- Fullscreen camera mode: camera feed replaces AI avatar as background
- Screen sharing: getDisplayMedia capture with permission dialog & privacy notice
- Subtitle mode: camera shrinks to corner, transcript takes center stage
- Warm orange theme matching Doubao's brand color
- Bottom controls: camera → screen share → mic → speaker → call/hang up
- Camera/mic toggles work independently of WebSocket connection
- No auto-reconnect on connection failure (prevents button flashing)
- Cyberpunk-to-orange redesign: cleaner aesthetic with frosted glass elements
New files: CompanionTopBar, ScenarioSelector, ScreenShareDialog,
useScreenShare, useCameraDevices
Co-Authored-By: Claude Big Pickle <noreply@anthropic.com>
- desktop/src/components/companion/CompanionControls.tsx +75 -39
- desktop/src/components/companion/CompanionTopBar.tsx +62 -0
- desktop/src/components/companion/CompanionTranscript.tsx +131 -39
- desktop/src/components/companion/CompanionVideoPanel.tsx +114 -83
- desktop/src/components/companion/ScenarioSelector.tsx +65 -0
- desktop/src/components/companion/ScreenShareDialog.tsx +90 -0
- desktop/src/hooks/useCameraDevices.ts +66 -0
- desktop/src/hooks/useCompanionWebSocket.ts +1 -10
- desktop/src/hooks/useScreenShare.ts +115 -0
- desktop/src/i18n/locales/en.ts +21 -0
- desktop/src/i18n/locales/zh.ts +22 -1
- desktop/src/pages/Companion.tsx +145 -8
- desktop/src/stores/companionStore.ts +118 -1
- desktop/src/types/companion.ts +9 -0
|
@@ -1,3 +1,4 @@
|
|
|
|
|
| 1 |
import { useTranslation } from '../../i18n'
|
| 2 |
import type { CompanionStatus } from '../../types/companion'
|
| 3 |
|
|
@@ -6,8 +7,10 @@ interface CompanionControlsProps {
|
|
| 6 |
micEnabled: boolean
|
| 7 |
cameraEnabled: boolean
|
| 8 |
generating: boolean
|
|
|
|
| 9 |
onToggleMic: () => void
|
| 10 |
onToggleCamera: () => void
|
|
|
|
| 11 |
onConnect: () => void
|
| 12 |
onDisconnect: () => void
|
| 13 |
onStop: () => void
|
|
@@ -19,8 +22,10 @@ export function CompanionControls({
|
|
| 19 |
micEnabled,
|
| 20 |
cameraEnabled,
|
| 21 |
generating,
|
|
|
|
| 22 |
onToggleMic,
|
| 23 |
onToggleCamera,
|
|
|
|
| 24 |
onConnect,
|
| 25 |
onDisconnect,
|
| 26 |
onStop,
|
|
@@ -28,84 +33,115 @@ export function CompanionControls({
|
|
| 28 |
}: CompanionControlsProps) {
|
| 29 |
const t = useTranslation()
|
| 30 |
const isConnected = status === 'connected'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
|
| 32 |
return (
|
| 33 |
-
<div className="companion-controls absolute bottom-
|
| 34 |
-
{/*
|
| 35 |
<button
|
| 36 |
type="button"
|
| 37 |
-
onClick={
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
micEnabled
|
| 41 |
? 'bg-white/10 text-white hover:bg-white/20'
|
| 42 |
-
: 'bg-
|
| 43 |
-
}
|
| 44 |
-
title={
|
| 45 |
>
|
| 46 |
<span className="material-symbols-outlined text-xl">
|
| 47 |
-
{
|
| 48 |
</span>
|
| 49 |
</button>
|
| 50 |
|
| 51 |
-
{/*
|
| 52 |
<button
|
| 53 |
type="button"
|
| 54 |
-
onClick={
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
title={cameraEnabled ? t('companion.cameraOn') : t('companion.cameraOff')}
|
| 62 |
>
|
| 63 |
<span className="material-symbols-outlined text-xl">
|
| 64 |
-
{
|
| 65 |
</span>
|
| 66 |
</button>
|
| 67 |
|
| 68 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
|
| 70 |
-
{/* Stop generation */}
|
| 71 |
{generating && (
|
| 72 |
<button
|
| 73 |
type="button"
|
| 74 |
onClick={onStop}
|
| 75 |
-
className="flex h-10 w-10 items-center justify-center rounded-
|
| 76 |
title="Stop"
|
| 77 |
>
|
| 78 |
<span className="material-symbols-outlined text-xl">stop</span>
|
| 79 |
</button>
|
| 80 |
)}
|
| 81 |
|
| 82 |
-
{/*
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 83 |
{isConnected ? (
|
| 84 |
<button
|
| 85 |
type="button"
|
| 86 |
-
onClick={
|
| 87 |
-
|
| 88 |
-
}
|
| 89 |
-
className="flex h-10 items-center gap-2 rounded-xl bg-red-500/20 px-4 text-red-400 hover:bg-red-500/30 transition-all text-sm font-medium"
|
| 90 |
>
|
| 91 |
-
<span className="material-symbols-outlined text-
|
| 92 |
-
{t('companion.disconnect')}
|
| 93 |
</button>
|
| 94 |
) : (
|
| 95 |
<button
|
| 96 |
type="button"
|
| 97 |
-
onClick={
|
| 98 |
-
onConnect()
|
| 99 |
-
// Resume audio context on user gesture
|
| 100 |
-
onResumeAudio()
|
| 101 |
-
}}
|
| 102 |
disabled={status === 'connecting'}
|
| 103 |
-
className="flex h-10 items-center
|
|
|
|
| 104 |
>
|
| 105 |
-
<span className="material-symbols-outlined text-
|
| 106 |
-
{status === 'connecting' ? 'hourglass_top' : '
|
| 107 |
</span>
|
| 108 |
-
{status === 'connecting' ? t('companion.status.connecting') : t('companion.connect')}
|
| 109 |
</button>
|
| 110 |
)}
|
| 111 |
</div>
|
|
|
|
| 1 |
+
import { useCallback, useState } from 'react'
|
| 2 |
import { useTranslation } from '../../i18n'
|
| 3 |
import type { CompanionStatus } from '../../types/companion'
|
| 4 |
|
|
|
|
| 7 |
micEnabled: boolean
|
| 8 |
cameraEnabled: boolean
|
| 9 |
generating: boolean
|
| 10 |
+
isSharing: boolean
|
| 11 |
onToggleMic: () => void
|
| 12 |
onToggleCamera: () => void
|
| 13 |
+
onScreenShareClick: () => void
|
| 14 |
onConnect: () => void
|
| 15 |
onDisconnect: () => void
|
| 16 |
onStop: () => void
|
|
|
|
| 22 |
micEnabled,
|
| 23 |
cameraEnabled,
|
| 24 |
generating,
|
| 25 |
+
isSharing,
|
| 26 |
onToggleMic,
|
| 27 |
onToggleCamera,
|
| 28 |
+
onScreenShareClick,
|
| 29 |
onConnect,
|
| 30 |
onDisconnect,
|
| 31 |
onStop,
|
|
|
|
| 33 |
}: CompanionControlsProps) {
|
| 34 |
const t = useTranslation()
|
| 35 |
const isConnected = status === 'connected'
|
| 36 |
+
const [speakerOn, setSpeakerOn] = useState(true)
|
| 37 |
+
|
| 38 |
+
const handleConnectClick = useCallback(() => {
|
| 39 |
+
onConnect()
|
| 40 |
+
onResumeAudio()
|
| 41 |
+
}, [onConnect, onResumeAudio])
|
| 42 |
|
| 43 |
return (
|
| 44 |
+
<div className="companion-controls absolute bottom-6 left-1/2 -translate-x-1/2 flex items-center gap-3 px-5 py-2.5 rounded-full bg-black/50 backdrop-blur-md border border-white/10 z-10">
|
| 45 |
+
{/* 1. Camera toggle — always clickable */}
|
| 46 |
<button
|
| 47 |
type="button"
|
| 48 |
+
onClick={onToggleCamera}
|
| 49 |
+
className={`flex h-10 w-10 items-center justify-center rounded-full transition-all ${
|
| 50 |
+
cameraEnabled
|
|
|
|
| 51 |
? 'bg-white/10 text-white hover:bg-white/20'
|
| 52 |
+
: 'bg-white/5 text-white/40 hover:text-white/60'
|
| 53 |
+
}`}
|
| 54 |
+
title={cameraEnabled ? t('companion.cameraOn') : t('companion.cameraOff')}
|
| 55 |
>
|
| 56 |
<span className="material-symbols-outlined text-xl">
|
| 57 |
+
{cameraEnabled ? 'videocam' : 'videocam_off'}
|
| 58 |
</span>
|
| 59 |
</button>
|
| 60 |
|
| 61 |
+
{/* 2. Screen share — always clickable */}
|
| 62 |
<button
|
| 63 |
type="button"
|
| 64 |
+
onClick={onScreenShareClick}
|
| 65 |
+
className={`flex h-10 w-10 items-center justify-center rounded-full transition-all ${
|
| 66 |
+
isSharing
|
| 67 |
+
? 'bg-orange-500/30 text-orange-300'
|
| 68 |
+
: 'bg-white/10 text-white hover:bg-white/20'
|
| 69 |
+
}`}
|
| 70 |
+
title={isSharing ? t('companion.screenShare.stop') : t('companion.screenShare.start')}
|
|
|
|
| 71 |
>
|
| 72 |
<span className="material-symbols-outlined text-xl">
|
| 73 |
+
{isSharing ? 'present_to_all' : 'screen_share'}
|
| 74 |
</span>
|
| 75 |
</button>
|
| 76 |
|
| 77 |
+
{/* 3. Mic toggle — always clickable */}
|
| 78 |
+
<button
|
| 79 |
+
type="button"
|
| 80 |
+
onClick={onToggleMic}
|
| 81 |
+
className={`flex h-10 w-10 items-center justify-center rounded-full transition-all ${
|
| 82 |
+
micEnabled
|
| 83 |
+
? 'bg-white/10 text-white hover:bg-white/20'
|
| 84 |
+
: 'bg-white/5 text-white/40 hover:text-white/60'
|
| 85 |
+
}`}
|
| 86 |
+
title={micEnabled ? t('companion.micOn') : t('companion.micOff')}
|
| 87 |
+
>
|
| 88 |
+
<span className="material-symbols-outlined text-xl">
|
| 89 |
+
{micEnabled ? 'mic' : 'mic_off'}
|
| 90 |
+
</span>
|
| 91 |
+
</button>
|
| 92 |
|
| 93 |
+
{/* Stop generation (when generating) */}
|
| 94 |
{generating && (
|
| 95 |
<button
|
| 96 |
type="button"
|
| 97 |
onClick={onStop}
|
| 98 |
+
className="flex h-10 w-10 items-center justify-center rounded-full bg-orange-500/30 text-orange-300 hover:bg-orange-500/50 transition-all"
|
| 99 |
title="Stop"
|
| 100 |
>
|
| 101 |
<span className="material-symbols-outlined text-xl">stop</span>
|
| 102 |
</button>
|
| 103 |
)}
|
| 104 |
|
| 105 |
+
{/* Separator */}
|
| 106 |
+
<div className="w-px h-7 bg-white/10" />
|
| 107 |
+
|
| 108 |
+
{/* 4. Speaker toggle */}
|
| 109 |
+
<button
|
| 110 |
+
type="button"
|
| 111 |
+
onClick={() => setSpeakerOn(!speakerOn)}
|
| 112 |
+
className={`flex h-10 w-10 items-center justify-center rounded-full transition-all ${
|
| 113 |
+
speakerOn
|
| 114 |
+
? 'bg-white/10 text-white hover:bg-white/20'
|
| 115 |
+
: 'bg-white/5 text-white/40'
|
| 116 |
+
}`}
|
| 117 |
+
title={t('companion.speaker')}
|
| 118 |
+
>
|
| 119 |
+
<span className="material-symbols-outlined text-xl">
|
| 120 |
+
{speakerOn ? 'volume_up' : 'volume_off'}
|
| 121 |
+
</span>
|
| 122 |
+
</button>
|
| 123 |
+
|
| 124 |
+
{/* 5. Call / Hang Up */}
|
| 125 |
{isConnected ? (
|
| 126 |
<button
|
| 127 |
type="button"
|
| 128 |
+
onClick={onDisconnect}
|
| 129 |
+
className="flex h-10 w-10 items-center justify-center rounded-full bg-red-500 text-white hover:bg-red-600 transition-all shadow-lg"
|
| 130 |
+
title={t('companion.hangUp')}
|
|
|
|
| 131 |
>
|
| 132 |
+
<span className="material-symbols-outlined text-xl">call_end</span>
|
|
|
|
| 133 |
</button>
|
| 134 |
) : (
|
| 135 |
<button
|
| 136 |
type="button"
|
| 137 |
+
onClick={handleConnectClick}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 138 |
disabled={status === 'connecting'}
|
| 139 |
+
className="flex h-10 w-10 items-center justify-center rounded-full bg-green-500 text-white hover:bg-green-600 transition-all disabled:opacity-40 shadow-lg"
|
| 140 |
+
title={t('companion.call')}
|
| 141 |
>
|
| 142 |
+
<span className="material-symbols-outlined text-xl">
|
| 143 |
+
{status === 'connecting' ? 'hourglass_top' : 'call'}
|
| 144 |
</span>
|
|
|
|
| 145 |
</button>
|
| 146 |
)}
|
| 147 |
</div>
|
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { useCallback } from 'react'
|
| 2 |
+
import { useTranslation } from '../../i18n'
|
| 3 |
+
import { useCompanionStore, SCENARIOS } from '../../stores/companionStore'
|
| 4 |
+
|
| 5 |
+
export function CompanionTopBar() {
|
| 6 |
+
const t = useTranslation()
|
| 7 |
+
const scenario = useCompanionStore((s) => s.scenario)
|
| 8 |
+
const subtitleEnabled = useCompanionStore((s) => s.subtitleEnabled)
|
| 9 |
+
|
| 10 |
+
const setScenarioPanelOpen = useCompanionStore((s) => s.setScenarioPanelOpen)
|
| 11 |
+
const setSubtitleEnabled = useCompanionStore((s) => s.setSubtitleEnabled)
|
| 12 |
+
|
| 13 |
+
const currentScenario = scenario
|
| 14 |
+
? SCENARIOS.find((s) => s.id === scenario)
|
| 15 |
+
: null
|
| 16 |
+
|
| 17 |
+
const handleScenarioClick = useCallback(() => {
|
| 18 |
+
setScenarioPanelOpen(true)
|
| 19 |
+
}, [setScenarioPanelOpen])
|
| 20 |
+
|
| 21 |
+
const handleSubtitleToggle = useCallback(() => {
|
| 22 |
+
setSubtitleEnabled(!subtitleEnabled)
|
| 23 |
+
}, [subtitleEnabled, setSubtitleEnabled])
|
| 24 |
+
|
| 25 |
+
return (
|
| 26 |
+
<div className="absolute top-0 left-0 right-0 z-30 flex items-center justify-between px-4 pt-4 pb-2">
|
| 27 |
+
{/* Left spacer */}
|
| 28 |
+
<div className="w-9" />
|
| 29 |
+
|
| 30 |
+
{/* Center: scenario selector capsule */}
|
| 31 |
+
<button
|
| 32 |
+
type="button"
|
| 33 |
+
onClick={handleScenarioClick}
|
| 34 |
+
className="flex items-center gap-1.5 px-3.5 h-8 rounded-full bg-black/30 backdrop-blur-md border border-white/10 hover:bg-white/10 transition-all text-white/70 hover:text-white"
|
| 35 |
+
>
|
| 36 |
+
{currentScenario ? (
|
| 37 |
+
<>
|
| 38 |
+
<span className="material-symbols-outlined text-base">{currentScenario.icon}</span>
|
| 39 |
+
<span className="text-xs font-medium">{currentScenario.name}</span>
|
| 40 |
+
</>
|
| 41 |
+
) : (
|
| 42 |
+
<span className="text-xs font-medium">{t('companion.scenario.select')}</span>
|
| 43 |
+
)}
|
| 44 |
+
<span className="material-symbols-outlined text-sm text-white/30">expand_more</span>
|
| 45 |
+
</button>
|
| 46 |
+
|
| 47 |
+
{/* Right: subtitle toggle */}
|
| 48 |
+
<button
|
| 49 |
+
type="button"
|
| 50 |
+
onClick={handleSubtitleToggle}
|
| 51 |
+
className={`flex h-8 w-8 items-center justify-center rounded-full backdrop-blur-md transition-all text-xs font-bold ${
|
| 52 |
+
subtitleEnabled
|
| 53 |
+
? 'bg-orange-500/40 text-orange-200'
|
| 54 |
+
: 'bg-black/30 text-white/40 hover:text-white/60'
|
| 55 |
+
}`}
|
| 56 |
+
title={t('companion.subtitle')}
|
| 57 |
+
>
|
| 58 |
+
字
|
| 59 |
+
</button>
|
| 60 |
+
</div>
|
| 61 |
+
)
|
| 62 |
+
}
|
|
@@ -1,5 +1,6 @@
|
|
| 1 |
import { useRef, useEffect, useState, useCallback } from 'react'
|
| 2 |
import { useTranslation } from '../../i18n'
|
|
|
|
| 3 |
|
| 4 |
interface CompanionTranscriptProps {
|
| 5 |
transcript: string
|
|
@@ -9,10 +10,18 @@ interface CompanionTranscriptProps {
|
|
| 9 |
disabled?: boolean
|
| 10 |
}
|
| 11 |
|
| 12 |
-
export function CompanionTranscript({
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
const scrollRef = useRef<HTMLDivElement>(null)
|
| 14 |
const [inputValue, setInputValue] = useState('')
|
| 15 |
const t = useTranslation()
|
|
|
|
|
|
|
| 16 |
|
| 17 |
useEffect(() => {
|
| 18 |
if (scrollRef.current) {
|
|
@@ -27,48 +36,131 @@ export function CompanionTranscript({ transcript, fullTranscript, generating, on
|
|
| 27 |
setInputValue('')
|
| 28 |
}, [inputValue, onSendText])
|
| 29 |
|
| 30 |
-
const handleKeyDown = useCallback(
|
| 31 |
-
|
| 32 |
-
e.
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
|
|
|
|
|
|
|
|
|
| 36 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
return (
|
| 38 |
<div className="companion-transcript absolute bottom-20 left-1/2 -translate-x-1/2 w-full max-w-2xl px-4">
|
| 39 |
-
{
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
<div
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
generating
|
| 44 |
-
? 'border-purple-500/50 shadow-[0_0_20px_rgba(147,51,234,0.2)]'
|
| 45 |
-
: 'border-white/10'
|
| 46 |
}`}
|
| 47 |
>
|
| 48 |
-
{fullTranscript && (
|
| 49 |
-
<div className="text-white/40 text-xs mb-2 whitespace-pre-wrap leading-relaxed">
|
| 50 |
-
{fullTranscript}
|
| 51 |
-
</div>
|
| 52 |
-
)}
|
| 53 |
-
{transcript && (
|
| 54 |
-
<div className="text-white/90 text-sm font-medium leading-relaxed">
|
| 55 |
-
{transcript}
|
| 56 |
-
<span className="inline-block w-1.5 h-4 ml-0.5 bg-purple-400 animate-pulse" />
|
| 57 |
-
</div>
|
| 58 |
-
)}
|
| 59 |
-
{!transcript && !fullTranscript && (
|
| 60 |
-
<div className="text-white/30 text-sm text-center py-2">
|
| 61 |
-
{t('companion.transcriptPlaceholder')}
|
| 62 |
-
</div>
|
| 63 |
-
)}
|
| 64 |
-
</div>
|
| 65 |
-
|
| 66 |
-
{/* Text input row */}
|
| 67 |
-
<div className={`flex items-center gap-2 rounded-2xl bg-black/40 backdrop-blur-md border px-4 py-2 transition-all duration-500 ${
|
| 68 |
-
generating
|
| 69 |
-
? 'border-purple-500/50 shadow-[0_0_20px_rgba(147,51,234,0.2)]'
|
| 70 |
-
: 'border-white/10'
|
| 71 |
-
}`}>
|
| 72 |
<input
|
| 73 |
type="text"
|
| 74 |
value={inputValue}
|
|
@@ -76,13 +168,13 @@ export function CompanionTranscript({ transcript, fullTranscript, generating, on
|
|
| 76 |
onKeyDown={handleKeyDown}
|
| 77 |
placeholder={t('companion.inputPlaceholder')}
|
| 78 |
disabled={disabled}
|
| 79 |
-
className="flex-1 bg-transparent text-sm text-white/
|
| 80 |
/>
|
| 81 |
<button
|
| 82 |
type="button"
|
| 83 |
onClick={handleSend}
|
| 84 |
disabled={disabled || !inputValue.trim()}
|
| 85 |
-
className="flex h-8 w-8 items-center justify-center rounded-lg bg-
|
| 86 |
title={t('companion.send')}
|
| 87 |
>
|
| 88 |
<span className="material-symbols-outlined text-lg">send</span>
|
|
|
|
| 1 |
import { useRef, useEffect, useState, useCallback } from 'react'
|
| 2 |
import { useTranslation } from '../../i18n'
|
| 3 |
+
import { useCompanionStore } from '../../stores/companionStore'
|
| 4 |
|
| 5 |
interface CompanionTranscriptProps {
|
| 6 |
transcript: string
|
|
|
|
| 10 |
disabled?: boolean
|
| 11 |
}
|
| 12 |
|
| 13 |
+
export function CompanionTranscript({
|
| 14 |
+
transcript,
|
| 15 |
+
fullTranscript,
|
| 16 |
+
generating,
|
| 17 |
+
onSendText,
|
| 18 |
+
disabled,
|
| 19 |
+
}: CompanionTranscriptProps) {
|
| 20 |
const scrollRef = useRef<HTMLDivElement>(null)
|
| 21 |
const [inputValue, setInputValue] = useState('')
|
| 22 |
const t = useTranslation()
|
| 23 |
+
const subtitleEnabled = useCompanionStore((s) => s.subtitleEnabled)
|
| 24 |
+
const hasContent = !!transcript || !!fullTranscript
|
| 25 |
|
| 26 |
useEffect(() => {
|
| 27 |
if (scrollRef.current) {
|
|
|
|
| 36 |
setInputValue('')
|
| 37 |
}, [inputValue, onSendText])
|
| 38 |
|
| 39 |
+
const handleKeyDown = useCallback(
|
| 40 |
+
(e: React.KeyboardEvent) => {
|
| 41 |
+
if (e.key === 'Enter' && !e.shiftKey) {
|
| 42 |
+
e.preventDefault()
|
| 43 |
+
handleSend()
|
| 44 |
+
}
|
| 45 |
+
},
|
| 46 |
+
[handleSend],
|
| 47 |
+
)
|
| 48 |
|
| 49 |
+
// When subtitles are on and there's content, show full-screen transcript
|
| 50 |
+
if (subtitleEnabled && hasContent) {
|
| 51 |
+
return (
|
| 52 |
+
<div className="absolute inset-0 z-5 flex flex-col">
|
| 53 |
+
{/* Scrollable transcript area - takes most of the screen */}
|
| 54 |
+
<div className="flex-1 flex items-center justify-center px-8 py-20">
|
| 55 |
+
<div
|
| 56 |
+
ref={scrollRef}
|
| 57 |
+
className="w-full max-w-2xl max-h-full overflow-y-auto scrollbar-thin"
|
| 58 |
+
>
|
| 59 |
+
{fullTranscript && (
|
| 60 |
+
<div className="text-white/50 text-sm mb-3 whitespace-pre-wrap leading-relaxed text-center">
|
| 61 |
+
{fullTranscript}
|
| 62 |
+
</div>
|
| 63 |
+
)}
|
| 64 |
+
{transcript && (
|
| 65 |
+
<div className="text-white/90 text-base font-medium leading-relaxed text-center">
|
| 66 |
+
{transcript}
|
| 67 |
+
<span className="inline-block w-1.5 h-4 ml-0.5 bg-orange-400 animate-pulse" />
|
| 68 |
+
</div>
|
| 69 |
+
)}
|
| 70 |
+
</div>
|
| 71 |
+
</div>
|
| 72 |
+
|
| 73 |
+
{/* Text input at bottom */}
|
| 74 |
+
<div className="absolute bottom-20 left-1/2 -translate-x-1/2 w-full max-w-md px-4">
|
| 75 |
+
<div className="flex items-center gap-2 rounded-full bg-black/50 backdrop-blur-md border border-white/10 px-4 py-2">
|
| 76 |
+
<input
|
| 77 |
+
type="text"
|
| 78 |
+
value={inputValue}
|
| 79 |
+
onChange={(e) => setInputValue(e.target.value)}
|
| 80 |
+
onKeyDown={handleKeyDown}
|
| 81 |
+
placeholder={t('companion.inputPlaceholder')}
|
| 82 |
+
disabled={disabled}
|
| 83 |
+
className="flex-1 bg-transparent text-sm text-white/80 placeholder-white/30 outline-none border-none disabled:opacity-40"
|
| 84 |
+
/>
|
| 85 |
+
<button
|
| 86 |
+
type="button"
|
| 87 |
+
onClick={handleSend}
|
| 88 |
+
disabled={disabled || !inputValue.trim()}
|
| 89 |
+
className="flex h-7 w-7 items-center justify-center rounded-full bg-orange-500/40 text-orange-200 hover:bg-orange-500/60 transition-all disabled:opacity-30"
|
| 90 |
+
title={t('companion.send')}
|
| 91 |
+
>
|
| 92 |
+
<span className="material-symbols-outlined text-sm">send</span>
|
| 93 |
+
</button>
|
| 94 |
+
</div>
|
| 95 |
+
</div>
|
| 96 |
+
</div>
|
| 97 |
+
)
|
| 98 |
+
}
|
| 99 |
+
|
| 100 |
+
// Subtitles on but no content yet - show compact version
|
| 101 |
+
if (subtitleEnabled && !hasContent) {
|
| 102 |
+
return (
|
| 103 |
+
<div className="companion-transcript absolute bottom-20 left-1/2 -translate-x-1/2 w-full max-w-md px-4">
|
| 104 |
+
<div
|
| 105 |
+
className={`flex items-center gap-2 rounded-full bg-black/40 backdrop-blur-md border border-white/10 px-4 py-2 transition-all duration-500 ${
|
| 106 |
+
generating ? 'border-orange-400/40' : ''
|
| 107 |
+
}`}
|
| 108 |
+
>
|
| 109 |
+
<input
|
| 110 |
+
type="text"
|
| 111 |
+
value={inputValue}
|
| 112 |
+
onChange={(e) => setInputValue(e.target.value)}
|
| 113 |
+
onKeyDown={handleKeyDown}
|
| 114 |
+
placeholder={t('companion.inputPlaceholder')}
|
| 115 |
+
disabled={disabled}
|
| 116 |
+
className="flex-1 bg-transparent text-sm text-white/80 placeholder-white/30 outline-none border-none disabled:opacity-40"
|
| 117 |
+
/>
|
| 118 |
+
<button
|
| 119 |
+
type="button"
|
| 120 |
+
onClick={handleSend}
|
| 121 |
+
disabled={disabled || !inputValue.trim()}
|
| 122 |
+
className="flex h-7 w-7 items-center justify-center rounded-full bg-orange-500/40 text-orange-200 hover:bg-orange-500/60 transition-all disabled:opacity-30"
|
| 123 |
+
title={t('companion.send')}
|
| 124 |
+
>
|
| 125 |
+
<span className="material-symbols-outlined text-sm">send</span>
|
| 126 |
+
</button>
|
| 127 |
+
</div>
|
| 128 |
+
</div>
|
| 129 |
+
)
|
| 130 |
+
}
|
| 131 |
+
|
| 132 |
+
// Subtitles off - show compact transcript bubble
|
| 133 |
return (
|
| 134 |
<div className="companion-transcript absolute bottom-20 left-1/2 -translate-x-1/2 w-full max-w-2xl px-4">
|
| 135 |
+
{hasContent && (
|
| 136 |
+
<div
|
| 137 |
+
ref={scrollRef}
|
| 138 |
+
className={`max-h-[120px] overflow-y-auto rounded-2xl bg-black/40 backdrop-blur-md border p-3 mb-2 scrollbar-thin transition-all duration-500 ${
|
| 139 |
+
generating
|
| 140 |
+
? 'border-orange-400/40'
|
| 141 |
+
: 'border-white/10'
|
| 142 |
+
}`}
|
| 143 |
+
>
|
| 144 |
+
{fullTranscript && !transcript && (
|
| 145 |
+
<div className="text-white/40 text-xs whitespace-pre-wrap leading-relaxed">
|
| 146 |
+
{fullTranscript}
|
| 147 |
+
</div>
|
| 148 |
+
)}
|
| 149 |
+
{transcript && (
|
| 150 |
+
<div className="text-white/80 text-sm font-medium leading-relaxed">
|
| 151 |
+
{transcript}
|
| 152 |
+
<span className="inline-block w-1.5 h-4 ml-0.5 bg-orange-400 animate-pulse" />
|
| 153 |
+
</div>
|
| 154 |
+
)}
|
| 155 |
+
</div>
|
| 156 |
+
)}
|
| 157 |
+
|
| 158 |
+
{/* Text input */}
|
| 159 |
<div
|
| 160 |
+
className={`flex items-center gap-2 rounded-2xl bg-black/40 backdrop-blur-md border px-4 py-2 transition-all duration-500 ${
|
| 161 |
+
generating ? 'border-orange-400/40' : 'border-white/10'
|
|
|
|
|
|
|
|
|
|
| 162 |
}`}
|
| 163 |
>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 164 |
<input
|
| 165 |
type="text"
|
| 166 |
value={inputValue}
|
|
|
|
| 168 |
onKeyDown={handleKeyDown}
|
| 169 |
placeholder={t('companion.inputPlaceholder')}
|
| 170 |
disabled={disabled}
|
| 171 |
+
className="flex-1 bg-transparent text-sm text-white/80 placeholder-white/30 outline-none border-none disabled:opacity-40"
|
| 172 |
/>
|
| 173 |
<button
|
| 174 |
type="button"
|
| 175 |
onClick={handleSend}
|
| 176 |
disabled={disabled || !inputValue.trim()}
|
| 177 |
+
className="flex h-8 w-8 items-center justify-center rounded-lg bg-orange-500/40 text-orange-200 hover:bg-orange-500/60 transition-all disabled:opacity-30"
|
| 178 |
title={t('companion.send')}
|
| 179 |
>
|
| 180 |
<span className="material-symbols-outlined text-lg">send</span>
|
|
@@ -8,6 +8,7 @@ interface CompanionVideoPanelProps {
|
|
| 8 |
speaking: boolean
|
| 9 |
generating: boolean
|
| 10 |
status: CompanionStatus
|
|
|
|
| 11 |
}
|
| 12 |
|
| 13 |
export function CompanionVideoPanel({
|
|
@@ -15,15 +16,20 @@ export function CompanionVideoPanel({
|
|
| 15 |
speaking,
|
| 16 |
generating,
|
| 17 |
status,
|
|
|
|
| 18 |
}: CompanionVideoPanelProps) {
|
| 19 |
const videoRef = useRef<HTMLVideoElement>(null)
|
| 20 |
const avatarFileRef = useRef<HTMLInputElement>(null)
|
| 21 |
-
const bgFileRef = useRef<HTMLInputElement>(null)
|
| 22 |
const t = useTranslation()
|
| 23 |
const avatarUrl = useCompanionStore((s) => s.avatarUrl)
|
| 24 |
const backgroundUrl = useCompanionStore((s) => s.backgroundUrl)
|
| 25 |
const setAvatarUrl = useCompanionStore((s) => s.setAvatarUrl)
|
| 26 |
-
const
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
|
| 28 |
useEffect(() => {
|
| 29 |
if (videoRef.current && webcamStream) {
|
|
@@ -35,10 +41,6 @@ export function CompanionVideoPanel({
|
|
| 35 |
avatarFileRef.current?.click()
|
| 36 |
}, [])
|
| 37 |
|
| 38 |
-
const handleBgPick = useCallback(() => {
|
| 39 |
-
bgFileRef.current?.click()
|
| 40 |
-
}, [])
|
| 41 |
-
|
| 42 |
const handleAvatarFile = useCallback(
|
| 43 |
(e: React.ChangeEvent<HTMLInputElement>) => {
|
| 44 |
const file = e.target.files?.[0]
|
|
@@ -46,23 +48,14 @@ export function CompanionVideoPanel({
|
|
| 46 |
const reader = new FileReader()
|
| 47 |
reader.onload = () => setAvatarUrl(reader.result as string)
|
| 48 |
reader.readAsDataURL(file)
|
| 49 |
-
// Reset so the same file can be re-selected
|
| 50 |
e.target.value = ''
|
| 51 |
},
|
| 52 |
[setAvatarUrl],
|
| 53 |
)
|
| 54 |
|
| 55 |
-
const
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
if (!file) return
|
| 59 |
-
const reader = new FileReader()
|
| 60 |
-
reader.onload = () => setBackgroundUrl(reader.result as string)
|
| 61 |
-
reader.readAsDataURL(file)
|
| 62 |
-
e.target.value = ''
|
| 63 |
-
},
|
| 64 |
-
[setBackgroundUrl],
|
| 65 |
-
)
|
| 66 |
|
| 67 |
const statusLabel = generating
|
| 68 |
? t('companion.speaking')
|
|
@@ -74,79 +67,107 @@ export function CompanionVideoPanel({
|
|
| 74 |
|
| 75 |
return (
|
| 76 |
<div className="companion-video-panel absolute inset-0 overflow-hidden">
|
| 77 |
-
{/* Background */}
|
| 78 |
{backgroundUrl ? (
|
| 79 |
<div
|
| 80 |
className="absolute inset-0 bg-cover bg-center transition-all duration-700"
|
| 81 |
style={{ backgroundImage: `url(${backgroundUrl})` }}
|
| 82 |
/>
|
| 83 |
) : (
|
| 84 |
-
<div className="absolute inset-0 bg-gradient-to-br from-
|
| 85 |
)}
|
| 86 |
|
| 87 |
-
{/*
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 104 |
|
| 105 |
-
{/* AI
|
| 106 |
-
|
| 107 |
-
<div
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 132 |
>
|
| 133 |
-
<span className="material-symbols-outlined text-
|
| 134 |
-
|
| 135 |
-
</span>
|
| 136 |
-
</div>
|
| 137 |
)}
|
| 138 |
</div>
|
| 139 |
-
{
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
onClick={handleAvatarPick}
|
| 143 |
-
className="absolute -bottom-1 -right-1 w-8 h-8 rounded-full bg-purple-600/80 hover:bg-purple-600 flex items-center justify-center transition-all opacity-0 group-hover:opacity-100 shadow-lg"
|
| 144 |
-
>
|
| 145 |
-
<span className="material-symbols-outlined text-sm text-white">edit</span>
|
| 146 |
-
</button>
|
| 147 |
</div>
|
| 148 |
-
|
| 149 |
-
</div>
|
| 150 |
<input
|
| 151 |
ref={avatarFileRef}
|
| 152 |
type="file"
|
|
@@ -156,8 +177,14 @@ export function CompanionVideoPanel({
|
|
| 156 |
/>
|
| 157 |
|
| 158 |
{/* Webcam PiP overlay */}
|
| 159 |
-
{webcamStream && (
|
| 160 |
-
<div
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 161 |
<video
|
| 162 |
ref={videoRef}
|
| 163 |
autoPlay
|
|
@@ -168,8 +195,12 @@ export function CompanionVideoPanel({
|
|
| 168 |
</div>
|
| 169 |
)}
|
| 170 |
|
| 171 |
-
{/* Connection status badge */}
|
| 172 |
-
<div
|
|
|
|
|
|
|
|
|
|
|
|
|
| 173 |
<span
|
| 174 |
className={`w-2 h-2 rounded-full ${
|
| 175 |
status === 'connected'
|
|
@@ -181,7 +212,7 @@ export function CompanionVideoPanel({
|
|
| 181 |
: 'bg-gray-500'
|
| 182 |
}`}
|
| 183 |
/>
|
| 184 |
-
<span className="text-white/
|
| 185 |
</div>
|
| 186 |
</div>
|
| 187 |
)
|
|
|
|
| 8 |
speaking: boolean
|
| 9 |
generating: boolean
|
| 10 |
status: CompanionStatus
|
| 11 |
+
onFlipCamera?: () => void
|
| 12 |
}
|
| 13 |
|
| 14 |
export function CompanionVideoPanel({
|
|
|
|
| 16 |
speaking,
|
| 17 |
generating,
|
| 18 |
status,
|
| 19 |
+
onFlipCamera,
|
| 20 |
}: CompanionVideoPanelProps) {
|
| 21 |
const videoRef = useRef<HTMLVideoElement>(null)
|
| 22 |
const avatarFileRef = useRef<HTMLInputElement>(null)
|
|
|
|
| 23 |
const t = useTranslation()
|
| 24 |
const avatarUrl = useCompanionStore((s) => s.avatarUrl)
|
| 25 |
const backgroundUrl = useCompanionStore((s) => s.backgroundUrl)
|
| 26 |
const setAvatarUrl = useCompanionStore((s) => s.setAvatarUrl)
|
| 27 |
+
const cameraFullscreen = useCompanionStore((s) => s.cameraFullscreen)
|
| 28 |
+
const cameraEnabled = useCompanionStore((s) => s.cameraEnabled)
|
| 29 |
+
const screenShareStream = useCompanionStore((s) => s.screenShareStream)
|
| 30 |
+
const transcript = useCompanionStore((s) => s.transcript)
|
| 31 |
+
const fullTranscript = useCompanionStore((s) => s.fullTranscript)
|
| 32 |
+
const subtitleEnabled = useCompanionStore((s) => s.subtitleEnabled)
|
| 33 |
|
| 34 |
useEffect(() => {
|
| 35 |
if (videoRef.current && webcamStream) {
|
|
|
|
| 41 |
avatarFileRef.current?.click()
|
| 42 |
}, [])
|
| 43 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
const handleAvatarFile = useCallback(
|
| 45 |
(e: React.ChangeEvent<HTMLInputElement>) => {
|
| 46 |
const file = e.target.files?.[0]
|
|
|
|
| 48 |
const reader = new FileReader()
|
| 49 |
reader.onload = () => setAvatarUrl(reader.result as string)
|
| 50 |
reader.readAsDataURL(file)
|
|
|
|
| 51 |
e.target.value = ''
|
| 52 |
},
|
| 53 |
[setAvatarUrl],
|
| 54 |
)
|
| 55 |
|
| 56 |
+
const isFullScreenCamera = cameraEnabled && cameraFullscreen && webcamStream
|
| 57 |
+
// When subtitle is on AND there's transcript content, shrink the main view
|
| 58 |
+
const hasTranscriptContent = subtitleEnabled && (!!transcript || !!fullTranscript)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 59 |
|
| 60 |
const statusLabel = generating
|
| 61 |
? t('companion.speaking')
|
|
|
|
| 67 |
|
| 68 |
return (
|
| 69 |
<div className="companion-video-panel absolute inset-0 overflow-hidden">
|
| 70 |
+
{/* Background - clean warm gradient */}
|
| 71 |
{backgroundUrl ? (
|
| 72 |
<div
|
| 73 |
className="absolute inset-0 bg-cover bg-center transition-all duration-700"
|
| 74 |
style={{ backgroundImage: `url(${backgroundUrl})` }}
|
| 75 |
/>
|
| 76 |
) : (
|
| 77 |
+
<div className="absolute inset-0 bg-gradient-to-br from-orange-950/80 via-slate-950 to-amber-950/60 transition-all duration-700" />
|
| 78 |
)}
|
| 79 |
|
| 80 |
+
{/* Fullscreen camera view */}
|
| 81 |
+
{isFullScreenCamera && (
|
| 82 |
+
<div className="absolute inset-0 bg-black">
|
| 83 |
+
<video
|
| 84 |
+
ref={videoRef}
|
| 85 |
+
autoPlay
|
| 86 |
+
muted
|
| 87 |
+
playsInline
|
| 88 |
+
className="h-full w-full object-cover scale-x-[-1]"
|
| 89 |
+
/>
|
| 90 |
+
{/* Flip camera overlay */}
|
| 91 |
+
<div className="absolute top-14 right-4 flex flex-col gap-2 z-20">
|
| 92 |
+
<button
|
| 93 |
+
type="button"
|
| 94 |
+
onClick={onFlipCamera}
|
| 95 |
+
className="flex h-10 w-10 items-center justify-center rounded-full bg-black/40 backdrop-blur-md text-white/70 hover:text-white hover:bg-black/60 transition-all"
|
| 96 |
+
title={t('companion.flipCamera')}
|
| 97 |
+
>
|
| 98 |
+
<span className="material-symbols-outlined text-xl">flip_camera_android</span>
|
| 99 |
+
</button>
|
| 100 |
+
</div>
|
| 101 |
+
</div>
|
| 102 |
+
)}
|
| 103 |
+
|
| 104 |
+
{/* Screen share overlay */}
|
| 105 |
+
{screenShareStream && !isFullScreenCamera && (
|
| 106 |
+
<div className="absolute inset-0 bg-black/60 flex items-center justify-center">
|
| 107 |
+
<div className="text-center">
|
| 108 |
+
<span className="material-symbols-outlined text-5xl text-orange-400/60 mb-3">present_to_all</span>
|
| 109 |
+
<p className="text-white/50 text-sm">正在共享屏幕</p>
|
| 110 |
+
</div>
|
| 111 |
+
</div>
|
| 112 |
+
)}
|
| 113 |
|
| 114 |
+
{/* AI Avatar - large center (default), small top-left when subtitles active */}
|
| 115 |
+
{!isFullScreenCamera && !screenShareStream && (
|
| 116 |
+
<div
|
| 117 |
+
className={`absolute flex flex-col items-center gap-3 transition-all duration-500 ${
|
| 118 |
+
hasTranscriptContent
|
| 119 |
+
? 'top-6 left-6 scale-[0.45] origin-top-left'
|
| 120 |
+
: 'left-1/2 top-1/2 -translate-x-1/2 -translate-y-1/2'
|
| 121 |
+
}`}
|
| 122 |
+
>
|
| 123 |
+
<div className="relative">
|
| 124 |
+
<div
|
| 125 |
+
className={`rounded-full transition-all duration-700 flex items-center justify-center overflow-hidden ${
|
| 126 |
+
hasTranscriptContent ? 'w-24 h-24' : 'w-36 h-36'
|
| 127 |
+
} ${
|
| 128 |
+
generating
|
| 129 |
+
? 'shadow-[0_0_60px_rgba(251,146,60,0.4)] ring-2 ring-orange-400/60'
|
| 130 |
+
: speaking
|
| 131 |
+
? 'shadow-[0_0_40px_rgba(251,146,60,0.25)] ring-2 ring-orange-400/30'
|
| 132 |
+
: 'shadow-[0_0_15px_rgba(255,255,255,0.08)] ring-1 ring-white/10'
|
| 133 |
+
}`}
|
| 134 |
+
style={{
|
| 135 |
+
background: generating
|
| 136 |
+
? 'radial-gradient(circle, rgba(251,146,60,0.25) 0%, rgba(249,115,22,0.1) 100%)'
|
| 137 |
+
: speaking
|
| 138 |
+
? 'radial-gradient(circle, rgba(251,146,60,0.15) 0%, rgba(249,115,22,0.05) 100%)'
|
| 139 |
+
: 'radial-gradient(circle, rgba(255,255,255,0.04) 0%, transparent 100%)',
|
| 140 |
+
}}
|
| 141 |
+
>
|
| 142 |
+
{avatarUrl ? (
|
| 143 |
+
<img src={avatarUrl} alt="avatar" className="w-full h-full object-cover" />
|
| 144 |
+
) : (
|
| 145 |
+
<div className="w-full h-full flex items-center justify-center">
|
| 146 |
+
{(speaking || generating) && (
|
| 147 |
+
<div className="absolute inset-0 rounded-full animate-ping opacity-10 bg-orange-400" />
|
| 148 |
+
)}
|
| 149 |
+
{/* Default female avatar silhouette */}
|
| 150 |
+
<div className="flex flex-col items-center justify-center text-white/50">
|
| 151 |
+
<span className="material-symbols-outlined text-5xl">face_6</span>
|
| 152 |
+
</div>
|
| 153 |
+
</div>
|
| 154 |
+
)}
|
| 155 |
+
</div>
|
| 156 |
+
{!hasTranscriptContent && (
|
| 157 |
+
<button
|
| 158 |
+
type="button"
|
| 159 |
+
onClick={handleAvatarPick}
|
| 160 |
+
className="absolute -bottom-1 -right-1 w-7 h-7 rounded-full bg-orange-500/70 hover:bg-orange-500 flex items-center justify-center transition-all opacity-0 group-hover:opacity-100 shadow-lg z-10"
|
| 161 |
>
|
| 162 |
+
<span className="material-symbols-outlined text-xs text-white">edit</span>
|
| 163 |
+
</button>
|
|
|
|
|
|
|
| 164 |
)}
|
| 165 |
</div>
|
| 166 |
+
{!hasTranscriptContent && (
|
| 167 |
+
<span className="text-white/40 text-xs font-medium tracking-wide">{statusLabel}</span>
|
| 168 |
+
)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 169 |
</div>
|
| 170 |
+
)}
|
|
|
|
| 171 |
<input
|
| 172 |
ref={avatarFileRef}
|
| 173 |
type="file"
|
|
|
|
| 177 |
/>
|
| 178 |
|
| 179 |
{/* Webcam PiP overlay */}
|
| 180 |
+
{webcamStream && !isFullScreenCamera && (
|
| 181 |
+
<div
|
| 182 |
+
className={`absolute rounded-2xl overflow-hidden border-2 border-white/15 shadow-2xl bg-black z-10 transition-all duration-500 ${
|
| 183 |
+
hasTranscriptContent
|
| 184 |
+
? 'top-4 right-4 w-[120px] h-[160px]'
|
| 185 |
+
: 'bottom-6 right-6 w-[160px] h-[220px]'
|
| 186 |
+
}`}
|
| 187 |
+
>
|
| 188 |
<video
|
| 189 |
ref={videoRef}
|
| 190 |
autoPlay
|
|
|
|
| 195 |
</div>
|
| 196 |
)}
|
| 197 |
|
| 198 |
+
{/* Connection status badge - top right (moved left when webcam PiP shows) */}
|
| 199 |
+
<div
|
| 200 |
+
className={`absolute top-4 flex items-center gap-2 px-3 py-1.5 rounded-full bg-black/40 backdrop-blur-sm z-10 ${
|
| 201 |
+
webcamStream && !isFullScreenCamera ? 'left-4' : 'right-4'
|
| 202 |
+
}`}
|
| 203 |
+
>
|
| 204 |
<span
|
| 205 |
className={`w-2 h-2 rounded-full ${
|
| 206 |
status === 'connected'
|
|
|
|
| 212 |
: 'bg-gray-500'
|
| 213 |
}`}
|
| 214 |
/>
|
| 215 |
+
<span className="text-white/50 text-xs">{status}</span>
|
| 216 |
</div>
|
| 217 |
</div>
|
| 218 |
)
|
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { useCallback } from 'react'
|
| 2 |
+
import { useTranslation } from '../../i18n'
|
| 3 |
+
import { useCompanionStore, SCENARIOS } from '../../stores/companionStore'
|
| 4 |
+
|
| 5 |
+
export function ScenarioSelector() {
|
| 6 |
+
const t = useTranslation()
|
| 7 |
+
const open = useCompanionStore((s) => s.scenarioPanelOpen)
|
| 8 |
+
const currentScenario = useCompanionStore((s) => s.scenario)
|
| 9 |
+
const setScenario = useCompanionStore((s) => s.setScenario)
|
| 10 |
+
const setScenarioPanelOpen = useCompanionStore((s) => s.setScenarioPanelOpen)
|
| 11 |
+
|
| 12 |
+
const handleSelect = useCallback(
|
| 13 |
+
(id: string) => {
|
| 14 |
+
setScenario(id)
|
| 15 |
+
},
|
| 16 |
+
[setScenario],
|
| 17 |
+
)
|
| 18 |
+
|
| 19 |
+
const handleClose = useCallback(() => {
|
| 20 |
+
setScenarioPanelOpen(false)
|
| 21 |
+
}, [setScenarioPanelOpen])
|
| 22 |
+
|
| 23 |
+
if (!open) return null
|
| 24 |
+
|
| 25 |
+
return (
|
| 26 |
+
<div className="absolute inset-0 z-40 flex items-start justify-center pt-20">
|
| 27 |
+
{/* Backdrop */}
|
| 28 |
+
<div className="absolute inset-0 bg-black/40 backdrop-sm" onClick={handleClose} />
|
| 29 |
+
|
| 30 |
+
{/* Grid panel */}
|
| 31 |
+
<div className="relative w-[420px] max-h-[70vh] p-5 rounded-2xl bg-black/60 backdrop-blur-xl border border-white/10 shadow-2xl">
|
| 32 |
+
<h3 className="text-white/70 text-sm font-medium mb-4 text-center">
|
| 33 |
+
{t('companion.scenario.select')}
|
| 34 |
+
</h3>
|
| 35 |
+
|
| 36 |
+
<div className="grid grid-cols-3 gap-2.5">
|
| 37 |
+
{SCENARIOS.map((scenario) => {
|
| 38 |
+
const isActive = currentScenario === scenario.id
|
| 39 |
+
return (
|
| 40 |
+
<button
|
| 41 |
+
key={scenario.id}
|
| 42 |
+
type="button"
|
| 43 |
+
onClick={() => handleSelect(scenario.id)}
|
| 44 |
+
className={`flex flex-col items-center gap-1.5 p-3 rounded-xl transition-all border ${
|
| 45 |
+
isActive
|
| 46 |
+
? 'bg-orange-500/20 border-orange-400/40'
|
| 47 |
+
: 'bg-white/[0.04] border-transparent hover:bg-white/[0.08]'
|
| 48 |
+
}`}
|
| 49 |
+
>
|
| 50 |
+
<span
|
| 51 |
+
className={`material-symbols-outlined text-2xl ${
|
| 52 |
+
isActive ? 'text-orange-300' : 'text-white/50'
|
| 53 |
+
}`}
|
| 54 |
+
>
|
| 55 |
+
{scenario.icon}
|
| 56 |
+
</span>
|
| 57 |
+
<span className="text-white/80 text-xs font-medium">{scenario.name}</span>
|
| 58 |
+
</button>
|
| 59 |
+
)
|
| 60 |
+
})}
|
| 61 |
+
</div>
|
| 62 |
+
</div>
|
| 63 |
+
</div>
|
| 64 |
+
)
|
| 65 |
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { useState, useCallback } from 'react'
|
| 2 |
+
import { useTranslation } from '../../i18n'
|
| 3 |
+
import { useCompanionStore } from '../../stores/companionStore'
|
| 4 |
+
|
| 5 |
+
interface ScreenShareDialogProps {
|
| 6 |
+
onStartScreenShare: () => void
|
| 7 |
+
}
|
| 8 |
+
|
| 9 |
+
export function ScreenShareDialog({ onStartScreenShare }: ScreenShareDialogProps) {
|
| 10 |
+
const t = useTranslation()
|
| 11 |
+
const open = useCompanionStore((s) => s.screenShareDialogOpen)
|
| 12 |
+
const setScreenShareDialogOpen = useCompanionStore((s) => s.setScreenShareDialogOpen)
|
| 13 |
+
const [sourceType, setSourceType] = useState<'screen' | 'window'>('screen')
|
| 14 |
+
|
| 15 |
+
const handleCancel = useCallback(() => {
|
| 16 |
+
setScreenShareDialogOpen(false)
|
| 17 |
+
}, [setScreenShareDialogOpen])
|
| 18 |
+
|
| 19 |
+
const handleStart = useCallback(() => {
|
| 20 |
+
setScreenShareDialogOpen(false)
|
| 21 |
+
onStartScreenShare()
|
| 22 |
+
}, [setScreenShareDialogOpen, onStartScreenShare])
|
| 23 |
+
|
| 24 |
+
if (!open) return null
|
| 25 |
+
|
| 26 |
+
return (
|
| 27 |
+
<div className="absolute inset-0 z-40 flex items-center justify-center">
|
| 28 |
+
{/* Backdrop */}
|
| 29 |
+
<div className="absolute inset-0 bg-black/50 backdrop-sm" onClick={handleCancel} />
|
| 30 |
+
|
| 31 |
+
{/* Dialog */}
|
| 32 |
+
<div className="relative w-[340px] p-5 rounded-2xl bg-black/70 backdrop-blur-xl border border-white/10 shadow-2xl">
|
| 33 |
+
<h3 className="text-white/80 text-sm font-medium mb-4 text-center">
|
| 34 |
+
{t('companion.screenShare.dialog.title')}
|
| 35 |
+
</h3>
|
| 36 |
+
|
| 37 |
+
{/* Source selector */}
|
| 38 |
+
<div className="mb-5 flex gap-2">
|
| 39 |
+
<button
|
| 40 |
+
type="button"
|
| 41 |
+
onClick={() => setSourceType('screen')}
|
| 42 |
+
className={`flex-1 flex flex-col items-center gap-1.5 p-3 rounded-xl border transition-all ${
|
| 43 |
+
sourceType === 'screen'
|
| 44 |
+
? 'bg-orange-500/20 border-orange-400/40'
|
| 45 |
+
: 'bg-white/[0.04] border-transparent hover:bg-white/[0.08]'
|
| 46 |
+
}`}
|
| 47 |
+
>
|
| 48 |
+
<span className="material-symbols-outlined text-xl text-white/60">monitor</span>
|
| 49 |
+
<span className="text-white/70 text-xs">{t('companion.screenShare.dialog.entireScreen')}</span>
|
| 50 |
+
</button>
|
| 51 |
+
<button
|
| 52 |
+
type="button"
|
| 53 |
+
onClick={() => setSourceType('window')}
|
| 54 |
+
className={`flex-1 flex flex-col items-center gap-1.5 p-3 rounded-xl border transition-all ${
|
| 55 |
+
sourceType === 'window'
|
| 56 |
+
? 'bg-orange-500/20 border-orange-400/40'
|
| 57 |
+
: 'bg-white/[0.04] border-transparent hover:bg-white/[0.08]'
|
| 58 |
+
}`}
|
| 59 |
+
>
|
| 60 |
+
<span className="material-symbols-outlined text-xl text-white/60">window</span>
|
| 61 |
+
<span className="text-white/70 text-xs">{t('companion.screenShare.dialog.window')}</span>
|
| 62 |
+
</button>
|
| 63 |
+
</div>
|
| 64 |
+
|
| 65 |
+
{/* Privacy notice */}
|
| 66 |
+
<p className="text-white/25 text-[11px] leading-relaxed mb-5">
|
| 67 |
+
{t('companion.screenShare.dialog.privacy')}
|
| 68 |
+
</p>
|
| 69 |
+
|
| 70 |
+
{/* Action buttons */}
|
| 71 |
+
<div className="flex gap-2.5">
|
| 72 |
+
<button
|
| 73 |
+
type="button"
|
| 74 |
+
onClick={handleCancel}
|
| 75 |
+
className="flex-1 h-9 rounded-xl bg-white/5 text-white/50 hover:bg-white/10 hover:text-white/70 transition-all text-xs font-medium"
|
| 76 |
+
>
|
| 77 |
+
{t('companion.screenShare.dialog.cancel')}
|
| 78 |
+
</button>
|
| 79 |
+
<button
|
| 80 |
+
type="button"
|
| 81 |
+
onClick={handleStart}
|
| 82 |
+
className="flex-1 h-9 rounded-xl bg-orange-500/30 text-orange-200 hover:bg-orange-500/50 transition-all text-xs font-medium"
|
| 83 |
+
>
|
| 84 |
+
{t('companion.screenShare.dialog.start')}
|
| 85 |
+
</button>
|
| 86 |
+
</div>
|
| 87 |
+
</div>
|
| 88 |
+
</div>
|
| 89 |
+
)
|
| 90 |
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { useState, useCallback } from 'react'
|
| 2 |
+
|
| 3 |
+
export function useCameraDevices() {
|
| 4 |
+
const [devices, setDevices] = useState<MediaDeviceInfo[]>([])
|
| 5 |
+
const [activeDeviceId, setActiveDeviceId] = useState<string | null>(null)
|
| 6 |
+
|
| 7 |
+
const enumerateCameras = useCallback(async () => {
|
| 8 |
+
try {
|
| 9 |
+
const allDevices = await navigator.mediaDevices.enumerateDevices()
|
| 10 |
+
const videoInputs = allDevices.filter((d) => d.kind === 'videoinput')
|
| 11 |
+
setDevices(videoInputs)
|
| 12 |
+
return videoInputs
|
| 13 |
+
} catch {
|
| 14 |
+
return []
|
| 15 |
+
}
|
| 16 |
+
}, [])
|
| 17 |
+
|
| 18 |
+
const switchCamera = useCallback(
|
| 19 |
+
async (deviceId: string): Promise<MediaStream | null> => {
|
| 20 |
+
try {
|
| 21 |
+
const stream = await navigator.mediaDevices.getUserMedia({
|
| 22 |
+
video: {
|
| 23 |
+
deviceId: { exact: deviceId },
|
| 24 |
+
width: { ideal: 640 },
|
| 25 |
+
height: { ideal: 480 },
|
| 26 |
+
},
|
| 27 |
+
audio: false,
|
| 28 |
+
})
|
| 29 |
+
setActiveDeviceId(deviceId)
|
| 30 |
+
return stream
|
| 31 |
+
} catch (err) {
|
| 32 |
+
console.error('Failed to switch camera:', err)
|
| 33 |
+
return null
|
| 34 |
+
}
|
| 35 |
+
},
|
| 36 |
+
[],
|
| 37 |
+
)
|
| 38 |
+
|
| 39 |
+
const getStreamForFacingMode = useCallback(
|
| 40 |
+
async (facingMode: 'user' | 'environment'): Promise<MediaStream | null> => {
|
| 41 |
+
try {
|
| 42 |
+
const stream = await navigator.mediaDevices.getUserMedia({
|
| 43 |
+
video: {
|
| 44 |
+
facingMode,
|
| 45 |
+
width: { ideal: 640 },
|
| 46 |
+
height: { ideal: 480 },
|
| 47 |
+
},
|
| 48 |
+
audio: false,
|
| 49 |
+
})
|
| 50 |
+
return stream
|
| 51 |
+
} catch (err) {
|
| 52 |
+
console.error('Failed to get camera with facing mode:', err)
|
| 53 |
+
return null
|
| 54 |
+
}
|
| 55 |
+
},
|
| 56 |
+
[],
|
| 57 |
+
)
|
| 58 |
+
|
| 59 |
+
return {
|
| 60 |
+
devices,
|
| 61 |
+
activeDeviceId,
|
| 62 |
+
enumerateCameras,
|
| 63 |
+
switchCamera,
|
| 64 |
+
getStreamForFacingMode,
|
| 65 |
+
}
|
| 66 |
+
}
|
|
@@ -57,20 +57,11 @@ export function useCompanionWebSocket(callbacks: Callbacks) {
|
|
| 57 |
wsRef.current = ws
|
| 58 |
|
| 59 |
ws.onopen = () => {
|
| 60 |
-
reconnectAttemptRef.current = 0
|
| 61 |
setStatus('connected')
|
| 62 |
}
|
| 63 |
|
| 64 |
ws.onclose = () => {
|
| 65 |
-
|
| 66 |
-
// Auto reconnect with exponential backoff
|
| 67 |
-
const attempt = reconnectAttemptRef.current
|
| 68 |
-
const delay = Math.min(1000 * Math.pow(2, attempt), 30000)
|
| 69 |
-
reconnectAttemptRef.current = attempt + 1
|
| 70 |
-
reconnectTimerRef.current = setTimeout(() => {
|
| 71 |
-
connect()
|
| 72 |
-
}, delay)
|
| 73 |
-
}
|
| 74 |
setStatus('disconnected')
|
| 75 |
}
|
| 76 |
|
|
|
|
| 57 |
wsRef.current = ws
|
| 58 |
|
| 59 |
ws.onopen = () => {
|
|
|
|
| 60 |
setStatus('connected')
|
| 61 |
}
|
| 62 |
|
| 63 |
ws.onclose = () => {
|
| 64 |
+
// No auto-reconnect — user clicks "call" to retry
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 65 |
setStatus('disconnected')
|
| 66 |
}
|
| 67 |
|
|
@@ -0,0 +1,115 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { useState, useRef, useCallback } from 'react'
|
| 2 |
+
|
| 3 |
+
export function useScreenShare() {
|
| 4 |
+
const [stream, setStream] = useState<MediaStream | null>(null)
|
| 5 |
+
const [isSharing, setIsSharing] = useState(false)
|
| 6 |
+
const [error, setError] = useState<string | null>(null)
|
| 7 |
+
const streamRef = useRef<MediaStream | null>(null)
|
| 8 |
+
const videoRef = useRef<HTMLVideoElement | null>(null)
|
| 9 |
+
const canvasRef = useRef<HTMLCanvasElement | null>(null)
|
| 10 |
+
|
| 11 |
+
const startScreenShare = useCallback(async () => {
|
| 12 |
+
try {
|
| 13 |
+
const mediaStream = await navigator.mediaDevices.getDisplayMedia({
|
| 14 |
+
video: true,
|
| 15 |
+
audio: false,
|
| 16 |
+
})
|
| 17 |
+
|
| 18 |
+
streamRef.current = mediaStream
|
| 19 |
+
setStream(mediaStream)
|
| 20 |
+
setIsSharing(true)
|
| 21 |
+
setError(null)
|
| 22 |
+
|
| 23 |
+
// Create hidden video + canvas for frame capture
|
| 24 |
+
const video = document.createElement('video')
|
| 25 |
+
video.srcObject = mediaStream
|
| 26 |
+
video.playsInline = true
|
| 27 |
+
video.muted = true
|
| 28 |
+
video.autoplay = true
|
| 29 |
+
videoRef.current = video
|
| 30 |
+
|
| 31 |
+
const canvas = document.createElement('canvas')
|
| 32 |
+
canvas.width = 640
|
| 33 |
+
canvas.height = 480
|
| 34 |
+
canvasRef.current = canvas
|
| 35 |
+
|
| 36 |
+
await video.play()
|
| 37 |
+
|
| 38 |
+
// When user stops sharing via the browser's built-in "Stop sharing" button
|
| 39 |
+
mediaStream.getVideoTracks()[0]?.addEventListener('ended', () => {
|
| 40 |
+
stopScreenShare()
|
| 41 |
+
})
|
| 42 |
+
} catch (err) {
|
| 43 |
+
const message = err instanceof Error ? err.message : String(err)
|
| 44 |
+
// User cancelled the permission dialog
|
| 45 |
+
if (message.includes('cancelled') || message.includes('Permission denied')) {
|
| 46 |
+
setError(null)
|
| 47 |
+
return
|
| 48 |
+
}
|
| 49 |
+
setError(`Screen share failed: ${message}`)
|
| 50 |
+
}
|
| 51 |
+
}, [])
|
| 52 |
+
|
| 53 |
+
const stopScreenShare = useCallback(() => {
|
| 54 |
+
if (streamRef.current) {
|
| 55 |
+
streamRef.current.getTracks().forEach((track) => track.stop())
|
| 56 |
+
streamRef.current = null
|
| 57 |
+
}
|
| 58 |
+
videoRef.current = null
|
| 59 |
+
canvasRef.current = null
|
| 60 |
+
setStream(null)
|
| 61 |
+
setIsSharing(false)
|
| 62 |
+
}, [])
|
| 63 |
+
|
| 64 |
+
const captureFrame = useCallback(async (): Promise<string | null> => {
|
| 65 |
+
const video = videoRef.current
|
| 66 |
+
const canvas = canvasRef.current
|
| 67 |
+
if (!video || !canvas || video.readyState < 2) return null
|
| 68 |
+
|
| 69 |
+
const ctx = canvas.getContext('2d')
|
| 70 |
+
if (!ctx) return null
|
| 71 |
+
|
| 72 |
+
// Maintain aspect ratio while fitting to canvas
|
| 73 |
+
const vw = video.videoWidth
|
| 74 |
+
const vh = video.videoHeight
|
| 75 |
+
if (vw === 0 || vh === 0) return null
|
| 76 |
+
|
| 77 |
+
const scale = Math.min(canvas.width / vw, canvas.height / vh)
|
| 78 |
+
const dw = vw * scale
|
| 79 |
+
const dh = vh * scale
|
| 80 |
+
const dx = (canvas.width - dw) / 2
|
| 81 |
+
const dy = (canvas.height - dh) / 2
|
| 82 |
+
|
| 83 |
+
ctx.clearRect(0, 0, canvas.width, canvas.height)
|
| 84 |
+
ctx.drawImage(video, dx, dy, dw, dh)
|
| 85 |
+
|
| 86 |
+
return new Promise((resolve) => {
|
| 87 |
+
canvas.toBlob(
|
| 88 |
+
(blob) => {
|
| 89 |
+
if (!blob) {
|
| 90 |
+
resolve(null)
|
| 91 |
+
return
|
| 92 |
+
}
|
| 93 |
+
const reader = new FileReader()
|
| 94 |
+
reader.onloadend = () => {
|
| 95 |
+
const result = reader.result as string
|
| 96 |
+
resolve(result.split(',')[1] ?? null)
|
| 97 |
+
}
|
| 98 |
+
reader.onerror = () => resolve(null)
|
| 99 |
+
reader.readAsDataURL(blob)
|
| 100 |
+
},
|
| 101 |
+
'image/jpeg',
|
| 102 |
+
0.7,
|
| 103 |
+
)
|
| 104 |
+
})
|
| 105 |
+
}, [])
|
| 106 |
+
|
| 107 |
+
return {
|
| 108 |
+
stream,
|
| 109 |
+
isSharing,
|
| 110 |
+
error,
|
| 111 |
+
captureFrame,
|
| 112 |
+
startScreenShare,
|
| 113 |
+
stopScreenShare,
|
| 114 |
+
}
|
| 115 |
+
}
|
|
@@ -79,6 +79,27 @@ export const en = {
|
|
| 79 |
'companion.speaking': 'Speaking...',
|
| 80 |
'companion.inputPlaceholder': 'Type a message...',
|
| 81 |
'companion.send': 'Send',
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
'sidebar.confirmDelete': 'Delete this session? This cannot be undone.',
|
| 83 |
'sidebar.batchManage': 'Batch manage',
|
| 84 |
'sidebar.batchSelectedCount': '{count} selected',
|
|
|
|
| 79 |
'companion.speaking': 'Speaking...',
|
| 80 |
'companion.inputPlaceholder': 'Type a message...',
|
| 81 |
'companion.send': 'Send',
|
| 82 |
+
'companion.scenario.select': 'Select Scenario',
|
| 83 |
+
'companion.scenario.simultaneous': 'Simultaneous Interpretation',
|
| 84 |
+
'companion.scenario.interview': 'Mock Interview',
|
| 85 |
+
'companion.scenario.english': 'English Practice',
|
| 86 |
+
'companion.scenario.singing': 'Singing',
|
| 87 |
+
'companion.scenario.idiom': 'Idiom Chain',
|
| 88 |
+
'companion.scenario.mood': 'Mood Treehole',
|
| 89 |
+
'companion.subtitle': 'Subtitles',
|
| 90 |
+
'companion.screenShare.start': 'Share Screen',
|
| 91 |
+
'companion.screenShare.stop': 'Stop Sharing',
|
| 92 |
+
'companion.screenShare.dialog.title': 'Start screen recording or sharing?',
|
| 93 |
+
'companion.screenShare.dialog.entireScreen': 'Entire Screen',
|
| 94 |
+
'companion.screenShare.dialog.window': 'Application Window',
|
| 95 |
+
'companion.screenShare.dialog.cancel': 'Cancel',
|
| 96 |
+
'companion.screenShare.dialog.start': 'Start',
|
| 97 |
+
'companion.screenShare.dialog.privacy': 'When sharing, recording, or casting content, the app can access everything displayed on your screen or played on your device. Please be careful not to leak passwords, payment info, messages, or other sensitive content.',
|
| 98 |
+
'companion.flipCamera': 'Flip Camera',
|
| 99 |
+
'companion.torch': 'Flashlight',
|
| 100 |
+
'companion.hangUp': 'Hang Up',
|
| 101 |
+
'companion.call': 'Call',
|
| 102 |
+
'companion.speaker': 'Speaker',
|
| 103 |
'sidebar.confirmDelete': 'Delete this session? This cannot be undone.',
|
| 104 |
'sidebar.batchManage': 'Batch manage',
|
| 105 |
'sidebar.batchSelectedCount': '{count} selected',
|
|
@@ -60,7 +60,7 @@ export const zh: Record<TranslationKey, string> = {
|
|
| 60 |
'sidebar.worktree': 'worktree',
|
| 61 |
'sidebar.sessionRunning': '会话运行中',
|
| 62 |
'sidebar.missingDir': '目录缺失',
|
| 63 |
-
'sidebar.companion': '
|
| 64 |
|
| 65 |
// ─── Companion ──────────────────────────────────────
|
| 66 |
'companion.connect': '连接',
|
|
@@ -81,6 +81,27 @@ export const zh: Record<TranslationKey, string> = {
|
|
| 81 |
'companion.speaking': '说话中...',
|
| 82 |
'companion.inputPlaceholder': '输入消息...',
|
| 83 |
'companion.send': '发送',
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 84 |
'sidebar.confirmDelete': '确定要删除这个会话吗?此操作不可撤销。',
|
| 85 |
'sidebar.batchManage': '批量管理',
|
| 86 |
'sidebar.batchSelectedCount': '已选 {count} 个',
|
|
|
|
| 60 |
'sidebar.worktree': 'worktree',
|
| 61 |
'sidebar.sessionRunning': '会话运行中',
|
| 62 |
'sidebar.missingDir': '目录缺失',
|
| 63 |
+
'sidebar.companion': '视频通话',
|
| 64 |
|
| 65 |
// ─── Companion ──────────────────────────────────────
|
| 66 |
'companion.connect': '连接',
|
|
|
|
| 81 |
'companion.speaking': '说话中...',
|
| 82 |
'companion.inputPlaceholder': '输入消息...',
|
| 83 |
'companion.send': '发送',
|
| 84 |
+
'companion.scenario.select': '选择场景',
|
| 85 |
+
'companion.scenario.simultaneous': '同声传译',
|
| 86 |
+
'companion.scenario.interview': '模拟面试',
|
| 87 |
+
'companion.scenario.english': '英语陪练',
|
| 88 |
+
'companion.scenario.singing': '唱歌',
|
| 89 |
+
'companion.scenario.idiom': '成语接龙',
|
| 90 |
+
'companion.scenario.mood': '心情树洞',
|
| 91 |
+
'companion.subtitle': '字幕',
|
| 92 |
+
'companion.screenShare.start': '屏幕共享',
|
| 93 |
+
'companion.screenShare.stop': '停止共享',
|
| 94 |
+
'companion.screenShare.dialog.title': '要开始录制或投放吗?',
|
| 95 |
+
'companion.screenShare.dialog.entireScreen': '整个屏幕',
|
| 96 |
+
'companion.screenShare.dialog.window': '特定应用窗口',
|
| 97 |
+
'companion.screenShare.dialog.cancel': '取消',
|
| 98 |
+
'companion.screenShare.dialog.start': '开始',
|
| 99 |
+
'companion.screenShare.dialog.privacy': '在分享、录制或投放内容时,应用可以访问屏幕上显示或设备中播放的所有内容。请务必小心操作,谨防密码、付款信息、消息、照片等内容泄露。',
|
| 100 |
+
'companion.flipCamera': '翻转摄像头',
|
| 101 |
+
'companion.torch': '手电筒',
|
| 102 |
+
'companion.hangUp': '挂断',
|
| 103 |
+
'companion.call': '呼叫',
|
| 104 |
+
'companion.speaker': '扬声器',
|
| 105 |
'sidebar.confirmDelete': '确定要删除这个会话吗?此操作不可撤销。',
|
| 106 |
'sidebar.batchManage': '批量管理',
|
| 107 |
'sidebar.batchSelectedCount': '已选 {count} 个',
|
|
@@ -4,9 +4,14 @@ import { useCompanionWebSocket } from '../hooks/useCompanionWebSocket'
|
|
| 4 |
import { useWebcam } from '../hooks/useWebcam'
|
| 5 |
import { useMicrophone } from '../hooks/useMicrophone'
|
| 6 |
import { useCompanionAudio } from '../hooks/useCompanionAudio'
|
|
|
|
|
|
|
| 7 |
import { CompanionVideoPanel } from '../components/companion/CompanionVideoPanel'
|
| 8 |
import { CompanionTranscript } from '../components/companion/CompanionTranscript'
|
| 9 |
import { CompanionControls } from '../components/companion/CompanionControls'
|
|
|
|
|
|
|
|
|
|
| 10 |
|
| 11 |
export function Companion() {
|
| 12 |
const status = useCompanionStore((s) => s.status)
|
|
@@ -16,12 +21,24 @@ export function Companion() {
|
|
| 16 |
const fullTranscript = useCompanionStore((s) => s.fullTranscript)
|
| 17 |
const micEnabled = useCompanionStore((s) => s.micEnabled)
|
| 18 |
const cameraEnabled = useCompanionStore((s) => s.cameraEnabled)
|
|
|
|
| 19 |
const error = useCompanionStore((s) => s.error)
|
|
|
|
|
|
|
| 20 |
const setMicEnabled = useCompanionStore((s) => s.setMicEnabled)
|
| 21 |
const setCameraEnabled = useCompanionStore((s) => s.setCameraEnabled)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
const reset = useCompanionStore((s) => s.reset)
|
| 23 |
const frameIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null)
|
|
|
|
| 24 |
|
|
|
|
| 25 |
const companionAudio = useCompanionAudio()
|
| 26 |
const onAudioReceived = useCallback(
|
| 27 |
(data: ArrayBuffer) => {
|
|
@@ -33,11 +50,50 @@ export function Companion() {
|
|
| 33 |
companionAudio.flush()
|
| 34 |
}, [companionAudio])
|
| 35 |
|
|
|
|
| 36 |
const ws = useCompanionWebSocket({ onAudioReceived, onDone })
|
| 37 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
useMicrophone(ws.connected && micEnabled, ws.sendAudio)
|
| 39 |
|
| 40 |
-
//
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
useEffect(() => {
|
| 42 |
if (ws.connected && cameraEnabled) {
|
| 43 |
frameIntervalRef.current = setInterval(async () => {
|
|
@@ -60,6 +116,41 @@ export function Companion() {
|
|
| 60 |
}
|
| 61 |
}, [ws.connected, cameraEnabled, webcam, ws])
|
| 62 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
const handleSendText = useCallback(
|
| 64 |
(text: string) => {
|
| 65 |
ws.sendText(text)
|
|
@@ -67,7 +158,6 @@ export function Companion() {
|
|
| 67 |
[ws],
|
| 68 |
)
|
| 69 |
|
| 70 |
-
// Resume audio context on user interaction
|
| 71 |
const handleResumeAudio = useCallback(() => {
|
| 72 |
companionAudio.resume()
|
| 73 |
}, [companionAudio])
|
|
@@ -77,26 +167,68 @@ export function Companion() {
|
|
| 77 |
}, [micEnabled, setMicEnabled])
|
| 78 |
|
| 79 |
const handleToggleCamera = useCallback(() => {
|
| 80 |
-
|
| 81 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
|
| 83 |
return (
|
| 84 |
<div className="relative h-full w-full overflow-hidden bg-black">
|
| 85 |
-
{/* Video background +
|
| 86 |
<CompanionVideoPanel
|
| 87 |
webcamStream={webcam.stream}
|
| 88 |
speaking={speaking}
|
| 89 |
generating={generating}
|
| 90 |
status={status}
|
|
|
|
| 91 |
/>
|
| 92 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 93 |
{/* Error overlay */}
|
| 94 |
{error && (
|
| 95 |
-
<div className="absolute top-
|
| 96 |
{error}
|
| 97 |
</div>
|
| 98 |
)}
|
| 99 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 100 |
{/* Transcript + text input */}
|
| 101 |
<CompanionTranscript
|
| 102 |
transcript={transcript}
|
|
@@ -106,17 +238,22 @@ export function Companion() {
|
|
| 106 |
disabled={status !== 'connected'}
|
| 107 |
/>
|
| 108 |
|
| 109 |
-
{/*
|
| 110 |
<CompanionControls
|
| 111 |
status={status}
|
| 112 |
micEnabled={micEnabled}
|
| 113 |
cameraEnabled={cameraEnabled}
|
| 114 |
generating={generating}
|
|
|
|
| 115 |
onToggleMic={handleToggleMic}
|
| 116 |
onToggleCamera={handleToggleCamera}
|
|
|
|
| 117 |
onConnect={ws.connect}
|
| 118 |
onDisconnect={() => {
|
| 119 |
ws.disconnect()
|
|
|
|
|
|
|
|
|
|
| 120 |
reset()
|
| 121 |
}}
|
| 122 |
onStop={ws.sendStop}
|
|
|
|
| 4 |
import { useWebcam } from '../hooks/useWebcam'
|
| 5 |
import { useMicrophone } from '../hooks/useMicrophone'
|
| 6 |
import { useCompanionAudio } from '../hooks/useCompanionAudio'
|
| 7 |
+
import { useScreenShare } from '../hooks/useScreenShare'
|
| 8 |
+
import { useCameraDevices } from '../hooks/useCameraDevices'
|
| 9 |
import { CompanionVideoPanel } from '../components/companion/CompanionVideoPanel'
|
| 10 |
import { CompanionTranscript } from '../components/companion/CompanionTranscript'
|
| 11 |
import { CompanionControls } from '../components/companion/CompanionControls'
|
| 12 |
+
import { CompanionTopBar } from '../components/companion/CompanionTopBar'
|
| 13 |
+
import { ScenarioSelector } from '../components/companion/ScenarioSelector'
|
| 14 |
+
import { ScreenShareDialog } from '../components/companion/ScreenShareDialog'
|
| 15 |
|
| 16 |
export function Companion() {
|
| 17 |
const status = useCompanionStore((s) => s.status)
|
|
|
|
| 21 |
const fullTranscript = useCompanionStore((s) => s.fullTranscript)
|
| 22 |
const micEnabled = useCompanionStore((s) => s.micEnabled)
|
| 23 |
const cameraEnabled = useCompanionStore((s) => s.cameraEnabled)
|
| 24 |
+
const cameraFacingMode = useCompanionStore((s) => s.cameraFacingMode)
|
| 25 |
const error = useCompanionStore((s) => s.error)
|
| 26 |
+
const scenario = useCompanionStore((s) => s.scenario)
|
| 27 |
+
const statusText = useCompanionStore((s) => s.statusText)
|
| 28 |
const setMicEnabled = useCompanionStore((s) => s.setMicEnabled)
|
| 29 |
const setCameraEnabled = useCompanionStore((s) => s.setCameraEnabled)
|
| 30 |
+
const setCameraFacingMode = useCompanionStore((s) => s.setCameraFacingMode)
|
| 31 |
+
const setCameraDevices = useCompanionStore((s) => s.setCameraDevices)
|
| 32 |
+
const setActiveCameraLabel = useCompanionStore((s) => s.setActiveCameraLabel)
|
| 33 |
+
const setCameraFullscreen = useCompanionStore((s) => s.setCameraFullscreen)
|
| 34 |
+
const setScreenShareStream = useCompanionStore((s) => s.setScreenShareStream)
|
| 35 |
+
const setScreenShareDialogOpen = useCompanionStore((s) => s.setScreenShareDialogOpen)
|
| 36 |
+
const setStatusText = useCompanionStore((s) => s.setStatusText)
|
| 37 |
const reset = useCompanionStore((s) => s.reset)
|
| 38 |
const frameIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null)
|
| 39 |
+
const screenShareIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null)
|
| 40 |
|
| 41 |
+
// Audio playback
|
| 42 |
const companionAudio = useCompanionAudio()
|
| 43 |
const onAudioReceived = useCallback(
|
| 44 |
(data: ArrayBuffer) => {
|
|
|
|
| 50 |
companionAudio.flush()
|
| 51 |
}, [companionAudio])
|
| 52 |
|
| 53 |
+
// WebSocket
|
| 54 |
const ws = useCompanionWebSocket({ onAudioReceived, onDone })
|
| 55 |
+
|
| 56 |
+
// Camera devices
|
| 57 |
+
const cameraDevices = useCameraDevices()
|
| 58 |
+
|
| 59 |
+
// Screen share
|
| 60 |
+
const screenShare = useScreenShare()
|
| 61 |
+
|
| 62 |
+
// Webcam runs independently (toggle on/off without needing connection)
|
| 63 |
+
const webcam = useWebcam(cameraEnabled)
|
| 64 |
+
// Mic only runs when connected (sends audio over WS)
|
| 65 |
useMicrophone(ws.connected && micEnabled, ws.sendAudio)
|
| 66 |
|
| 67 |
+
// Enumerate cameras when camera is enabled
|
| 68 |
+
useEffect(() => {
|
| 69 |
+
if (cameraEnabled) {
|
| 70 |
+
cameraDevices.enumerateCameras().then((devices) => {
|
| 71 |
+
setCameraDevices(devices)
|
| 72 |
+
if (devices.length > 0) {
|
| 73 |
+
setActiveCameraLabel(devices[0]?.label ?? null)
|
| 74 |
+
}
|
| 75 |
+
})
|
| 76 |
+
}
|
| 77 |
+
}, [cameraEnabled, cameraDevices, setCameraDevices, setActiveCameraLabel])
|
| 78 |
+
|
| 79 |
+
// Update status text based on state
|
| 80 |
+
useEffect(() => {
|
| 81 |
+
if (status === 'connected') {
|
| 82 |
+
if (generating) {
|
| 83 |
+
setStatusText('AI 说话中...')
|
| 84 |
+
} else if (speaking) {
|
| 85 |
+
setStatusText('请说话')
|
| 86 |
+
} else {
|
| 87 |
+
setStatusText('你可以开始说话')
|
| 88 |
+
}
|
| 89 |
+
} else if (status === 'connecting') {
|
| 90 |
+
setStatusText('连接中...')
|
| 91 |
+
} else {
|
| 92 |
+
setStatusText('')
|
| 93 |
+
}
|
| 94 |
+
}, [status, generating, speaking, setStatusText])
|
| 95 |
+
|
| 96 |
+
// Periodic webcam frame capture (only send when connected)
|
| 97 |
useEffect(() => {
|
| 98 |
if (ws.connected && cameraEnabled) {
|
| 99 |
frameIntervalRef.current = setInterval(async () => {
|
|
|
|
| 116 |
}
|
| 117 |
}, [ws.connected, cameraEnabled, webcam, ws])
|
| 118 |
|
| 119 |
+
// Periodic screen share frame capture
|
| 120 |
+
useEffect(() => {
|
| 121 |
+
if (ws.connected && screenShare.isSharing) {
|
| 122 |
+
screenShareIntervalRef.current = setInterval(async () => {
|
| 123 |
+
const frame = await screenShare.captureFrame()
|
| 124 |
+
if (frame) {
|
| 125 |
+
ws.sendFrame(frame)
|
| 126 |
+
}
|
| 127 |
+
}, 1000)
|
| 128 |
+
} else {
|
| 129 |
+
if (screenShareIntervalRef.current) {
|
| 130 |
+
clearInterval(screenShareIntervalRef.current)
|
| 131 |
+
screenShareIntervalRef.current = null
|
| 132 |
+
}
|
| 133 |
+
}
|
| 134 |
+
return () => {
|
| 135 |
+
if (screenShareIntervalRef.current) {
|
| 136 |
+
clearInterval(screenShareIntervalRef.current)
|
| 137 |
+
screenShareIntervalRef.current = null
|
| 138 |
+
}
|
| 139 |
+
}
|
| 140 |
+
}, [ws.connected, screenShare.isSharing, screenShare, ws])
|
| 141 |
+
|
| 142 |
+
// Send scenario context when connecting
|
| 143 |
+
useEffect(() => {
|
| 144 |
+
if (ws.connected && scenario) {
|
| 145 |
+
ws.sendText(`[Scenario: ${scenario}]`)
|
| 146 |
+
}
|
| 147 |
+
}, [ws.connected]) // only send on connect
|
| 148 |
+
|
| 149 |
+
// Sync screen share stream to store
|
| 150 |
+
useEffect(() => {
|
| 151 |
+
setScreenShareStream(screenShare.stream)
|
| 152 |
+
}, [screenShare.stream, setScreenShareStream])
|
| 153 |
+
|
| 154 |
const handleSendText = useCallback(
|
| 155 |
(text: string) => {
|
| 156 |
ws.sendText(text)
|
|
|
|
| 158 |
[ws],
|
| 159 |
)
|
| 160 |
|
|
|
|
| 161 |
const handleResumeAudio = useCallback(() => {
|
| 162 |
companionAudio.resume()
|
| 163 |
}, [companionAudio])
|
|
|
|
| 167 |
}, [micEnabled, setMicEnabled])
|
| 168 |
|
| 169 |
const handleToggleCamera = useCallback(() => {
|
| 170 |
+
const next = !cameraEnabled
|
| 171 |
+
setCameraEnabled(next)
|
| 172 |
+
// When camera turns on, show as fullscreen background
|
| 173 |
+
if (next) {
|
| 174 |
+
setCameraFullscreen(true)
|
| 175 |
+
} else {
|
| 176 |
+
setCameraFullscreen(false)
|
| 177 |
+
}
|
| 178 |
+
}, [cameraEnabled, setCameraEnabled, setCameraFullscreen])
|
| 179 |
+
|
| 180 |
+
const handleFlipCamera = useCallback(async () => {
|
| 181 |
+
const nextMode = cameraFacingMode === 'user' ? 'environment' : 'user'
|
| 182 |
+
setCameraFacingMode(nextMode)
|
| 183 |
+
await cameraDevices.getStreamForFacingMode(nextMode)
|
| 184 |
+
}, [cameraFacingMode, setCameraFacingMode, cameraDevices])
|
| 185 |
+
|
| 186 |
+
const handleScreenShareClick = useCallback(() => {
|
| 187 |
+
if (screenShare.isSharing) {
|
| 188 |
+
screenShare.stopScreenShare()
|
| 189 |
+
} else {
|
| 190 |
+
setScreenShareDialogOpen(true)
|
| 191 |
+
}
|
| 192 |
+
}, [screenShare, setScreenShareDialogOpen])
|
| 193 |
+
|
| 194 |
+
const handleStartScreenShare = useCallback(async () => {
|
| 195 |
+
await screenShare.startScreenShare()
|
| 196 |
+
}, [screenShare])
|
| 197 |
|
| 198 |
return (
|
| 199 |
<div className="relative h-full w-full overflow-hidden bg-black">
|
| 200 |
+
{/* Video background + content */}
|
| 201 |
<CompanionVideoPanel
|
| 202 |
webcamStream={webcam.stream}
|
| 203 |
speaking={speaking}
|
| 204 |
generating={generating}
|
| 205 |
status={status}
|
| 206 |
+
onFlipCamera={handleFlipCamera}
|
| 207 |
/>
|
| 208 |
|
| 209 |
+
{/* Top bar with scenario selector + subtitle toggle */}
|
| 210 |
+
<CompanionTopBar />
|
| 211 |
+
|
| 212 |
+
{/* Scenario selection panel overlay */}
|
| 213 |
+
<ScenarioSelector />
|
| 214 |
+
|
| 215 |
+
{/* Screen share dialog overlay */}
|
| 216 |
+
<ScreenShareDialog onStartScreenShare={handleStartScreenShare} />
|
| 217 |
+
|
| 218 |
{/* Error overlay */}
|
| 219 |
{error && (
|
| 220 |
+
<div className="absolute top-14 left-1/2 -translate-x-1/2 px-4 py-2 rounded-xl bg-red-500/20 border border-red-500/30 text-red-300 text-sm z-20">
|
| 221 |
{error}
|
| 222 |
</div>
|
| 223 |
)}
|
| 224 |
|
| 225 |
+
{/* Status text bar */}
|
| 226 |
+
{statusText && (
|
| 227 |
+
<div className="absolute bottom-[88px] left-1/2 -translate-x-1/2 z-10">
|
| 228 |
+
<span className="text-white/30 text-xs tracking-widest">{statusText}</span>
|
| 229 |
+
</div>
|
| 230 |
+
)}
|
| 231 |
+
|
| 232 |
{/* Transcript + text input */}
|
| 233 |
<CompanionTranscript
|
| 234 |
transcript={transcript}
|
|
|
|
| 238 |
disabled={status !== 'connected'}
|
| 239 |
/>
|
| 240 |
|
| 241 |
+
{/* Bottom controls */}
|
| 242 |
<CompanionControls
|
| 243 |
status={status}
|
| 244 |
micEnabled={micEnabled}
|
| 245 |
cameraEnabled={cameraEnabled}
|
| 246 |
generating={generating}
|
| 247 |
+
isSharing={screenShare.isSharing}
|
| 248 |
onToggleMic={handleToggleMic}
|
| 249 |
onToggleCamera={handleToggleCamera}
|
| 250 |
+
onScreenShareClick={handleScreenShareClick}
|
| 251 |
onConnect={ws.connect}
|
| 252 |
onDisconnect={() => {
|
| 253 |
ws.disconnect()
|
| 254 |
+
if (screenShare.isSharing) {
|
| 255 |
+
screenShare.stopScreenShare()
|
| 256 |
+
}
|
| 257 |
reset()
|
| 258 |
}}
|
| 259 |
onStop={ws.sendStop}
|
|
@@ -1,5 +1,5 @@
|
|
| 1 |
import { create } from 'zustand'
|
| 2 |
-
import type { CompanionStatus } from '../types/companion'
|
| 3 |
|
| 4 |
const AVATAR_KEY = 'companion-avatar-url'
|
| 5 |
const BACKGROUND_KEY = 'companion-background-url'
|
|
@@ -24,6 +24,57 @@ function persist(key: string, value: string | null) {
|
|
| 24 |
}
|
| 25 |
}
|
| 26 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
interface CompanionStore {
|
| 28 |
status: CompanionStatus
|
| 29 |
serverUrl: string
|
|
@@ -38,6 +89,30 @@ interface CompanionStore {
|
|
| 38 |
avatarUrl: string | null
|
| 39 |
backgroundUrl: string | null
|
| 40 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
setServerUrl: (url: string) => void
|
| 42 |
setVoiceName: (name: string) => void
|
| 43 |
setStatus: (status: CompanionStatus) => void
|
|
@@ -52,6 +127,18 @@ interface CompanionStore {
|
|
| 52 |
setAvatarUrl: (url: string | null) => void
|
| 53 |
setBackgroundUrl: (url: string | null) => void
|
| 54 |
reset: () => void
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
}
|
| 56 |
|
| 57 |
const DEFAULT_SERVER_URL = 'ws://127.0.0.1:8889/ws/companion'
|
|
@@ -70,6 +157,18 @@ export const useCompanionStore = create<CompanionStore>((set) => ({
|
|
| 70 |
avatarUrl: loadPersisted(AVATAR_KEY),
|
| 71 |
backgroundUrl: loadPersisted(BACKGROUND_KEY),
|
| 72 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
setServerUrl: (url) => set({ serverUrl: url }),
|
| 74 |
setVoiceName: (name) => set({ voiceName: name }),
|
| 75 |
setStatus: (status) => set({ status, error: status === 'error' ? undefined : null }),
|
|
@@ -98,5 +197,23 @@ export const useCompanionStore = create<CompanionStore>((set) => ({
|
|
| 98 |
generating: false,
|
| 99 |
transcript: '',
|
| 100 |
error: null,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
}),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
}))
|
|
|
|
| 1 |
import { create } from 'zustand'
|
| 2 |
+
import type { CompanionStatus, ScenarioOption } from '../types/companion'
|
| 3 |
|
| 4 |
const AVATAR_KEY = 'companion-avatar-url'
|
| 5 |
const BACKGROUND_KEY = 'companion-background-url'
|
|
|
|
| 24 |
}
|
| 25 |
}
|
| 26 |
|
| 27 |
+
export const SCENARIOS: ScenarioOption[] = [
|
| 28 |
+
{
|
| 29 |
+
id: 'interview',
|
| 30 |
+
name: '模拟面试',
|
| 31 |
+
nameEn: 'Mock Interview',
|
| 32 |
+
icon: 'work_history',
|
| 33 |
+
description: 'AI 模拟考官提问,支持上传简历',
|
| 34 |
+
descriptionEn: 'AI mock interviewer with resume support',
|
| 35 |
+
},
|
| 36 |
+
{
|
| 37 |
+
id: 'english',
|
| 38 |
+
name: '英语陪练',
|
| 39 |
+
nameEn: 'English Practice',
|
| 40 |
+
icon: 'translate',
|
| 41 |
+
description: '全英文对话,提升口语能力',
|
| 42 |
+
descriptionEn: 'Full English conversation practice',
|
| 43 |
+
},
|
| 44 |
+
{
|
| 45 |
+
id: 'singing',
|
| 46 |
+
name: '唱歌',
|
| 47 |
+
nameEn: 'Singing',
|
| 48 |
+
icon: 'music_note',
|
| 49 |
+
description: 'AI 陪你唱歌互动',
|
| 50 |
+
descriptionEn: 'Sing with AI',
|
| 51 |
+
},
|
| 52 |
+
{
|
| 53 |
+
id: 'translate',
|
| 54 |
+
name: '同声传译',
|
| 55 |
+
nameEn: 'Simultaneous Interpretation',
|
| 56 |
+
icon: 'language',
|
| 57 |
+
description: '实时翻译多国语言',
|
| 58 |
+
descriptionEn: 'Real-time multi-language translation',
|
| 59 |
+
},
|
| 60 |
+
{
|
| 61 |
+
id: 'idiom',
|
| 62 |
+
name: '成语接龙',
|
| 63 |
+
nameEn: 'Idiom Chain',
|
| 64 |
+
icon: 'abc',
|
| 65 |
+
description: '文字游戏互动',
|
| 66 |
+
descriptionEn: 'Chinese idiom word game',
|
| 67 |
+
},
|
| 68 |
+
{
|
| 69 |
+
id: 'mood',
|
| 70 |
+
name: '心情树洞',
|
| 71 |
+
nameEn: 'Mood Treehole',
|
| 72 |
+
icon: 'forest',
|
| 73 |
+
description: '情感陪伴与倾诉',
|
| 74 |
+
descriptionEn: 'Emotional companion',
|
| 75 |
+
},
|
| 76 |
+
]
|
| 77 |
+
|
| 78 |
interface CompanionStore {
|
| 79 |
status: CompanionStatus
|
| 80 |
serverUrl: string
|
|
|
|
| 89 |
avatarUrl: string | null
|
| 90 |
backgroundUrl: string | null
|
| 91 |
|
| 92 |
+
// New: scenario
|
| 93 |
+
scenario: string | null
|
| 94 |
+
scenarioPanelOpen: boolean
|
| 95 |
+
|
| 96 |
+
// New: subtitle
|
| 97 |
+
subtitleEnabled: boolean
|
| 98 |
+
|
| 99 |
+
// New: camera devices
|
| 100 |
+
cameraFacingMode: 'user' | 'environment'
|
| 101 |
+
cameraDevices: MediaDeviceInfo[]
|
| 102 |
+
activeCameraLabel: string | null
|
| 103 |
+
|
| 104 |
+
// New: screen share
|
| 105 |
+
screenShareStream: MediaStream | null
|
| 106 |
+
|
| 107 |
+
// New: screen share dialog
|
| 108 |
+
screenShareDialogOpen: boolean
|
| 109 |
+
|
| 110 |
+
// New: status text
|
| 111 |
+
statusText: string
|
| 112 |
+
|
| 113 |
+
// New: fullscreen camera mode
|
| 114 |
+
cameraFullscreen: boolean
|
| 115 |
+
|
| 116 |
setServerUrl: (url: string) => void
|
| 117 |
setVoiceName: (name: string) => void
|
| 118 |
setStatus: (status: CompanionStatus) => void
|
|
|
|
| 127 |
setAvatarUrl: (url: string | null) => void
|
| 128 |
setBackgroundUrl: (url: string | null) => void
|
| 129 |
reset: () => void
|
| 130 |
+
|
| 131 |
+
// New actions
|
| 132 |
+
setScenario: (id: string | null) => void
|
| 133 |
+
setScenarioPanelOpen: (open: boolean) => void
|
| 134 |
+
setSubtitleEnabled: (enabled: boolean) => void
|
| 135 |
+
setCameraFacingMode: (mode: 'user' | 'environment') => void
|
| 136 |
+
setCameraDevices: (devices: MediaDeviceInfo[]) => void
|
| 137 |
+
setActiveCameraLabel: (label: string | null) => void
|
| 138 |
+
setScreenShareStream: (stream: MediaStream | null) => void
|
| 139 |
+
setScreenShareDialogOpen: (open: boolean) => void
|
| 140 |
+
setStatusText: (text: string) => void
|
| 141 |
+
setCameraFullscreen: (fullscreen: boolean) => void
|
| 142 |
}
|
| 143 |
|
| 144 |
const DEFAULT_SERVER_URL = 'ws://127.0.0.1:8889/ws/companion'
|
|
|
|
| 157 |
avatarUrl: loadPersisted(AVATAR_KEY),
|
| 158 |
backgroundUrl: loadPersisted(BACKGROUND_KEY),
|
| 159 |
|
| 160 |
+
// New defaults
|
| 161 |
+
scenario: null,
|
| 162 |
+
scenarioPanelOpen: false,
|
| 163 |
+
subtitleEnabled: true,
|
| 164 |
+
cameraFacingMode: 'user',
|
| 165 |
+
cameraDevices: [],
|
| 166 |
+
activeCameraLabel: null,
|
| 167 |
+
screenShareStream: null,
|
| 168 |
+
screenShareDialogOpen: false,
|
| 169 |
+
statusText: '你可以开始说话',
|
| 170 |
+
cameraFullscreen: false,
|
| 171 |
+
|
| 172 |
setServerUrl: (url) => set({ serverUrl: url }),
|
| 173 |
setVoiceName: (name) => set({ voiceName: name }),
|
| 174 |
setStatus: (status) => set({ status, error: status === 'error' ? undefined : null }),
|
|
|
|
| 197 |
generating: false,
|
| 198 |
transcript: '',
|
| 199 |
error: null,
|
| 200 |
+
scenario: null,
|
| 201 |
+
subtitleEnabled: true,
|
| 202 |
+
cameraFacingMode: 'user',
|
| 203 |
+
screenShareStream: null,
|
| 204 |
+
statusText: '你可以开始说话',
|
| 205 |
+
cameraFullscreen: false,
|
| 206 |
}),
|
| 207 |
+
|
| 208 |
+
// New action implementations
|
| 209 |
+
setScenario: (id) => set({ scenario: id, scenarioPanelOpen: false }),
|
| 210 |
+
setScenarioPanelOpen: (open) => set({ scenarioPanelOpen: open }),
|
| 211 |
+
setSubtitleEnabled: (enabled) => set({ subtitleEnabled: enabled }),
|
| 212 |
+
setCameraFacingMode: (mode) => set({ cameraFacingMode: mode }),
|
| 213 |
+
setCameraDevices: (devices) => set({ cameraDevices: devices }),
|
| 214 |
+
setActiveCameraLabel: (label) => set({ activeCameraLabel: label }),
|
| 215 |
+
setScreenShareStream: (stream) => set({ screenShareStream: stream }),
|
| 216 |
+
setScreenShareDialogOpen: (open) => set({ screenShareDialogOpen: open }),
|
| 217 |
+
setStatusText: (text) => set({ statusText: text }),
|
| 218 |
+
setCameraFullscreen: (fullscreen) => set({ cameraFullscreen: fullscreen }),
|
| 219 |
}))
|
|
@@ -1,5 +1,14 @@
|
|
| 1 |
export type CompanionStatus = 'disconnected' | 'connecting' | 'connected' | 'error'
|
| 2 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
export type CompanionConfig = {
|
| 4 |
serverUrl: string
|
| 5 |
voiceName: string
|
|
|
|
| 1 |
export type CompanionStatus = 'disconnected' | 'connecting' | 'connected' | 'error'
|
| 2 |
|
| 3 |
+
export type ScenarioOption = {
|
| 4 |
+
id: string
|
| 5 |
+
name: string
|
| 6 |
+
nameEn: string
|
| 7 |
+
icon: string
|
| 8 |
+
description: string
|
| 9 |
+
descriptionEn: string
|
| 10 |
+
}
|
| 11 |
+
|
| 12 |
export type CompanionConfig = {
|
| 13 |
serverUrl: string
|
| 14 |
voiceName: string
|