chenbhao Claude Big Pickle commited on
Commit
3aea50a
·
1 Parent(s): a457004

feat: Doubao-style video call with scenario selection, screen share & fullscreen camera

Browse files

Redesign the companion (视频通话) page to match Doubao's video calling experience:
- Scenario Selection: 6 scenarios (模拟面试, 英语陪练, 唱歌, 同声传译, 成语接龙, 心情树洞) in a floating grid
- Fullscreen camera mode: camera feed replaces AI avatar as background
- Screen sharing: getDisplayMedia capture with permission dialog & privacy notice
- Subtitle mode: camera shrinks to corner, transcript takes center stage
- Warm orange theme matching Doubao's brand color
- Bottom controls: camera → screen share → mic → speaker → call/hang up
- Camera/mic toggles work independently of WebSocket connection
- No auto-reconnect on connection failure (prevents button flashing)
- Cyberpunk-to-orange redesign: cleaner aesthetic with frosted glass elements

New files: CompanionTopBar, ScenarioSelector, ScreenShareDialog,
useScreenShare, useCameraDevices

Co-Authored-By: Claude Big Pickle <noreply@anthropic.com>

desktop/src/components/companion/CompanionControls.tsx CHANGED
@@ -1,3 +1,4 @@
 
1
  import { useTranslation } from '../../i18n'
2
  import type { CompanionStatus } from '../../types/companion'
3
 
@@ -6,8 +7,10 @@ interface CompanionControlsProps {
6
  micEnabled: boolean
7
  cameraEnabled: boolean
8
  generating: boolean
 
9
  onToggleMic: () => void
10
  onToggleCamera: () => void
 
11
  onConnect: () => void
12
  onDisconnect: () => void
13
  onStop: () => void
@@ -19,8 +22,10 @@ export function CompanionControls({
19
  micEnabled,
20
  cameraEnabled,
21
  generating,
 
22
  onToggleMic,
23
  onToggleCamera,
 
24
  onConnect,
25
  onDisconnect,
26
  onStop,
@@ -28,84 +33,115 @@ export function CompanionControls({
28
  }: CompanionControlsProps) {
29
  const t = useTranslation()
30
  const isConnected = status === 'connected'
 
 
 
 
 
 
31
 
32
  return (
33
- <div className="companion-controls absolute bottom-4 left-1/2 -translate-x-1/2 flex items-center gap-3 px-6 py-3 rounded-2xl bg-black/50 backdrop-blur-md border border-white/10">
34
- {/* Mic toggle */}
35
  <button
36
  type="button"
37
- onClick={onToggleMic}
38
- disabled={!isConnected}
39
- className={`flex h-10 w-10 items-center justify-center rounded-xl transition-all ${
40
- micEnabled
41
  ? 'bg-white/10 text-white hover:bg-white/20'
42
- : 'bg-red-500/20 text-red-400'
43
- } disabled:opacity-40`}
44
- title={micEnabled ? t('companion.micOn') : t('companion.micOff')}
45
  >
46
  <span className="material-symbols-outlined text-xl">
47
- {micEnabled ? 'mic' : 'mic_off'}
48
  </span>
49
  </button>
50
 
51
- {/* Camera toggle */}
52
  <button
53
  type="button"
54
- onClick={onToggleCamera}
55
- disabled={!isConnected}
56
- className={`flex h-10 w-10 items-center justify-center rounded-xl transition-all ${
57
- cameraEnabled
58
- ? 'bg-white/10 text-white hover:bg-white/20'
59
- : 'bg-red-500/20 text-red-400'
60
- } disabled:opacity-40`}
61
- title={cameraEnabled ? t('companion.cameraOn') : t('companion.cameraOff')}
62
  >
63
  <span className="material-symbols-outlined text-xl">
64
- {cameraEnabled ? 'videocam' : 'videocam_off'}
65
  </span>
66
  </button>
67
 
68
- <div className="w-px h-8 bg-white/10" />
 
 
 
 
 
 
 
 
 
 
 
 
 
 
69
 
70
- {/* Stop generation */}
71
  {generating && (
72
  <button
73
  type="button"
74
  onClick={onStop}
75
- className="flex h-10 w-10 items-center justify-center rounded-xl bg-red-500/30 text-red-400 hover:bg-red-500/50 transition-all"
76
  title="Stop"
77
  >
78
  <span className="material-symbols-outlined text-xl">stop</span>
79
  </button>
80
  )}
81
 
82
- {/* Connect / Disconnect */}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
83
  {isConnected ? (
84
  <button
85
  type="button"
86
- onClick={() => {
87
- onDisconnect()
88
- }}
89
- className="flex h-10 items-center gap-2 rounded-xl bg-red-500/20 px-4 text-red-400 hover:bg-red-500/30 transition-all text-sm font-medium"
90
  >
91
- <span className="material-symbols-outlined text-lg">link_off</span>
92
- {t('companion.disconnect')}
93
  </button>
94
  ) : (
95
  <button
96
  type="button"
97
- onClick={() => {
98
- onConnect()
99
- // Resume audio context on user gesture
100
- onResumeAudio()
101
- }}
102
  disabled={status === 'connecting'}
103
- className="flex h-10 items-center gap-2 rounded-xl bg-purple-600/30 px-4 text-purple-300 hover:bg-purple-600/50 transition-all text-sm font-medium disabled:opacity-40"
 
104
  >
105
- <span className="material-symbols-outlined text-lg">
106
- {status === 'connecting' ? 'hourglass_top' : 'link'}
107
  </span>
108
- {status === 'connecting' ? t('companion.status.connecting') : t('companion.connect')}
109
  </button>
110
  )}
111
  </div>
 
1
+ import { useCallback, useState } from 'react'
2
  import { useTranslation } from '../../i18n'
3
  import type { CompanionStatus } from '../../types/companion'
4
 
 
7
  micEnabled: boolean
8
  cameraEnabled: boolean
9
  generating: boolean
10
+ isSharing: boolean
11
  onToggleMic: () => void
12
  onToggleCamera: () => void
13
+ onScreenShareClick: () => void
14
  onConnect: () => void
15
  onDisconnect: () => void
16
  onStop: () => void
 
22
  micEnabled,
23
  cameraEnabled,
24
  generating,
25
+ isSharing,
26
  onToggleMic,
27
  onToggleCamera,
28
+ onScreenShareClick,
29
  onConnect,
30
  onDisconnect,
31
  onStop,
 
33
  }: CompanionControlsProps) {
34
  const t = useTranslation()
35
  const isConnected = status === 'connected'
36
+ const [speakerOn, setSpeakerOn] = useState(true)
37
+
38
+ const handleConnectClick = useCallback(() => {
39
+ onConnect()
40
+ onResumeAudio()
41
+ }, [onConnect, onResumeAudio])
42
 
43
  return (
44
+ <div className="companion-controls absolute bottom-6 left-1/2 -translate-x-1/2 flex items-center gap-3 px-5 py-2.5 rounded-full bg-black/50 backdrop-blur-md border border-white/10 z-10">
45
+ {/* 1. Camera toggle — always clickable */}
46
  <button
47
  type="button"
48
+ onClick={onToggleCamera}
49
+ className={`flex h-10 w-10 items-center justify-center rounded-full transition-all ${
50
+ cameraEnabled
 
51
  ? 'bg-white/10 text-white hover:bg-white/20'
52
+ : 'bg-white/5 text-white/40 hover:text-white/60'
53
+ }`}
54
+ title={cameraEnabled ? t('companion.cameraOn') : t('companion.cameraOff')}
55
  >
56
  <span className="material-symbols-outlined text-xl">
57
+ {cameraEnabled ? 'videocam' : 'videocam_off'}
58
  </span>
59
  </button>
60
 
61
+ {/* 2. Screen share — always clickable */}
62
  <button
63
  type="button"
64
+ onClick={onScreenShareClick}
65
+ className={`flex h-10 w-10 items-center justify-center rounded-full transition-all ${
66
+ isSharing
67
+ ? 'bg-orange-500/30 text-orange-300'
68
+ : 'bg-white/10 text-white hover:bg-white/20'
69
+ }`}
70
+ title={isSharing ? t('companion.screenShare.stop') : t('companion.screenShare.start')}
 
71
  >
72
  <span className="material-symbols-outlined text-xl">
73
+ {isSharing ? 'present_to_all' : 'screen_share'}
74
  </span>
75
  </button>
76
 
77
+ {/* 3. Mic toggle — always clickable */}
78
+ <button
79
+ type="button"
80
+ onClick={onToggleMic}
81
+ className={`flex h-10 w-10 items-center justify-center rounded-full transition-all ${
82
+ micEnabled
83
+ ? 'bg-white/10 text-white hover:bg-white/20'
84
+ : 'bg-white/5 text-white/40 hover:text-white/60'
85
+ }`}
86
+ title={micEnabled ? t('companion.micOn') : t('companion.micOff')}
87
+ >
88
+ <span className="material-symbols-outlined text-xl">
89
+ {micEnabled ? 'mic' : 'mic_off'}
90
+ </span>
91
+ </button>
92
 
93
+ {/* Stop generation (when generating) */}
94
  {generating && (
95
  <button
96
  type="button"
97
  onClick={onStop}
98
+ className="flex h-10 w-10 items-center justify-center rounded-full bg-orange-500/30 text-orange-300 hover:bg-orange-500/50 transition-all"
99
  title="Stop"
100
  >
101
  <span className="material-symbols-outlined text-xl">stop</span>
102
  </button>
103
  )}
104
 
105
+ {/* Separator */}
106
+ <div className="w-px h-7 bg-white/10" />
107
+
108
+ {/* 4. Speaker toggle */}
109
+ <button
110
+ type="button"
111
+ onClick={() => setSpeakerOn(!speakerOn)}
112
+ className={`flex h-10 w-10 items-center justify-center rounded-full transition-all ${
113
+ speakerOn
114
+ ? 'bg-white/10 text-white hover:bg-white/20'
115
+ : 'bg-white/5 text-white/40'
116
+ }`}
117
+ title={t('companion.speaker')}
118
+ >
119
+ <span className="material-symbols-outlined text-xl">
120
+ {speakerOn ? 'volume_up' : 'volume_off'}
121
+ </span>
122
+ </button>
123
+
124
+ {/* 5. Call / Hang Up */}
125
  {isConnected ? (
126
  <button
127
  type="button"
128
+ onClick={onDisconnect}
129
+ className="flex h-10 w-10 items-center justify-center rounded-full bg-red-500 text-white hover:bg-red-600 transition-all shadow-lg"
130
+ title={t('companion.hangUp')}
 
131
  >
132
+ <span className="material-symbols-outlined text-xl">call_end</span>
 
133
  </button>
134
  ) : (
135
  <button
136
  type="button"
137
+ onClick={handleConnectClick}
 
 
 
 
138
  disabled={status === 'connecting'}
139
+ className="flex h-10 w-10 items-center justify-center rounded-full bg-green-500 text-white hover:bg-green-600 transition-all disabled:opacity-40 shadow-lg"
140
+ title={t('companion.call')}
141
  >
142
+ <span className="material-symbols-outlined text-xl">
143
+ {status === 'connecting' ? 'hourglass_top' : 'call'}
144
  </span>
 
145
  </button>
146
  )}
147
  </div>
desktop/src/components/companion/CompanionTopBar.tsx ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { useCallback } from 'react'
2
+ import { useTranslation } from '../../i18n'
3
+ import { useCompanionStore, SCENARIOS } from '../../stores/companionStore'
4
+
5
+ export function CompanionTopBar() {
6
+ const t = useTranslation()
7
+ const scenario = useCompanionStore((s) => s.scenario)
8
+ const subtitleEnabled = useCompanionStore((s) => s.subtitleEnabled)
9
+
10
+ const setScenarioPanelOpen = useCompanionStore((s) => s.setScenarioPanelOpen)
11
+ const setSubtitleEnabled = useCompanionStore((s) => s.setSubtitleEnabled)
12
+
13
+ const currentScenario = scenario
14
+ ? SCENARIOS.find((s) => s.id === scenario)
15
+ : null
16
+
17
+ const handleScenarioClick = useCallback(() => {
18
+ setScenarioPanelOpen(true)
19
+ }, [setScenarioPanelOpen])
20
+
21
+ const handleSubtitleToggle = useCallback(() => {
22
+ setSubtitleEnabled(!subtitleEnabled)
23
+ }, [subtitleEnabled, setSubtitleEnabled])
24
+
25
+ return (
26
+ <div className="absolute top-0 left-0 right-0 z-30 flex items-center justify-between px-4 pt-4 pb-2">
27
+ {/* Left spacer */}
28
+ <div className="w-9" />
29
+
30
+ {/* Center: scenario selector capsule */}
31
+ <button
32
+ type="button"
33
+ onClick={handleScenarioClick}
34
+ className="flex items-center gap-1.5 px-3.5 h-8 rounded-full bg-black/30 backdrop-blur-md border border-white/10 hover:bg-white/10 transition-all text-white/70 hover:text-white"
35
+ >
36
+ {currentScenario ? (
37
+ <>
38
+ <span className="material-symbols-outlined text-base">{currentScenario.icon}</span>
39
+ <span className="text-xs font-medium">{currentScenario.name}</span>
40
+ </>
41
+ ) : (
42
+ <span className="text-xs font-medium">{t('companion.scenario.select')}</span>
43
+ )}
44
+ <span className="material-symbols-outlined text-sm text-white/30">expand_more</span>
45
+ </button>
46
+
47
+ {/* Right: subtitle toggle */}
48
+ <button
49
+ type="button"
50
+ onClick={handleSubtitleToggle}
51
+ className={`flex h-8 w-8 items-center justify-center rounded-full backdrop-blur-md transition-all text-xs font-bold ${
52
+ subtitleEnabled
53
+ ? 'bg-orange-500/40 text-orange-200'
54
+ : 'bg-black/30 text-white/40 hover:text-white/60'
55
+ }`}
56
+ title={t('companion.subtitle')}
57
+ >
58
+
59
+ </button>
60
+ </div>
61
+ )
62
+ }
desktop/src/components/companion/CompanionTranscript.tsx CHANGED
@@ -1,5 +1,6 @@
1
  import { useRef, useEffect, useState, useCallback } from 'react'
2
  import { useTranslation } from '../../i18n'
 
3
 
4
  interface CompanionTranscriptProps {
5
  transcript: string
@@ -9,10 +10,18 @@ interface CompanionTranscriptProps {
9
  disabled?: boolean
10
  }
11
 
12
- export function CompanionTranscript({ transcript, fullTranscript, generating, onSendText, disabled }: CompanionTranscriptProps) {
 
 
 
 
 
 
13
  const scrollRef = useRef<HTMLDivElement>(null)
14
  const [inputValue, setInputValue] = useState('')
15
  const t = useTranslation()
 
 
16
 
17
  useEffect(() => {
18
  if (scrollRef.current) {
@@ -27,48 +36,131 @@ export function CompanionTranscript({ transcript, fullTranscript, generating, on
27
  setInputValue('')
28
  }, [inputValue, onSendText])
29
 
30
- const handleKeyDown = useCallback((e: React.KeyboardEvent) => {
31
- if (e.key === 'Enter' && !e.shiftKey) {
32
- e.preventDefault()
33
- handleSend()
34
- }
35
- }, [handleSend])
 
 
 
36
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
37
  return (
38
  <div className="companion-transcript absolute bottom-20 left-1/2 -translate-x-1/2 w-full max-w-2xl px-4">
39
- {/* Transcript display */}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
40
  <div
41
- ref={scrollRef}
42
- className={`max-h-[150px] overflow-y-auto rounded-2xl bg-black/40 backdrop-blur-md border p-4 mb-2 scrollbar-thin transition-all duration-500 ${
43
- generating
44
- ? 'border-purple-500/50 shadow-[0_0_20px_rgba(147,51,234,0.2)]'
45
- : 'border-white/10'
46
  }`}
47
  >
48
- {fullTranscript && (
49
- <div className="text-white/40 text-xs mb-2 whitespace-pre-wrap leading-relaxed">
50
- {fullTranscript}
51
- </div>
52
- )}
53
- {transcript && (
54
- <div className="text-white/90 text-sm font-medium leading-relaxed">
55
- {transcript}
56
- <span className="inline-block w-1.5 h-4 ml-0.5 bg-purple-400 animate-pulse" />
57
- </div>
58
- )}
59
- {!transcript && !fullTranscript && (
60
- <div className="text-white/30 text-sm text-center py-2">
61
- {t('companion.transcriptPlaceholder')}
62
- </div>
63
- )}
64
- </div>
65
-
66
- {/* Text input row */}
67
- <div className={`flex items-center gap-2 rounded-2xl bg-black/40 backdrop-blur-md border px-4 py-2 transition-all duration-500 ${
68
- generating
69
- ? 'border-purple-500/50 shadow-[0_0_20px_rgba(147,51,234,0.2)]'
70
- : 'border-white/10'
71
- }`}>
72
  <input
73
  type="text"
74
  value={inputValue}
@@ -76,13 +168,13 @@ export function CompanionTranscript({ transcript, fullTranscript, generating, on
76
  onKeyDown={handleKeyDown}
77
  placeholder={t('companion.inputPlaceholder')}
78
  disabled={disabled}
79
- className="flex-1 bg-transparent text-sm text-white/90 placeholder-white/30 outline-none border-none disabled:opacity-40"
80
  />
81
  <button
82
  type="button"
83
  onClick={handleSend}
84
  disabled={disabled || !inputValue.trim()}
85
- className="flex h-8 w-8 items-center justify-center rounded-lg bg-purple-600/40 text-purple-300 hover:bg-purple-600/60 transition-all disabled:opacity-30"
86
  title={t('companion.send')}
87
  >
88
  <span className="material-symbols-outlined text-lg">send</span>
 
1
  import { useRef, useEffect, useState, useCallback } from 'react'
2
  import { useTranslation } from '../../i18n'
3
+ import { useCompanionStore } from '../../stores/companionStore'
4
 
5
  interface CompanionTranscriptProps {
6
  transcript: string
 
10
  disabled?: boolean
11
  }
12
 
13
+ export function CompanionTranscript({
14
+ transcript,
15
+ fullTranscript,
16
+ generating,
17
+ onSendText,
18
+ disabled,
19
+ }: CompanionTranscriptProps) {
20
  const scrollRef = useRef<HTMLDivElement>(null)
21
  const [inputValue, setInputValue] = useState('')
22
  const t = useTranslation()
23
+ const subtitleEnabled = useCompanionStore((s) => s.subtitleEnabled)
24
+ const hasContent = !!transcript || !!fullTranscript
25
 
26
  useEffect(() => {
27
  if (scrollRef.current) {
 
36
  setInputValue('')
37
  }, [inputValue, onSendText])
38
 
39
+ const handleKeyDown = useCallback(
40
+ (e: React.KeyboardEvent) => {
41
+ if (e.key === 'Enter' && !e.shiftKey) {
42
+ e.preventDefault()
43
+ handleSend()
44
+ }
45
+ },
46
+ [handleSend],
47
+ )
48
 
49
+ // When subtitles are on and there's content, show full-screen transcript
50
+ if (subtitleEnabled && hasContent) {
51
+ return (
52
+ <div className="absolute inset-0 z-5 flex flex-col">
53
+ {/* Scrollable transcript area - takes most of the screen */}
54
+ <div className="flex-1 flex items-center justify-center px-8 py-20">
55
+ <div
56
+ ref={scrollRef}
57
+ className="w-full max-w-2xl max-h-full overflow-y-auto scrollbar-thin"
58
+ >
59
+ {fullTranscript && (
60
+ <div className="text-white/50 text-sm mb-3 whitespace-pre-wrap leading-relaxed text-center">
61
+ {fullTranscript}
62
+ </div>
63
+ )}
64
+ {transcript && (
65
+ <div className="text-white/90 text-base font-medium leading-relaxed text-center">
66
+ {transcript}
67
+ <span className="inline-block w-1.5 h-4 ml-0.5 bg-orange-400 animate-pulse" />
68
+ </div>
69
+ )}
70
+ </div>
71
+ </div>
72
+
73
+ {/* Text input at bottom */}
74
+ <div className="absolute bottom-20 left-1/2 -translate-x-1/2 w-full max-w-md px-4">
75
+ <div className="flex items-center gap-2 rounded-full bg-black/50 backdrop-blur-md border border-white/10 px-4 py-2">
76
+ <input
77
+ type="text"
78
+ value={inputValue}
79
+ onChange={(e) => setInputValue(e.target.value)}
80
+ onKeyDown={handleKeyDown}
81
+ placeholder={t('companion.inputPlaceholder')}
82
+ disabled={disabled}
83
+ className="flex-1 bg-transparent text-sm text-white/80 placeholder-white/30 outline-none border-none disabled:opacity-40"
84
+ />
85
+ <button
86
+ type="button"
87
+ onClick={handleSend}
88
+ disabled={disabled || !inputValue.trim()}
89
+ className="flex h-7 w-7 items-center justify-center rounded-full bg-orange-500/40 text-orange-200 hover:bg-orange-500/60 transition-all disabled:opacity-30"
90
+ title={t('companion.send')}
91
+ >
92
+ <span className="material-symbols-outlined text-sm">send</span>
93
+ </button>
94
+ </div>
95
+ </div>
96
+ </div>
97
+ )
98
+ }
99
+
100
+ // Subtitles on but no content yet - show compact version
101
+ if (subtitleEnabled && !hasContent) {
102
+ return (
103
+ <div className="companion-transcript absolute bottom-20 left-1/2 -translate-x-1/2 w-full max-w-md px-4">
104
+ <div
105
+ className={`flex items-center gap-2 rounded-full bg-black/40 backdrop-blur-md border border-white/10 px-4 py-2 transition-all duration-500 ${
106
+ generating ? 'border-orange-400/40' : ''
107
+ }`}
108
+ >
109
+ <input
110
+ type="text"
111
+ value={inputValue}
112
+ onChange={(e) => setInputValue(e.target.value)}
113
+ onKeyDown={handleKeyDown}
114
+ placeholder={t('companion.inputPlaceholder')}
115
+ disabled={disabled}
116
+ className="flex-1 bg-transparent text-sm text-white/80 placeholder-white/30 outline-none border-none disabled:opacity-40"
117
+ />
118
+ <button
119
+ type="button"
120
+ onClick={handleSend}
121
+ disabled={disabled || !inputValue.trim()}
122
+ className="flex h-7 w-7 items-center justify-center rounded-full bg-orange-500/40 text-orange-200 hover:bg-orange-500/60 transition-all disabled:opacity-30"
123
+ title={t('companion.send')}
124
+ >
125
+ <span className="material-symbols-outlined text-sm">send</span>
126
+ </button>
127
+ </div>
128
+ </div>
129
+ )
130
+ }
131
+
132
+ // Subtitles off - show compact transcript bubble
133
  return (
134
  <div className="companion-transcript absolute bottom-20 left-1/2 -translate-x-1/2 w-full max-w-2xl px-4">
135
+ {hasContent && (
136
+ <div
137
+ ref={scrollRef}
138
+ className={`max-h-[120px] overflow-y-auto rounded-2xl bg-black/40 backdrop-blur-md border p-3 mb-2 scrollbar-thin transition-all duration-500 ${
139
+ generating
140
+ ? 'border-orange-400/40'
141
+ : 'border-white/10'
142
+ }`}
143
+ >
144
+ {fullTranscript && !transcript && (
145
+ <div className="text-white/40 text-xs whitespace-pre-wrap leading-relaxed">
146
+ {fullTranscript}
147
+ </div>
148
+ )}
149
+ {transcript && (
150
+ <div className="text-white/80 text-sm font-medium leading-relaxed">
151
+ {transcript}
152
+ <span className="inline-block w-1.5 h-4 ml-0.5 bg-orange-400 animate-pulse" />
153
+ </div>
154
+ )}
155
+ </div>
156
+ )}
157
+
158
+ {/* Text input */}
159
  <div
160
+ className={`flex items-center gap-2 rounded-2xl bg-black/40 backdrop-blur-md border px-4 py-2 transition-all duration-500 ${
161
+ generating ? 'border-orange-400/40' : 'border-white/10'
 
 
 
162
  }`}
163
  >
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
164
  <input
165
  type="text"
166
  value={inputValue}
 
168
  onKeyDown={handleKeyDown}
169
  placeholder={t('companion.inputPlaceholder')}
170
  disabled={disabled}
171
+ className="flex-1 bg-transparent text-sm text-white/80 placeholder-white/30 outline-none border-none disabled:opacity-40"
172
  />
173
  <button
174
  type="button"
175
  onClick={handleSend}
176
  disabled={disabled || !inputValue.trim()}
177
+ className="flex h-8 w-8 items-center justify-center rounded-lg bg-orange-500/40 text-orange-200 hover:bg-orange-500/60 transition-all disabled:opacity-30"
178
  title={t('companion.send')}
179
  >
180
  <span className="material-symbols-outlined text-lg">send</span>
desktop/src/components/companion/CompanionVideoPanel.tsx CHANGED
@@ -8,6 +8,7 @@ interface CompanionVideoPanelProps {
8
  speaking: boolean
9
  generating: boolean
10
  status: CompanionStatus
 
11
  }
12
 
13
  export function CompanionVideoPanel({
@@ -15,15 +16,20 @@ export function CompanionVideoPanel({
15
  speaking,
16
  generating,
17
  status,
 
18
  }: CompanionVideoPanelProps) {
19
  const videoRef = useRef<HTMLVideoElement>(null)
20
  const avatarFileRef = useRef<HTMLInputElement>(null)
21
- const bgFileRef = useRef<HTMLInputElement>(null)
22
  const t = useTranslation()
23
  const avatarUrl = useCompanionStore((s) => s.avatarUrl)
24
  const backgroundUrl = useCompanionStore((s) => s.backgroundUrl)
25
  const setAvatarUrl = useCompanionStore((s) => s.setAvatarUrl)
26
- const setBackgroundUrl = useCompanionStore((s) => s.setBackgroundUrl)
 
 
 
 
 
27
 
28
  useEffect(() => {
29
  if (videoRef.current && webcamStream) {
@@ -35,10 +41,6 @@ export function CompanionVideoPanel({
35
  avatarFileRef.current?.click()
36
  }, [])
37
 
38
- const handleBgPick = useCallback(() => {
39
- bgFileRef.current?.click()
40
- }, [])
41
-
42
  const handleAvatarFile = useCallback(
43
  (e: React.ChangeEvent<HTMLInputElement>) => {
44
  const file = e.target.files?.[0]
@@ -46,23 +48,14 @@ export function CompanionVideoPanel({
46
  const reader = new FileReader()
47
  reader.onload = () => setAvatarUrl(reader.result as string)
48
  reader.readAsDataURL(file)
49
- // Reset so the same file can be re-selected
50
  e.target.value = ''
51
  },
52
  [setAvatarUrl],
53
  )
54
 
55
- const handleBgFile = useCallback(
56
- (e: React.ChangeEvent<HTMLInputElement>) => {
57
- const file = e.target.files?.[0]
58
- if (!file) return
59
- const reader = new FileReader()
60
- reader.onload = () => setBackgroundUrl(reader.result as string)
61
- reader.readAsDataURL(file)
62
- e.target.value = ''
63
- },
64
- [setBackgroundUrl],
65
- )
66
 
67
  const statusLabel = generating
68
  ? t('companion.speaking')
@@ -74,79 +67,107 @@ export function CompanionVideoPanel({
74
 
75
  return (
76
  <div className="companion-video-panel absolute inset-0 overflow-hidden">
77
- {/* Background */}
78
  {backgroundUrl ? (
79
  <div
80
  className="absolute inset-0 bg-cover bg-center transition-all duration-700"
81
  style={{ backgroundImage: `url(${backgroundUrl})` }}
82
  />
83
  ) : (
84
- <div className="absolute inset-0 bg-gradient-to-br from-slate-900 via-slate-800 to-slate-900 transition-all duration-700" />
85
  )}
86
 
87
- {/* Background edit button */}
88
- <button
89
- type="button"
90
- onClick={handleBgPick}
91
- className="absolute top-4 left-4 flex items-center gap-1.5 px-2.5 py-1.5 rounded-full bg-black/30 backdrop-blur-sm text-white/50 hover:text-white/80 hover:bg-black/50 transition-all text-xs"
92
- title="Change background"
93
- >
94
- <span className="material-symbols-outlined text-sm">image</span>
95
- Background
96
- </button>
97
- <input
98
- ref={bgFileRef}
99
- type="file"
100
- accept="image/*"
101
- className="hidden"
102
- onChange={handleBgFile}
103
- />
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
104
 
105
- {/* AI Presence indicator */}
106
- <div className="absolute left-1/2 top-1/2 -translate-x-1/2 -translate-y-1/2 flex flex-col items-center gap-4">
107
- <div className="relative group">
108
- <div
109
- className={`w-32 h-32 rounded-full transition-all duration-500 flex items-center justify-center overflow-hidden ${
110
- generating
111
- ? 'shadow-[0_0_60px_rgba(147,51,234,0.4)] animate-pulse ring-2 ring-purple-500/50'
112
- : speaking
113
- ? 'shadow-[0_0_30px_rgba(59,130,246,0.3)] ring-2 ring-blue-500/30'
114
- : ''
115
- }`}
116
- >
117
- {avatarUrl ? (
118
- <img
119
- src={avatarUrl}
120
- alt="avatar"
121
- className="w-full h-full object-cover"
122
- />
123
- ) : (
124
- <div
125
- className={`w-full h-full flex items-center justify-center ${
126
- generating
127
- ? 'bg-purple-600/30'
128
- : speaking
129
- ? 'bg-blue-500/20'
130
- : 'bg-gray-600/10'
131
- }`}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
132
  >
133
- <span className="material-symbols-outlined text-5xl text-white/60">
134
- {generating ? 'record_voice_over' : 'psychology'}
135
- </span>
136
- </div>
137
  )}
138
  </div>
139
- {/* Avatar edit overlay */}
140
- <button
141
- type="button"
142
- onClick={handleAvatarPick}
143
- className="absolute -bottom-1 -right-1 w-8 h-8 rounded-full bg-purple-600/80 hover:bg-purple-600 flex items-center justify-center transition-all opacity-0 group-hover:opacity-100 shadow-lg"
144
- >
145
- <span className="material-symbols-outlined text-sm text-white">edit</span>
146
- </button>
147
  </div>
148
- <span className="text-white/40 text-sm font-medium">{statusLabel}</span>
149
- </div>
150
  <input
151
  ref={avatarFileRef}
152
  type="file"
@@ -156,8 +177,14 @@ export function CompanionVideoPanel({
156
  />
157
 
158
  {/* Webcam PiP overlay */}
159
- {webcamStream && (
160
- <div className="absolute bottom-6 right-6 w-[180px] h-[240px] rounded-2xl overflow-hidden border-2 border-white/20 shadow-2xl bg-black">
 
 
 
 
 
 
161
  <video
162
  ref={videoRef}
163
  autoPlay
@@ -168,8 +195,12 @@ export function CompanionVideoPanel({
168
  </div>
169
  )}
170
 
171
- {/* Connection status badge */}
172
- <div className="absolute top-4 right-4 flex items-center gap-2 px-3 py-1.5 rounded-full bg-black/40 backdrop-blur-sm">
 
 
 
 
173
  <span
174
  className={`w-2 h-2 rounded-full ${
175
  status === 'connected'
@@ -181,7 +212,7 @@ export function CompanionVideoPanel({
181
  : 'bg-gray-500'
182
  }`}
183
  />
184
- <span className="text-white/60 text-xs">{status}</span>
185
  </div>
186
  </div>
187
  )
 
8
  speaking: boolean
9
  generating: boolean
10
  status: CompanionStatus
11
+ onFlipCamera?: () => void
12
  }
13
 
14
  export function CompanionVideoPanel({
 
16
  speaking,
17
  generating,
18
  status,
19
+ onFlipCamera,
20
  }: CompanionVideoPanelProps) {
21
  const videoRef = useRef<HTMLVideoElement>(null)
22
  const avatarFileRef = useRef<HTMLInputElement>(null)
 
23
  const t = useTranslation()
24
  const avatarUrl = useCompanionStore((s) => s.avatarUrl)
25
  const backgroundUrl = useCompanionStore((s) => s.backgroundUrl)
26
  const setAvatarUrl = useCompanionStore((s) => s.setAvatarUrl)
27
+ const cameraFullscreen = useCompanionStore((s) => s.cameraFullscreen)
28
+ const cameraEnabled = useCompanionStore((s) => s.cameraEnabled)
29
+ const screenShareStream = useCompanionStore((s) => s.screenShareStream)
30
+ const transcript = useCompanionStore((s) => s.transcript)
31
+ const fullTranscript = useCompanionStore((s) => s.fullTranscript)
32
+ const subtitleEnabled = useCompanionStore((s) => s.subtitleEnabled)
33
 
34
  useEffect(() => {
35
  if (videoRef.current && webcamStream) {
 
41
  avatarFileRef.current?.click()
42
  }, [])
43
 
 
 
 
 
44
  const handleAvatarFile = useCallback(
45
  (e: React.ChangeEvent<HTMLInputElement>) => {
46
  const file = e.target.files?.[0]
 
48
  const reader = new FileReader()
49
  reader.onload = () => setAvatarUrl(reader.result as string)
50
  reader.readAsDataURL(file)
 
51
  e.target.value = ''
52
  },
53
  [setAvatarUrl],
54
  )
55
 
56
+ const isFullScreenCamera = cameraEnabled && cameraFullscreen && webcamStream
57
+ // When subtitle is on AND there's transcript content, shrink the main view
58
+ const hasTranscriptContent = subtitleEnabled && (!!transcript || !!fullTranscript)
 
 
 
 
 
 
 
 
59
 
60
  const statusLabel = generating
61
  ? t('companion.speaking')
 
67
 
68
  return (
69
  <div className="companion-video-panel absolute inset-0 overflow-hidden">
70
+ {/* Background - clean warm gradient */}
71
  {backgroundUrl ? (
72
  <div
73
  className="absolute inset-0 bg-cover bg-center transition-all duration-700"
74
  style={{ backgroundImage: `url(${backgroundUrl})` }}
75
  />
76
  ) : (
77
+ <div className="absolute inset-0 bg-gradient-to-br from-orange-950/80 via-slate-950 to-amber-950/60 transition-all duration-700" />
78
  )}
79
 
80
+ {/* Fullscreen camera view */}
81
+ {isFullScreenCamera && (
82
+ <div className="absolute inset-0 bg-black">
83
+ <video
84
+ ref={videoRef}
85
+ autoPlay
86
+ muted
87
+ playsInline
88
+ className="h-full w-full object-cover scale-x-[-1]"
89
+ />
90
+ {/* Flip camera overlay */}
91
+ <div className="absolute top-14 right-4 flex flex-col gap-2 z-20">
92
+ <button
93
+ type="button"
94
+ onClick={onFlipCamera}
95
+ className="flex h-10 w-10 items-center justify-center rounded-full bg-black/40 backdrop-blur-md text-white/70 hover:text-white hover:bg-black/60 transition-all"
96
+ title={t('companion.flipCamera')}
97
+ >
98
+ <span className="material-symbols-outlined text-xl">flip_camera_android</span>
99
+ </button>
100
+ </div>
101
+ </div>
102
+ )}
103
+
104
+ {/* Screen share overlay */}
105
+ {screenShareStream && !isFullScreenCamera && (
106
+ <div className="absolute inset-0 bg-black/60 flex items-center justify-center">
107
+ <div className="text-center">
108
+ <span className="material-symbols-outlined text-5xl text-orange-400/60 mb-3">present_to_all</span>
109
+ <p className="text-white/50 text-sm">正在共享屏幕</p>
110
+ </div>
111
+ </div>
112
+ )}
113
 
114
+ {/* AI Avatar - large center (default), small top-left when subtitles active */}
115
+ {!isFullScreenCamera && !screenShareStream && (
116
+ <div
117
+ className={`absolute flex flex-col items-center gap-3 transition-all duration-500 ${
118
+ hasTranscriptContent
119
+ ? 'top-6 left-6 scale-[0.45] origin-top-left'
120
+ : 'left-1/2 top-1/2 -translate-x-1/2 -translate-y-1/2'
121
+ }`}
122
+ >
123
+ <div className="relative">
124
+ <div
125
+ className={`rounded-full transition-all duration-700 flex items-center justify-center overflow-hidden ${
126
+ hasTranscriptContent ? 'w-24 h-24' : 'w-36 h-36'
127
+ } ${
128
+ generating
129
+ ? 'shadow-[0_0_60px_rgba(251,146,60,0.4)] ring-2 ring-orange-400/60'
130
+ : speaking
131
+ ? 'shadow-[0_0_40px_rgba(251,146,60,0.25)] ring-2 ring-orange-400/30'
132
+ : 'shadow-[0_0_15px_rgba(255,255,255,0.08)] ring-1 ring-white/10'
133
+ }`}
134
+ style={{
135
+ background: generating
136
+ ? 'radial-gradient(circle, rgba(251,146,60,0.25) 0%, rgba(249,115,22,0.1) 100%)'
137
+ : speaking
138
+ ? 'radial-gradient(circle, rgba(251,146,60,0.15) 0%, rgba(249,115,22,0.05) 100%)'
139
+ : 'radial-gradient(circle, rgba(255,255,255,0.04) 0%, transparent 100%)',
140
+ }}
141
+ >
142
+ {avatarUrl ? (
143
+ <img src={avatarUrl} alt="avatar" className="w-full h-full object-cover" />
144
+ ) : (
145
+ <div className="w-full h-full flex items-center justify-center">
146
+ {(speaking || generating) && (
147
+ <div className="absolute inset-0 rounded-full animate-ping opacity-10 bg-orange-400" />
148
+ )}
149
+ {/* Default female avatar silhouette */}
150
+ <div className="flex flex-col items-center justify-center text-white/50">
151
+ <span className="material-symbols-outlined text-5xl">face_6</span>
152
+ </div>
153
+ </div>
154
+ )}
155
+ </div>
156
+ {!hasTranscriptContent && (
157
+ <button
158
+ type="button"
159
+ onClick={handleAvatarPick}
160
+ className="absolute -bottom-1 -right-1 w-7 h-7 rounded-full bg-orange-500/70 hover:bg-orange-500 flex items-center justify-center transition-all opacity-0 group-hover:opacity-100 shadow-lg z-10"
161
  >
162
+ <span className="material-symbols-outlined text-xs text-white">edit</span>
163
+ </button>
 
 
164
  )}
165
  </div>
166
+ {!hasTranscriptContent && (
167
+ <span className="text-white/40 text-xs font-medium tracking-wide">{statusLabel}</span>
168
+ )}
 
 
 
 
 
169
  </div>
170
+ )}
 
171
  <input
172
  ref={avatarFileRef}
173
  type="file"
 
177
  />
178
 
179
  {/* Webcam PiP overlay */}
180
+ {webcamStream && !isFullScreenCamera && (
181
+ <div
182
+ className={`absolute rounded-2xl overflow-hidden border-2 border-white/15 shadow-2xl bg-black z-10 transition-all duration-500 ${
183
+ hasTranscriptContent
184
+ ? 'top-4 right-4 w-[120px] h-[160px]'
185
+ : 'bottom-6 right-6 w-[160px] h-[220px]'
186
+ }`}
187
+ >
188
  <video
189
  ref={videoRef}
190
  autoPlay
 
195
  </div>
196
  )}
197
 
198
+ {/* Connection status badge - top right (moved left when webcam PiP shows) */}
199
+ <div
200
+ className={`absolute top-4 flex items-center gap-2 px-3 py-1.5 rounded-full bg-black/40 backdrop-blur-sm z-10 ${
201
+ webcamStream && !isFullScreenCamera ? 'left-4' : 'right-4'
202
+ }`}
203
+ >
204
  <span
205
  className={`w-2 h-2 rounded-full ${
206
  status === 'connected'
 
212
  : 'bg-gray-500'
213
  }`}
214
  />
215
+ <span className="text-white/50 text-xs">{status}</span>
216
  </div>
217
  </div>
218
  )
desktop/src/components/companion/ScenarioSelector.tsx ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { useCallback } from 'react'
2
+ import { useTranslation } from '../../i18n'
3
+ import { useCompanionStore, SCENARIOS } from '../../stores/companionStore'
4
+
5
+ export function ScenarioSelector() {
6
+ const t = useTranslation()
7
+ const open = useCompanionStore((s) => s.scenarioPanelOpen)
8
+ const currentScenario = useCompanionStore((s) => s.scenario)
9
+ const setScenario = useCompanionStore((s) => s.setScenario)
10
+ const setScenarioPanelOpen = useCompanionStore((s) => s.setScenarioPanelOpen)
11
+
12
+ const handleSelect = useCallback(
13
+ (id: string) => {
14
+ setScenario(id)
15
+ },
16
+ [setScenario],
17
+ )
18
+
19
+ const handleClose = useCallback(() => {
20
+ setScenarioPanelOpen(false)
21
+ }, [setScenarioPanelOpen])
22
+
23
+ if (!open) return null
24
+
25
+ return (
26
+ <div className="absolute inset-0 z-40 flex items-start justify-center pt-20">
27
+ {/* Backdrop */}
28
+ <div className="absolute inset-0 bg-black/40 backdrop-sm" onClick={handleClose} />
29
+
30
+ {/* Grid panel */}
31
+ <div className="relative w-[420px] max-h-[70vh] p-5 rounded-2xl bg-black/60 backdrop-blur-xl border border-white/10 shadow-2xl">
32
+ <h3 className="text-white/70 text-sm font-medium mb-4 text-center">
33
+ {t('companion.scenario.select')}
34
+ </h3>
35
+
36
+ <div className="grid grid-cols-3 gap-2.5">
37
+ {SCENARIOS.map((scenario) => {
38
+ const isActive = currentScenario === scenario.id
39
+ return (
40
+ <button
41
+ key={scenario.id}
42
+ type="button"
43
+ onClick={() => handleSelect(scenario.id)}
44
+ className={`flex flex-col items-center gap-1.5 p-3 rounded-xl transition-all border ${
45
+ isActive
46
+ ? 'bg-orange-500/20 border-orange-400/40'
47
+ : 'bg-white/[0.04] border-transparent hover:bg-white/[0.08]'
48
+ }`}
49
+ >
50
+ <span
51
+ className={`material-symbols-outlined text-2xl ${
52
+ isActive ? 'text-orange-300' : 'text-white/50'
53
+ }`}
54
+ >
55
+ {scenario.icon}
56
+ </span>
57
+ <span className="text-white/80 text-xs font-medium">{scenario.name}</span>
58
+ </button>
59
+ )
60
+ })}
61
+ </div>
62
+ </div>
63
+ </div>
64
+ )
65
+ }
desktop/src/components/companion/ScreenShareDialog.tsx ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { useState, useCallback } from 'react'
2
+ import { useTranslation } from '../../i18n'
3
+ import { useCompanionStore } from '../../stores/companionStore'
4
+
5
+ interface ScreenShareDialogProps {
6
+ onStartScreenShare: () => void
7
+ }
8
+
9
+ export function ScreenShareDialog({ onStartScreenShare }: ScreenShareDialogProps) {
10
+ const t = useTranslation()
11
+ const open = useCompanionStore((s) => s.screenShareDialogOpen)
12
+ const setScreenShareDialogOpen = useCompanionStore((s) => s.setScreenShareDialogOpen)
13
+ const [sourceType, setSourceType] = useState<'screen' | 'window'>('screen')
14
+
15
+ const handleCancel = useCallback(() => {
16
+ setScreenShareDialogOpen(false)
17
+ }, [setScreenShareDialogOpen])
18
+
19
+ const handleStart = useCallback(() => {
20
+ setScreenShareDialogOpen(false)
21
+ onStartScreenShare()
22
+ }, [setScreenShareDialogOpen, onStartScreenShare])
23
+
24
+ if (!open) return null
25
+
26
+ return (
27
+ <div className="absolute inset-0 z-40 flex items-center justify-center">
28
+ {/* Backdrop */}
29
+ <div className="absolute inset-0 bg-black/50 backdrop-sm" onClick={handleCancel} />
30
+
31
+ {/* Dialog */}
32
+ <div className="relative w-[340px] p-5 rounded-2xl bg-black/70 backdrop-blur-xl border border-white/10 shadow-2xl">
33
+ <h3 className="text-white/80 text-sm font-medium mb-4 text-center">
34
+ {t('companion.screenShare.dialog.title')}
35
+ </h3>
36
+
37
+ {/* Source selector */}
38
+ <div className="mb-5 flex gap-2">
39
+ <button
40
+ type="button"
41
+ onClick={() => setSourceType('screen')}
42
+ className={`flex-1 flex flex-col items-center gap-1.5 p-3 rounded-xl border transition-all ${
43
+ sourceType === 'screen'
44
+ ? 'bg-orange-500/20 border-orange-400/40'
45
+ : 'bg-white/[0.04] border-transparent hover:bg-white/[0.08]'
46
+ }`}
47
+ >
48
+ <span className="material-symbols-outlined text-xl text-white/60">monitor</span>
49
+ <span className="text-white/70 text-xs">{t('companion.screenShare.dialog.entireScreen')}</span>
50
+ </button>
51
+ <button
52
+ type="button"
53
+ onClick={() => setSourceType('window')}
54
+ className={`flex-1 flex flex-col items-center gap-1.5 p-3 rounded-xl border transition-all ${
55
+ sourceType === 'window'
56
+ ? 'bg-orange-500/20 border-orange-400/40'
57
+ : 'bg-white/[0.04] border-transparent hover:bg-white/[0.08]'
58
+ }`}
59
+ >
60
+ <span className="material-symbols-outlined text-xl text-white/60">window</span>
61
+ <span className="text-white/70 text-xs">{t('companion.screenShare.dialog.window')}</span>
62
+ </button>
63
+ </div>
64
+
65
+ {/* Privacy notice */}
66
+ <p className="text-white/25 text-[11px] leading-relaxed mb-5">
67
+ {t('companion.screenShare.dialog.privacy')}
68
+ </p>
69
+
70
+ {/* Action buttons */}
71
+ <div className="flex gap-2.5">
72
+ <button
73
+ type="button"
74
+ onClick={handleCancel}
75
+ className="flex-1 h-9 rounded-xl bg-white/5 text-white/50 hover:bg-white/10 hover:text-white/70 transition-all text-xs font-medium"
76
+ >
77
+ {t('companion.screenShare.dialog.cancel')}
78
+ </button>
79
+ <button
80
+ type="button"
81
+ onClick={handleStart}
82
+ className="flex-1 h-9 rounded-xl bg-orange-500/30 text-orange-200 hover:bg-orange-500/50 transition-all text-xs font-medium"
83
+ >
84
+ {t('companion.screenShare.dialog.start')}
85
+ </button>
86
+ </div>
87
+ </div>
88
+ </div>
89
+ )
90
+ }
desktop/src/hooks/useCameraDevices.ts ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { useState, useCallback } from 'react'
2
+
3
+ export function useCameraDevices() {
4
+ const [devices, setDevices] = useState<MediaDeviceInfo[]>([])
5
+ const [activeDeviceId, setActiveDeviceId] = useState<string | null>(null)
6
+
7
+ const enumerateCameras = useCallback(async () => {
8
+ try {
9
+ const allDevices = await navigator.mediaDevices.enumerateDevices()
10
+ const videoInputs = allDevices.filter((d) => d.kind === 'videoinput')
11
+ setDevices(videoInputs)
12
+ return videoInputs
13
+ } catch {
14
+ return []
15
+ }
16
+ }, [])
17
+
18
+ const switchCamera = useCallback(
19
+ async (deviceId: string): Promise<MediaStream | null> => {
20
+ try {
21
+ const stream = await navigator.mediaDevices.getUserMedia({
22
+ video: {
23
+ deviceId: { exact: deviceId },
24
+ width: { ideal: 640 },
25
+ height: { ideal: 480 },
26
+ },
27
+ audio: false,
28
+ })
29
+ setActiveDeviceId(deviceId)
30
+ return stream
31
+ } catch (err) {
32
+ console.error('Failed to switch camera:', err)
33
+ return null
34
+ }
35
+ },
36
+ [],
37
+ )
38
+
39
+ const getStreamForFacingMode = useCallback(
40
+ async (facingMode: 'user' | 'environment'): Promise<MediaStream | null> => {
41
+ try {
42
+ const stream = await navigator.mediaDevices.getUserMedia({
43
+ video: {
44
+ facingMode,
45
+ width: { ideal: 640 },
46
+ height: { ideal: 480 },
47
+ },
48
+ audio: false,
49
+ })
50
+ return stream
51
+ } catch (err) {
52
+ console.error('Failed to get camera with facing mode:', err)
53
+ return null
54
+ }
55
+ },
56
+ [],
57
+ )
58
+
59
+ return {
60
+ devices,
61
+ activeDeviceId,
62
+ enumerateCameras,
63
+ switchCamera,
64
+ getStreamForFacingMode,
65
+ }
66
+ }
desktop/src/hooks/useCompanionWebSocket.ts CHANGED
@@ -57,20 +57,11 @@ export function useCompanionWebSocket(callbacks: Callbacks) {
57
  wsRef.current = ws
58
 
59
  ws.onopen = () => {
60
- reconnectAttemptRef.current = 0
61
  setStatus('connected')
62
  }
63
 
64
  ws.onclose = () => {
65
- if (!manualDisconnectRef.current) {
66
- // Auto reconnect with exponential backoff
67
- const attempt = reconnectAttemptRef.current
68
- const delay = Math.min(1000 * Math.pow(2, attempt), 30000)
69
- reconnectAttemptRef.current = attempt + 1
70
- reconnectTimerRef.current = setTimeout(() => {
71
- connect()
72
- }, delay)
73
- }
74
  setStatus('disconnected')
75
  }
76
 
 
57
  wsRef.current = ws
58
 
59
  ws.onopen = () => {
 
60
  setStatus('connected')
61
  }
62
 
63
  ws.onclose = () => {
64
+ // No auto-reconnect — user clicks "call" to retry
 
 
 
 
 
 
 
 
65
  setStatus('disconnected')
66
  }
67
 
desktop/src/hooks/useScreenShare.ts ADDED
@@ -0,0 +1,115 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { useState, useRef, useCallback } from 'react'
2
+
3
+ export function useScreenShare() {
4
+ const [stream, setStream] = useState<MediaStream | null>(null)
5
+ const [isSharing, setIsSharing] = useState(false)
6
+ const [error, setError] = useState<string | null>(null)
7
+ const streamRef = useRef<MediaStream | null>(null)
8
+ const videoRef = useRef<HTMLVideoElement | null>(null)
9
+ const canvasRef = useRef<HTMLCanvasElement | null>(null)
10
+
11
+ const startScreenShare = useCallback(async () => {
12
+ try {
13
+ const mediaStream = await navigator.mediaDevices.getDisplayMedia({
14
+ video: true,
15
+ audio: false,
16
+ })
17
+
18
+ streamRef.current = mediaStream
19
+ setStream(mediaStream)
20
+ setIsSharing(true)
21
+ setError(null)
22
+
23
+ // Create hidden video + canvas for frame capture
24
+ const video = document.createElement('video')
25
+ video.srcObject = mediaStream
26
+ video.playsInline = true
27
+ video.muted = true
28
+ video.autoplay = true
29
+ videoRef.current = video
30
+
31
+ const canvas = document.createElement('canvas')
32
+ canvas.width = 640
33
+ canvas.height = 480
34
+ canvasRef.current = canvas
35
+
36
+ await video.play()
37
+
38
+ // When user stops sharing via the browser's built-in "Stop sharing" button
39
+ mediaStream.getVideoTracks()[0]?.addEventListener('ended', () => {
40
+ stopScreenShare()
41
+ })
42
+ } catch (err) {
43
+ const message = err instanceof Error ? err.message : String(err)
44
+ // User cancelled the permission dialog
45
+ if (message.includes('cancelled') || message.includes('Permission denied')) {
46
+ setError(null)
47
+ return
48
+ }
49
+ setError(`Screen share failed: ${message}`)
50
+ }
51
+ }, [])
52
+
53
+ const stopScreenShare = useCallback(() => {
54
+ if (streamRef.current) {
55
+ streamRef.current.getTracks().forEach((track) => track.stop())
56
+ streamRef.current = null
57
+ }
58
+ videoRef.current = null
59
+ canvasRef.current = null
60
+ setStream(null)
61
+ setIsSharing(false)
62
+ }, [])
63
+
64
+ const captureFrame = useCallback(async (): Promise<string | null> => {
65
+ const video = videoRef.current
66
+ const canvas = canvasRef.current
67
+ if (!video || !canvas || video.readyState < 2) return null
68
+
69
+ const ctx = canvas.getContext('2d')
70
+ if (!ctx) return null
71
+
72
+ // Maintain aspect ratio while fitting to canvas
73
+ const vw = video.videoWidth
74
+ const vh = video.videoHeight
75
+ if (vw === 0 || vh === 0) return null
76
+
77
+ const scale = Math.min(canvas.width / vw, canvas.height / vh)
78
+ const dw = vw * scale
79
+ const dh = vh * scale
80
+ const dx = (canvas.width - dw) / 2
81
+ const dy = (canvas.height - dh) / 2
82
+
83
+ ctx.clearRect(0, 0, canvas.width, canvas.height)
84
+ ctx.drawImage(video, dx, dy, dw, dh)
85
+
86
+ return new Promise((resolve) => {
87
+ canvas.toBlob(
88
+ (blob) => {
89
+ if (!blob) {
90
+ resolve(null)
91
+ return
92
+ }
93
+ const reader = new FileReader()
94
+ reader.onloadend = () => {
95
+ const result = reader.result as string
96
+ resolve(result.split(',')[1] ?? null)
97
+ }
98
+ reader.onerror = () => resolve(null)
99
+ reader.readAsDataURL(blob)
100
+ },
101
+ 'image/jpeg',
102
+ 0.7,
103
+ )
104
+ })
105
+ }, [])
106
+
107
+ return {
108
+ stream,
109
+ isSharing,
110
+ error,
111
+ captureFrame,
112
+ startScreenShare,
113
+ stopScreenShare,
114
+ }
115
+ }
desktop/src/i18n/locales/en.ts CHANGED
@@ -79,6 +79,27 @@ export const en = {
79
  'companion.speaking': 'Speaking...',
80
  'companion.inputPlaceholder': 'Type a message...',
81
  'companion.send': 'Send',
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
82
  'sidebar.confirmDelete': 'Delete this session? This cannot be undone.',
83
  'sidebar.batchManage': 'Batch manage',
84
  'sidebar.batchSelectedCount': '{count} selected',
 
79
  'companion.speaking': 'Speaking...',
80
  'companion.inputPlaceholder': 'Type a message...',
81
  'companion.send': 'Send',
82
+ 'companion.scenario.select': 'Select Scenario',
83
+ 'companion.scenario.simultaneous': 'Simultaneous Interpretation',
84
+ 'companion.scenario.interview': 'Mock Interview',
85
+ 'companion.scenario.english': 'English Practice',
86
+ 'companion.scenario.singing': 'Singing',
87
+ 'companion.scenario.idiom': 'Idiom Chain',
88
+ 'companion.scenario.mood': 'Mood Treehole',
89
+ 'companion.subtitle': 'Subtitles',
90
+ 'companion.screenShare.start': 'Share Screen',
91
+ 'companion.screenShare.stop': 'Stop Sharing',
92
+ 'companion.screenShare.dialog.title': 'Start screen recording or sharing?',
93
+ 'companion.screenShare.dialog.entireScreen': 'Entire Screen',
94
+ 'companion.screenShare.dialog.window': 'Application Window',
95
+ 'companion.screenShare.dialog.cancel': 'Cancel',
96
+ 'companion.screenShare.dialog.start': 'Start',
97
+ 'companion.screenShare.dialog.privacy': 'When sharing, recording, or casting content, the app can access everything displayed on your screen or played on your device. Please be careful not to leak passwords, payment info, messages, or other sensitive content.',
98
+ 'companion.flipCamera': 'Flip Camera',
99
+ 'companion.torch': 'Flashlight',
100
+ 'companion.hangUp': 'Hang Up',
101
+ 'companion.call': 'Call',
102
+ 'companion.speaker': 'Speaker',
103
  'sidebar.confirmDelete': 'Delete this session? This cannot be undone.',
104
  'sidebar.batchManage': 'Batch manage',
105
  'sidebar.batchSelectedCount': '{count} selected',
desktop/src/i18n/locales/zh.ts CHANGED
@@ -60,7 +60,7 @@ export const zh: Record<TranslationKey, string> = {
60
  'sidebar.worktree': 'worktree',
61
  'sidebar.sessionRunning': '会话运行中',
62
  'sidebar.missingDir': '目录缺失',
63
- 'sidebar.companion': 'AI 伴侣',
64
 
65
  // ─── Companion ──────────────────────────────────────
66
  'companion.connect': '连接',
@@ -81,6 +81,27 @@ export const zh: Record<TranslationKey, string> = {
81
  'companion.speaking': '说话中...',
82
  'companion.inputPlaceholder': '输入消息...',
83
  'companion.send': '发送',
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
84
  'sidebar.confirmDelete': '确定要删除这个会话吗?此操作不可撤销。',
85
  'sidebar.batchManage': '批量管理',
86
  'sidebar.batchSelectedCount': '已选 {count} 个',
 
60
  'sidebar.worktree': 'worktree',
61
  'sidebar.sessionRunning': '会话运行中',
62
  'sidebar.missingDir': '目录缺失',
63
+ 'sidebar.companion': '视频通话',
64
 
65
  // ─── Companion ──────────────────────────────────────
66
  'companion.connect': '连接',
 
81
  'companion.speaking': '说话中...',
82
  'companion.inputPlaceholder': '输入消息...',
83
  'companion.send': '发送',
84
+ 'companion.scenario.select': '选择场景',
85
+ 'companion.scenario.simultaneous': '同声传译',
86
+ 'companion.scenario.interview': '模拟面试',
87
+ 'companion.scenario.english': '英语陪练',
88
+ 'companion.scenario.singing': '唱歌',
89
+ 'companion.scenario.idiom': '成语接龙',
90
+ 'companion.scenario.mood': '心情树洞',
91
+ 'companion.subtitle': '字幕',
92
+ 'companion.screenShare.start': '屏幕共享',
93
+ 'companion.screenShare.stop': '停止共享',
94
+ 'companion.screenShare.dialog.title': '要开始录制或投放吗?',
95
+ 'companion.screenShare.dialog.entireScreen': '整个屏幕',
96
+ 'companion.screenShare.dialog.window': '特定应用窗口',
97
+ 'companion.screenShare.dialog.cancel': '取消',
98
+ 'companion.screenShare.dialog.start': '开始',
99
+ 'companion.screenShare.dialog.privacy': '在分享、录制或投放内容时,应用可以访问屏幕上显示或设备中播放的所有内容。请务必小心操作,谨防密码、付款信息、消息、照片等内容泄露。',
100
+ 'companion.flipCamera': '翻转摄像头',
101
+ 'companion.torch': '手电筒',
102
+ 'companion.hangUp': '挂断',
103
+ 'companion.call': '呼叫',
104
+ 'companion.speaker': '扬声器',
105
  'sidebar.confirmDelete': '确定要删除这个会话吗?此操作不可撤销。',
106
  'sidebar.batchManage': '批量管理',
107
  'sidebar.batchSelectedCount': '已选 {count} 个',
desktop/src/pages/Companion.tsx CHANGED
@@ -4,9 +4,14 @@ import { useCompanionWebSocket } from '../hooks/useCompanionWebSocket'
4
  import { useWebcam } from '../hooks/useWebcam'
5
  import { useMicrophone } from '../hooks/useMicrophone'
6
  import { useCompanionAudio } from '../hooks/useCompanionAudio'
 
 
7
  import { CompanionVideoPanel } from '../components/companion/CompanionVideoPanel'
8
  import { CompanionTranscript } from '../components/companion/CompanionTranscript'
9
  import { CompanionControls } from '../components/companion/CompanionControls'
 
 
 
10
 
11
  export function Companion() {
12
  const status = useCompanionStore((s) => s.status)
@@ -16,12 +21,24 @@ export function Companion() {
16
  const fullTranscript = useCompanionStore((s) => s.fullTranscript)
17
  const micEnabled = useCompanionStore((s) => s.micEnabled)
18
  const cameraEnabled = useCompanionStore((s) => s.cameraEnabled)
 
19
  const error = useCompanionStore((s) => s.error)
 
 
20
  const setMicEnabled = useCompanionStore((s) => s.setMicEnabled)
21
  const setCameraEnabled = useCompanionStore((s) => s.setCameraEnabled)
 
 
 
 
 
 
 
22
  const reset = useCompanionStore((s) => s.reset)
23
  const frameIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null)
 
24
 
 
25
  const companionAudio = useCompanionAudio()
26
  const onAudioReceived = useCallback(
27
  (data: ArrayBuffer) => {
@@ -33,11 +50,50 @@ export function Companion() {
33
  companionAudio.flush()
34
  }, [companionAudio])
35
 
 
36
  const ws = useCompanionWebSocket({ onAudioReceived, onDone })
37
- const webcam = useWebcam(ws.connected && cameraEnabled)
 
 
 
 
 
 
 
 
 
38
  useMicrophone(ws.connected && micEnabled, ws.sendAudio)
39
 
40
- // Periodic frame capture when connected and camera enabled
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
41
  useEffect(() => {
42
  if (ws.connected && cameraEnabled) {
43
  frameIntervalRef.current = setInterval(async () => {
@@ -60,6 +116,41 @@ export function Companion() {
60
  }
61
  }, [ws.connected, cameraEnabled, webcam, ws])
62
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
63
  const handleSendText = useCallback(
64
  (text: string) => {
65
  ws.sendText(text)
@@ -67,7 +158,6 @@ export function Companion() {
67
  [ws],
68
  )
69
 
70
- // Resume audio context on user interaction
71
  const handleResumeAudio = useCallback(() => {
72
  companionAudio.resume()
73
  }, [companionAudio])
@@ -77,26 +167,68 @@ export function Companion() {
77
  }, [micEnabled, setMicEnabled])
78
 
79
  const handleToggleCamera = useCallback(() => {
80
- setCameraEnabled(!cameraEnabled)
81
- }, [cameraEnabled, setCameraEnabled])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
82
 
83
  return (
84
  <div className="relative h-full w-full overflow-hidden bg-black">
85
- {/* Video background + PiP */}
86
  <CompanionVideoPanel
87
  webcamStream={webcam.stream}
88
  speaking={speaking}
89
  generating={generating}
90
  status={status}
 
91
  />
92
 
 
 
 
 
 
 
 
 
 
93
  {/* Error overlay */}
94
  {error && (
95
- <div className="absolute top-4 left-1/2 -translate-x-1/2 px-4 py-2 rounded-xl bg-red-500/20 border border-red-500/30 text-red-300 text-sm">
96
  {error}
97
  </div>
98
  )}
99
 
 
 
 
 
 
 
 
100
  {/* Transcript + text input */}
101
  <CompanionTranscript
102
  transcript={transcript}
@@ -106,17 +238,22 @@ export function Companion() {
106
  disabled={status !== 'connected'}
107
  />
108
 
109
- {/* Controls */}
110
  <CompanionControls
111
  status={status}
112
  micEnabled={micEnabled}
113
  cameraEnabled={cameraEnabled}
114
  generating={generating}
 
115
  onToggleMic={handleToggleMic}
116
  onToggleCamera={handleToggleCamera}
 
117
  onConnect={ws.connect}
118
  onDisconnect={() => {
119
  ws.disconnect()
 
 
 
120
  reset()
121
  }}
122
  onStop={ws.sendStop}
 
4
  import { useWebcam } from '../hooks/useWebcam'
5
  import { useMicrophone } from '../hooks/useMicrophone'
6
  import { useCompanionAudio } from '../hooks/useCompanionAudio'
7
+ import { useScreenShare } from '../hooks/useScreenShare'
8
+ import { useCameraDevices } from '../hooks/useCameraDevices'
9
  import { CompanionVideoPanel } from '../components/companion/CompanionVideoPanel'
10
  import { CompanionTranscript } from '../components/companion/CompanionTranscript'
11
  import { CompanionControls } from '../components/companion/CompanionControls'
12
+ import { CompanionTopBar } from '../components/companion/CompanionTopBar'
13
+ import { ScenarioSelector } from '../components/companion/ScenarioSelector'
14
+ import { ScreenShareDialog } from '../components/companion/ScreenShareDialog'
15
 
16
  export function Companion() {
17
  const status = useCompanionStore((s) => s.status)
 
21
  const fullTranscript = useCompanionStore((s) => s.fullTranscript)
22
  const micEnabled = useCompanionStore((s) => s.micEnabled)
23
  const cameraEnabled = useCompanionStore((s) => s.cameraEnabled)
24
+ const cameraFacingMode = useCompanionStore((s) => s.cameraFacingMode)
25
  const error = useCompanionStore((s) => s.error)
26
+ const scenario = useCompanionStore((s) => s.scenario)
27
+ const statusText = useCompanionStore((s) => s.statusText)
28
  const setMicEnabled = useCompanionStore((s) => s.setMicEnabled)
29
  const setCameraEnabled = useCompanionStore((s) => s.setCameraEnabled)
30
+ const setCameraFacingMode = useCompanionStore((s) => s.setCameraFacingMode)
31
+ const setCameraDevices = useCompanionStore((s) => s.setCameraDevices)
32
+ const setActiveCameraLabel = useCompanionStore((s) => s.setActiveCameraLabel)
33
+ const setCameraFullscreen = useCompanionStore((s) => s.setCameraFullscreen)
34
+ const setScreenShareStream = useCompanionStore((s) => s.setScreenShareStream)
35
+ const setScreenShareDialogOpen = useCompanionStore((s) => s.setScreenShareDialogOpen)
36
+ const setStatusText = useCompanionStore((s) => s.setStatusText)
37
  const reset = useCompanionStore((s) => s.reset)
38
  const frameIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null)
39
+ const screenShareIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null)
40
 
41
+ // Audio playback
42
  const companionAudio = useCompanionAudio()
43
  const onAudioReceived = useCallback(
44
  (data: ArrayBuffer) => {
 
50
  companionAudio.flush()
51
  }, [companionAudio])
52
 
53
+ // WebSocket
54
  const ws = useCompanionWebSocket({ onAudioReceived, onDone })
55
+
56
+ // Camera devices
57
+ const cameraDevices = useCameraDevices()
58
+
59
+ // Screen share
60
+ const screenShare = useScreenShare()
61
+
62
+ // Webcam runs independently (toggle on/off without needing connection)
63
+ const webcam = useWebcam(cameraEnabled)
64
+ // Mic only runs when connected (sends audio over WS)
65
  useMicrophone(ws.connected && micEnabled, ws.sendAudio)
66
 
67
+ // Enumerate cameras when camera is enabled
68
+ useEffect(() => {
69
+ if (cameraEnabled) {
70
+ cameraDevices.enumerateCameras().then((devices) => {
71
+ setCameraDevices(devices)
72
+ if (devices.length > 0) {
73
+ setActiveCameraLabel(devices[0]?.label ?? null)
74
+ }
75
+ })
76
+ }
77
+ }, [cameraEnabled, cameraDevices, setCameraDevices, setActiveCameraLabel])
78
+
79
+ // Update status text based on state
80
+ useEffect(() => {
81
+ if (status === 'connected') {
82
+ if (generating) {
83
+ setStatusText('AI 说话中...')
84
+ } else if (speaking) {
85
+ setStatusText('请说话')
86
+ } else {
87
+ setStatusText('你可以开始说话')
88
+ }
89
+ } else if (status === 'connecting') {
90
+ setStatusText('连接中...')
91
+ } else {
92
+ setStatusText('')
93
+ }
94
+ }, [status, generating, speaking, setStatusText])
95
+
96
+ // Periodic webcam frame capture (only send when connected)
97
  useEffect(() => {
98
  if (ws.connected && cameraEnabled) {
99
  frameIntervalRef.current = setInterval(async () => {
 
116
  }
117
  }, [ws.connected, cameraEnabled, webcam, ws])
118
 
119
+ // Periodic screen share frame capture
120
+ useEffect(() => {
121
+ if (ws.connected && screenShare.isSharing) {
122
+ screenShareIntervalRef.current = setInterval(async () => {
123
+ const frame = await screenShare.captureFrame()
124
+ if (frame) {
125
+ ws.sendFrame(frame)
126
+ }
127
+ }, 1000)
128
+ } else {
129
+ if (screenShareIntervalRef.current) {
130
+ clearInterval(screenShareIntervalRef.current)
131
+ screenShareIntervalRef.current = null
132
+ }
133
+ }
134
+ return () => {
135
+ if (screenShareIntervalRef.current) {
136
+ clearInterval(screenShareIntervalRef.current)
137
+ screenShareIntervalRef.current = null
138
+ }
139
+ }
140
+ }, [ws.connected, screenShare.isSharing, screenShare, ws])
141
+
142
+ // Send scenario context when connecting
143
+ useEffect(() => {
144
+ if (ws.connected && scenario) {
145
+ ws.sendText(`[Scenario: ${scenario}]`)
146
+ }
147
+ }, [ws.connected]) // only send on connect
148
+
149
+ // Sync screen share stream to store
150
+ useEffect(() => {
151
+ setScreenShareStream(screenShare.stream)
152
+ }, [screenShare.stream, setScreenShareStream])
153
+
154
  const handleSendText = useCallback(
155
  (text: string) => {
156
  ws.sendText(text)
 
158
  [ws],
159
  )
160
 
 
161
  const handleResumeAudio = useCallback(() => {
162
  companionAudio.resume()
163
  }, [companionAudio])
 
167
  }, [micEnabled, setMicEnabled])
168
 
169
  const handleToggleCamera = useCallback(() => {
170
+ const next = !cameraEnabled
171
+ setCameraEnabled(next)
172
+ // When camera turns on, show as fullscreen background
173
+ if (next) {
174
+ setCameraFullscreen(true)
175
+ } else {
176
+ setCameraFullscreen(false)
177
+ }
178
+ }, [cameraEnabled, setCameraEnabled, setCameraFullscreen])
179
+
180
+ const handleFlipCamera = useCallback(async () => {
181
+ const nextMode = cameraFacingMode === 'user' ? 'environment' : 'user'
182
+ setCameraFacingMode(nextMode)
183
+ await cameraDevices.getStreamForFacingMode(nextMode)
184
+ }, [cameraFacingMode, setCameraFacingMode, cameraDevices])
185
+
186
+ const handleScreenShareClick = useCallback(() => {
187
+ if (screenShare.isSharing) {
188
+ screenShare.stopScreenShare()
189
+ } else {
190
+ setScreenShareDialogOpen(true)
191
+ }
192
+ }, [screenShare, setScreenShareDialogOpen])
193
+
194
+ const handleStartScreenShare = useCallback(async () => {
195
+ await screenShare.startScreenShare()
196
+ }, [screenShare])
197
 
198
  return (
199
  <div className="relative h-full w-full overflow-hidden bg-black">
200
+ {/* Video background + content */}
201
  <CompanionVideoPanel
202
  webcamStream={webcam.stream}
203
  speaking={speaking}
204
  generating={generating}
205
  status={status}
206
+ onFlipCamera={handleFlipCamera}
207
  />
208
 
209
+ {/* Top bar with scenario selector + subtitle toggle */}
210
+ <CompanionTopBar />
211
+
212
+ {/* Scenario selection panel overlay */}
213
+ <ScenarioSelector />
214
+
215
+ {/* Screen share dialog overlay */}
216
+ <ScreenShareDialog onStartScreenShare={handleStartScreenShare} />
217
+
218
  {/* Error overlay */}
219
  {error && (
220
+ <div className="absolute top-14 left-1/2 -translate-x-1/2 px-4 py-2 rounded-xl bg-red-500/20 border border-red-500/30 text-red-300 text-sm z-20">
221
  {error}
222
  </div>
223
  )}
224
 
225
+ {/* Status text bar */}
226
+ {statusText && (
227
+ <div className="absolute bottom-[88px] left-1/2 -translate-x-1/2 z-10">
228
+ <span className="text-white/30 text-xs tracking-widest">{statusText}</span>
229
+ </div>
230
+ )}
231
+
232
  {/* Transcript + text input */}
233
  <CompanionTranscript
234
  transcript={transcript}
 
238
  disabled={status !== 'connected'}
239
  />
240
 
241
+ {/* Bottom controls */}
242
  <CompanionControls
243
  status={status}
244
  micEnabled={micEnabled}
245
  cameraEnabled={cameraEnabled}
246
  generating={generating}
247
+ isSharing={screenShare.isSharing}
248
  onToggleMic={handleToggleMic}
249
  onToggleCamera={handleToggleCamera}
250
+ onScreenShareClick={handleScreenShareClick}
251
  onConnect={ws.connect}
252
  onDisconnect={() => {
253
  ws.disconnect()
254
+ if (screenShare.isSharing) {
255
+ screenShare.stopScreenShare()
256
+ }
257
  reset()
258
  }}
259
  onStop={ws.sendStop}
desktop/src/stores/companionStore.ts CHANGED
@@ -1,5 +1,5 @@
1
  import { create } from 'zustand'
2
- import type { CompanionStatus } from '../types/companion'
3
 
4
  const AVATAR_KEY = 'companion-avatar-url'
5
  const BACKGROUND_KEY = 'companion-background-url'
@@ -24,6 +24,57 @@ function persist(key: string, value: string | null) {
24
  }
25
  }
26
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
  interface CompanionStore {
28
  status: CompanionStatus
29
  serverUrl: string
@@ -38,6 +89,30 @@ interface CompanionStore {
38
  avatarUrl: string | null
39
  backgroundUrl: string | null
40
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
41
  setServerUrl: (url: string) => void
42
  setVoiceName: (name: string) => void
43
  setStatus: (status: CompanionStatus) => void
@@ -52,6 +127,18 @@ interface CompanionStore {
52
  setAvatarUrl: (url: string | null) => void
53
  setBackgroundUrl: (url: string | null) => void
54
  reset: () => void
 
 
 
 
 
 
 
 
 
 
 
 
55
  }
56
 
57
  const DEFAULT_SERVER_URL = 'ws://127.0.0.1:8889/ws/companion'
@@ -70,6 +157,18 @@ export const useCompanionStore = create<CompanionStore>((set) => ({
70
  avatarUrl: loadPersisted(AVATAR_KEY),
71
  backgroundUrl: loadPersisted(BACKGROUND_KEY),
72
 
 
 
 
 
 
 
 
 
 
 
 
 
73
  setServerUrl: (url) => set({ serverUrl: url }),
74
  setVoiceName: (name) => set({ voiceName: name }),
75
  setStatus: (status) => set({ status, error: status === 'error' ? undefined : null }),
@@ -98,5 +197,23 @@ export const useCompanionStore = create<CompanionStore>((set) => ({
98
  generating: false,
99
  transcript: '',
100
  error: null,
 
 
 
 
 
 
101
  }),
 
 
 
 
 
 
 
 
 
 
 
 
102
  }))
 
1
  import { create } from 'zustand'
2
+ import type { CompanionStatus, ScenarioOption } from '../types/companion'
3
 
4
  const AVATAR_KEY = 'companion-avatar-url'
5
  const BACKGROUND_KEY = 'companion-background-url'
 
24
  }
25
  }
26
 
27
+ export const SCENARIOS: ScenarioOption[] = [
28
+ {
29
+ id: 'interview',
30
+ name: '模拟面试',
31
+ nameEn: 'Mock Interview',
32
+ icon: 'work_history',
33
+ description: 'AI 模拟考官提问,支持上传简历',
34
+ descriptionEn: 'AI mock interviewer with resume support',
35
+ },
36
+ {
37
+ id: 'english',
38
+ name: '英语陪练',
39
+ nameEn: 'English Practice',
40
+ icon: 'translate',
41
+ description: '全英文对话,提升口语能力',
42
+ descriptionEn: 'Full English conversation practice',
43
+ },
44
+ {
45
+ id: 'singing',
46
+ name: '唱歌',
47
+ nameEn: 'Singing',
48
+ icon: 'music_note',
49
+ description: 'AI 陪你唱歌互动',
50
+ descriptionEn: 'Sing with AI',
51
+ },
52
+ {
53
+ id: 'translate',
54
+ name: '同声传译',
55
+ nameEn: 'Simultaneous Interpretation',
56
+ icon: 'language',
57
+ description: '实时翻译多国语言',
58
+ descriptionEn: 'Real-time multi-language translation',
59
+ },
60
+ {
61
+ id: 'idiom',
62
+ name: '成语接龙',
63
+ nameEn: 'Idiom Chain',
64
+ icon: 'abc',
65
+ description: '文字游戏互动',
66
+ descriptionEn: 'Chinese idiom word game',
67
+ },
68
+ {
69
+ id: 'mood',
70
+ name: '心情树洞',
71
+ nameEn: 'Mood Treehole',
72
+ icon: 'forest',
73
+ description: '情感陪伴与倾诉',
74
+ descriptionEn: 'Emotional companion',
75
+ },
76
+ ]
77
+
78
  interface CompanionStore {
79
  status: CompanionStatus
80
  serverUrl: string
 
89
  avatarUrl: string | null
90
  backgroundUrl: string | null
91
 
92
+ // New: scenario
93
+ scenario: string | null
94
+ scenarioPanelOpen: boolean
95
+
96
+ // New: subtitle
97
+ subtitleEnabled: boolean
98
+
99
+ // New: camera devices
100
+ cameraFacingMode: 'user' | 'environment'
101
+ cameraDevices: MediaDeviceInfo[]
102
+ activeCameraLabel: string | null
103
+
104
+ // New: screen share
105
+ screenShareStream: MediaStream | null
106
+
107
+ // New: screen share dialog
108
+ screenShareDialogOpen: boolean
109
+
110
+ // New: status text
111
+ statusText: string
112
+
113
+ // New: fullscreen camera mode
114
+ cameraFullscreen: boolean
115
+
116
  setServerUrl: (url: string) => void
117
  setVoiceName: (name: string) => void
118
  setStatus: (status: CompanionStatus) => void
 
127
  setAvatarUrl: (url: string | null) => void
128
  setBackgroundUrl: (url: string | null) => void
129
  reset: () => void
130
+
131
+ // New actions
132
+ setScenario: (id: string | null) => void
133
+ setScenarioPanelOpen: (open: boolean) => void
134
+ setSubtitleEnabled: (enabled: boolean) => void
135
+ setCameraFacingMode: (mode: 'user' | 'environment') => void
136
+ setCameraDevices: (devices: MediaDeviceInfo[]) => void
137
+ setActiveCameraLabel: (label: string | null) => void
138
+ setScreenShareStream: (stream: MediaStream | null) => void
139
+ setScreenShareDialogOpen: (open: boolean) => void
140
+ setStatusText: (text: string) => void
141
+ setCameraFullscreen: (fullscreen: boolean) => void
142
  }
143
 
144
  const DEFAULT_SERVER_URL = 'ws://127.0.0.1:8889/ws/companion'
 
157
  avatarUrl: loadPersisted(AVATAR_KEY),
158
  backgroundUrl: loadPersisted(BACKGROUND_KEY),
159
 
160
+ // New defaults
161
+ scenario: null,
162
+ scenarioPanelOpen: false,
163
+ subtitleEnabled: true,
164
+ cameraFacingMode: 'user',
165
+ cameraDevices: [],
166
+ activeCameraLabel: null,
167
+ screenShareStream: null,
168
+ screenShareDialogOpen: false,
169
+ statusText: '你可以开始说话',
170
+ cameraFullscreen: false,
171
+
172
  setServerUrl: (url) => set({ serverUrl: url }),
173
  setVoiceName: (name) => set({ voiceName: name }),
174
  setStatus: (status) => set({ status, error: status === 'error' ? undefined : null }),
 
197
  generating: false,
198
  transcript: '',
199
  error: null,
200
+ scenario: null,
201
+ subtitleEnabled: true,
202
+ cameraFacingMode: 'user',
203
+ screenShareStream: null,
204
+ statusText: '你可以开始说话',
205
+ cameraFullscreen: false,
206
  }),
207
+
208
+ // New action implementations
209
+ setScenario: (id) => set({ scenario: id, scenarioPanelOpen: false }),
210
+ setScenarioPanelOpen: (open) => set({ scenarioPanelOpen: open }),
211
+ setSubtitleEnabled: (enabled) => set({ subtitleEnabled: enabled }),
212
+ setCameraFacingMode: (mode) => set({ cameraFacingMode: mode }),
213
+ setCameraDevices: (devices) => set({ cameraDevices: devices }),
214
+ setActiveCameraLabel: (label) => set({ activeCameraLabel: label }),
215
+ setScreenShareStream: (stream) => set({ screenShareStream: stream }),
216
+ setScreenShareDialogOpen: (open) => set({ screenShareDialogOpen: open }),
217
+ setStatusText: (text) => set({ statusText: text }),
218
+ setCameraFullscreen: (fullscreen) => set({ cameraFullscreen: fullscreen }),
219
  }))
desktop/src/types/companion.ts CHANGED
@@ -1,5 +1,14 @@
1
  export type CompanionStatus = 'disconnected' | 'connecting' | 'connected' | 'error'
2
 
 
 
 
 
 
 
 
 
 
3
  export type CompanionConfig = {
4
  serverUrl: string
5
  voiceName: string
 
1
  export type CompanionStatus = 'disconnected' | 'connecting' | 'connected' | 'error'
2
 
3
+ export type ScenarioOption = {
4
+ id: string
5
+ name: string
6
+ nameEn: string
7
+ icon: string
8
+ description: string
9
+ descriptionEn: string
10
+ }
11
+
12
  export type CompanionConfig = {
13
  serverUrl: string
14
  voiceName: string