From 38bcc6e96bb1e612ce17a5ce426d2e97a334d80e Mon Sep 17 00:00:00 2001
From: Adam <x_1337@outlook.com>
Date: Fri, 4 Jul 2025 23:04:51 -0700
Subject: [PATCH] Enable voice (#1639)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

# READ CAREFULLY THEN REMOVE

Remove bullet points that are not relevant.

PLEASE REFRAIN FROM USING AI TO WRITE YOUR CODE AND PR DESCRIPTION. IF YOU DO USE AI TO WRITE YOUR CODE PLEASE PROVIDE A DESCRIPTION AND REVIEW IT CAREFULLY. MAKE SURE YOU UNDERSTAND THE CODE YOU ARE SUBMITTING USING AI.

- Pull requests that do not follow these guidelines will be closed without review or comment.
- If you use AI to write your PR description your pr will be close without review or comment.
- If you are unsure about anything, feel free to ask for clarification.

## Description

Please provide a clear description of your changes.

---

## Type of Change

Please delete options that are not relevant.

- [ ] 🐛 Bug fix (non-breaking change which fixes an issue)
- [ ] ✨ New feature (non-breaking change which adds functionality)
- [ ] 💥 Breaking change (fix or feature with breaking changes)
- [ ] 📝 Documentation update
- [ ] 🎨 UI/UX improvement
- [ ] 🔒 Security enhancement
- [ ] ⚡ Performance improvement

## Areas Affected

Please check all that apply:

- [ ] Email Integration (Gmail, IMAP, etc.)
- [ ] User Interface/Experience
- [ ] Authentication/Authorization
- [ ] Data Storage/Management
- [ ] API Endpoints
- [ ] Documentation
- [ ] Testing Infrastructure
- [ ] Development Workflow
- [ ] Deployment/Infrastructure

## Testing Done

Describe the tests you've done:

- [ ] Unit tests added/updated
- [ ] Integration tests added/updated
- [ ] Manual testing performed
- [ ] Cross-browser testing (if UI changes)
- [ ] Mobile responsiveness verified (if UI changes)

## Security Considerations

For changes involving data or authentication:

- [ ] No sensitive data is exposed
- [ ] Authentication checks are in place
- [ ] Input validation is implemented
- [ ] Rate limiting is considered (if applicable)

## Checklist

- [ ] I have read the [CONTRIBUTING](https://github.com/Mail-0/Zero/blob/staging/.github/CONTRIBUTING.md) document
- [ ] My code follows the project's style guidelines
- [ ] I have performed a self-review of my code
- [ ] I have commented my code, particularly in complex areas
- [ ] I have updated the documentation
- [ ] My changes generate no new warnings
- [ ] I have added tests that prove my fix/feature works
- [ ] All tests pass locally
- [ ] Any dependent changes are merged and published

## Additional Notes

Add any other context about the pull request here.

## Screenshots/Recordings

Add screenshots or recordings here if applicable.

---

_By submitting this pull request, I confirm that my contribution is made under the terms of the project's license._
---
 apps/mail/app/(routes)/layout.tsx        |  2 --
 apps/mail/app/(routes)/mail/layout.tsx   |  2 --
 apps/mail/components/create/ai-chat.tsx  |  6 +++++-
 apps/mail/lib/hotkeys/global-hotkeys.tsx |  5 ++---
 apps/mail/providers/voice-provider.tsx   | 24 +++++++++---------------
 5 files changed, 16 insertions(+), 23 deletions(-)

diff --git a/apps/mail/app/(routes)/layout.tsx b/apps/mail/app/(routes)/layout.tsx
index 8e3f11f89..1bc11bcf9 100644
--- a/apps/mail/app/(routes)/layout.tsx
+++ b/apps/mail/app/(routes)/layout.tsx
@@ -6,13 +6,11 @@ import { Outlet } from 'react-router';
 export default function Layout() {
   return (
     <CommandPaletteProvider>
-      {/* <VoiceProvider> */}
       <HotkeyProviderWrapper>
         <div className="relative flex max-h-screen w-full overflow-hidden">
           <Outlet />
         </div>
       </HotkeyProviderWrapper>
-      {/* </VoiceProvider> */}
     </CommandPaletteProvider>
   );
 }
diff --git a/apps/mail/app/(routes)/mail/layout.tsx b/apps/mail/app/(routes)/mail/layout.tsx
index 50c81c51e..d1ea00286 100644
--- a/apps/mail/app/(routes)/mail/layout.tsx
+++ b/apps/mail/app/(routes)/mail/layout.tsx
@@ -8,7 +8,6 @@ import type { Route } from './+types/layout';
 
 export default function MailLayout() {
   return (
-    // <VoiceProvider>
     <HotkeyProviderWrapper>
       <AppSidebar />
       <div className="bg-sidebar dark:bg-sidebar w-full">
@@ -17,6 +16,5 @@ export default function MailLayout() {
       <OnboardingWrapper />
       <NotificationProvider />
     </HotkeyProviderWrapper>
-    // </VoiceProvider>
   );
 }
diff --git a/apps/mail/components/create/ai-chat.tsx b/apps/mail/components/create/ai-chat.tsx
index ce0b00030..50faed6b7 100644
--- a/apps/mail/components/create/ai-chat.tsx
+++ b/apps/mail/components/create/ai-chat.tsx
@@ -2,6 +2,7 @@ import { Tooltip, TooltipContent, TooltipTrigger } from '../ui/tooltip';
 import { Popover, PopoverContent, PopoverTrigger } from '../ui/popover';
 import { Avatar, AvatarFallback, AvatarImage } from '../ui/avatar';
 import { useAIFullScreen, useAISidebar } from '../ui/ai-sidebar';
+import { VoiceProvider } from '@/providers/voice-provider';
 import useComposeEditor from '@/hooks/use-compose-editor';
 import { useRef, useCallback, useEffect } from 'react';
 import { Markdown } from '@react-email/components';
@@ -10,6 +11,7 @@ import { TextShimmer } from '../ui/text-shimmer';
 import { useThread } from '@/hooks/use-threads';
 import { MailLabels } from '../mail/mail-list';
 import { cn, getEmailLogo } from '@/lib/utils';
+import { VoiceButton } from '../voice-button';
 import { EditorContent } from '@tiptap/react';
 import { CurvedArrow } from '../icons/icons';
 import { Tools } from '../../types/tools';
@@ -426,7 +428,9 @@ export function AIChat({
             </div>
             <div className="grid">
               <div className="flex justify-end gap-1">
-                {/* <VoiceButton /> */}
+                <VoiceProvider>
+                  <VoiceButton />
+                </VoiceProvider>
                 <button
                   form="ai-chat-form"
                   type="submit"
diff --git a/apps/mail/lib/hotkeys/global-hotkeys.tsx b/apps/mail/lib/hotkeys/global-hotkeys.tsx
index 13e99392e..b3cb6b394 100644
--- a/apps/mail/lib/hotkeys/global-hotkeys.tsx
+++ b/apps/mail/lib/hotkeys/global-hotkeys.tsx
@@ -1,14 +1,13 @@
 import { useCommandPalette } from '@/components/context/command-palette-context';
 import { useOptimisticActions } from '@/hooks/use-optimistic-actions';
 import { keyboardShortcuts } from '@/config/shortcuts';
-import { useVoice } from '@/providers/voice-provider';
 import { useShortcuts } from './use-hotkey-utils';
 import { useQueryState } from 'nuqs';
 
 export function GlobalHotkeys() {
-  const [composeOpen, setComposeOpen] = useQueryState('isComposeOpen');
+  const [, setComposeOpen] = useQueryState('isComposeOpen');
   const { clearAllFilters } = useCommandPalette();
-  const [isCommandPaletteOpen, setIsCommandPaletteOpen] = useQueryState('isCommandPaletteOpen');
+  const [, setIsCommandPaletteOpen] = useQueryState('isCommandPaletteOpen');
   const { undoLastAction } = useOptimisticActions();
   const scope = 'global';
 
diff --git a/apps/mail/providers/voice-provider.tsx b/apps/mail/providers/voice-provider.tsx
index f11494fad..ca7cae822 100644
--- a/apps/mail/providers/voice-provider.tsx
+++ b/apps/mail/providers/voice-provider.tsx
@@ -15,7 +15,7 @@ interface VoiceContextType {
 
   startConversation: (context?: any) => Promise<void>;
   endConversation: () => Promise<void>;
-  requestPermission: () => Promise<void>;
+  requestPermission: () => Promise<boolean>;
   sendContext: (context: any) => void;
 }
 
@@ -30,18 +30,6 @@ export function VoiceProvider({ children }: { children: ReactNode }) {
   const [isOpen, setOpen] = useState(false);
   const [currentContext, setCurrentContext] = useState<any>(null);
 
-  useEffect(() => {
-    if (!session) return;
-
-    navigator.mediaDevices
-      .getUserMedia({ audio: true })
-      .then((stream) => {
-        setHasPermission(true);
-        stream.getTracks().forEach((track) => track.stop());
-      })
-      .catch(() => setHasPermission(false));
-  }, [session]);
-
   const conversation = useConversation({
     onConnect: () => {
       setIsInitializing(false);
@@ -87,15 +75,17 @@ export function VoiceProvider({ children }: { children: ReactNode }) {
       stream.getTracks().forEach((track) => track.stop());
       setHasPermission(true);
       setErrorMessage('');
+      return true;
     } catch {
       setErrorMessage('Microphone access denied. Please enable microphone permissions.');
+      return false;
     }
   };
 
   const startConversation = async (context?: any) => {
     if (!hasPermission) {
-      await requestPermission();
-      if (!hasPermission) return;
+      const result = await requestPermission();
+      if (!result) return;
     }
 
     try {
@@ -110,6 +100,10 @@ export function VoiceProvider({ children }: { children: ReactNode }) {
 
       await conversation.startSession({
         agentId: agentId,
+        onMessage: (message) => {
+          // TODO: Handle message, ideally send it to ai chat agent or show it somewhere on the screen?
+          console.log('message', message);
+        },
         dynamicVariables: {
           user_name: session?.user.name.split(' ')[0] || 'User',
           user_email: session?.user.email || '',