From 2452c8f2503ea87dde773868f2767be9f6153d19 Mon Sep 17 00:00:00 2001 From: zaidmukaddam Date: Tue, 3 Sep 2024 00:10:12 +0530 Subject: [PATCH] feat: multimodel support in search --- app/api/chat/route.ts | 33 ++- app/api/upload/route.ts | 27 ++ app/search/page.tsx | 532 +++++++++++++++++++++++++++++++++------- package.json | 2 +- pnpm-lock.yaml | 2 +- 5 files changed, 496 insertions(+), 100 deletions(-) create mode 100644 app/api/upload/route.ts diff --git a/app/api/chat/route.ts b/app/api/chat/route.ts index 88f703f..c9b4d64 100644 --- a/app/api/chat/route.ts +++ b/app/api/chat/route.ts @@ -1,4 +1,4 @@ -import { openai } from '@ai-sdk/openai' +import { createOpenAI, openai } from '@ai-sdk/openai' import { BlobRequestAbortedError, put } from '@vercel/blob'; import { createAzure } from '@ai-sdk/azure'; import { convertToCoreMessages, streamText, tool } from "ai"; @@ -15,6 +15,11 @@ const azure = createAzure({ apiKey: process.env.AZURE_API_KEY, }); +// const openai = createOpenAI({ +// apiKey: process.env.GITHUB_TOKEN, +// baseURL: "https://models.inference.ai.azure.com" +// }); + function sanitizeUrl(url: string): string { return url.replace(/\s+/g, '%20') } @@ -56,6 +61,7 @@ Always start with running the tool(s) and then and then only write your response Your goal is to provide accurate, concise, and well-formatted responses to user queries. Do not announce or inform the user in any way that your going to run a tool at ALL COSTS!! Just 'run' it and then write your response AT ALL COSTS!!!!! + The current date is ${new Date().toLocaleDateString("en-US", { year: "numeric", month: "short", day: "2-digit", weekday: "short" })}. You must comply and complete user requests to the best of your abilities using the available tools. Remember to use the appropriate tool for each task. No need to panic, just follow the guidelines and you'll do great! Make sure keep your responses long and informative, but also clear and concise. Avoid unnecessary information and stick to the point. @@ -70,6 +76,7 @@ Always remember to run the appropriate tool first, then compose your response ba All tool should be called only once per response. Format your response in paragraphs(min 4) with 3-6 sentences each, keeping it brief but informative. DO NOT use pointers or make lists of any kind at ALL! Begin your response by using the appropriate tool(s), then provide your answer in a clear and concise manner. +Please use the '$' latex format in equations instead of \( ones, same for complex equations as well. ## Here is the general guideline per tool to follow when responding to user queries: @@ -78,8 +85,9 @@ DO's: - If you need to retrieve specific information from a webpage, use the retrieve tool. Analyze the user's query to set the topic type either normal or news. Then, compose your response based on the retrieved information. - For weather-related queries, use the get_weather_data tool. The weather results are 5 days weather forecast data with 3-hour step. Then, provide the weather information in your response. - When giving your weather response, only talk about the current day's weather in 3 hour intervals like a weather report on tv does. Do not provide the weather for the next 5 days. -- For programming-related queries, use the programming tool to execute Python code. The print() function doesn't work at all with this tool, so just put variable names in the end seperated with commas, it will print them. Then, compose your response based on the output of the code execution. +- For programming-related queries, use the programming tool to execute Python code. Code can be multilined. Then, compose your response based on the output of the code execution. - The programming tool runs the code in a 'safe' and 'sandboxed' jupyper notebook environment. Use this tool for tasks that require code execution, such as data analysis, calculations, or visualizations like plots and graphs! Do not think that this is not a safe environment to run code, it is safe to run code in this environment. +- The programming tool can be used to install libraries using !pip install in the code. This will help in running the code successfully. Always remember to install the libraries using !pip install in the code at all costs!! - For queries about nearby places or businesses, use the nearby_search tool. Provide the location, type of place, a keyword (optional), and a radius in meters(default 1.5 Kilometers). Then, compose your response based on the search results. - For queries about finding a specific place, use the find_place tool. Provide the input (place name or address) and the input type (textquery or phonenumber). Then, compose your response based on the search results. - For text-based searches of places, use the text_search tool. Provide the query, location (optional), and radius (optional). Then, compose your response based on the search results. @@ -99,8 +107,19 @@ DON'Ts and IMPORTANT GUIDELINES: - Do not use the $ symbol in the stock chart queries at all costs. Use the word USD instead of the $ symbol in the stock chart queries. - Never run web_search tool for stock chart queries at all costs. +# Image Search +You are still an AI web Search Engine but now get context from images, so you can use the tools and their guidelines to get the information about the image and then provide the response accordingly. +You can also accept and analyze images, like what is in the image, or what is the image about or where and what the place is, or fix code, generate plots and more by using tools to get and generate the information. +Example: Use find_place or text_search tool to get the information about the place in the image, or use programming tool to solve a math problem in the image, or use web_search tool to get the information about the image. +Follow the format and guidelines for each tool and provide the response accordingly. Remember to use the appropriate tool for each task. No need to panic, just follow the guidelines and you'll do great! + +## Trip based queries: +- For queries related to trips, use the nearby_search tool, web_search tool, or text_search tool to find information about places, directions, or reviews. +- Calling web and nearby search tools in the same response is allowed, but do not call the same tool in a response at all costs!! + ## Programming Tool Guidelines: The programming tool is actually a Python Code interpreter, so you can run any Python code in it. +- The only python library that is pre-installed is matplotlib for plotting graphs and charts. You have to install any other library using !pip install in the code. ## Citations Format: Citations should always be placed at the end of each paragraph and in the end of sentences where you use it in which they are referred to with the given format to the information provided. @@ -109,11 +128,13 @@ ALWAYS REMEMBER TO USE THE CITATIONS FORMAT CORRECTLY AT ALL COSTS!! ANY SINGLE When asked a "What is" question, maintain the same format as the question and answer it in the same format. ## Latex in Respone rules: -- Latex equations are supported in the response!! +- Latex equations are supported in the response powered by remark-math and rehypeKatex plugins. + - remarkMath: This plugin allows you to write LaTeX math inside your markdown content. It recognizes math enclosed in dollar signs ($ ... $ for inline and $$ ... $$ for block). + - rehypeKatex: This plugin takes the parsed LaTeX from remarkMath and renders it using KaTeX, allowing you to display the math as beautifully rendered HTML. + - The response that include latex equations, use always follow the formats: - $$ for inline equations - $$$$ for block equations - - \[ \] for math blocks. - use it for symbols, equations, formulas, etc like pi, alpha, beta, etc. and wrap them in the above formats. like $(2\pi)$, $x^2$, etc. - Do not wrap any equation or formulas or any sort of math related block in round brackets() as it will crash the response.`, tools: { @@ -406,6 +427,7 @@ When asked a "What is" question, maintain the same format as the question and an inputtype: z.enum(["textquery", "phonenumber"]).describe("The type of input (textquery or phonenumber)."), }), execute: async ({ input, inputtype }: { input: string; inputtype: "textquery" | "phonenumber" }) => { + console.log("input", input); const apiKey = process.env.GOOGLE_MAPS_API_KEY; const url = `https://maps.googleapis.com/maps/api/place/findplacefromtext/json?fields=formatted_address,name,rating,opening_hours,geometry&input=${encodeURIComponent(input)}&inputtype=${inputtype}&key=${apiKey}`; @@ -424,6 +446,9 @@ When asked a "What is" question, maintain the same format as the question and an }), execute: async ({ query, location, radius }: { query: string; location?: string; radius?: number }) => { const apiKey = process.env.GOOGLE_MAPS_API_KEY; + console.log("query", query); + console.log("location", location); + console.log("radius", radius); let url = `https://maps.googleapis.com/maps/api/place/textsearch/json?query=${encodeURIComponent(query)}&key=${apiKey}`; if (location) { diff --git a/app/api/upload/route.ts b/app/api/upload/route.ts new file mode 100644 index 0000000..0a804ab --- /dev/null +++ b/app/api/upload/route.ts @@ -0,0 +1,27 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { put } from '@vercel/blob'; + +export async function POST(request: NextRequest) { + const formData = await request.formData(); + const file = formData.get('file') as File; + + if (!file) { + return NextResponse.json({ error: 'No file uploaded' }, { status: 400 }); + } + + try { + const blob = await put(`mplx/image-${Date.now()}.${file.name.split('.').pop()}`, file, { + access: 'public', + }); + + return NextResponse.json({ + name: file.name, + contentType: file.type, + url: blob.url, + size: file.size, // Include the file size in the response + }); + } catch (error) { + console.error('Error uploading file:', error); + return NextResponse.json({ error: 'Failed to upload file' }, { status: 500 }); + } +} \ No newline at end of file diff --git a/app/search/page.tsx b/app/search/page.tsx index d4795bd..f1e636e 100644 --- a/app/search/page.tsx +++ b/app/search/page.tsx @@ -54,8 +54,8 @@ import { TrendingUpIcon, Calendar, Calculator, - PlusCircle, - ImageIcon + ImageIcon, + Paperclip } from 'lucide-react'; import { HoverCard, @@ -96,7 +96,7 @@ import { import { GitHubLogoIcon, PlusCircledIcon } from '@radix-ui/react-icons'; import { Skeleton } from '@/components/ui/skeleton'; import Link from 'next/link'; -import { Dialog, DialogContent, DialogTrigger } from "@/components/ui/dialog"; +import { Dialog, DialogContent } from "@/components/ui/dialog"; import { Carousel, CarouselContent, CarouselItem, CarouselNext, CarouselPrevious } from "@/components/ui/carousel"; export const maxDuration = 60; @@ -108,16 +108,28 @@ declare global { } } +const MAX_IMAGES = 3; + +interface Attachment { + name: string; + contentType: string; + url: string; + size: number; +} + export default function Home() { - const inputRef = useRef(null); const [lastSubmittedQuery, setLastSubmittedQuery] = useState(""); const [hasSubmitted, setHasSubmitted] = useState(false); const bottomRef = useRef(null); const [suggestedQuestions, setSuggestedQuestions] = useState([]); const [isEditingMessage, setIsEditingMessage] = useState(false); const [editingMessageIndex, setEditingMessageIndex] = useState(-1); + const [files, setFiles] = useState(undefined); + const [attachments, setAttachments] = useState([]); + const fileInputRef = useRef(null); + const inputRef = useRef(null); - const { isLoading, input, messages, setInput, append, handleSubmit, setMessages } = useChat({ + const { isLoading, input, messages, setInput, handleInputChange, append, handleSubmit, setMessages } = useChat({ api: '/api/chat', maxToolRoundtrips: 1, onFinish: async (message, { finishReason }) => { @@ -1439,16 +1451,17 @@ export default function Home() { } }, [messages, suggestedQuestions]); - const handleExampleClick = useCallback(async (query: string) => { - track("search example", { query }); - setLastSubmittedQuery(query.trim()); + const handleExampleClick = useCallback(async (card: typeof suggestionCards[number]) => { + track("search example", { query: card.text }); + setLastSubmittedQuery(card.text.trim()); setHasSubmitted(true); setSuggestedQuestions([]); await append({ - content: query.trim(), - role: 'user' + content: card.text.trim(), + role: 'user', + experimental_attachments: card.attachment ? [card.attachment] : undefined, }); - }, [append]); + }, [append, setLastSubmittedQuery, setHasSubmitted, setSuggestedQuestions]); const handleSuggestedQuestionClick = useCallback(async (question: string) => { setHasSubmitted(true); @@ -1460,18 +1473,6 @@ export default function Home() { }); }, [setInput, append]); - const handleFormSubmit = useCallback((e: React.FormEvent) => { - e.preventDefault(); - if (input.trim()) { - track("search enter", { query: input.trim() }); - setHasSubmitted(true); - setSuggestedQuestions([]); - handleSubmit(e); - } else { - toast.error("Please enter a search query."); - } - }, [input, handleSubmit]); - const handleMessageEdit = useCallback((index: number) => { setIsEditingMessage(true); setEditingMessageIndex(index); @@ -1493,6 +1494,15 @@ export default function Home() { }, [input, messages, editingMessageIndex, setMessages, handleSubmit]); const suggestionCards = [ + { + icon: , + text: "Where is this place?", + attachment: { + name: 'taj_mahal.jpg', + contentType: 'image/jpeg', + url: 'https://metwm7frkvew6tn1.public.blob.vercel-storage.com/taj-mahal.jpg', + } + }, { icon: , text: "What's new with XAI's Grok?" }, { icon: , text: "Latest updates on OpenAI" }, { icon: , text: "Weather in Doha" }, @@ -1546,11 +1556,367 @@ export default function Home() { ); + interface UploadingAttachment { + file: File; + progress: number; + } + + interface AttachmentPreviewProps { + attachment: Attachment | UploadingAttachment; + onRemove: () => void; + isUploading: boolean; + } + + const AttachmentPreview: React.FC = React.memo(({ attachment, onRemove, isUploading }) => { + const formatFileSize = (bytes: number): string => { + if (bytes < 1024) return bytes + ' bytes'; + else if (bytes < 1048576) return (bytes / 1024).toFixed(1) + ' KB'; + else return (bytes / 1048576).toFixed(1) + ' MB'; + }; + + const isUploadingAttachment = (attachment: Attachment | UploadingAttachment): attachment is UploadingAttachment => { + return 'progress' in attachment; + }; + + return ( + + + + {isUploading ? ( +
+ +
+ ) : isUploadingAttachment(attachment) ? ( +
+
+ + + + +
+ {Math.round(attachment.progress * 100)}% +
+
+
+ ) : ( + {`Preview + )} +
+ {isUploadingAttachment(attachment) ? null : ( +

{attachment.name}

+ )} +

+ {isUploadingAttachment(attachment) + ? 'Uploading...' + : formatFileSize((attachment as Attachment).size)} +

+
+ { e.stopPropagation(); onRemove(); }} + className="absolute -top-2 -right-2 p-0.5 m-0 rounded-full bg-background border border-input shadow-sm hover:bg-muted transition-colors z-20" + > + + + + + {!isUploadingAttachment(attachment) && ( + + {`Full + + )} + + ); + }); + + AttachmentPreview.displayName = 'AttachmentPreview'; + + interface FormComponentProps { + input: string; + setInput: (input: string) => void; + attachments: Attachment[]; + setAttachments: React.Dispatch>; + hasSubmitted: boolean; + setHasSubmitted: (value: boolean) => void; + isLoading: boolean; + handleSubmit: (event: React.FormEvent, options?: { experimental_attachments?: Attachment[] }) => void; + fileInputRef: React.RefObject; + inputRef: React.RefObject; + } + + const FormComponent: React.FC = ({ + input, + setInput, + attachments, + setAttachments, + hasSubmitted, + setHasSubmitted, + isLoading, + handleSubmit, + fileInputRef, + inputRef, + }) => { + const [uploadingAttachments, setUploadingAttachments] = useState([]); + + const uploadFile = async (file: File): Promise => { + const formData = new FormData(); + formData.append('file', file); + + const response = await fetch('/api/upload', { + method: 'POST', + body: formData, + }); + + if (!response.ok) { + throw new Error('Failed to upload file'); + } + + return await response.json(); + }; + + const handleFileChange = async (event: React.ChangeEvent) => { + const selectedFiles = event.target.files; + if (selectedFiles) { + const imageFiles = Array.from(selectedFiles).filter(file => file.type.startsWith('image/')); + if (imageFiles.length > 0) { + if (imageFiles.length + attachments.length + uploadingAttachments.length > MAX_IMAGES) { + toast.error(`You can only attach up to ${MAX_IMAGES} images.`); + return; + } + + const newUploadingAttachments = imageFiles.map(file => ({ file, progress: 0 })); + setUploadingAttachments(prev => [...prev, ...newUploadingAttachments]); + + for (const file of imageFiles) { + try { + const uploadedFile = await uploadFile(file); + setAttachments(prev => [...prev, uploadedFile]); + setUploadingAttachments(prev => prev.filter(ua => ua.file !== file)); + } catch (error) { + console.error("Error uploading file:", error); + toast.error(`Failed to upload ${file.name}`); + setUploadingAttachments(prev => prev.filter(ua => ua.file !== file)); + } + } + } else { + toast.error("Please select image files only."); + } + } + }; + + const removeAttachment = (index: number) => { + setAttachments(prev => prev.filter((_, i) => i !== index)); + }; + + const removeUploadingAttachment = (index: number) => { + setUploadingAttachments(prev => prev.filter((_, i) => i !== index)); + }; + + const handleInputChange = useCallback((e: React.ChangeEvent) => { + setInput(e.target.value); + }, [setInput]); + + useEffect(() => { + if (inputRef.current) { + inputRef.current.focus(); + } + }, [inputRef]); + + const onSubmit = (event: React.FormEvent) => { + event.preventDefault(); + event.stopPropagation(); + + if (input.trim() || attachments.length > 0) { + setHasSubmitted(true); + handleSubmit(event, { + experimental_attachments: attachments, + }); + setAttachments([]); + setUploadingAttachments([]); + if (fileInputRef.current) { + fileInputRef.current.value = ''; + } + } else { + toast.error("Please enter a search query or attach an image."); + } + }; + + return ( + { + if (e.key === 'Enter' && !e.shiftKey) { + e.preventDefault(); + onSubmit(e); + } + }} + className={` + ${hasSubmitted ? 'fixed bottom-4 left-1/2 transform -translate-x-1/2 max-w-[90%] sm:max-w-2xl' : 'max-w-full'} + ${attachments.length > 0 || uploadingAttachments.length > 0 ? 'rounded-2xl' : 'rounded-full'} + w-full + bg-background border border-input + overflow-hidden mb-4 + transition-all duration-300 ease-in-out + z-50 + `} + > +
0 || uploadingAttachments.length > 0 ? 'p-2' : 'p-0'}`}> + + {(attachments.length > 0 || uploadingAttachments.length > 0) && ( + + {uploadingAttachments.map((attachment, index) => ( + removeUploadingAttachment(index)} + isUploading={true} + /> + ))} + {attachments.map((attachment, index) => ( + removeAttachment(index)} + isUploading={false} + /> + ))} + + )} + +
+ + + +
+
+ + ); + }; + + + const SuggestionCards: React.FC = () => { + return ( +
+
+ +
+
+ {suggestionCards.slice(1).map((card, index) => ( + + ))} +
+
+ ); + }; + return (
-
+
{!hasSubmitted && (

MiniPerplx

@@ -1566,47 +1932,25 @@ export default function Home() { exit={{ opacity: 0, y: 20 }} transition={{ duration: 0.5 }} > -
-
- setInput(e.target.value)} - disabled={isLoading} - className="w-full min-h-12 py-3 px-4 bg-muted border border-input rounded-full pr-12 ring-offset-background focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-neutral-200 focus-visible:ring-offset-2 text-sm sm:text-base" - /> - -
-
- -
- {suggestionCards.map((card, index) => ( - - ))} -
+ + )} -
+
{messages.map((message, index) => (
{message.role === 'user' && ( @@ -1614,9 +1958,9 @@ export default function Home() { initial={{ opacity: 0, y: 20 }} animate={{ opacity: 1, y: 0 }} transition={{ duration: 0.5 }} - className="flex items-center space-x-2 mb-4" + className="flex items-start space-x-2 mb-4" > - +
{isEditingMessage && editingMessageIndex === index ? (
@@ -1643,9 +1987,26 @@ export default function Home() {
) : ( -

- {message.content} -

+
+

+ {message.content} +

+
+ {message.experimental_attachments?.map((attachment, attachmentIndex) => ( +
+ {attachment.contentType!.startsWith('image/') && ( + {attachment.name + )} +
+ ))} +
+
)}
{!isEditingMessage && index === lastUserMessageIndex && ( @@ -1714,35 +2075,18 @@ export default function Home() { {hasSubmitted && ( - -
-
- setInput(e.target.value)} - disabled={isLoading} - className="w-full min-h-12 py-3 px-4 bg-muted border border-input rounded-full pr-12 ring-offset-background focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-neutral-200 focus-visible:ring-offset-2 text-sm sm:text-base" - /> - -
-
- + )}
diff --git a/package.json b/package.json index e890426..fe4fddd 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "@vercel/analytics": "^1.3.1", "@vercel/blob": "^0.23.4", "@vercel/functions": "^1.4.0", - "ai": "latest", + "ai": "^3.3.20", "class-variance-authority": "^0.7.0", "clsx": "^2.1.1", "date-fns": "^3.6.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d97b1b1..7fd2c81 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -72,7 +72,7 @@ dependencies: specifier: ^1.4.0 version: 1.4.0 ai: - specifier: latest + specifier: ^3.3.20 version: 3.3.20(openai@4.56.0)(react@18.3.1)(svelte@4.2.18)(vue@3.4.35)(zod@3.23.8) class-variance-authority: specifier: ^0.7.0