Project: Building a Private Desktop AI Assistant

In this project, we will build a private desktop chat terminal. The frontend uses a clean React interface, communicating with a local backend that forwards requests to your offline Ollama service.

1. System Topology

graph LR
    A[React Desktop App] -->|POST /api/chat| B[Local Node Server Proxy]
    B -->|SDK port 11434| C[Ollama Llama/Qwen]

2. Implementing the Backend Gateway Route

To bypass CORS restrictions when making requests directly from frontend browsers, proxy queries through your local server API:

// app/api/local-chat/route.ts
import { NextResponse } from "next/server";
import ollama from "@ollama/ollama";

export async function POST(req: Request) {
  try {
    const { messages } = await req.json();

    const responseStream = await ollama.chat({
      model: "qwen2.5",
      messages,
      stream: true, // Enables token streaming
    });

    const stream = new ReadableStream({
      async start(controller) {
        const encoder = new TextEncoder();
        
        for await (const part of responseStream) {
          const text = part.message.content;
          if (text) {
            controller.enqueue(encoder.encode(text));
          }
        }
        controller.close();
      },
    });

    return new Response(stream, {
      headers: {
        "Content-Type": "text/event-stream",
        "Cache-Control": "no-cache",
      },
    });
  } catch (err: any) {
    return NextResponse.json({ error: err.message }, { status: 500 });
  }
}

3. Implementing the React Desktop Chat UI

Write the chat console component that consumes the local server stream:

// app/chat/DesktopConsole.tsx
"use client";

import React, { useState } from "react";

export default function DesktopConsole() {
  const [messages, setMessages] = useState<{ role: string; content: string }[]>([]);
  const [input, setInput] = useState("");
  const [loading, setLoading] = useState(false);

  async function handleSend() {
    if (!input.trim() || loading) return;

    const userMessage = { role: "user", content: input };
    const updatedHistory = [...messages, userMessage];
    
    setMessages(updatedHistory);
    setInput("");
    setLoading(true);

    try {
      const response = await fetch("/api/local-chat", {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({ messages: updatedHistory }),
      });

      const reader = response.body?.getReader();
      const decoder = new TextDecoder();
      
      if (!reader) return;

      const assistantMessagePlaceholder = { role: "assistant", content: "" };
      setMessages((prev) => [...prev, assistantMessagePlaceholder]);

      while (true) {
        const { done, value } = await reader.read();
        if (done) break;

        const token = decoder.decode(value);
        setMessages((prev) => {
          const next = [...prev];
          const lastIndex = next.length - 1;
          next[lastIndex].content += token; // Append stream character
          return next;
        });
      }
    } catch (err) {
      console.error("Local inference network error:", err);
    } finally {
      setLoading(false);
    }
  }

  return (
    <div className="max-w-2xl mx-auto border rounded-3xl bg-white shadow-lg flex flex-col h-[500px] mt-12 p-6 justify-between">
      <div className="flex-1 overflow-y-auto space-y-4 pr-2">
        {messages.map((msg, idx) => (
          <div key={idx} className={`p-3 rounded-2xl text-sm max-w-lg ${msg.role === "user" ? "bg-blue-600 text-white ml-auto" : "bg-gray-100 text-gray-800"}`}>
            <strong>{msg.role === "user" ? "You" : "Local AI"}:</strong>
            <p className="mt-1 whitespace-pre-wrap">{msg.content}</p>
          </div>
        ))}
      </div>

      <div className="mt-4 flex gap-2">
        <input
          value={input}
          onChange={(e) => setInput(e.target.value)}
          onKeyDown={(e) => e.key === "Enter" && handleSend()}
          className="flex-1 border px-4 py-2.5 rounded-xl text-sm focus:outline-none"
          placeholder="Type private prompt..."
        />
        <button onClick={handleSend} className="bg-blue-600 text-white font-semibold px-4 py-2 rounded-xl text-sm transition hover:bg-blue-700">
          Send
        </button>
      </div>
    </div>
  );
}