基于 Google Gemini 多模态视觉模型的图像智能识别与结构化提取实战

客户端请求
  │
  ├── 1. 图片 URL (JSON 格式) ──► 服务端 fetch 下载为 ArrayBuffer
  │                                    │
  └── 2. 本地文件 (FormData 格式) ────► 直接读取 File 二进制流
                                       │
                                       ▼ 转换为 Base64 InlineData
                              { inlineData: { data: '...', mimeType: 'image/png' } }
                                       │
                                       ▼ 注入 Prompt 与 JSON Schema
                              Gemini 视觉大模型 (gemini-1.5-flash)
                                       │
                                       ▼
                       结构化 JSON 或 SSE 打字机流式输出

// src/routes/vision.ts
import { Hono } from 'hono';
import { zValidator } from '@hono/zod-validator';
import { z } from 'zod';
import { GoogleGenerativeAI } from '@google/generative-ai';
 
const visionRoute = new Hono();
const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY!);
const model = genAI.getGenerativeModel({ model: 'gemini-1.5-flash' });
 
visionRoute.post(
  '/image-url',
  zValidator(
    'json',
    z.object({
      imageUrl: z.string().url('必须提供合法的图片 URL'),
      prompt: z.string().default('请详细描述这张图片的内容，并提取所有可见的文本。'),
      mimeType: z.enum(['image/jpeg', 'image/png', 'image/webp']).default('image/jpeg'),
    })
  ),
  async (c) => {
    const { imageUrl, prompt, mimeType } = c.req.valid('json');
 
    try {
      // 1. 服务端拉取远程图片并转为二进制
      const resp = await fetch(imageUrl);
      if (!resp.ok) throw new Error(`拉取图片失败: ${resp.statusText}`);
      
      const buffer = await resp.arrayBuffer();
      const base64Data = Buffer.from(buffer).toString('base64');
 
      // 2. 构造多模态 Payload 调用模型
      const result = await model.generateContent([
        {
          inlineData: {
            data: base64Data,
            mimeType,
          },
        },
        prompt,
      ]);
 
      return c.json({
        success: true,
        text: result.response.text(),
      });
    } catch (error: any) {
      return c.json({ success: false, error: error.message }, 500);
    }
  }
);

import { SchemaType } from '@google/generative-ai';
 
// 定义带 JSON Schema 的发票专用解析模型
const invoiceModel = genAI.getGenerativeModel({
  model: 'gemini-1.5-flash',
  generationConfig: {
    responseMimeType: 'application/json',
    responseSchema: {
      type: SchemaType.OBJECT,
      properties: {
        merchant: { type: SchemaType.STRING, description: '商家或开票方名称' },
        date: { type: SchemaType.STRING, description: '开票日期 YYYY-MM-DD' },
        totalAmount: { type: SchemaType.NUMBER, description: '总消费金额' },
        currency: { type: SchemaType.STRING, description: '币种 (如 CNY, USD)' },
        items: {
          type: SchemaType.ARRAY,
          items: {
            type: SchemaType.OBJECT,
            properties: {
              name: { type: SchemaType.STRING },
              price: { type: SchemaType.NUMBER },
              quantity: { type: SchemaType.INTEGER },
            },
          },
        },
      },
      required: ['merchant', 'totalAmount'],
    },
  },
});
 
visionRoute.post('/upload-invoice', async (c) => {
  const body = await c.req.parseBody();
  const file = body['file'];
 
  if (!(file instanceof File)) {
    return c.json({ error: '必须上传有效的文件' }, 400);
  }
 
  const arrayBuffer = await file.arrayBuffer();
  const base64Data = Buffer.from(arrayBuffer).toString('base64');
 
  const result = await invoiceModel.generateContent([
    {
      inlineData: {
        data: base64Data,
        mimeType: file.type || 'image/png',
      },
    },
    '请提取发票中的商家、日期、明细和总金额。',
  ]);
 
  // 直接解析返回的结构化 JSON
  const parsedInvoice = JSON.parse(result.response.text());
  return c.json({ success: true, data: parsedInvoice });
});