crm-ai-demo/bridge/index_kb.js

// 客服知识库向量化：读取 customer-service-kb/*.md，按 "### Q:" 问答块切分，
// 用 SiliconFlow Qwen3-Embedding-0.6B(1024维) 生成向量，存入共享 postgres
// products 库的 kb_docs 表，供 bridge 做 RAG 检索回答客户政策类问题。
//
// 切块策略(遵循知识库 README 建议)：
//   - 每个文件先解析 YAML frontmatter(title/category/keywords/source)
//   - 每个 "### Q: ..." 到下一个 "### Q:" 之间为一个 chunk(含问题+答案)
//   - embedding 文本 = keywords + Q + A，提升召回
//   - 额外保留 title/category 便于过滤/引用
//
// 运行: node index_kb.js   (PG_URL 默认 postgres://postgres:postgres@localhost:5432/products)
const fs = require('fs');
const path = require('path');
const { Pool } = require('pg');

const EMB = JSON.parse(fs.readFileSync(path.join(__dirname, 'kb.config.json'), 'utf8'));
const MODEL = EMB.model || 'text-embedding-v4';
const DIM = EMB.dim || 1024;
const PG_URL = process.env.PG_URL || 'postgres://postgres:postgres@localhost:5432/products';
const pool = new Pool({ connectionString: PG_URL });

const KB_DIR = path.join(__dirname, '..', 'customer-service-kb');

// ---------- 解析 frontmatter(支持多行 YAML 数组，如 source_url) ----------
function parseFrontmatter(text) {
  const m = text.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
  if (!m) return { meta: {}, body: text };
  const meta = {};
  const fm = m[1].split('\n');
  for (let i = 0; i < fm.length; i++) {
    const kv = fm[i].match(/^(\w+):\s*(.*)$/);
    if (!kv) continue;
    const key = kv[1];
    let v = kv[2].trim();
    if (v.startsWith('[')) {
      // 单行数组 [a, b, c]
      v = v.replace(/[\[\]]/g, '').split(',').map((s) => s.trim().replace(/^["']|["']$/g, '')).filter(Boolean);
    } else if (v === '') {
      // 多行数组：后续缩进的 - 开头行
      const arr = [];
      while (i + 1 < fm.length && /^\s+-\s+/.test(fm[i + 1])) {
        i++;
        let item = fm[i].replace(/^\s+-\s+/, '').trim().replace(/^["']|["']$/g, '');
        if (item) arr.push(item);
      }
      v = arr;
    } else {
      v = v.replace(/^["']|["']$/g, '');
    }
    meta[key] = v;
  }
  return { meta, body: m[2] };
}

// ---------- 按 Q: 切块 ----------
function splitChunks(body) {
  // 去掉文件顶部的 "> 引用块"(章节说明)，但保留答案里的内容
  const lines = body.split('\n');
  const chunks = [];
  let cur = null;
  let heading = null; // 文件内 "## " 一级标题(章节名)
  for (const line of lines) {
    const h2 = line.match(/^##\s+(.*)$/);
    if (h2) { heading = h2[1].trim(); continue; }
    const q = line.match(/^###\s+Q:\s*(.*)$/);
    if (q) {
      if (cur) chunks.push(cur);
      cur = { question: q[1].trim(), answer: '', section: heading || '' };
    } else if (cur) {
      cur.answer += line + '\n';
    }
  }
  if (cur) chunks.push(cur);
  // 清理答案首尾空白
  return chunks.map((c) => ({ ...c, answer: c.answer.trim() }));
}

// 章节 → 来源 URL 映射：dropshipping KB 各章节对应不同 faq 子页，
// 避免"物流"问题附上 dropshipping 首页/about-us 等无关链接。
const SECTION_URL_MAP = {
  'Value proposition': ['https://dropshipping.yehwang.com/', 'https://dropshipping.yehwang.com/about-us'],
  'Joining & requirements': ['https://dropshipping.yehwang.com/faq/register'],
  'How it works (4 steps)': ['https://dropshipping.yehwang.com/'],
  'Brands & catalog': ['https://dropshipping.yehwang.com/'],
  'Pricing & margins': ['https://dropshipping.yehwang.com/'],
  'Packaging & branding': ['https://dropshipping.yehwang.com/'],
  'Orders, minimum & products': ['https://dropshipping.yehwang.com/faq/order'],
  'Payment & VAT': ['https://dropshipping.yehwang.com/faq/payment'],
  'Shipping & delivery': ['https://dropshipping.yehwang.com/faq/shipment'],
  'Returns': ['https://dropshipping.yehwang.com/faq/returns'],
};

// ---------- embedding ----------
async function embedText(text) {
  const resp = await fetch(`${EMB.baseURL}/embeddings`, {
    method: 'POST',
    headers: { Authorization: `Bearer ${EMB.apiKey}`, 'Content-Type': 'application/json' },
    body: JSON.stringify({ model: MODEL, input: text, encoding_format: 'float', dimensions: DIM }),
  });
  const d = await resp.json();
  if (!d.data || !d.data[0] || !d.data[0].embedding) {
    throw new Error('embedText err: ' + JSON.stringify(d).slice(0, 200));
  }
  return d.data[0].embedding;
}

const vecStr = (v) => `[${v.join(',')}]`;

async function main() {
  // 1. 建表
  await pool.query(`CREATE EXTENSION IF NOT EXISTS vector`);
  await pool.query(`
    CREATE TABLE IF NOT EXISTS kb_docs (
      id SERIAL PRIMARY KEY,
      source_file TEXT,
      title TEXT,
      category TEXT,
      section TEXT,
      question TEXT,
      answer TEXT,
      keywords TEXT[],
      source_urls TEXT[],
      text_vec vector(${DIM})
    )
  `);
  // 重建表：模型/维度/向量空间变了(0.6B -> v4)，旧向量不可复用，直接 DROP 重建
  await pool.query('DROP TABLE IF EXISTS kb_docs');
  await pool.query(`
    CREATE TABLE kb_docs (
      id SERIAL PRIMARY KEY,
      source_file TEXT,
      title TEXT,
      category TEXT,
      section TEXT,
      question TEXT,
      answer TEXT,
      keywords TEXT[],
      source_urls TEXT[],
      text_vec vector(${DIM})
    )
  `);
  // HNSW 索引：小数据集召回质量优于 ivfflat，且无需调 lists 参数。
  // 检索用余弦距离(<=>)，故显式指定 vector_cosine_ops。
  await pool.query(`CREATE INDEX kb_docs_vec_idx ON kb_docs USING hnsw (text_vec vector_cosine_ops)`);
  console.log(`建表完成, model=${MODEL}, dim=${DIM}`);

  // 2. 收集文件(排除 README/index/_source)
  const files = fs.readdirSync(KB_DIR)
    .filter((f) => f.endsWith('.md') && /^\d+/.test(f))
    .sort();
  console.log(`发现 ${files.length} 个知识库文件: ${files.join(', ')}`);

  let total = 0;
  for (const file of files) {
    const raw = fs.readFileSync(path.join(KB_DIR, file), 'utf8');
    const { meta, body } = parseFrontmatter(raw);
    const chunks = splitChunks(body);
    console.log(`\n[${file}] title="${meta.title}" category="${meta.category}" -> ${chunks.length} chunks`);

    for (const ch of chunks) {
      const keywords = Array.isArray(meta.keywords) ? meta.keywords : [];
      // 来源 URL：优先用章节级映射(如 dropshipping 各章节对应不同 faq 子页)，再兜底文件级
      let sourceUrls = [];
      if (ch.section && SECTION_URL_MAP[ch.section]) {
        sourceUrls = SECTION_URL_MAP[ch.section];
      } else if (Array.isArray(meta.source_url)) {
        sourceUrls = meta.source_url.filter((u) => /^https?:\/\//.test(u));
      }
      // embedding 文本：keywords + 问题 + 答案，最大化召回
      const embText = [
        keywords.length ? keywords.join(' ') : '',
        ch.question,
        ch.answer,
      ].filter(Boolean).join('\n');
      const vec = await embedText(embText);
      await pool.query(
        `INSERT INTO kb_docs (source_file, title, category, section, question, answer, keywords, source_urls, text_vec)
         VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9::vector)`,
        [file, meta.title || '', meta.category || '', ch.section, ch.question, ch.answer, keywords, sourceUrls, vecStr(vec)]
      );
      total++;
      process.stdout.write(`  [${total}] Q: ${ch.question.slice(0, 60)}${sourceUrls.length ? '  src=' + sourceUrls.length : ''}\n`);
    }
  }
  console.log(`\n上传完成，共 ${total} 个 chunk`);

  // 3. 测试检索
  const tq = await embedText('What is the minimum order value? Can I pay by bank transfer?');
  const ts = await pool.query(
    `SELECT question, category, 1 - (text_vec <=> $1::vector) AS score
     FROM kb_docs ORDER BY text_vec <=> $1::vector LIMIT 5`, [vecStr(tq)]
  );
  console.log('\n检索 "What is the minimum order value? bank transfer":');
  ts.rows.forEach((r, i) => console.log(`  ${i + 1}. [${r.category}] score=${Number(r.score).toFixed(3)} Q: ${r.question}`));

  const tq2 = await embedText('meine Bestellung stornieren Rückgabe kaputt'); // 德语
  const ts2 = await pool.query(
    `SELECT question, category, 1 - (text_vec <=> $1::vector) AS score
     FROM kb_docs ORDER BY text_vec <=> $1::vector LIMIT 3`, [vecStr(tq2)]
  );
  console.log('\n检索(德语) "meine Bestellung stornieren Rückgabe kaputt":');
  ts2.rows.forEach((r, i) => console.log(`  ${i + 1}. [${r.category}] score=${Number(r.score).toFixed(3)} Q: ${r.question}`));

  await pool.end();
}
main().catch((e) => { console.error('ERR', e.message); process.exit(1); });