207 lines
8.4 KiB
JavaScript
207 lines
8.4 KiB
JavaScript
|
// 客服知识库向量化：读取 customer-service-kb/*.md，按 "### Q:" 问答块切分，
|
|||
|
|
// 用 SiliconFlow Qwen3-Embedding-0.6B(1024维) 生成向量，存入共享 postgres
|
|||
|
|
// products 库的 kb_docs 表，供 bridge 做 RAG 检索回答客户政策类问题。
|
|||
|
|
//
|
|||
|
|
// 切块策略(遵循知识库 README 建议)：
|
|||
|
|
// - 每个文件先解析 YAML frontmatter(title/category/keywords/source)
|
|||
|
|
// - 每个 "### Q: ..." 到下一个 "### Q:" 之间为一个 chunk(含问题+答案)
|
|||
|
|
// - embedding 文本 = keywords + Q + A，提升召回
|
|||
|
|
// - 额外保留 title/category 便于过滤/引用
|
|||
|
|
//
|
|||
|
|
// 运行: node index_kb.js (PG_URL 默认 postgres://postgres:postgres@localhost:5432/products)
|
|||
|
|
const fs = require('fs');
|
|||
|
|
const path = require('path');
|
|||
|
|
const { Pool } = require('pg');
|
|||
|
|
|
|||
|
|
const EMB = JSON.parse(fs.readFileSync(path.join(__dirname, 'kb.config.json'), 'utf8'));
|
|||
|
|
const MODEL = EMB.model || 'text-embedding-v4';
|
|||
|
|
const DIM = EMB.dim || 1024;
|
|||
|
|
const PG_URL = process.env.PG_URL || 'postgres://postgres:postgres@localhost:5432/products';
|
|||
|
|
const pool = new Pool({ connectionString: PG_URL });
|
|||
|
|
|
|||
|
|
const KB_DIR = path.join(__dirname, '..', 'customer-service-kb');
|
|||
|
|
|
|||
|
|
// ---------- 解析 frontmatter(支持多行 YAML 数组，如 source_url) ----------
|
|||
|
|
function parseFrontmatter(text) {
|
|||
|
|
const m = text.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
|
|||
|
|
if (!m) return { meta: {}, body: text };
|
|||
|
|
const meta = {};
|
|||
|
|
const fm = m[1].split('\n');
|
|||
|
|
for (let i = 0; i < fm.length; i++) {
|
|||
|
|
const kv = fm[i].match(/^(\w+):\s*(.*)$/);
|
|||
|
|
if (!kv) continue;
|
|||
|
|
const key = kv[1];
|
|||
|
|
let v = kv[2].trim();
|
|||
|
|
if (v.startsWith('[')) {
|
|||
|
|
// 单行数组 [a, b, c]
|
|||
|
|
v = v.replace(/[\[\]]/g, '').split(',').map((s) => s.trim().replace(/^["']|["']$/g, '')).filter(Boolean);
|
|||
|
|
} else if (v === '') {
|
|||
|
|
// 多行数组：后续缩进的 - 开头行
|
|||
|
|
const arr = [];
|
|||
|
|
while (i + 1 < fm.length && /^\s+-\s+/.test(fm[i + 1])) {
|
|||
|
|
i++;
|
|||
|
|
let item = fm[i].replace(/^\s+-\s+/, '').trim().replace(/^["']|["']$/g, '');
|
|||
|
|
if (item) arr.push(item);
|
|||
|
|
}
|
|||
|
|
v = arr;
|
|||
|
|
} else {
|
|||
|
|
v = v.replace(/^["']|["']$/g, '');
|
|||
|
|
}
|
|||
|
|
meta[key] = v;
|
|||
|
|
}
|
|||
|
|
return { meta, body: m[2] };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ---------- 按 Q: 切块 ----------
|
|||
|
|
function splitChunks(body) {
|
|||
|
|
// 去掉文件顶部的 "> 引用块"(章节说明)，但保留答案里的内容
|
|||
|
|
const lines = body.split('\n');
|
|||
|
|
const chunks = [];
|
|||
|
|
let cur = null;
|
|||
|
|
let heading = null; // 文件内 "## " 一级标题(章节名)
|
|||
|
|
for (const line of lines) {
|
|||
|
|
const h2 = line.match(/^##\s+(.*)$/);
|
|||
|
|
if (h2) { heading = h2[1].trim(); continue; }
|
|||
|
|
const q = line.match(/^###\s+Q:\s*(.*)$/);
|
|||
|
|
if (q) {
|
|||
|
|
if (cur) chunks.push(cur);
|
|||
|
|
cur = { question: q[1].trim(), answer: '', section: heading || '' };
|
|||
|
|
} else if (cur) {
|
|||
|
|
cur.answer += line + '\n';
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if (cur) chunks.push(cur);
|
|||
|
|
// 清理答案首尾空白
|
|||
|
|
return chunks.map((c) => ({ ...c, answer: c.answer.trim() }));
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 章节 → 来源 URL 映射：dropshipping KB 各章节对应不同 faq 子页，
|
|||
|
|
// 避免"物流"问题附上 dropshipping 首页/about-us 等无关链接。
|
|||
|
|
const SECTION_URL_MAP = {
|
|||
|
|
'Value proposition': ['https://dropshipping.yehwang.com/', 'https://dropshipping.yehwang.com/about-us'],
|
|||
|
|
'Joining & requirements': ['https://dropshipping.yehwang.com/faq/register'],
|
|||
|
|
'How it works (4 steps)': ['https://dropshipping.yehwang.com/'],
|
|||
|
|
'Brands & catalog': ['https://dropshipping.yehwang.com/'],
|
|||
|
|
'Pricing & margins': ['https://dropshipping.yehwang.com/'],
|
|||
|
|
'Packaging & branding': ['https://dropshipping.yehwang.com/'],
|
|||
|
|
'Orders, minimum & products': ['https://dropshipping.yehwang.com/faq/order'],
|
|||
|
|
'Payment & VAT': ['https://dropshipping.yehwang.com/faq/payment'],
|
|||
|
|
'Shipping & delivery': ['https://dropshipping.yehwang.com/faq/shipment'],
|
|||
|
|
'Returns': ['https://dropshipping.yehwang.com/faq/returns'],
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
// ---------- embedding ----------
|
|||
|
|
async function embedText(text) {
|
|||
|
|
const resp = await fetch(`${EMB.baseURL}/embeddings`, {
|
|||
|
|
method: 'POST',
|
|||
|
|
headers: { Authorization: `Bearer ${EMB.apiKey}`, 'Content-Type': 'application/json' },
|
|||
|
|
body: JSON.stringify({ model: MODEL, input: text, encoding_format: 'float', dimensions: DIM }),
|
|||
|
|
});
|
|||
|
|
const d = await resp.json();
|
|||
|
|
if (!d.data || !d.data[0] || !d.data[0].embedding) {
|
|||
|
|
throw new Error('embedText err: ' + JSON.stringify(d).slice(0, 200));
|
|||
|
|
}
|
|||
|
|
return d.data[0].embedding;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const vecStr = (v) => `[${v.join(',')}]`;
|
|||
|
|
|
|||
|
|
async function main() {
|
|||
|
|
// 1. 建表
|
|||
|
|
await pool.query(`CREATE EXTENSION IF NOT EXISTS vector`);
|
|||
|
|
await pool.query(`
|
|||
|
|
CREATE TABLE IF NOT EXISTS kb_docs (
|
|||
|
|
id SERIAL PRIMARY KEY,
|
|||
|
|
source_file TEXT,
|
|||
|
|
title TEXT,
|
|||
|
|
category TEXT,
|
|||
|
|
section TEXT,
|
|||
|
|
question TEXT,
|
|||
|
|
answer TEXT,
|
|||
|
|
keywords TEXT[],
|
|||
|
|
source_urls TEXT[],
|
|||
|
|
text_vec vector(${DIM})
|
|||
|
|
)
|
|||
|
|
`);
|
|||
|
|
// 重建表：模型/维度/向量空间变了(0.6B -> v4)，旧向量不可复用，直接 DROP 重建
|
|||
|
|
await pool.query('DROP TABLE IF EXISTS kb_docs');
|
|||
|
|
await pool.query(`
|
|||
|
|
CREATE TABLE kb_docs (
|
|||
|
|
id SERIAL PRIMARY KEY,
|
|||
|
|
source_file TEXT,
|
|||
|
|
title TEXT,
|
|||
|
|
category TEXT,
|
|||
|
|
section TEXT,
|
|||
|
|
question TEXT,
|
|||
|
|
answer TEXT,
|
|||
|
|
keywords TEXT[],
|
|||
|
|
source_urls TEXT[],
|
|||
|
|
text_vec vector(${DIM})
|
|||
|
|
)
|
|||
|
|
`);
|
|||
|
|
// HNSW 索引：小数据集召回质量优于 ivfflat，且无需调 lists 参数。
|
|||
|
|
// 检索用余弦距离(<=>)，故显式指定 vector_cosine_ops。
|
|||
|
|
await pool.query(`CREATE INDEX kb_docs_vec_idx ON kb_docs USING hnsw (text_vec vector_cosine_ops)`);
|
|||
|
|
console.log(`建表完成, model=${MODEL}, dim=${DIM}`);
|
|||
|
|
|
|||
|
|
// 2. 收集文件(排除 README/index/_source)
|
|||
|
|
const files = fs.readdirSync(KB_DIR)
|
|||
|
|
.filter((f) => f.endsWith('.md') && /^\d+/.test(f))
|
|||
|
|
.sort();
|
|||
|
|
console.log(`发现 ${files.length} 个知识库文件: ${files.join(', ')}`);
|
|||
|
|
|
|||
|
|
let total = 0;
|
|||
|
|
for (const file of files) {
|
|||
|
|
const raw = fs.readFileSync(path.join(KB_DIR, file), 'utf8');
|
|||
|
|
const { meta, body } = parseFrontmatter(raw);
|
|||
|
|
const chunks = splitChunks(body);
|
|||
|
|
console.log(`\n[${file}] title="${meta.title}" category="${meta.category}" -> ${chunks.length} chunks`);
|
|||
|
|
|
|||
|
|
for (const ch of chunks) {
|
|||
|
|
const keywords = Array.isArray(meta.keywords) ? meta.keywords : [];
|
|||
|
|
// 来源 URL：优先用章节级映射(如 dropshipping 各章节对应不同 faq 子页)，再兜底文件级
|
|||
|
|
let sourceUrls = [];
|
|||
|
|
if (ch.section && SECTION_URL_MAP[ch.section]) {
|
|||
|
|
sourceUrls = SECTION_URL_MAP[ch.section];
|
|||
|
|
} else if (Array.isArray(meta.source_url)) {
|
|||
|
|
sourceUrls = meta.source_url.filter((u) => /^https?:\/\//.test(u));
|
|||
|
|
}
|
|||
|
|
// embedding 文本：keywords + 问题 + 答案，最大化召回
|
|||
|
|
const embText = [
|
|||
|
|
keywords.length ? keywords.join(' ') : '',
|
|||
|
|
ch.question,
|
|||
|
|
ch.answer,
|
|||
|
|
].filter(Boolean).join('\n');
|
|||
|
|
const vec = await embedText(embText);
|
|||
|
|
await pool.query(
|
|||
|
|
`INSERT INTO kb_docs (source_file, title, category, section, question, answer, keywords, source_urls, text_vec)
|
|||
|
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9::vector)`,
|
|||
|
|
[file, meta.title || '', meta.category || '', ch.section, ch.question, ch.answer, keywords, sourceUrls, vecStr(vec)]
|
|||
|
|
);
|
|||
|
|
total++;
|
|||
|
|
process.stdout.write(` [${total}] Q: ${ch.question.slice(0, 60)}${sourceUrls.length ? ' src=' + sourceUrls.length : ''}\n`);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
console.log(`\n上传完成，共 ${total} 个 chunk`);
|
|||
|
|
|
|||
|
|
// 3. 测试检索
|
|||
|
|
const tq = await embedText('What is the minimum order value? Can I pay by bank transfer?');
|
|||
|
|
const ts = await pool.query(
|
|||
|
|
`SELECT question, category, 1 - (text_vec <=> $1::vector) AS score
|
|||
|
|
FROM kb_docs ORDER BY text_vec <=> $1::vector LIMIT 5`, [vecStr(tq)]
|
|||
|
|
);
|
|||
|
|
console.log('\n检索 "What is the minimum order value? bank transfer":');
|
|||
|
|
ts.rows.forEach((r, i) => console.log(` ${i + 1}. [${r.category}] score=${Number(r.score).toFixed(3)} Q: ${r.question}`));
|
|||
|
|
|
|||
|
|
const tq2 = await embedText('meine Bestellung stornieren Rückgabe kaputt'); // 德语
|
|||
|
|
const ts2 = await pool.query(
|
|||
|
|
`SELECT question, category, 1 - (text_vec <=> $1::vector) AS score
|
|||
|
|
FROM kb_docs ORDER BY text_vec <=> $1::vector LIMIT 3`, [vecStr(tq2)]
|
|||
|
|
);
|
|||
|
|
console.log('\n检索(德语) "meine Bestellung stornieren Rückgabe kaputt":');
|
|||
|
|
ts2.rows.forEach((r, i) => console.log(` ${i + 1}. [${r.category}] score=${Number(r.score).toFixed(3)} Q: ${r.question}`));
|
|||
|
|
|
|||
|
|
await pool.end();
|
|||
|
|
}
|
|||
|
|
main().catch((e) => { console.error('ERR', e.message); process.exit(1); });
|