如何用低代码搭建训练一个专业知识库问答GPT机器人

function splitDocuments(files, chunkSize) {
	let docSize = chunkSize || 1000;
	let textString = '';
	let index = 0;
	let documents = [];

	for(let i = 0, len = files.length; i < len; i++) {

		if(files[i] && files[i].content) {
			textString = files[i].content;
		}
		else {
			textString = fs.readFileSync(files[i], "utf8");
		}
		textString = textString.replace(/\n|\r/g, " ").replace(/<.*?>/g,"")

	    let start = 0;
	    while (start < textString.length) {
	      const end = start + docSize;
	      const chunk = textString.slice(start, end);
	      documents.push({ docIndex: index++, fileIndex: files[i].fileIndex, filename: files[i].filename || files[i], content: chunk });
	      start = end;
		}
  	}
	  fs.writeFileSync("./docstore.json", JSON.stringify(documents));

  	return documents;
}

async function initVector(sql, docs){
    const maxElements = docs.length || 500; // 最多处理500个
    for (let j = 0; j < maxElements; j++ ) {
        const input = docs[j].content;
        const filename = docs[j].filename;
        const fileIndex = docs[j].fileIndex
        const docIndex = docs[j].docIndex

        // 通过根据训练日志返回断点docIndex，调整 docIndex 的值，确保从断点继续向量化
        if(docIndex >= 0 &&  docIndex < 1000 ){
            log("start embedding fileIndex: ", fileIndex, 'docIndex: ', docIndex, "filename:", filename);
            const embedding = await embedding(input);
            const embeddingArr = "[" + embedding + "]";
            const metadata = { filename, "doclength": maxElements, index: j };

            const insertRet = await sql`
              INSERT INTO documents ( content, appcode, metadata, embedding )
              VALUES
              ( ${input}, 'wedadoc',  ${metadata}, ${embeddingArr} )`
            await delay(1000); // 如果embedding API并发请求限制，可设置随机数sleep
        }
        else {
          continue;
        }
    }
    return true;
}

async function embedding (text) {
    const raw_text = text.replace(/\n|\r/g, " ");
    const embeddingResponse = await fetch(
        OPENAI_URL + "/v1/embeddings",
        {
          method: "POST",
          headers: {
            "Authorization": `Bearer ${OPENAI_API_KEY}`,
            "Content-Type": "application/json"
          },
          body: JSON.stringify({
            input: raw_text,
            model: "text-embedding-ada-002"
          })
        }
    );
    const embeddingData = await embeddingResponse.json();    
    const [{ embedding }] = embeddingData.data;
    log({embedding});
    return embedding;
}

async function searchKnn(question, k, sql){
    const embedding = await embedding(question);
    const embeddingArr = "[" + embedding + "]";
    const result = await sql`SELECT * FROM match_documents(${embeddingArr},'wedadoc', 0.1, ${k})`
    return result;
}

async function getChatGPT (query, documents){
    let contextText = "";
    if (documents) {
        for (let i = 0; i < documents.length; i++) {
            const document = documents[i];
            const content = document.content;  
            const url = encodeURI(document.metadata['filename']);
            contextText += `${content.trim()}\n SOURCE: ${url}\n---\n`;
        }
    }
    const systemContent = `You are a helpful assistant. When given CONTEXT you answer questions using only that information,and you always format your output in markdown. `;
    const userMessage = `CONTEXT:
      ${contextText}
      USER QUESTION: 
      ${query}`;
    const messages = [
        {
          role: "system",
          content: systemContent
        },
        {
          role: "user",
          content: userMessage
        }
    ];
  
    const chatResponse = await fetch(
        OPENAI_URL + "/v1/chat/completions",
        {
            method: "POST",
            headers: {
                "Authorization": `Bearer ${OPENAI_API_KEY}`,
                "Content-Type": "application/json"
            },
            body: JSON.stringify({
                "model": "gpt-3.5-turbo", 
                "messages": messages,
                "temperature": 0.3, 
                "max_tokens": 2000,
            })
        }
    );
    return await chatResponse.json();
}

create table documents (
  id bigserial primary key,
  content text, -- corresponds to Document.pageContent
  metadata json, -- corresponds to Document.metadata
  embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed
);

create or replace function match_documents (
  query_embedding vector(1536),
  similarity_threshold float,
  match_count int
)
returns table (
  id bigint,
  content text,
  metadata json,
  similarity float
)
language plpgsql
as $$
begin
  return query
  select
    documents.id,
    documents.content,
    documents.metadata,
    1 - (documents.embedding <=> query_embedding) as similarity
  from documents
  where 1 - (documents.embedding <=> query_embedding) > similarity_threshold  
  order by documents.embedding <=> query_embedding
  limit match_count;
end;
$$;