选择最适合数据的嵌入模型：OpenAI 和开源多语言嵌入的对比测试

 from llama_index.readers.web import SimpleWebPageReader
 from llama_index.core.node_parser import SentenceSplitter
 
 language = "EN"
 url_doc = "https://eur-lex.europa.eu/legal-content/"+language+"/TXT/HTML/?uri=CELEX:52021PC0206"
 
 documents = SimpleWebPageReader(html_to_text=True).load_data([url_doc])
 
 parser = SentenceSplitter(chunk_size=1000)
 nodes = parser.get_nodes_from_documents(documents, show_progress=True)

 prompts={}
 prompts["EN"] = """\
 Context information is below.
 
 ---------------------
 {context_str}
 ---------------------
 
 Given the context information and not prior knowledge, generate only questions based on the below query.
 
 You are a Teacher/ Professor. Your task is to setup {num_questions_per_chunk} questions for an upcoming quiz/examination.
 The questions should be diverse in nature across the document. Restrict the questions to the context information provided."
 """

 from llama_index.llms import OpenAI
 from llama_index.legacy.finetuning import generate_qa_embedding_pairs
 
 qa_dataset = generate_qa_embedding_pairs(
     llm=OpenAI(model="gpt-3.5-turbo-0125",additional_kwargs={'seed':42}),
     nodes=nodes,
     qa_generate_prompt_tmpl = prompts[language],
     num_questions_per_chunk=2
 )

 def evaluate(dataset, embed_model, insert_batch_size=1000, top_k=5):
     # Get corpus, queries, and relevant documents from the qa_dataset object
     corpus = dataset.corpus
     queries = dataset.queries
     relevant_docs = dataset.relevant_docs
 
     # Create TextNode objects for each document in the corpus and create a VectorStoreIndex to efficiently store and retrieve embeddings
     nodes = [TextNode(id_=id_, text=text) for id_, text in corpus.items()]
     index = VectorStoreIndex(
         nodes, embed_model=embed_model, insert_batch_size=insert_batch_size
     )
     retriever = index.as_retriever(similarity_top_k=top_k)
 
     # Prepare to collect evaluation results
     eval_results = []
 
     # Iterate over each query in the dataset to evaluate retrieval performance
     for query_id, query in tqdm(queries.items()):
         # Retrieve the top_k most similar documents for the current query and extract the IDs of the retrieved documents
         retrieved_nodes = retriever.retrieve(query)
         retrieved_ids = [node.node.node_id for node in retrieved_nodes]
 
         # Check if the expected document was among the retrieved documents
         expected_id = relevant_docs[query_id][0]
         is_hit = expected_id in retrieved_ids  # assume 1 relevant doc per query
 
         # Calculate the Mean Reciprocal Rank (MRR) and append to results
         if is_hit:
             rank = retrieved_ids.index(expected_id) + 1
             mrr = 1 / rank
         else:
             mrr = 0
         eval_results.append(mrr)
 
     # Return the average MRR across all queries as the final evaluation metric
     return np.average(eval_results)

 from llama_index.embeddings.openai import OpenAIEmbedding
 
 embed_model = OpenAIEmbedding(model=model_spec['model_name'],
                               dimensions=model_spec['dimensions'])

 embeddings_model_spec = {
 }
 
 embeddings_model_spec['OAI-Large-256']={'model_name':'text-embedding-3-large','dimensions':256}
 embeddings_model_spec['OAI-Large-3072']={'model_name':'text-embedding-3-large','dimensions':3072}
 embeddings_model_spec['OAI-Small']={'model_name':'text-embedding-3-small','dimensions':1536}
 embeddings_model_spec['OAI-ada-002']={'model_name':'text-embedding-ada-002','dimensions':None}
 
 results = []
 
 languages = ["EN", "FR", "CS", "HU"]
 
 # Loop through all languages
 for language in languages:
 
     # Load dataset
     file_name=language+"_dataset.json"
     qa_dataset = EmbeddingQAFinetuneDataset.from_json(file_name)
 
     # Loop through all models
     for model_name, model_spec in embeddings_model_spec.items():
 
         # Get model
         embed_model = OpenAIEmbedding(model=model_spec['model_name'],
                                       dimensions=model_spec['dimensions'])
 
         # Assess embedding score (in terms of MRR)
         score = evaluate(qa_dataset, embed_model)
 
         results.append([language, model_name, score])
 
 df_results = pd.DataFrame(results, columns = ["Language" ,"Embedding model", "MRR"])

 embeddings_model_spec = {
 }
 
 embeddings_model_spec['E5-mistral-7b']={'model_name':'intfloat/e5-mistral-7b-instruct','max_length':32768, 'pooling_type':'last_token', 
                                         'normalize': True, 'batch_size':1, 'kwargs': {'load_in_4bit':True, 'bnb_4bit_compute_dtype':torch.float16}}
 embeddings_model_spec['ML-E5-large']={'model_name':'intfloat/multilingual-e5-large','max_length':512, 'pooling_type':'mean', 
                                       'normalize': True, 'batch_size':1, 'kwargs': {'device_map': 'cuda', 'torch_dtype':torch.float16}}
 embeddings_model_spec['BGE-M3']={'model_name':'BAAI/bge-m3','max_length':8192, 'pooling_type':'cls', 
                                  'normalize': True, 'batch_size':1, 'kwargs': {'device_map': 'cuda', 'torch_dtype':torch.float16}}
 embeddings_model_spec['Nomic-Embed']={'model_name':'nomic-ai/nomic-embed-text-v1','max_length':8192, 'pooling_type':'mean', 
                                       'normalize': True, 'batch_size':1, 'kwargs': {'device_map': 'cuda', 'trust_remote_code' : True}}
 
 results = []
 
 languages = ["EN", "FR", "CS", "HU"]
 
 # Loop through all models
 for model_name, model_spec in embeddings_model_spec.items():
 
     print("Processing model : "+str(model_spec))
 
     # Get model
     tokenizer = AutoTokenizer.from_pretrained(model_spec['model_name'])
     embed_model = AutoModel.from_pretrained(model_spec['model_name'], **model_spec['kwargs'])
         
     if model_name=="Nomic-Embed":
         embed_model.to('cuda')
 
     # Loop through all languages
     for language in languages:
 
         # Load dataset
         file_name=language+"_dataset.json"
         qa_dataset = EmbeddingQAFinetuneDataset.from_json(file_name)
 
         start_time_assessment=time.time()
 
         # Assess embedding score (in terms of hit rate at k=5)
         score = evaluate(qa_dataset, tokenizer, embed_model, model_spec['normalize'], model_spec['max_length'], model_spec['pooling_type'])
 
         # Get duration of score assessment
         duration_assessment = time.time()-start_time_assessment
 
         results.append([language, model_name, score, duration_assessment])
 
 df_results = pd.DataFrame(results, columns = ["Language" ,"Embedding model", "MRR", "Duration"])