福布斯 AI 50 榜单中唯一开源向量数据库：Weaviate

git clone https://github.com/soulteary/weaviate-quickstart.git
cd weaviate-quickstart

[
  {
"SolarTerms":"立春",
"Title":"木兰花·立春日作",
"Author":"陆游",
"Poem":"春盘春酒年年好，试戴银旛判醉倒。今朝一岁大家添，不是人间偏我老。",
"Description":"《月令七十二候集解》有载：“立，始建也。春气始而建立也。”在生生不息的春风中，一年的序幕由此开启。我们用一颗丰盈而善良的心，向着春意盎然的天地间走去。立春丨一候东风解冻，二候蜇虫始振，三候鱼陟负冰。"
},
{
"SolarTerms":"雨水",
"Title":"早春呈水部张十八员外",
"Author":"韩愈",
"Poem":"天街小雨润如酥，草色遥看近却无。最是一年春好处，绝胜烟柳满皇都。",
"Description":"雨水是二十四节气之中的第二个节气，此时，气温回升、冰雪融化、降水增多，故取名为雨水。我们将要迎接的，不仅有温暖的晨曦，更是一场被东风吹来的，新的、充满生机的甘霖。雨水丨一候獭祭鱼；二候鸿雁来；三候草木萌动。"
},
{
"SolarTerms":"惊蛰",
"Title":"走笔谢孟谏议寄新茶",
"Author":"卢仝",
"Poem":"闻道新年入山里，蛰虫惊动春风起。天子须尝阳羡茶，百草不敢先开花。",
"Description":"惊蛰，古称“启蛰”，预示着生命在这一刻重生。春的希望就在眼前，惊蛰过后，万物复苏，一派生机勃勃的景色。让我们且听风吟，且闻鸟语，邂逅最美的花开！惊蛰丨一候桃始华；二候仓庚（黄鹂）鸣；三候鹰化为鸠。"
},
{
"SolarTerms":"春分",
"Title":"春日",
"Author":"朱熹",
"Poem":"胜日寻芳泗水滨，无边光景一时新。等闲识得东风面，万紫千红总是春。",
"Description":"春分，古时又称为“日中”、“仲春之月”。自此，进入春和日丽、万红千翠争媚时节。好花不常看，好景不常在，趁着风和日丽出门踏青，莫要辜负了好春光。春分丨一候元鸟至；二候雷乃发声；三候始电。"
},
{
"SolarTerms":"清明",
"Title":"清明",
"Author":"杜牧",
"Poem":"清明时节雨纷纷，路上行人欲断魂。借问酒家何处有，牧童遥指杏花村。",
"Description":"清明是为春天而来，它将蓬勃的生机暗自酝酿。清明节，我们缅怀逝去的家人。清明节，我们思念远方的亲人。生活无遗憾，人世有牵挂！清明丨一候桐始华；二候田鼠化为鹌；三候虹始见。"
},
{
"SolarTerms":"谷雨",
"Title":"晚春田园杂兴",
"Author":"范成大",
"Poem":"谷雨如丝复似尘，煮瓶浮蜡正尝新。牡丹破萼樱桃熟，未许飞花减却春。",
"Description":"雨我公田，雨其谷于水，播种时节到了。谷雨已至，愿你工作中一“谷”作气，事业高升；朋友间“谷”道热肠，人缘美好；生活中欢欣“谷”舞，快乐舒畅。谷雨丨一候萍始生；二候鸣鸠拂其羽；三候为戴任降于桑。"
},
{
"SolarTerms":"立夏",
"Title":"初夏",
"Author":"朱淑真",
"Poem":"竹摇清影罩幽窗，两两时禽噪夕阳。谢却海棠飞尽絮，困人天气日初长。",
"Description":"立夏表示即将告别春天，是夏天的开始。立夏之美，在于希望。愿你所有的烦恼，都将随风而去，幸福的感觉似夏花弥漫！立夏丨一候蝼蝈鸣；二候蚯蚓出；三候王瓜生。"
},
{
"SolarTerms":"小满",
"Title":"自桃川至辰州绝句四十有二",
"Author":"赵蕃",
"Poem":"一春多雨慧当悭，今岁还防似去年。玉历检来知小满，又愁阴久碍蚕眠。",
"Description":"小满，是一年中最有智慧的节气。小有所得，小有所望。我们的生活开始安稳顺遂，有沉静的增长，有从容的精进，小有知足，小有可期。小满丨一候苦菜秀；二候靡草死；三候麦秋至。"
},
{
"SolarTerms":"芒种",
"Title":"梅雨五绝（其二）",
"Author":"范成大",
"Poem":"乙酉甲申雷雨惊，乘除却贺芒种晴。插秧先插蚤籼稻，少忍数旬蒸米成。",
"Description":"“芒种”也称为“忙种”“忙着种”，预示着要开始忙碌的田间生活。收获过后，又要种下新一轮，人们种下希望的同时，也种下了期待。仲夏至此始，未来皆可期！芒种丨一候螳螂生；二候鵙（jú）始鸣；三候反舌无声。"
},
{
"SolarTerms":"夏至",
"Title":"积雨辋川庄作",
"Author":"王维",
"Poem":"积雨空林烟火迟，蒸藜炊黍饷东菑。漠漠水田飞白鹭，阴阴夏木啭黄鹂。",
"Description":"夏为大，至为极，万物到此壮大繁茂到极点、阳气也达到极致，所以是一年中夜最短、昼最长的一天。愿你在炎暑，有凉风袭人；愿你在夏日，有良人送爽；愿你向阳而立，拥抱无限生机！夏至丨一候鹿角解；二候蝉始鸣；三候半夏生。"
},
{
"SolarTerms":"小暑",
"Title":"赠别王侍御赴上都",
"Author":"韩翃",
"Poem":"相思掩泣复何如，公子门前人渐疏。幸有心期当小暑，葛衣纱帽望回车。",
"Description":"小暑，表示季夏时节的正式开始。天气开始炎热，但还没到最热，故称“小暑”。小暑时节，心静自然凉。眼前无长物，窗下有清风。散热有心静，凉生为室空。小暑丨一候温风至；二候蟋蟀居宇；三候鹰始鸷。"
},
{
"SolarTerms":"大暑",
"Title":"鹧鸪天",
"Author":"晁补之",
"Poem":"吉梦灵蛇朱夏宜。佳辰阿母会瑶池。竹风荷雨来消暑，玉李冰瓜可疗饥。",
"Description":"在骄阳的侵袭下，一年之中最热的时光，也就此拉开序幕。大暑时节，去见想见的人，去做想做的事，在前行的路上不断的遇见美好，遇见幸福，遇见心中期待的一切。大暑丨一候腐草为萤；二候土润溽暑；三候大雨时行。"
},
{
"SolarTerms":"立秋",
"Title":"城中晚夏思山",
"Author":"齐己",
"Poem":"葛衣沾汗功虽健，纸扇摇风力甚卑。苦热恨无行脚处，微凉喜到立秋时。",
"Description":"立秋是秋季的第一个节气，意味着秋天真正开始了。秋天是思念的季节，思念在远方的亲人，思念外出未归的子女，思念相处一生的伴侣。愿我们能在这个收获的季节，让心灵充盈，满载而归！立秋丨一候凉风至；二候白露生；三候寒蝉鸣。"
},
{
"SolarTerms":"处暑",
"Title":"早秋山中作",
"Author":"王维",
"Poem":"草间蛩响临秋急，山里蝉声薄暮悲。寂寞柴门人不到，空林独与白云期。",
"Description":"暑气将于这一天结束，秋意冉冉，从而开始一年之中最美好的秋高气爽的时节。白天要注意防暑，夜晚要记得添衣。暑去秋来，愿君务必珍重，秋日安康！处暑丨一候鹰乃祭鸟；二候天地始肃；三候禾乃登。"
},
{
"SolarTerms":"白露",
"Title":"金陵城西楼月下吟",
"Author":"李白",
"Poem":"金陵夜寂凉风发，独上高楼望吴越。白云映水摇空城，白露垂珠滴秋月。",
"Description":"白露秋分夜，一夜凉一夜；一场秋雨一场凉，一场白露一场霜；白露白茫茫，谷子满田黄。成熟与收获，尽数来到眼前。沉重的汗水凝结成沉甸甸的果实，人们喜悦，感念，分享。白露丨一候鸿雁来；二候元鸟归；三候群鸟养羞。"
},
{
"SolarTerms":"秋分",
"Title":"秋词二首（其二）",
"Author":"刘禹锡",
"Poem":"山明水净夜来霜，数树深红出浅黄。试上高楼清入骨，岂如春色嗾人狂。",
"Description":"秋分，天高云淡，气净风轻，是一个浪漫的节气。秋云飘逸，秋水如镜，处处迷人，令人陶醉。正是邀三五好友快意潇洒，观秋之华彩、赏世间美景的好时候。秋分丨一候雷始收声；二候蛰虫坯户；三候水始涸。"
},
{
"SolarTerms":"寒露",
"Title":"秋兴八首·其七",
"Author":"杜甫",
"Poem":"波漂菰米沈云黑，露冷莲房坠粉红。关塞极天唯鸟道，江湖满地一渔翁。",
"Description":"《月令七十二候集解》中说：九月节，露气寒冷，将凝结也。又是一年寒露至，又是一年秋意浓。最美的秋在眼前，最美的风景在身边，用心珍惜方能长久！寒露丨一候鸿雁来宾；二候雀入大水为蛤；三候菊有黄华。"
},
{
"SolarTerms":"霜降",
"Title":"村夜",
"Author":"白居易",
"Poem":"霜草苍苍虫切切，村南村北行人绝。独出前门望野田，月明荞麦花如雪。",
"Description":"从霜降当天起，暮秋已至天气凉，万物秋收冬藏。好柿成霜，喜从天降。活在当下，不留遗憾。岁岁年年，愿你安暖！霜降丨一候豺乃祭兽；二候草木黄落；三候蜇虫咸俯。"
},
{
"SolarTerms":"立冬",
"Title":"初冬夜饮",
"Author":"杜牧",
"Poem":"淮阳多病偶求欢，客袖侵霜与烛盘。砌下梨花一堆雪，明年谁此凭阑干？",
"Description":"立冬，代表着冬季开始，万物收藏，规避寒冷。一岁年华，已近尾声。在今年的最后一季里，定要修身养心，持善悦己！立冬丨一候水始冰；二候地始冻；三候雉入大水为蜃。"
},
{
"SolarTerms":"小雪",
"Title":"小雪",
"Author":"戴叔伦",
"Poem":"花雪随风不厌看，更多还肯失林峦。愁人正在书窗下，一片飞来一片寒。",
"Description":"小雪时雪还未盛，故称“小雪”。人生实苦，善待自己。在这个季节里，如果累了，就放松心情，再冷的天气，也要温暖自己的心！小雪丨一候虹藏不见；二候天气上升地气下降；三候闭塞而成冬。"
},
{
"SolarTerms":"大雪",
"Title":"北风行",
"Author":"李白",
"Poem":"燕山雪花大如席，片片吹落轩辕台。幽州思妇十二月，停歌罢笑双蛾摧。",
"Description":"从这天起，大雪来临，寒冬已至。天雪鬃呼啸。待早上一睁眼，冰封的窗户就亮得耀目。风中闻草木，雪里见江山。想要记起和忘掉一切，只需这一场大雪。大雪丨一候鹃鸥不鸣；二候虎始交；三候荔挺出。"
},
{
"SolarTerms":"冬至",
"Title":"邯郸冬至夜思家",
"Author":"白居易",
"Poem":"邯郸驿里逢冬至，抱膝灯前影伴身。想得家中夜深坐，还应说着远行人。",
"Description":"冬至，古称“至日”，“至”是极致的意思，冬藏之气至此而极。依着血缘的眷念，漂泊的游子都将踏上归家的旅途，冬至的温暖也安抚着每一个思归的心。冬至丨一候蚯蚓结；二候麋角解；三候水泉动。"
},
{
"SolarTerms":"小寒",
"Title":"冬夜寄温飞卿",
"Author":"鱼玄机",
"Poem":"苦思搜诗灯下吟，不眠长夜怕寒衾。满庭木叶愁风起，透幌纱窗惜月沉。",
"Description":"小寒一过，就进入“出门冰上走”的三九天了。小寒，是一个充满希望的时节——此时旧岁近暮，新岁即将登场，坚毅的鸿雁已先开始启程北飞了。万物开始复苏，生机已然蠢动。小寒丨一候雁北乡，二候鹊始巢，三候雉始鸲。"
},
{
"SolarTerms":"大寒",
"Title":"寒夜",
"Author":"杜耒",
"Poem":"寒夜客来茶当酒，竹炉汤沸火初红。寻常一样窗前月，才有梅花便不同。",
"Description":"大寒是这一年里的最后一个节气，也是一年中阴阳转换的重要时机。大寒过后，又是一年新的轮回。愿下个轮回，依旧有你相伴！大寒丨一候鸡乳；二候征鸟厉疾；三候水泽腹坚。"
}
]

conda create -n weaviate python=3.9 -y
conda activate weaviate

import torch.nn.functional as F
from transformers importAutoTokenizer,AutoModel

# 定义输入内容
input_texts =[
"天气好热，哪里有卖冰棍的",
"今天好冷，该多穿两件",
"夏天",
"冬天"
]

# 指定预训练模型，在线模型 "thenlper/gte-base-zh"，此处使用本地目录中的模型
model_id ="./thenlper/gte-small"

# 加载分词器和模型
tokenizer =AutoTokenizer.from_pretrained(model_id)
model =AutoModel.from_pretrained(model_id)

# 对输入内容进行分词、编码
batch_dict = tokenizer(input_texts, max_length=512, padding=True, truncation=True, return_tensors='pt')

# 获取 Embeddings
outputs = model(**batch_dict)
embeddings = outputs.last_hidden_state[:,0]

# 标准化 embeddings，使用 L2 归一化，使其长度为 1
embeddings = F.normalize(embeddings, p=2, dim=1)
# 计算相似度，选择第一个元素，和除了第一个元素进行比较
scores =(embeddings[:1]@ embeddings[1:].T)*100
# 打印结果分数
print(scores.tolist())

pip install torch transformers

# python usage1.py
[[80.17919921875, 82.7370376586914, 84.4228286743164]]

from sentence_transformers importSentenceTransformer
from sentence_transformers.util import cos_sim

# 定义输入内容
sentences =[
"天气好热，哪里有卖冰棍的",
"今天好冷，该多穿两件",
"夏天",
"冬天"
]

# 指定预训练模型，在线模型 "thenlper/gte-base-zh"，此处使用本地目录中的模型
model_id ="./thenlper/gte-small"

# 加载 SentenceTransformer 模型
model =SentenceTransformer(model_id)
# 获取向量
embeddings = model.encode(sentences)
# 计算第一个句子（index 0）和第二个句子（index 1）的嵌入向量之间的余弦相似度，并打印结果
print(cos_sim(embeddings[0], embeddings[1]))

pip install sentence_transformers

# python usage2.py
tensor([[0.8448]])

version: "3.4"
services:
weaviate:
command:
---host
-0.0.0.0
---port
-"8080"
---scheme
-http
image:semitechnologies/weaviate:1.25.5
ports:
-8086:8080
-50051:50051
volumes:
-./weaviate_data:/var/lib/weaviate
restart:on-failure:0
environment:
QUERY_DEFAULTS_LIMIT:25
# https://weaviate.io/developers/weaviate/configuration/authentication
AUTHENTICATION_APIKEY_ENABLED:"true"
AUTHENTICATION_APIKEY_ALLOWED_KEYS:"soulteary,password"
AUTHENTICATION_APIKEY_USERS:"soulteary,user@lab.io"
AUTHORIZATION_ADMINLIST_ENABLED:"true"
AUTHORIZATION_ADMINLIST_USERS:"soulteary,user@lab.io"
PERSISTENCE_DATA_PATH:"/var/lib/weaviate"
CLUSTER_HOSTNAME:"node1"
ENABLE_MODULES:"text2vec-transformers"
# Support enabled modules, ENABLE_MODULES: "text2vec-cohere,text2vec-huggingface,text2vec-palm,text2vec-openai,generative-openai,generative-cohere,generative-palm,ref2vec-centroid,reranker-cohere,qna-openai"
# Only store vectors, DEFAULT_VECTORIZER_MODULE: "none"
DEFAULT_VECTORIZER_MODULE:"text2vec-transformers"
TRANSFORMERS_INFERENCE_API:"http://t2v-transformers:8080"

t2v-transformers:
image:soulteary/t2v-transformers:2024.06.27
environment:
# set to 1 to enable
ENABLE_CUDA:0
ports:
-9090:8080

package main

import(
"context"
"encoding/json"
"fmt"
"os"
"time"

"github.com/weaviate/weaviate-go-client/v4/weaviate"
"github.com/weaviate/weaviate-go-client/v4/weaviate/auth"
"github.com/weaviate/weaviate-go-client/v4/weaviate/graphql"
"github.com/weaviate/weaviate/entities/models"
)

func init(){
    os.Setenv("WEAVIATE_INSTANCE_URL","localhost:8086")
    os.Setenv("WEAVIATE_SCHEME","http")
    os.Setenv("WEAVIATE_API_KEY","soulteary")
}

func CreateClient(host string, scheme string, key string)(*weaviate.Client,error){
    cfg := weaviate.Config{
Host:       host,
Scheme:     scheme,
AuthConfig: auth.ApiKey{Value: key},
}

    client, err := weaviate.NewClient(cfg)
if err !=nil{
returnnil, err
}

return client,nil
}

func CreateDB(client *weaviate.Client)error{
    classObj :=&models.Class{
Class:"TraditionalFestival",
Vectorizer:"text2vec-transformers",
}

// add the schema
    err := client.Schema().ClassCreator().WithClass(classObj).Do(context.Background())
if err !=nil{
return err
}

    buf, err := os.ReadFile("./traditional-festival.json")
if err !=nil{
return err
}

var items []map[string]string
    err = json.Unmarshal(buf,&items)
if err !=nil{
return err
}

    objects :=make([]*models.Object,len(items))
for i :=range items {
        objects[i]=&models.Object{
Class:"TraditionalFestival",
Properties:map[string]any{
"SolarTerms":  items[i]["SolarTerms"],
"Title":       items[i]["Title"],
"Author":      items[i]["Author"],
"Poem":        items[i]["Poem"],
"Description": items[i]["Description"],
},
}
}

    batchRes, err := client.Batch().ObjectsBatcher().WithObjects(objects...).Do(context.Background())
if err !=nil{
return err
}
for _, res :=range batchRes {
if res.Result.Errors!=nil{
return fmt.Errorf("error: %v", res.Result.Errors)
}
}

returnnil
}

func Query(client *weaviate.Client, queries []string)error{
    st := time.Now()

    fields :=[]graphql.Field{
{Name:"solarTerms"},
{Name:"title"},
{Name:"author"},
{Name:"poem"},
{Name:"description"},
}

    nearText := client.GraphQL().
NearTextArgBuilder().
WithConcepts(queries)

    result, err := client.GraphQL().Get().
WithClassName("TraditionalFestival").
WithFields(fields...).
WithNearText(nearText).
WithLimit(2).
Do(context.Background())
if err !=nil{
return err
}

    fmt.Println()
    fmt.Println("Time:", time.Since(st))
    fmt.Println()

    buf, err := json.Marshal(result)
if err !=nil{
return err
}

    fmt.Printf("%v",string(buf))

returnnil
}

func main(){
    hostURL := os.Getenv("WEAVIATE_INSTANCE_URL")
    scheme := os.Getenv("WEAVIATE_SCHEME")
    apikey := os.Getenv("WEAVIATE_API_KEY")

// create a client
    client, err :=CreateClient(hostURL, scheme, apikey)
if err !=nil{
panic(err)
}

// try to create a database
    err =CreateDB(client)
if err !=nil{
        fmt.Println("CreateDB", err)
}

// try to query
    queries :=[]string{"夏天吃瓜，冰棍也行吧"}
    fmt.Println("Queries", queries)

    err =Query(client, queries)
if err !=nil{
        fmt.Println("Query", err)
}
}

# cd weaviate-quickstart
# go mod tidy

go: downloading gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
go: downloading github.com/kr/pretty v0.3.1
go: downloading github.com/rogpeppe/go-internal v1.11.0
go: downloading github.com/kr/text v0.2.0

Queries [夏天吃瓜，冰棍也行吧]

Time: 13.65638ms

{"data":{"Get":{"TraditionalFestival":[{"author":"杜牧","description":"立冬，代表着冬季开始，万物收藏，规避寒冷。一岁年华，已近尾声。在今年的最后一季里，定要修身养心，持善悦己！立冬丨一候水始冰；二候地始冻；三候雉入大水为蜃。","poem":"淮阳多病偶求欢，客袖侵霜与烛盘。砌下梨花一堆雪，明年谁此凭阑干？","solarTerms":"立冬","title":"初冬夜饮"},{"author":"韩翃","description":"小暑，表示季夏时节的正式开始。天气开始炎热，但还没到最热，故称“小暑”。小暑时节，心静自然凉。眼前无长物，窗下有清风。散热有心静，凉生为室空。小暑丨一候温风至；二候蟋蟀居宇；三候鹰始鸷。","poem":"相思掩泣复何如，公子门前人渐疏。幸有心期当小暑，葛衣纱帽望回车。","solarTerms":"小暑","title":"赠别王侍御赴上都"}]}}}

...
  t2v-transformers:
image:soulteary/t2v-transformers:2024.06.27
environment:
# set to 1 to enable
ENABLE_CUDA:0
ports:
-9090:8080

FROM semitechnologies/transformers-inference:custom

COPY ./thenlper/gte-small /app/models/model

docker build -t soulteary/t2v-transformers:2024.06.27 .

version: "3.4"
services:
weaviate:
# 省略 ...
environment:
# 省略 ...
CLUSTER_HOSTNAME:"node1"
ENABLE_MODULES:"text2vec-transformers"
DEFAULT_VECTORIZER_MODULE:"text2vec-transformers"
TRANSFORMERS_INFERENCE_API:"http://t2v-transformers:8080"

t2v-transformers:
image:soulteary/t2v-transformers:2024.06.27
environment:
# set to 1 to enable
ENABLE_CUDA:0
ports:
-9090:8080

    objects := make([]*models.Object,len(items))
for i :=range items {
        objects[i]=&models.Object{
Class:"TraditionalFestival",
Properties:map[string]any{
"SolarTerms":  items[i]["SolarTerms"],
"Title":       items[i]["Title"],
"Author":      items[i]["Author"],
"Poem":        items[i]["Poem"],
"Description": items[i]["Description"],
},
}
}

    batchRes, err := client.Batch().ObjectsBatcher().WithObjects(objects...).Do(context.Background())

[
  {
"SolarTerms":"立春",
"Title":"木兰花·立春日作",
"Author":"陆游",
"Poem":"春盘春酒年年好，试戴银旛判醉倒。今朝一岁大家添，不是人间偏我老。",
"Description":"《月令七十二候集解》有载：“立，始建也。春气始而建立也。”在生生不息的春风中，一年的序幕由此开启。我们用一颗丰盈而善良的心，向着春意盎然的天地间走去。立春丨一候东风解冻，二候蜇虫始振，三候鱼陟负冰。"
},
...
]

where := filters.Where().
WithPath([]string{"author"}).
WithOperator(filters.Equal).
WithValueString("陆游")

result, err := client.GraphQL().Get().
WithClassName("TraditionalFestival").
WithFields(fields...).
WithNearText(nearText).
WithLimit(2).
WithWhere(where).
Do(context.Background())

# go run main.go
Queries [夏天吃瓜，冰棍也行吧]

Time: 16.928206ms

{"data":{"Get":{"TraditionalFestival":[{"author":"陆游","description":"《月令七十二候集解》有载：“立，始建也。春气始而建立也。”在生生不息的春风中，一年的序幕由此开启。我们用一颗丰盈而善良的心，向着春意盎然的天地间走去。立春丨一候东风解冻，二候蜇虫始振，三候鱼陟负冰。","poem":"春盘春酒年年好，试戴银旛判醉倒。今朝一岁大家添，不是人间偏我老。","solarTerms":"立春","title":"木兰花·立春日作"}]}}}%        

du -hs weaviate_data                       
112K weaviate_data

du -hs traditional-festival.json 
12K traditional-festival.json

du -hs weaviate_data
556K weaviate_data