KubeSphere 最佳实战：KubeSphere 部署向量数据库 Milvus（GitHub Star 28.5k）实战指南

mkdir test-data
fio --rw=write --ioengine=sync --fdatasync=1 --directory=test-data --size=2200m --bs=2300 --name=mytest

kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: test-nfs-fio-pvc
spec:
  storageClassName: nfs-sc
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 1Gi

kubectl apply -f test-nfs-fio-pvc.yaml

kind: Pod
apiVersion: v1
metadata:
  name: test-nfs-fio-pod
spec:
  containers:
  - name: test-nfs-fio-pod
    image: openebs/tests-fio:latest
    command:
      - "/bin/sh"
    args:
      - "-c"
      - "touch /mnt/SUCCESS && sleep 86400"
    volumeMounts:
      - name: nfs-fio-pvc
        mountPath: "/mnt"
  restartPolicy: "Never"
  volumes:
    - name: nfs-fio-pvc
      persistentVolumeClaim:
        claimName: test-nfs-fio-pvc

kubectl apply -f test-nfs-fio-pod.yaml

kubectl exec -it test-nfs-fio-pod -- /bin/bash

root@test-nfs-fio-pod:/# df -h
Filesystem                                                                                   Size  Used Avail Use% Mounted on
overlay                                                                                      100G  6.5G   94G   7% /
tmpfs                                                                                         64M     0   64M   0% /dev
tmpfs                                                                                        7.6G     0  7.6G   0% /sys/fs/cgroup
192.168.9.97:/datanfs/k8s/default-test-nfs-fio-pvc-pvc-7158aa17-f003-47a2-b6d6-07c69d014178  100G  746M  100G   1% /mnt
/dev/mapper/openeuler-root                                                                    35G  2.3G   31G   7% /etc/hosts
/dev/mapper/data-lvdata                                                                      100G  6.5G   94G   7% /etc/hostname
shm                                                                                           64M     0   64M   0% /dev/shm
tmpfs                                                                                         14G   12K   14G   1% /run/secrets/kubernetes.io/serviceaccount
tmpfs                                                                                        7.6G     0  7.6G   0% /proc/acpi
tmpfs                                                                                        7.6G     0  7.6G   0% /proc/scsi
tmpfs                                                                                        7.6G     0  7.6G   0% /sys/firmware

mkdir /mnt/test-data
fio --rw=write --ioengine=sync --fdatasync=1 --directory=/mnt/test-data --size=2200m --bs=2300 --name=nfstest

root@test-nfs-fio-pod:/# fio --rw=write --ioengine=sync --fdatasync=1 --directory=/mnt/test-data --size=2200m --bs=2300 --name=nfstest
nfstest: (g=0): rw=write, bs=2300-2300/2300-2300/2300-2300, ioengine=sync, iodepth=1
fio-2.2.10
Starting 1 process
nfstest: Laying out IO file(s) (1 file(s) / 2200MB)
Jobs: 1 (f=1): [W(1)] [100.0% done] [0KB/956KB/0KB /s] [0/426/0 iops] [eta 00m:00s]
nfstest: (groupid=0, jobs=1): err= 0: pid=26: Tue Jul 23 02:55:35 2024
  write: io=2199.2MB, bw=870003B/s, iops=378, runt=2651558msec
    clat (usec): min=4, max=87472, avg=224.30, stdev=309.99
     lat (usec): min=4, max=87473, avg=224.71, stdev=310.00
    clat percentiles (usec):
     |  1.00th=[    7],  5.00th=[    8], 10.00th=[    8], 20.00th=[    9],
     | 30.00th=[   10], 40.00th=[   12], 50.00th=[  302], 60.00th=[  338],
     | 70.00th=[  366], 80.00th=[  398], 90.00th=[  442], 95.00th=[  482],
     | 99.00th=[  588], 99.50th=[  700], 99.90th=[ 1848], 99.95th=[ 3120],
     | 99.99th=[ 7008]
    bw (KB  /s): min=    1, max= 1078, per=100.00%, avg=873.03, stdev=221.69
    lat (usec) : 10=25.84%, 20=16.33%, 50=1.64%, 100=0.03%, 250=0.23%
    lat (usec) : 500=52.30%, 750=3.19%, 1000=0.15%
    lat (msec) : 2=0.20%, 4=0.05%, 10=0.03%, 20=0.01%, 50=0.01%
    lat (msec) : 100=0.01%
  cpu          : usr=0.38%, sys=1.56%, ctx=1567342, majf=0, minf=12
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=1002985/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1

Run status group 0 (all jobs):
  WRITE: io=2199.2MB, aggrb=849KB/s, minb=849KB/s, maxb=849KB/s, mint=2651558msec, maxt=2651558msec

helm repo add milvus https://zilliztech.github.io/milvus-helm/

helm repo update milvus				

helm install my-release milvus/milvus

helm install --namespace opsxlab --create-namespace opsxlab milvus/milvus \
    --set image.all.repository="milvusdb/milvus" \
    --set image.tools.repository="milvusdb/milvus-config-tool" \
    --set etcd.image.registry="docker.io" \
    --set etcd.image.repository="milvusdb/etcd" \
    --set pulsar.images.broker.repository="milvusdb/pulsar" \
    --set pulsar.images.autorecovery.repository="milvusdb/pulsar" \
    --set pulsar.images.zookeeper.repository="milvusdb/pulsar" \
    --set pulsar.images.bookie.repository="milvusdb/pulsar" \
    --set pulsar.images.proxy.repository="milvusdb/pulsar" \
    --set pulsar.images.pulsar_manager.repository="milvusdb/pulsar-manager" \
    --set pulsar.pulsar_metadata.image.repository="milvusdb/pulsar" \
    --set minio.image.repository="milvusdb/minio" \
    --set etcd.persistence.storageClass="nfs-sc" \
    --set minio.persistence.storageClass="nfs-sc" \
    --set pulsar.zookeeper.volumes.data.storageClassName="nfs-sc" \
    --set pulsar.bookkeeper.volumes.journal.storageClassName="nfs-sc"

kubectl get pods -n opsxlab

$ kubectl get pods -n opsxlab
NAME                                         READY   STATUS      RESTARTS        AGE
opsxlab-etcd-0                               1/1     Running     0               8m9s
opsxlab-etcd-1                               1/1     Running     0               8m20s
opsxlab-etcd-2                               1/1     Running     0               11m
opsxlab-milvus-datacoord-6cd875684d-fbkgx    1/1     Running     6 (9m19s ago)   13m
opsxlab-milvus-datanode-5c5558cbc7-r24vj     1/1     Running     6 (9m40s ago)   13m
opsxlab-milvus-indexcoord-f48d66647-52crb    1/1     Running     0               13m
opsxlab-milvus-indexnode-c6979d59b-rvm5p     1/1     Running     6 (9m44s ago)   13m
opsxlab-milvus-proxy-79997676f9-hftdf        1/1     Running     6 (9m22s ago)   13m
opsxlab-milvus-querycoord-5d94f97dc4-tv52n   1/1     Running     6 (9m43s ago)   13m
opsxlab-milvus-querynode-59b9bd7c8b-7qljj    1/1     Running     6 (9m26s ago)   13m
opsxlab-milvus-rootcoord-745b9fbb68-mr7st    1/1     Running     6 (9m36s ago)   13m
opsxlab-minio-0                              1/1     Running     0               13m
opsxlab-minio-1                              1/1     Running     0               13m
opsxlab-minio-2                              1/1     Running     0               13m
opsxlab-minio-3                              1/1     Running     0               13m
opsxlab-pulsar-bookie-0                      1/1     Running     0               13m
opsxlab-pulsar-bookie-1                      1/1     Running     0               13m
opsxlab-pulsar-bookie-2                      1/1     Running     0               13m
opsxlab-pulsar-bookie-init-7l4kh             0/1     Completed   0               13m
opsxlab-pulsar-broker-0                      1/1     Running     0               13m
opsxlab-pulsar-proxy-0                       1/1     Running     0               13m
opsxlab-pulsar-pulsar-init-ntbh8             0/1     Completed   0               13m
opsxlab-pulsar-recovery-0                    1/1     Running     0               13m
opsxlab-pulsar-zookeeper-0                   1/1     Running     0               13m
opsxlab-pulsar-zookeeper-1                   1/1     Running     0               13m
opsxlab-pulsar-zookeeper-2                   1/1     Running     0               12m

extraConfigFiles:
  user.yaml: |+
    common:
      security:
        authorizationEnabled: true

helm upgrade opsxlab milvus/milvus --values custom-milvus.yaml -n opsxlab

kind: Service
apiVersion: v1
metadata:
  name: opsxlab-milvus-external
  namespace: opsxlab
  labels:
    app: opsxlab-milvus-external
    app.kubernetes.io/name: milvus
spec:
  ports:
    - name: milvus
      protocol: TCP
      port: 19530
      targetPort: 19530
      nodePort: 31011
  selector:
    app.kubernetes.io/instance: opsxlab
    app.kubernetes.io/name: milvus
    component: proxy
  clusterIP: 
  type: NodePort

kubectl apply -f milvus-external-svc.yaml

wget https://raw.githubusercontent.com/milvus-io/pymilvus/master/examples/hello_milvus.py

kind: Pod
apiVersion: v1
metadata:
  name: test-milvus-pod
spec:
  containers:
  - name: test-milvus-pod
    image: opsxlab/milvus-hello:v2.3.7-amd64
    command:
      - "/bin/sh"
    args:
      - "-c"
      - "sleep 86400"
  restartPolicy: "Never"

kubectl apply -f test-milvus-pod.yam

kubectl exec -it test-milvus-pod -- /bin/bash

# 原内容（32行）
 connections.connect("default", host="localhost", port="19530")

# 修改为
connections.connect("default", host="192.168.9.91", port="31011", user="root", password='Milvus')

python hello_milvus.py

root@test-milvus-pod:/app# python hello_milvus.py

=== start connecting to Milvus     ===

Does collection hello_milvus exist in Milvus: False

=== Create collection `hello_milvus` ===


=== Start inserting entities       ===

Number of entities in Milvus: 3001

=== Start Creating index IVF_FLAT  ===


=== Start loading                  ===


=== Start searching based on vector similarity ===

hit: id: 2998, distance: 0.0, entity: {'random': 0.9728033590489911}, random field: 0.9728033590489911
hit: id: 999, distance: 0.09934990108013153, entity: {'random': 0.9519034206569449}, random field: 0.9519034206569449
hit: id: 1310, distance: 0.10135537385940552, entity: {'random': 0.26669865443188623}, random field: 0.26669865443188623
hit: id: 2999, distance: 0.0, entity: {'random': 0.02316334456872482}, random field: 0.02316334456872482
hit: id: 2502, distance: 0.13083189725875854, entity: {'random': 0.9289998713260136}, random field: 0.9289998713260136
hit: id: 2669, distance: 0.1590736210346222, entity: {'random': 0.6080847854541138}, random field: 0.6080847854541138
search latency = 0.2742s

=== Start querying with `random > 0.5` ===

query result:
-{'random': 0.6378742006852851, 'embeddings': [np.float32(0.8367804), np.float32(0.20963514), np.float32(0.6766955), np.float32(0.39746654), np.float32(0.8180806), np.float32(0.1201905), np.float32(0.9467144), np.float32(0.6947491)], 'pk': '0'}
search latency = 0.2361s
query pagination(limit=4):
        [{'random': 0.6378742006852851, 'pk': '0'}, {'random': 0.5763523024650556, 'pk': '100'}, {'random': 0.9425935891639464, 'pk': '1000'}, {'random': 0.7893211256191387, 'pk': '1001'}]
query pagination(offset=1, limit=3):
        [{'random': 0.5763523024650556, 'pk': '100'}, {'random': 0.9425935891639464, 'pk': '1000'}, {'random': 0.7893211256191387, 'pk': '1001'}]

=== Start hybrid searching with `random > 0.5` ===

hit: id: 2998, distance: 0.0, entity: {'random': 0.9728033590489911}, random field: 0.9728033590489911
hit: id: 999, distance: 0.09934990108013153, entity: {'random': 0.9519034206569449}, random field: 0.9519034206569449
hit: id: 1553, distance: 0.12913644313812256, entity: {'random': 0.7723335927084438}, random field: 0.7723335927084438
hit: id: 2502, distance: 0.13083189725875854, entity: {'random': 0.9289998713260136}, random field: 0.9289998713260136
hit: id: 2669, distance: 0.1590736210346222, entity: {'random': 0.6080847854541138}, random field: 0.6080847854541138
hit: id: 2628, distance: 0.1914074569940567, entity: {'random': 0.940077754658375}, random field: 0.940077754658375
search latency = 0.2026s

=== Start deleting with expr `pk in ["0" , "1"]` ===

query before delete by expr=`pk in ["0" , "1"]` -> result:
-{'embeddings': [np.float32(0.8367804), np.float32(0.20963514), np.float32(0.6766955), np.float32(0.39746654), np.float32(0.8180806), np.float32(0.1201905), np.float32(0.9467144), np.float32(0.6947491)], 'pk': '0', 'random': 0.6378742006852851}
-{'embeddings': [np.float32(0.27875876), np.float32(0.95355743), np.float32(0.976228), np.float32(0.54545516), np.float32(0.16776836), np.float32(0.82360446), np.float32(0.65080017), np.float32(0.21096307)], 'pk': '1', 'random': 0.43925103574669633}

query after delete by expr=`pk in ["0" , "1"]` -> result: []

=== Drop collection `hello_milvus` ===

apiVersion: v1
kind: Service
metadata:
  name: milvus-attu-external
  namespace: opsxlab
  labels:
    app: attu
spec:
  type: NodePort
  clusterIP:
  ports:
  - name: attu
    protocol: TCP
    port: 3000
    targetPort: 3000
    nodePort: 31012
  selector:
    app: attu

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: milvus-attu
  namespace: opsxlab
  labels:
    app: attu
spec:
  replicas: 1
  selector:
    matchLabels:
      app: attu
  template:
    metadata:
      labels:
        app: attu
    spec:
      containers:
      - name: attu
        image: zilliz/attu:v2.4
        imagePullPolicy: IfNotPresent
        ports:
        - name: attu
          containerPort: 3000
          protocol: TCP
        env:
        - name: MILVUS_URL
          value: "opsxlab-milvus:19530"

kubectl apply -f milvus-attu.yaml