利用人工智能和机器人技术实现复杂的自动化任务！

User:
"These are frames from a video that I want to upload. Generate a compelling description that I can upload along with the video."

#GPT-4V对图像的描述处理
GPT4:
"🐺 Survival of the Fittest: An Epic Tale in the Snow ❄️ - Witness the intense drama of nature as a pack of wolves face off against mighty bison in a harsh winter landscape. This raw footage captures the essence of the wild where every creature fights for survival. With each frame, experience the tension, the strategy, and the sheer force exerted in this life-or-death struggle. See nature's true colors in this gripping encounter on the snowy plains. 🦬"
 
Remember to respect wildlife and nature. This video may contain scenes that some viewers might find intense or distressing, but they depict natural animal behaviors important for ecological studies and understanding the reality of life in the wilderness.

import io
import os
from enum import Enum
from typing import Protocol
 
import openai
import speech_recognition as sr
from pydub import AudioSegment
from pydub.playback import play

class Interface(Protocol):
    def input(self, prefix: str = "") -> str:
        return prefix + self._input_impl()
 
    def _input_impl(self) -> str:
        ...
 
    def output(self, message: str) -> None:
        ...

class Audio(Interface):
    def __init__(self):
        self.r = sr.Recognizer()
        self.mic = sr.Microphone()
        # openai-whisper API key
        self.client = openai.OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))

 def _input_impl(self) -> str:
        print("Please tell me your command.")
        with self.mic as source:
            self.r.adjust_for_ambient_noise(source)
            audio = self.r.listen(source)
 
        try:
            return self.r.recognize_whisper(audio, language="japanese")
 
        except sr.UnknownValueError:
            print("could not understand audio")
        except sr.RequestError as e:
            print("Could not request results from Google Speech Recognition service; {0}".format(e))

import cv2
import numpy as np
import supervision as sv
import torch
from segment_anything import SamAutomaticMaskGenerator, sam_model_registry
 
from .utils import download_sam_model_to_cache

#将图像转化为RGB格式
        image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
#对图像处理，进行目标检测和标记的绘制
        sam_result = self.mask_generator.generate(image_rgb)
        detections = sv.Detections.from_sam(sam_result=sam_result)
        height, width, _ = image.shape
        image_area = height * width
 
        min_area_mask = (detections.area / image_area) > self.MIN_AREA_PERCENTAGE
        max_area_mask = (detections.area / image_area) < self.MAX_AREA_PERCENTAGE
        detections = detections[min_area_mask & max_area_mask]
        
        
        #返回图片和检测的信息的结果
        labels = [str(i) for i in range(len(detections))]
        annotated_image = mask_annotator.annotate(scene=image_rgb.copy(), detections=detections)
        annotated_image = label_annotator.annotate(scene=annotated_image, detections=detections, labels=labels)
        return annotated_image, detections
 

def prepare_inputs(message: str, image: np.ndarray) -> dict:
    # # Path to your image
    # image_path = "temp.jpg"
    # # Getting the base64 string
    base64_image = encode_image_from_cv2(image)
 
    payload = {
        "model": "gpt-4-vision-preview",
        "messages": [
            {"role": "system", "content": [metaprompt]},
            {
                "role": "user",
                "content": [
                    {
                        "type": "text",
                        "text": message,
                    },
                    {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}},
                ],
            },
        ],
        "max_tokens": 800,
    }
 
    return payload
 
 
def request_gpt4v(message: str, image: np.ndarray) -> str:
    payload = prepare_inputs(message, image)
    response = requests.post("https://api.openai.com/v1/chat/completions", headers=headers, json=payload)
    res = response.json()["choices"][0]["message"]["content"]
    return res

    def move_to_object(self, object_no: int, speed: Optional[float] = None) -> None:
        object_no = self._check_and_correct_object_no(object_no)
        print("[MyCobotController] Move to Object No. {}".format(object_no))
        detection = (
            np.array([-self._detections[object_no][0], -self._detections[object_no][1]]) + self.capture_coord.pos[:2]
        )
        print("[MyCobotController] Object pos:", detection[0], detection[1])
        self.move_to_xy(detection[0], detection[1], speed)

    def grab(self, speed: Optional[float] = None) -> None:
        print("[MyCobotController] Grab to Object")
        current_pos = self.current_coords().pos
        self.move_to_z(self.object_height + self.end_effector_height, speed)
        self._mycobot.set_basic_output(self._suction_pin, 0)
        time.sleep(2)
        self.move_to_z(current_pos[2], speed)

    def move_to_place(self, place_name: str, speed: Optional[float] = None) -> None:
        print("[MyCobotController] Move to Place {}".format(place_name))
        self._current_position = self.positions[place_name]
        self._mycobot.sync_send_angles(
            np.array(self._current_position) + self.calc_gravity_compensation(self._current_position),
            speed or self._default_speed,
            self._command_timeout,
        )
        print("Current coords: {}".format(self.current_coords()))