视频画中画方案

import UIKit
import AVKit
import CoreFoundation
import TXLiteAVSDK_Professional

class PipVC: UIViewController {
    let trtcCloud = TRTCCloud()
    var pipController: AVPictureInPictureController?
    var combinedPixelBuffer: CVPixelBuffer?
    let pixelBufferLock = DispatchQueue(label: "com.demo.pip")
    var pipDisplayLayer: AVSampleBufferDisplayLayer!
}

func enterTrtcRoom() {
    let params = TRTCParams()
    params.sdkAppId = UInt32(SDKAppID)
    params.roomId = UInt32(roomId)
    params.userId = userId
    params.role = .audience
    params.userSig = GenerateTestUserSig.genTestUserSig(identifier: userId) as String

    trtcCloud.addDelegate(self)
    trtcCloud.enterRoom(params, appScene: .LIVE)   
}

func setupAudioSession() {
    do {
        try AVAudioSession.sharedInstance().setCategory(.playback)
    } catch let error {
        print("+> error: \\(error)")
        return
    }

    do {
        try AVAudioSession.sharedInstance().setActive(true)
    } catch let error {
        print("+> error: \\(error)")
        return
    }
}

func enableBGDecode() {
    let param: [String : Any] = ["api": "enableBackgroundDecoding",
                                 "params": ["enable":true]]
    if let jsonData = try? JSONSerialization.data(withJSONObject: param, options: .fragmentsAllowed) {
        let paramJsonString = String.init(data: jsonData, encoding: .utf8) ?? ""
        trtcCloud.callExperimentalAPI(paramJsonString)
    }
}

func setupPipController() {
    let screenWidth = UIScreen.main.bounds.width
    let videoHeight = screenWidth / 2 / 9 * 16
    
    pipDisplayLayer = AVSampleBufferDisplayLayer()
    pipDisplayLayer.frame = CGRect(x: 0, y: 0, width: screenWidth, height: videoHeight) // Adjust size as needed
    pipDisplayLayer.videoGravity = .resizeAspect
    pipDisplayLayer.isOpaque = true
    pipDisplayLayer.backgroundColor = CGColor(red: 0, green: 0, blue: 0, alpha: 1)
    view.layer.addSublayer(pipDisplayLayer)

    if AVPictureInPictureController.isPictureInPictureSupported() {
        let contentSource = AVPictureInPictureController.ContentSource(
            sampleBufferDisplayLayer: pipDisplayLayer,
            playbackDelegate: self
        )
        pipController = AVPictureInPictureController(contentSource: contentSource)
        pipController?.delegate = self
        pipController?.canStartPictureInPictureAutomaticallyFromInline = true
    } else {
        print("+> error")
    }
}

func displayPixelBuffer(_ pixelBuffer: CVPixelBuffer, in layer: AVSampleBufferDisplayLayer) {
    var timing = CMSampleTimingInfo.init(duration: .invalid,
                                         presentationTimeStamp: .invalid,
                                         decodeTimeStamp: .invalid)
    var videoInfo: CMVideoFormatDescription? = nil
    var result = CMVideoFormatDescriptionCreateForImageBuffer(allocator: nil,
                                                              imageBuffer: pixelBuffer,
                                                              formatDescriptionOut: &videoInfo)
    if result != 0 {
        return
    }
    guard let videoInfo = videoInfo else {
        return
    }
    var sampleBuffer: CMSampleBuffer? = nil
    result = CMSampleBufferCreateForImageBuffer(allocator: kCFAllocatorDefault,
                                                imageBuffer: pixelBuffer,
                                                dataReady: true,
                                                makeDataReadyCallback: nil,
                                                refcon: nil,
                                                formatDescription: videoInfo,
                                                sampleTiming: &timing,
                                                sampleBufferOut: &sampleBuffer)
    if result != 0 {
        return
    }
    guard let sampleBuffer = sampleBuffer else {
        return
    }
    guard let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer,
                                                                    createIfNecessary: true) else {
        return
    }
    CFDictionarySetValue(
        unsafeBitCast(CFArrayGetValueAtIndex(attachments, 0), to: CFMutableDictionary.self),
        Unmanaged.passUnretained(kCMSampleAttachmentKey_DisplayImmediately).toOpaque(),
        Unmanaged.passUnretained(kCFBooleanTrue).toOpaque())
        layer.enqueue(sampleBuffer)
    if layer.status == .failed {
        if let error = layer.error as? NSError {
            if error.code == -11847 {
                print("+> error")
            }
        }
    }    
}

extension PipVC: TRTCVideoRenderDelegate {
    func onRenderVideoFrame(_ frame: TRTCVideoFrame, userId: String?, streamType: TRTCVideoStreamType) {
        guard let newPixelBuffer = frame.pixelBuffer else { print("+> error"); return}
        pixelBufferLock.sync {
            if combinedPixelBuffer == nil {
                createCombinedPixelBuffer(from: newPixelBuffer)
            }
            if userId == "left" {
                updateCombinedPixelBuffer(with: newPixelBuffer, forLeft: true)
            } else {
                updateCombinedPixelBuffer(with: newPixelBuffer, forLeft: false)
            }
        }

        if let combinedBuffer = combinedPixelBuffer {
            DispatchQueue.main.async {
                self.displayPixelBuffer(combinedBuffer, in: self.pipDisplayLayer)
            }
        }
    }
}

// 关闭画中画
pipController?.stopPictureInPicture()
// 开启画中画
pipController?.startPictureInPicture()

<activity
    android:name="com.tencent.mlvb.pictureinpicture.PictureInPictureActivity"
    android:theme="@style/Theme.AppCompat.Light.NoActionBar"
    android:configChanges="screenSize|smallestScreenSize|screenLayout|orientation"
    android:supportsPictureInPicture="true"

private void startPictureInPicture() {
    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
        PictureInPictureParams.Builder pictureInPictureBuilder = new PictureInPictureParams.Builder();
        Rational aspectRatio = new Rational(mVideoView.getWidth(), mVideoView.getHeight());
        pictureInPictureBuilder.setAspectRatio(aspectRatio);
        //进入画中画
        enterPictureInPictureMode(pictureInPictureBuilder.build());
    } else {
        Toast.makeText(this, R.string.picture_in_picture_not_supported, Toast.LENGTH_SHORT).show();
    }
}

@Override
public void onPictureInPictureModeChanged(boolean isInPictureInPictureMode, Configuration configuration) {
    super.onPictureInPictureModeChanged(isInPictureInPictureMode, configuration);
    if (isInPictureInPictureMode) {
       //进入画中画后，需要隐藏的view
    } else{
       //退出画中画后，需要显示的view
    }
}

// mLivePlayer 对应左边的视频View（TXCloudVideoView），设置了V2TXLiveFillModeFit
mLivePlayer.startLivePlay(PLAY_URL);
mLivePlayer.setRenderFillMode(V2TXLiveDef.V2TXLiveFillMode.V2TXLiveFillModeFit);

// mLivePlayer2 对应右边的视频View（TXCloudVideoView）
mLivePlayer2.startLivePlay(PLAY_URL);

<com.tencent.rtmp.ui.TXCloudVideoView
    android:id="@+id/video_view"
    android:layout_width="192dp"
    android:layout_height="108dp"
    android:layout_alignParentStart="true"
    android:background="#00BCD4"/>

<com.tencent.rtmp.ui.TXCloudVideoView
    android:id="@+id/video_view2"
    android:layout_width="192dp"
    android:layout_height="108dp"
    android:layout_alignTop="@+id/video_view"
    android:layout_toEndOf="@+id/video_view"
    android:background="#3F51B5"/>

@Override
public void onPictureInPictureModeChanged(boolean isInPictureInPictureMode, Configuration configuration) {
    super.onPictureInPictureModeChanged(isInPictureInPictureMode, configuration);
    if (isInPictureInPictureMode) {
        // 设置mVideoView 的宽为 100dp
        RelativeLayout.LayoutParams layoutParams = (RelativeLayout.LayoutParams) mVideoView.getLayoutParams();
        layoutParams.width = (int) TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_DIP, 100, getResources().getDisplayMetrics());
    } else {
        // 退出画中画，还原video_view的宽度
        RelativeLayout.LayoutParams layoutParams = (RelativeLayout.LayoutParams) mVideoView.getLayoutParams();
        layoutParams.width = (int) TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_DIP, 192, getResources().getDisplayMetrics());
    }
}

trtcCloud.callExperimentalAPI(jsonEncode({
    "api": "enablePictureInPictureFloatingWindow",
    "params": {"enable": true}
}));

var pipCode = await _livePlayer!.enablePictureInPicture(true);
if (pipCode != V2TXLIVE_OK) {
  print("error: $pipCode");
}

final channel = MethodChannel('flutter_ios_pip_demo');

await channel.invokeMethod('enablePip', {
                            'marginTop': appBarHeight + topSafeAreaHeight,
                            'pkLeft': pkLeftUserId,
                            'pkRight': pkRightUserId,
                          });