我有一个应用程序,可以播放从用户库中选择的视频。我打算这个应用程序最终能够做的是渲染一个覆盖在视频(当它正在播放),然后输出到一个新的媒体文件。为了做到这一点,我需要捕获解码的帧,以便我可以渲染这个覆盖和输出到一个文件,一旦视频播放已经结束。
这是我使用AVFoundation的第一个应用程序,我花了一两天的时间试图通过谷歌和苹果文档找到实现这一目标的方法,我还以为我在AVPlayerItemVideoOutput对象中有一些东西。但是,从来不执行委托回调。
我发现AVPlayerItemVideoOutput必须在AVPlayerItem处于状态readyToPlay之后创建。因此,在我的PlayerUIView的初始值中,我向AVPlayerItem添加了一个观察者,以观察它的状态。
init(frame: CGRect, url: Binding<URL?>) {
_url = url
// Setup the player
player = AVPlayer(url: url.wrappedValue!)
super.init(frame: frame)
playerLayer.player = player
playerLayer.videoGravity = .resizeAspect
layer.addSublayer(playerLayer)
//displayLink = CADisplayLink()
// Setup looping
player.actionAtItemEnd = .none
NotificationCenter.default.addObserver(self,
selector: #selector(playerItemDidReachEnd(notification:)),
name: .AVPlayerItemDidPlayToEndTime,
object: player.currentItem)
player.currentItem?.addObserver(self, forKeyPath: #keyPath(AVPlayerItem.status), options: [.old, .new], context: nil)
// Start the movie
player.play()
}我在中间创建了一个CADisplayLink --注释掉了--因为我看到它可以在某种程度上被用于这个,但不完全确定它应该如何做或应该做什么。我还关心的是,它从显示的视频中获取帧,而不是从实际解码的视频帧中获取帧,这就是我想要的。
当状态第一次设置为readyToPlay时,我创建并添加AVPlayerItemVideoOutput。
override func observeValue(forKeyPath keyPath: String?, of object: Any?, change: [NSKeyValueChangeKey : Any]?, context: UnsafeMutableRawPointer?) {
if let item = object as? AVPlayerItem {
if item.status == AVPlayerItem.Status.readyToPlay && item.outputs.count == 0 {
let settings = [ String(kCVPixelBufferPixelFormatTypeKey): kCVPixelFormatType_24RGB ]
let output = AVPlayerItemVideoOutput(pixelBufferAttributes: settings)
output.setDelegate(PlayerOutput(output: output), queue: DispatchQueue(label: ""))
player.currentItem?.add(output)
}
}
}在委托PlayerOutput上,我希望在新帧可用时得到通知。此时,我将访问AVPlayerItemVideoOutput对象以访问像素缓冲区。
class PlayerOutput : NSObject, AVPlayerItemOutputPullDelegate {
func outputMediaDataWillChange(_ sender: AVPlayerItemOutput) {
let videoOutput = sender as! AVPlayerItemVideoOutput
let newPixelBuff = videoOutput.hasNewPixelBuffer(forItemTime: CMTime(seconds: 1, preferredTimescale: 10))
}
}然而,这个回调是永远不会进行的。我在代码中设置了一个断点,它永远不会被击中。从AVFoundation中其他地方的命名和类似代码中,我认为每一个新帧都会被击中,这样我就可以访问缓冲区中的帧,但我没有看到任何事情发生。我错过了什么或者做错了什么吗?
我有一种感觉,我没有很好地使用/理解这些类以及它们的用途,但在命名上类似于AVCaptureVideoDataOutput等类,我在应用程序的其他地方成功地实现了这些类,它们的工作方式似乎不太一样。很难找到做我想用AVPlayer做的事情的例子。
编辑:当前代码的工作示例:
import SwiftUI
import AVFoundation
struct CustomCameraPhotoView: View {
@State private var image: Image?
@State private var showingCustomCamera = false
@State private var showImagePicker = false
@State private var inputImage: UIImage?
@State private var url: URL?
var body: some View {
ZStack {
if url != nil
{
PlayerView(url: $url)
}
else
{
Button(action: {
self.showImagePicker = true
}) {
Text("Select a Video").foregroundColor(.white).font(.headline)
}
}
}.edgesIgnoringSafeArea(.all)
.sheet(isPresented: $showImagePicker,
onDismiss: loadImage) {
PhotoCaptureView(showImagePicker: self.$showImagePicker, image: self.$image, url: self.$url)
}.edgesIgnoringSafeArea(.leading).edgesIgnoringSafeArea(.trailing)
}
func loadImage() {
guard let inputImage = inputImage else { return }
image = Image(uiImage: inputImage)
}
}
struct PlayerView: UIViewControllerRepresentable {
@Binding var url: URL?
func updateUIViewController(_ uiView: UIViewController, context: UIViewControllerRepresentableContext<PlayerView>) {
}
func makeCoordinator() -> PlayerCoordinator{
//Make Coordinator which will commnicate with the ImagePickerViewController
PlayerCoordinator()
}
func makeUIViewController(context: Context) -> UIViewController {
let view = PlayerUIView(frame: .zero, url: $url)
let controller = PlayerController()
controller.view = view
return controller
}
}
class PlayerCoordinator : NSObject, UINavigationControllerDelegate {
}
class PlayerController: UIViewController {
override var shouldAutorotate: Bool {
return false
}
override var supportedInterfaceOrientations: UIInterfaceOrientationMask {
return .all
}
}
class PlayerUIView: UIView {
private let playerLayer = AVPlayerLayer()
private var playerOutput = PlayerOutput()
private let _myVideoOutputQueue = DispatchQueue(label: "VideoFrames", qos: .background, attributes: .concurrent, autoreleaseFrequency: .workItem, target: nil)
var displayLink: CADisplayLink?
var player: AVPlayer
@Binding var url: URL?
required init?(coder: NSCoder) {
fatalError("init(coder:) has not been implemented")
}
init(frame: CGRect, url: Binding<URL?>) {
_url = url
// Setup the player
player = AVPlayer(url: url.wrappedValue!)
super.init(frame: frame)
playerLayer.player = player
playerLayer.videoGravity = .resizeAspect
layer.addSublayer(playerLayer)
let settings = [ String(kCVPixelBufferPixelFormatTypeKey): kCVPixelFormatType_32BGRA ]
let output = AVPlayerItemVideoOutput(pixelBufferAttributes: settings)
output.setDelegate(self.playerOutput, queue: self._myVideoOutputQueue)
player.currentItem?.add(output)
//displayLink = CADisplayLink()
// Setup looping
player.actionAtItemEnd = .none
NotificationCenter.default.addObserver(self,
selector: #selector(playerItemDidReachEnd(notification:)),
name: .AVPlayerItemDidPlayToEndTime,
object: player.currentItem)
player.currentItem?.addObserver(self, forKeyPath: #keyPath(AVPlayerItem.status), options: [.old, .new], context: nil)
// Start the movie
player.play()
}
override func observeValue(forKeyPath keyPath: String?, of object: Any?, change: [NSKeyValueChangeKey : Any]?, context: UnsafeMutableRawPointer?) {
if let item = object as? AVPlayerItem {
if item.status == AVPlayerItem.Status.readyToPlay && item.outputs.count == 0 {
}
}
}
@objc
func playerItemDidReachEnd(notification: Notification) {
self.url = nil
}
override func layoutSubviews() {
super.layoutSubviews()
playerLayer.frame = bounds
}
class PlayerOutput : NSObject, AVPlayerItemOutputPullDelegate {
func outputMediaDataWillChange(_ sender: AVPlayerItemOutput) {
let videoOutput = sender as! AVPlayerItemVideoOutput
let newPixelBuff = videoOutput.hasNewPixelBuffer(forItemTime: CMTime(seconds: 1, preferredTimescale: 10))
}
}
}
struct ImagePicker : UIViewControllerRepresentable {
@Binding var isShown : Bool
@Binding var image : Image?
@Binding var url : URL?
func updateUIViewController(_ uiViewController: UIImagePickerController, context: UIViewControllerRepresentableContext<ImagePicker>)
{
//Update UIViewcontrolleer Method
}
func makeCoordinator() -> ImagePickerCoordinator{
//Make Coordinator which will commnicate with the ImagePickerViewController
ImagePickerCoordinator(isShown: $isShown, image: $image, url: $url)
}
func makeUIViewController(context: UIViewControllerRepresentableContext<ImagePicker>) -> UIImagePickerController
{
let picker = UIImagePickerController()
picker.sourceType = .photoLibrary
picker.delegate = context.coordinator
picker.mediaTypes = ["public.movie"]
picker.videoQuality = .typeHigh
return picker
}
}
class ImagePickerCoordinator : NSObject, UINavigationControllerDelegate, UIImagePickerControllerDelegate{
@Binding var isShown : Bool
@Binding var image : Image?
@Binding var url: URL?
init(isShown : Binding<Bool>, image: Binding<Image?>, url: Binding<URL?>) {
_isShown = isShown
_image = image
_url = url
}
//Selected Image
func imagePickerController(_ picker: UIImagePickerController, didFinishPickingMediaWithInfo info: [UIImagePickerController.InfoKey : Any]) {
let uiImage = info[UIImagePickerController.InfoKey.mediaURL] as! URL
url = uiImage
//image = Image(uiImage: uiImage)
isShown = false
}
//Image selection got cancel
func imagePickerControllerDidCancel(_ picker: UIImagePickerController) {
isShown = false
}
}
struct PhotoCaptureView: View {
@Binding var showImagePicker : Bool
@Binding var image : Image?
@Binding var url : URL?
var body: some View {
ImagePicker(isShown: $showImagePicker, image: $image, url: $url)
}
}
struct ContentView_Previews: PreviewProvider {
static var previews: some View {
CustomCameraPhotoView()
}
}发布于 2020-06-26 09:42:24
编辑:,你是对的,怀疑outputMediaDataWillChange和AVCaptureVideoDataOutput完全不同。它不是每帧调用,而是指示一些球员状态的变化(我有点不清楚到底是什么情况)。有两个选项(至少)读取和修改视频输出。
videoComposition添加到当前的playerItem中,在其中可以使用CoreImage工具来调整视频(例如,添加一个覆盖层)。在这个例子中,我对输出应用了一个简单的模糊过滤器,但是CoreImage允许您做更复杂的事情。let blurFilter = CIFilter(name: "CIGaussianBlur")
if let playerItem = player.currentItem {
let asset = playerItem.asset
playerItem.videoComposition = AVMutableVideoComposition(asset: asset) { (filteringRequest) in
let source = filteringRequest.sourceImage
blurFilter?.setValue(source, forKey: kCIInputImageKey)
filteringRequest.finish(with: blurFilter?.outputImage ?? source, context: nil)
}
}CADisplayLink。显示链接允许您将操作与显示刷新速率同步。您可以以下列方式从视频输出中获取帧:lazy var displayLink: CADisplayLink = CADisplayLink(target: self, selector: #selector(displayLinkDidRefresh(link:)))
init(frame: CGRect, url: Binding<URL?>) {
...
// activate the displayLink
displayLink.add(to: .main, forMode: .common)
...
}
@objc func displayLinkDidRefresh(link: CADisplayLink) {
guard let videoOutput = self.videoOutput else { return }
let itemTime = player.currentTime()
if videoOutput.hasNewPixelBuffer(forItemTime: itemTime) {
var presentationItemTime: CMTime = .zero
if let pixelBuffer = videoOutput.copyPixelBuffer(forItemTime: itemTime, itemTimeForDisplay: &presentationItemTime) {
// process the pixelbuffer here
}
}
}完全极小示例:
import SwiftUI
import AVFoundation
struct CustomCameraPhotoView: View {
@State private var image: Image?
@State private var showingCustomCamera = false
@State private var showImagePicker = false
@State private var inputImage: UIImage?
@State private var url: URL?
var body: some View {
ZStack {
if url != nil {
PlayerView(url: $url)
} else {
Button(action: {
self.showImagePicker = true
}) {
Text("Select a Video")
.font(.headline)
}
}
} .edgesIgnoringSafeArea(.all)
.sheet(isPresented: $showImagePicker,
onDismiss: loadImage) {
PhotoCaptureView(showImagePicker: self.$showImagePicker, image: self.$image, url: self.$url)
}
.edgesIgnoringSafeArea(.leading).edgesIgnoringSafeArea(.trailing)
}
func loadImage() {
guard let inputImage = inputImage else { return }
image = Image(uiImage: inputImage)
}
}
struct PlayerView: UIViewControllerRepresentable {
@Binding var url: URL?
func updateUIViewController(_ uiView: UIViewController, context: UIViewControllerRepresentableContext<PlayerView>) {
}
func makeUIViewController(context: Context) -> UIViewController {
let view = PlayerUIView(frame: .zero, url: $url)
let controller = PlayerController()
controller.view = view
return controller
}
}
class PlayerController: UIViewController {
override var shouldAutorotate: Bool { false }
override var supportedInterfaceOrientations: UIInterfaceOrientationMask { .all }
}
class PlayerUIView: UIView {
private let playerLayer = AVPlayerLayer()
private let _myVideoOutputQueue = DispatchQueue(label: "VideoFrames", qos: .background, attributes: .concurrent, autoreleaseFrequency: .workItem, target: nil)
lazy var displayLink: CADisplayLink = CADisplayLink(target: self, selector: #selector(displayLinkDidRefresh(link:)))
var player: AVPlayer
var videoOutput: AVPlayerItemVideoOutput
@Binding var url: URL?
required init?(coder: NSCoder) {
fatalError("init(coder:) has not been implemented")
}
init(frame: CGRect, url: Binding<URL?>) {
_url = url
// Setup the player
player = AVPlayer(url: url.wrappedValue!)
let settings = [ String(kCVPixelBufferPixelFormatTypeKey): kCVPixelFormatType_32BGRA ]
let output = AVPlayerItemVideoOutput(pixelBufferAttributes: settings)
self.videoOutput = output
super.init(frame: frame)
playerLayer.player = player
playerLayer.videoGravity = .resizeAspect
layer.addSublayer(playerLayer)
attachVideoComposition()
player.currentItem?.add(output)
displayLink.add(to: .main, forMode: .common)
// Start the movie
player.play()
}
private func attachVideoComposition() {
let blurFilter = CIFilter(name: "CIGaussianBlur")
if let playerItem = player.currentItem {
let asset = playerItem.asset
playerItem.videoComposition = AVMutableVideoComposition(asset: asset) { (filteringRequest) in
let source = filteringRequest.sourceImage
blurFilter?.setValue(source, forKey: kCIInputImageKey)
// Apply CoreImage provessing here
filteringRequest.finish(with: blurFilter?.outputImage ?? source, context: nil)
}
}
}
override func observeValue(forKeyPath keyPath: String?, of object: Any?, change: [NSKeyValueChangeKey : Any]?, context: UnsafeMutableRawPointer?) {
if let item = object as? AVPlayerItem {
if item.status == AVPlayerItem.Status.readyToPlay && item.outputs.count == 0 {
}
}
}
override func layoutSubviews() {
super.layoutSubviews()
playerLayer.frame = bounds
}
@objc func displayLinkDidRefresh(link: CADisplayLink) {
let itemTime = player.currentTime()
if videoOutput.hasNewPixelBuffer(forItemTime: itemTime) {
var presentationItemTime: CMTime = .zero
if let pixelBuffer = videoOutput.copyPixelBuffer(forItemTime: itemTime, itemTimeForDisplay: &presentationItemTime) {
// process the pixelbuffer here
print(pixelBuffer)
}
}
}
}
struct ImagePicker : UIViewControllerRepresentable {
@Binding var isShown : Bool
@Binding var image : Image?
@Binding var url : URL?
func updateUIViewController(_ uiViewController: UIImagePickerController, context: UIViewControllerRepresentableContext<ImagePicker>) {
//Update UIViewcontrolleer Method
}
func makeCoordinator() -> ImagePickerCoordinator{
ImagePickerCoordinator(isShown: $isShown, image: $image, url: $url)
}
func makeUIViewController(context: UIViewControllerRepresentableContext<ImagePicker>) -> UIImagePickerController {
let picker = UIImagePickerController()
picker.sourceType = .photoLibrary
picker.delegate = context.coordinator
picker.mediaTypes = ["public.movie"]
picker.videoQuality = .typeHigh
return picker
}
}
class ImagePickerCoordinator : NSObject, UINavigationControllerDelegate, UIImagePickerControllerDelegate {
@Binding var isShown : Bool
@Binding var image : Image?
@Binding var url: URL?
init(isShown : Binding<Bool>, image: Binding<Image?>, url: Binding<URL?>) {
_isShown = isShown
_image = image
_url = url
}
func imagePickerController(_ picker: UIImagePickerController, didFinishPickingMediaWithInfo info: [UIImagePickerController.InfoKey : Any]) {
url = info[UIImagePickerController.InfoKey.mediaURL] as? URL
isShown = false
}
func imagePickerControllerDidCancel(_ picker: UIImagePickerController) {
isShown = false
}
}
struct PhotoCaptureView: View {
@Binding var showImagePicker : Bool
@Binding var image : Image?
@Binding var url : URL?
var body: some View {
ImagePicker(isShown: $showImagePicker, image: $image, url: $url)
}
}
struct ContentView_Previews: PreviewProvider {
static var previews: some View {
CustomCameraPhotoView()
}
}原始答案:代表总是(?)定义为弱成员。见文档。在调用委托之前,PlayerOutput对象将有退出作用域。使某个对象的PlayerOutput成员在回放过程中处于活动状态,您的代码应该按原样工作。
https://stackoverflow.com/questions/62537172
复制相似问题