干货 | 移动端使用OpenGL转场特效的音视频合成应用

Y = 0.299R + 0.587G + 0.114BU = -0.147R - 0.289G + 0.436BV = 0.615R - 0.515G - 0.100B R = Y + 1.14VG = Y - 0.39U - 0.58VB = Y + 2.03U

一般常见的视频编码格式有H26x系列和MPEG系列。
H26x（1/2/3/4/5）系列由ITU（International Telecommunication Union）国际电传视讯联盟主导。
MPEG（1/2/3/4）系列由MPEG（Moving Picture Experts Group， ISO旗下的组织）主导。

    val width = 720
    val height = 1280
    val bitrate = 5000
    val encodeType = "video/avc"
    //配置用于编码的MediaCodec
    val mCodec = MediaCodec.createEncoderByType(encodeType)
    val outputFormat = MediaFormat.createVideoFormat(encodeType, width, height)
    outputFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitrate)
    outputFormat.setInteger(MediaFormat.KEY_FRAME_RATE, DEFAULT_ENCODE_FRAME_RATE)
    outputFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1)
    outputFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface)
    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
        outputFormat.setInteger(
            MediaFormat.KEY_BITRATE_MODE,
            MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CQ
        )
    }
    codec.configure(outputFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
    //这一步很关键，这一步得到的surface后面将会用到
    val mSurface = codec.createInputSurface()
    mCodec.start()
    val mOutputBuffers = mCodec.outputBuffers
    val mInputBuffers = mCodec.inputBuffers

//1，创建 EGLDisplay
val mEGLDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY)
// 2，初始化 EGLDisplay
val version = IntArray(2)
EGL14.eglInitialize(mEGLDisplay, version, 0, version, 1)
// 3，初始化EGLConfig，EGLContext上下文
val config :EGLConfig? = null
if (mEGLContext === EGL14.EGL_NO_CONTEXT) {
    var renderableType = EGL14.EGL_OPENGL_ES2_BIT
        val attrList = intArrayOf(
            EGL14.EGL_RED_SIZE, 8,
            EGL14.EGL_GREEN_SIZE, 8,
            EGL14.EGL_BLUE_SIZE, 8,
            EGL14.EGL_ALPHA_SIZE, 8,
            EGL14.EGL_RENDERABLE_TYPE, renderableType,
            EGL14.EGL_NONE, 0,
            EGL14.EGL_NONE
        )
        //配置Android指定的标记
        if (flags and FLAG_RECORDABLE != 0) {
            attrList[attrList.size - 3] = EGL_RECORDABLE_ANDROID
            attrList[attrList.size - 2] = 1
        }
        val configs = arrayOfNulls<EGLConfig>(1)
        val numConfigs = IntArray(1)

        //获取可用的EGL配置列表
        if (!EGL14.eglChooseConfig(mEGLDisplay, attrList, 0,
                configs, 0, configs.size,
                numConfigs, 0)) {
            configs[0]
        }
    val attr2List = intArrayOf(EGL14.EGL_CONTEXT_CLIENT_VERSION, 2, EGL14.EGL_NONE)
    val context = EGL14.eglCreateContext(
        mEGLDisplay, config, sharedContext,
        attr2List, 0
    )
    mEGLConfig = config
    mEGLContext = context
}
//这里还需要创建一个EGL用于输出的surface，这里的参数就可以传入上一小节介绍的利用MeddiaCodec创建的Surface
 fun createWindowSurface(surface: Any): EGLSurface {
        val surfaceAttr = intArrayOf(EGL14.EGL_NONE)

        val eglSurface = EGL14.eglCreateWindowSurface(
                                        mEGLDisplay, mEGLConfig, surface,
                                        surfaceAttr, 0)

        if (eglSurface == null) {
            throw RuntimeException("Surface was null")
        }

        return eglSurface
    }

//创建一个MediaMuxer，需要指定输出保存的路径，和输出保存的格式。
val mediaMuxer = MediaMuxer(path, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4)
//根据MediaFormat添加媒体轨道
mediaMuxer.addTrack(MediaFormat(...))
//将输入的数据，根据指定的轨道保存到指定的文件路径中。
muxer.writeSampleData(currentTrackIndex, inputBuffer, bufferInfo);
//结合上面的所说的使用MediaCodec获取到已编码的数据
//当前帧的信息
var mBufferInfo = MediaCodec.BufferInfo()
//编码输出缓冲区
var mOutputBuffers: Array<ByteBuffer>? = null
//获取到编码输出的索引，写到指定的保存路径
val index = mCodec.dequeueOutputBuffer(mBufferInfo, 1000)
muxer.writeSampleData(currentTrackIndex,mOutputBuffers[index],mBufferInfo)

 //根据指定文件路径创建MediaExtractor
 val mediaExtractor = MediaExtractor(...)
 //为MediaExtractor选择好对应的媒体轨道
 mediaExtractor.selectTrack(...)
 //读取一帧的数据
 val inputBuffer = ByteBuffer.allocate(...)
 mediaExtractor.readSampleData(inputBuffer, 0)
 //进入下一帧
 mediaExtractor.advance()
 //MediaExtractor读取到的音频数据可以使用MediaMuxer的writeSampleData方法写入到指定的文件中

CMSampleBufferRef sampleBuffer = [assetReaderAudioOutput copyNextSampleBuffer];

CVPixelBufferRef pixelBuffer = NULL;
    CVPixelBufferPoolCreatePixelBuffer(NULL, self.inputPixelBufferAdptor.pixelBufferPool,&pixelBuffer);

if(self.inputPixelBufferAdptor.assetWriterInput.isReadyForMoreMediaData) {
    BOOL success = [self.inputPixelBufferAdptor appendPixelBuffer:newPixelBuffer withPresentationTime:self.currentSampleTime];    

    if (success) {
        NSLog(@"append buffer success");
    }
}

AVAssetWriter *assetWriter = [[AVAssetWriter alloc] initWithURL:[NSURL fileURLWithPath:outFilePath] fileType:AVFileTypeMPEG4 error:&outError];

CGSize size = CGSizeMake(480, 960);

NSDictionary *videoSetDic = [NSDictionary dictionaryWithObjectsAndKeys:AVVideoCodecTypeH264,AVVideoCodecKey,
[NSNumber numberWithInt:size.width],AVVideoWidthKey,[NSNumber numberWithInt:size.height],AVVideoHeightKey,nil];

AVAssetWriterInput *videoWriterInput = [[AVAssetWriterInput alloc] initWithMediaType:AVMediaTypeVideo outputSettings:videoSetDic];

//将读取的图片内容添加到assetWriter                                              
if ([assetWriter canAddInput:videoWriterInput]) {
    [assetWriter addInput:videoWriterInput];
}

NSDictionary *pixelBufferAttributes = [NSDictionary dictionaryWithObjectsAndKeys:[NSNumber numberWithInt:kCVPixelFormatType_32BGRA],kCVPixelBufferPixelFormatTypeKey,nil];

AVAssetWriterInputPixelBufferAdaptor *adaptor = [AVAssetWriterInputPixelBufferAdaptor assetWriterInputPixelBufferAdaptorWithAssetWriterInput:videoWriterInput pixelBufferAttributes:pixelBufferAttributes];

//创建音频资源
AVURLAsset *audioAsset = [[AVURLAsset alloc] initWithURL:audioUrl options:nil];
//创建音频Track
AVAssetTrack *assetAudioTrack = [audioAsset tracksWithMediaType:AVMediaTypeAudio].firstObject;
//创建读取器 
AVAssetReader *assetReader = [[AVAssetReader alloc] initWithAsset:audioAsset error:&error];
//读取音频track中的数据
NSDictionary *audioSettings = @{AVFormatIDKey : [NSNumber numberWithUnsignedInt:kAudioFormatLinearPCM]};

AVAssetReaderTrackOutput *assetReaderAudioOutput = [AVAssetReaderTrackOutput assetReaderTrackOutputWithTrack:assetAudioTrack outputSettings: audioSettings];
//向接收器添加assetReaderAudioOutput输出
if ([assetReader canAddOutput:assetReaderAudioOutput]) {
    [assetReader addOutput:assetReaderAudioOutput];
}

//音频通道数据，设置音频的比特率、采样率的通道数
AudioChannelLayout acl;
bzero( &acl, sizeof(acl));
acl.mChannelLayoutTag = kAudioChannelLayoutTag_Stereo;

NSData *channelLayoutAsData = [NSData dataWithBytes:&acl length:offsetof(AudioChannelLayout, acl)];

NSDictionary *audioSettings = @{AVFormatIDKey:[NSNumber numberWithUnsignedInt:kAudioFormatMPEG4AAC],AVEncoderBitRateKey:[NSNumber numberWithInteger:128000], AVSampleRateKey:[NSNumber numberWithInteger:44100], AVChannelLayoutKey:channelLayoutAsData,AVNumberOfChannelsKey : [NSNumber numberWithUnsignedInteger:2]};
//创建音频的assetWriterAudioInput，将读取的音频内容添加到assetWriter
AVAssetWriterInput *assetWriterAudioInput = [AVAssetWriterInput assetWriterInputWithMediaType:[assetAudioTrack mediaType] outputSettings: audioSettings];

if ([assetWriter canAddInput:assetWriterAudioInput]) {
    [assetWriter addInput:assetWriterAudioInput];
}

//Writer开始进行写入流程
[assetWriter startSessionAtSourceTime:kCMTimeZero];

-(void)renderToTextureWithVertices:(const GLfloat *)vertices textureCoordinates:(const GLfloat *)textureCoordinates;

-(BOOL)appendPixelBuffer:(CVPixelBufferRef)pixelBuffer withPresentationTime:(CMTime)presentationTime;

- (id)initWithVertexShaderFromString:(NSString *)vertexShaderString fragmentShaderFromString:(NSString *)fragmentShaderString;

precision highp float;
varying highp vec2 textureCoordinate;
uniform sampler2D imageTexture;
uniform sampler2D imageTexture2;
uniform mediump vec4 v4Param1;
float progress = v4Param1.x;
void main()
{
    vec4 color1 = texture2D(imageTexture, textureCoordinate);
    vec4 color2 = texture2D(imageTextur2, textureCoordinate);
    gl_FragColor = mix(color1, color2, step(1.0-textureCoordinate.x,progress));
}

- (void)newFrameReadyAtTime:(CMTime)frameTime atIndex:(NSInteger)textureIndex;

- (void)renderToTextureWithVertices:(const GLfloat *)vertices textureCoordinates:(const GLfloat *)textureCoordinates;

- (void)informTargetsAboutNewFrameAtTime:(CMTime)frameTime;

- (void)newFrameReadyAtTime:(CMTime)frameTime atIndex:(NSInteger)textureIndex {
    static const GLfloat imageVertices[] = {
        -1.0f, -1.0f,
        1.0f, -1.0f,
        -1.0f,  1.0f,
        1.0f,  1.0f,
    };
    [self renderToTextureWithVertices:imageVertices textureCoordinates:[[self class] textureCoordinatesForRotation:inputRotation]];
}

- (void)renderToTextureWithVertices:(const GLfloat *)vertices textureCoordinates:(const GLfloat *)textureCoordinates {
glClearColor(backgroundColorRed, backgroundColorGreen, backgroundColorBlue, 1.0);
glClear(GL_COLOR_BUFFER_BIT);
glActiveTexture(GL_TEXTURE5);
glBindTexture(GL_TEXTURE_2D, [firstInputFramebuffer texture]);
glUniform1i(filterInputTextureUniform, 5);
glVertexAttribPointer(filterPositionAttribute, 2, GL_FLOAT, 0, 0, [self adjustVertices:vertices]);
glVertexAttribPointer(filterTextureCoordinateAttribute, 2, GL_FLOAT, 0, 0, textureCoordinates);
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
glFinish();
CVPixelBufferRef pixel_buffer = NULL;
CVReturn status = CVPixelBufferPoolCreatePixelBuffer(NULL, [self.videoPixelBufferAdaptor pixelBufferPool], &pixel_buffer);
if ((pixel_buffer == NULL) || (status != kCVReturnSuccess)) {
    CVPixelBufferRelease(pixel_buffer);
    return;
} else {
    CVPixelBufferLockBaseAddress(pixel_buffer, 0);
    GLubyte *pixelBufferData = (GLubyte *)CVPixelBufferGetBaseAddress(pixel_buffer);
    glReadPixels(0, 0, self.sizeOfFBO.width, self.sizeOfFBO.height, GL_RGBA, GL_UNSIGNED_BYTE, pixelBufferData);
CVPixelBufferUnlockBaseAddress(pixel_buffer, 0);
        }
    }
}

CVPixelBufferLockBaseAddress(pixelBuffer, 0);
if (self.assetWriter.status != AVAssetWriterStatusWriting) {
    [self.assetWriter startWriting];
}
[self.assetWriter startSessionAtSourceTime:frameTime];
if (self.assetWriter.status == AVAssetWriterStatusWriting) {
    if (CMTIME_IS_NUMERIC(frameTime) == NO)  {
        CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
        return;
    }
    //确定写操作是否已完成、失败或已取消
    if ([self.videoPixelBufferAdaptor appendPixelBuffer:pixelBufferwithPresentationTime:frameTime]) {                                                    
        NSLog(@"%f", CMTimeGetSeconds(frameTime));
    }
}else {
    NSLog(@"status:%d", self.assetWriter.status);
    }
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);