translate video_filter.c into English

translate video_filter.c into English
pull/166/head
xufulong 5 years ago
parent 639c0082bf
commit 2c65398174
  1. 126
      app/src/main/cpp/video_filter.c

@ -63,23 +63,23 @@ int got_frame;
AVCodecContext *audioCodecCtx; AVCodecContext *audioCodecCtx;
jboolean playAudio = JNI_TRUE; jboolean playAudio = JNI_TRUE;
//const char *filter_descr = "lutyuv='u=128:v=128'";//黑白 //const char *filter_descr = "lutyuv='u=128:v=128'";//black white
//const char *filter_descr = "hue='h=60:s=-3'";//hue滤镜 //const char *filter_descr = "hue='h=60:s=-3'";//hue
//const char *filter_descr = "vflip";//上下反序 //const char *filter_descr = "vflip";//up or down
//const char *filter_descr = "hflip";//左右反序 //const char *filter_descr = "hflip";//left or right
//const char *filter_descr = "rotate=90";//旋转90° //const char *filter_descr = "rotate=90";//rotate
//const char *filter_descr = "colorbalance=bs=0.3";//添加蓝色背景 //const char *filter_descr = "colorbalance=bs=0.3";//balance
//const char *filter_descr = "drawbox=x=100:y=100:w=100:h=100:color=pink@0.5'";//绘制矩形 //const char *filter_descr = "drawbox=x=100:y=100:w=100:h=100:color=pink@0.5'";//rectangle
//const char *filter_descr = "drawgrid=w=iw/3:h=ih/3:t=2:c=white@0.5";//九宫格分割 //const char *filter_descr = "drawgrid=w=iw/3:h=ih/3:t=2:c=white@0.5";//grid
//const char *filter_descr = "edgedetect=low=0.1:high=0.4";//边缘检测 //const char *filter_descr = "edgedetect=low=0.1:high=0.4";//edge detection
//const char *filter_descr = "lutrgb='r=0:g=0'";//去掉红色、绿色分量,只保留蓝色 //const char *filter_descr = "lutrgb='r=0:g=0'";//lut
//const char *filter_descr = "noise=alls=20:allf=t+u";//添加噪声 //const char *filter_descr = "noise=alls=20:allf=t+u";//add noise
//const char *filter_descr = "vignette='PI/4+random(1)*PI/50':eval=frame";//闪烁装饰 //const char *filter_descr = "vignette='PI/4+random(1)*PI/50':eval=frame";//flash
//const char *filter_descr = "gblur=sigma=0.5:steps=1:planes=1:sigmaV=1";//高斯模糊 //const char *filter_descr = "gblur=sigma=0.5:steps=1:planes=1:sigmaV=1";//Gauss blur
//const char *filter_descr = "drawtext=fontfile='arial.ttf':fontcolor=green:fontsize=30:text='Hello world'";//绘制文字 //const char *filter_descr = "drawtext=fontfile='arial.ttf':fontcolor=green:fontsize=30:text='Hello world'";//draw text
//const char *filter_descr = "movie=my_logo.png[wm];[in][wm]overlay=5:5[out]";//添加图片水印 //const char *filter_descr = "movie=my_logo.png[wm];[in][wm]overlay=5:5[out]";//add watermark
//初始化滤波器 //init filter
int init_filters(const char *filters_descr) { int init_filters(const char *filters_descr) {
char args[512]; char args[512];
int ret = 0; int ret = 0;
@ -149,24 +149,20 @@ int init_filters(const char *filters_descr) {
return ret; return ret;
} }
//初始化视频解码器与播放器 //init player
int open_input(JNIEnv *env, const char *file_name, jobject surface) { int open_input(JNIEnv *env, const char *file_name, jobject surface) {
LOGI(TAG, "open file:%s\n", file_name); LOGI(TAG, "open file:%s\n", file_name);
//注册所有组件
av_register_all(); av_register_all();
//分配上下文
pFormatCtx = avformat_alloc_context(); pFormatCtx = avformat_alloc_context();
//打开视频文件
if (avformat_open_input(&pFormatCtx, file_name, NULL, NULL) != 0) { if (avformat_open_input(&pFormatCtx, file_name, NULL, NULL) != 0) {
LOGE(TAG, "Couldn't open file:%s\n", file_name); LOGE(TAG, "Couldn't open file:%s\n", file_name);
return -1; return -1;
} }
//检索多媒体流信息
if (avformat_find_stream_info(pFormatCtx, NULL) < 0) { if (avformat_find_stream_info(pFormatCtx, NULL) < 0) {
LOGE(TAG, "Couldn't find stream information."); LOGE(TAG, "Couldn't find stream information.");
return -1; return -1;
} }
//寻找视频流的第一帧
int i; int i;
for (i = 0; i < pFormatCtx->nb_streams; i++) { for (i = 0; i < pFormatCtx->nb_streams; i++) {
if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO
@ -179,9 +175,7 @@ int open_input(JNIEnv * env, const char* file_name, jobject surface){
return -1; return -1;
} }
//获取codec上下文指针
pCodecCtx = pFormatCtx->streams[video_stream_index]->codec; pCodecCtx = pFormatCtx->streams[video_stream_index]->codec;
//寻找视频流的解码器
AVCodec *pCodec = avcodec_find_decoder(pCodecCtx->codec_id); AVCodec *pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
if (pCodec == NULL) { if (pCodec == NULL) {
LOGE(TAG, "couldn't find Codec."); LOGE(TAG, "couldn't find Codec.");
@ -191,25 +185,22 @@ int open_input(JNIEnv * env, const char* file_name, jobject surface){
LOGE(TAG, "Couldn't open codec."); LOGE(TAG, "Couldn't open codec.");
return -1; return -1;
} }
// 获取native window
nativeWindow = ANativeWindow_fromSurface(env, surface);
// 设置native window的buffer大小,可自动拉伸 nativeWindow = ANativeWindow_fromSurface(env, surface);
ANativeWindow_setBuffersGeometry(nativeWindow, pCodecCtx->width, pCodecCtx->height, WINDOW_FORMAT_RGBA_8888); ANativeWindow_setBuffersGeometry(nativeWindow, pCodecCtx->width, pCodecCtx->height,
//申请内存 WINDOW_FORMAT_RGBA_8888);
pFrame = av_frame_alloc(); pFrame = av_frame_alloc();
pFrameRGBA = av_frame_alloc(); pFrameRGBA = av_frame_alloc();
if (pFrameRGBA == NULL || pFrame == NULL) { if (pFrameRGBA == NULL || pFrame == NULL) {
LOGE(TAG, "Couldn't allocate video frame."); LOGE(TAG, "Couldn't allocate video frame.");
return -1; return -1;
} }
// buffer中数据用于渲染,且格式为RGBA int numBytes = av_image_get_buffer_size(AV_PIX_FMT_RGBA, pCodecCtx->width, pCodecCtx->height,
int numBytes=av_image_get_buffer_size(AV_PIX_FMT_RGBA, pCodecCtx->width, pCodecCtx->height, 1); 1);
buffer = (uint8_t *) av_malloc(numBytes * sizeof(uint8_t)); buffer = (uint8_t *) av_malloc(numBytes * sizeof(uint8_t));
av_image_fill_arrays(pFrameRGBA->data, pFrameRGBA->linesize, buffer, AV_PIX_FMT_RGBA, av_image_fill_arrays(pFrameRGBA->data, pFrameRGBA->linesize, buffer, AV_PIX_FMT_RGBA,
pCodecCtx->width, pCodecCtx->height, 1); pCodecCtx->width, pCodecCtx->height, 1);
// 由于解码出来的帧格式不是RGBA的,在渲染之前需要进行格式转换
sws_ctx = sws_getContext(pCodecCtx->width, sws_ctx = sws_getContext(pCodecCtx->width,
pCodecCtx->height, pCodecCtx->height,
pCodecCtx->pix_fmt, pCodecCtx->pix_fmt,
@ -224,9 +215,7 @@ int open_input(JNIEnv * env, const char* file_name, jobject surface){
return 0; return 0;
} }
//初始化音频解码器与播放器
int init_audio(JNIEnv *env, jclass jthiz) { int init_audio(JNIEnv *env, jclass jthiz) {
//获取音频流索引位置
int i; int i;
for (i = 0; i < pFormatCtx->nb_streams; i++) { for (i = 0; i < pFormatCtx->nb_streams; i++) {
if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO) { if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO) {
@ -235,32 +224,23 @@ int init_audio(JNIEnv * env, jclass jthiz){
} }
} }
//获取音频解码器
audioCodecCtx = pFormatCtx->streams[audio_stream_index]->codec; audioCodecCtx = pFormatCtx->streams[audio_stream_index]->codec;
AVCodec *codec = avcodec_find_decoder(audioCodecCtx->codec_id); AVCodec *codec = avcodec_find_decoder(audioCodecCtx->codec_id);
if (codec == NULL) { if (codec == NULL) {
LOGE(TAG, "无法获取音频解码器"); LOGE(TAG, "could not find audio decoder");
return -1; return -1;
} }
//打开音频解码器
if (avcodec_open2(audioCodecCtx, codec, NULL) < 0) { if (avcodec_open2(audioCodecCtx, codec, NULL) < 0) {
LOGE(TAG, "无法打开音频解码器"); LOGE(TAG, "could not open audio decoder");
return -1; return -1;
} }
//frame->16bit 44100 PCM 统一音频采样格式与采样率
audio_swr_ctx = swr_alloc();
//输入的采样格式 audio_swr_ctx = swr_alloc();
enum AVSampleFormat in_sample_fmt = audioCodecCtx->sample_fmt; enum AVSampleFormat in_sample_fmt = audioCodecCtx->sample_fmt;
//输出采样格式16bit PCM
out_sample_fmt = AV_SAMPLE_FMT_S16; out_sample_fmt = AV_SAMPLE_FMT_S16;
//输入采样率
int in_sample_rate = audioCodecCtx->sample_rate; int in_sample_rate = audioCodecCtx->sample_rate;
//输出采样率
int out_sample_rate = in_sample_rate; int out_sample_rate = in_sample_rate;
//声道布局(2个声道,默认立体声stereo)
uint64_t in_ch_layout = audioCodecCtx->channel_layout; uint64_t in_ch_layout = audioCodecCtx->channel_layout;
//输出的声道布局(立体声)
uint64_t out_ch_layout = AV_CH_LAYOUT_STEREO; uint64_t out_ch_layout = AV_CH_LAYOUT_STEREO;
swr_alloc_set_opts(audio_swr_ctx, swr_alloc_set_opts(audio_swr_ctx,
@ -268,8 +248,6 @@ int init_audio(JNIEnv * env, jclass jthiz){
in_ch_layout, in_sample_fmt, in_sample_rate, in_ch_layout, in_sample_fmt, in_sample_rate,
0, NULL); 0, NULL);
swr_init(audio_swr_ctx); swr_init(audio_swr_ctx);
//输出的声道个数
out_channel_nb = av_get_channel_layout_nb_channels(out_ch_layout); out_channel_nb = av_get_channel_layout_nb_channels(out_ch_layout);
jclass player_class = (*env)->GetObjectClass(env, jthiz); jclass player_class = (*env)->GetObjectClass(env, jthiz);
@ -277,50 +255,40 @@ int init_audio(JNIEnv * env, jclass jthiz){
LOGE(TAG, "player_class not found..."); LOGE(TAG, "player_class not found...");
return -1; return -1;
} }
//AudioTrack对象 jmethodID audio_track_method = (*env)->GetMethodID(env, player_class, "createAudioTrack",
jmethodID audio_track_method = (*env)->GetMethodID(env,player_class,"createAudioTrack","(II)Landroid/media/AudioTrack;"); "(II)Landroid/media/AudioTrack;");
if (!audio_track_method) { if (!audio_track_method) {
LOGE(TAG, "audio_track_method not found..."); LOGE(TAG, "audio_track_method not found...");
return -1; return -1;
} }
audio_track = (*env)->CallObjectMethod(env,jthiz,audio_track_method,out_sample_rate,out_channel_nb); audio_track = (*env)->CallObjectMethod(env, jthiz, audio_track_method, out_sample_rate,
out_channel_nb);
//调用play方法
jclass audio_track_class = (*env)->GetObjectClass(env, audio_track); jclass audio_track_class = (*env)->GetObjectClass(env, audio_track);
jmethodID audio_track_play_mid = (*env)->GetMethodID(env, audio_track_class, "play", "()V"); jmethodID audio_track_play_mid = (*env)->GetMethodID(env, audio_track_class, "play", "()V");
(*env)->CallVoidMethod(env, audio_track, audio_track_play_mid); (*env)->CallVoidMethod(env, audio_track, audio_track_play_mid);
//获取write()方法
audio_track_write_mid = (*env)->GetMethodID(env, audio_track_class, "write", "([BII)I"); audio_track_write_mid = (*env)->GetMethodID(env, audio_track_class, "write", "([BII)I");
//16bit 44100 PCM 数据
out_buffer = (uint8_t *) av_malloc(MAX_AUDIO_FRAME_SIZE); out_buffer = (uint8_t *) av_malloc(MAX_AUDIO_FRAME_SIZE);
return 0; return 0;
} }
int play_audio(JNIEnv *env, AVPacket *packet, AVFrame *frame) { int play_audio(JNIEnv *env, AVPacket *packet, AVFrame *frame) {
//解码
int ret = avcodec_decode_audio4(audioCodecCtx, frame, &got_frame, packet); int ret = avcodec_decode_audio4(audioCodecCtx, frame, &got_frame, packet);
if (ret < 0) { if (ret < 0) {
return ret; return ret;
} }
//解码一帧成功
if (got_frame > 0) { if (got_frame > 0) {
//音频格式转换 swr_convert(audio_swr_ctx, &out_buffer, MAX_AUDIO_FRAME_SIZE,
swr_convert(audio_swr_ctx, &out_buffer, MAX_AUDIO_FRAME_SIZE,(const uint8_t **)frame->data,frame->nb_samples); (const uint8_t **) frame->data, frame->nb_samples);
int out_buffer_size = av_samples_get_buffer_size(NULL, out_channel_nb, int out_buffer_size = av_samples_get_buffer_size(NULL, out_channel_nb,
frame->nb_samples, out_sample_fmt, 1); frame->nb_samples, out_sample_fmt, 1);
jbyteArray audio_sample_array = (*env)->NewByteArray(env, out_buffer_size); jbyteArray audio_sample_array = (*env)->NewByteArray(env, out_buffer_size);
jbyte *sample_byte_array = (*env)->GetByteArrayElements(env, audio_sample_array, NULL); jbyte *sample_byte_array = (*env)->GetByteArrayElements(env, audio_sample_array, NULL);
//拷贝缓冲数据
memcpy(sample_byte_array, out_buffer, (size_t) out_buffer_size); memcpy(sample_byte_array, out_buffer, (size_t) out_buffer_size);
//释放数组
(*env)->ReleaseByteArrayElements(env, audio_sample_array, sample_byte_array, 0); (*env)->ReleaseByteArrayElements(env, audio_sample_array, sample_byte_array, 0);
//调用AudioTrack的write方法进行播放
(*env)->CallIntMethod(env, audio_track, audio_track_write_mid, (*env)->CallIntMethod(env, audio_track, audio_track_write_mid,
audio_sample_array, 0, out_buffer_size); audio_sample_array, 0, out_buffer_size);
//释放局部引用
(*env)->DeleteLocalRef(env, audio_sample_array); (*env)->DeleteLocalRef(env, audio_sample_array);
usleep(1000);//1000 * 16 usleep(1000);//1000 * 16
} }
@ -332,14 +300,14 @@ VIDEO_PLAYER_FUNC(jint, filter, jstring filePath, jobject surface, jstring filte
int ret; int ret;
const char *file_name = (*env)->GetStringUTFChars(env, filePath, JNI_FALSE); const char *file_name = (*env)->GetStringUTFChars(env, filePath, JNI_FALSE);
const char *filter_descr = (*env)->GetStringUTFChars(env, filterDescr, JNI_FALSE); const char *filter_descr = (*env)->GetStringUTFChars(env, filterDescr, JNI_FALSE);
//打开输入文件 //open input file
if (!is_playing) { if (!is_playing) {
LOGI(TAG, "open_input..."); LOGI(TAG, "open_input...");
if ((ret = open_input(env, file_name, surface)) < 0) { if ((ret = open_input(env, file_name, surface)) < 0) {
LOGE(TAG, "Couldn't allocate video frame."); LOGE(TAG, "Couldn't allocate video frame.");
goto end; goto end;
} }
//注册滤波器 //register filter
avfilter_register_all(); avfilter_register_all();
filter_frame = av_frame_alloc(); filter_frame = av_frame_alloc();
if (filter_frame == NULL) { if (filter_frame == NULL) {
@ -347,7 +315,7 @@ VIDEO_PLAYER_FUNC(jint, filter, jstring filePath, jobject surface, jstring filte
ret = -1; ret = -1;
goto end; goto end;
} }
//初始化音频解码器 //init audio decoder
if ((ret = init_audio(env, thiz)) < 0) { if ((ret = init_audio(env, thiz)) < 0) {
LOGE(TAG, "Couldn't init_audio."); LOGE(TAG, "Couldn't init_audio.");
goto end; goto end;
@ -355,7 +323,7 @@ VIDEO_PLAYER_FUNC(jint, filter, jstring filePath, jobject surface, jstring filte
} }
//初始化滤波器 //init filter
if ((ret = init_filters(filter_descr)) < 0) { if ((ret = init_filters(filter_descr)) < 0) {
LOGE(TAG, "init_filter error, ret=%d\n", ret); LOGE(TAG, "init_filter error, ret=%d\n", ret);
goto end; goto end;
@ -366,36 +334,35 @@ VIDEO_PLAYER_FUNC(jint, filter, jstring filePath, jobject surface, jstring filte
AVPacket packet; AVPacket packet;
while (av_read_frame(pFormatCtx, &packet) >= 0 && !release) { while (av_read_frame(pFormatCtx, &packet) >= 0 && !release) {
//切换滤波器,退出当初播放 //switch filter
if (again) { if (again) {
goto again; goto again;
} }
//判断是否为视频流 //is video stream or not
if (packet.stream_index == video_stream_index) { if (packet.stream_index == video_stream_index) {
//对该帧进行解码 //对该帧进行解码
avcodec_decode_video2(pCodecCtx, pFrame, &frameFinished, &packet); avcodec_decode_video2(pCodecCtx, pFrame, &frameFinished, &packet);
if (frameFinished) { if (frameFinished) {
//把解码后视频帧添加到filter_graph //add frame to filter_graph
if (av_buffersrc_add_frame_flags(buffersrc_ctx, pFrame, AV_BUFFERSRC_FLAG_KEEP_REF) < 0) { if (av_buffersrc_add_frame_flags(buffersrc_ctx, pFrame,
AV_BUFFERSRC_FLAG_KEEP_REF) < 0) {
LOGE(TAG, "Error while feeding the filter_graph\n"); LOGE(TAG, "Error while feeding the filter_graph\n");
break; break;
} }
//把滤波后的视频帧从filter graph取出来 //take frame from filter graph
ret = av_buffersink_get_frame(buffersink_ctx, filter_frame); ret = av_buffersink_get_frame(buffersink_ctx, filter_frame);
if (ret >= 0) { if (ret >= 0) {
// lock native window // lock native window
ANativeWindow_lock(nativeWindow, &windowBuffer, 0); ANativeWindow_lock(nativeWindow, &windowBuffer, 0);
// 格式转换 // convert
sws_scale(sws_ctx, (uint8_t const *const *) filter_frame->data, sws_scale(sws_ctx, (uint8_t const *const *) filter_frame->data,
filter_frame->linesize, 0, pCodecCtx->height, filter_frame->linesize, 0, pCodecCtx->height,
pFrameRGBA->data, pFrameRGBA->linesize); pFrameRGBA->data, pFrameRGBA->linesize);
// 获取stride
uint8_t *dst = windowBuffer.bits; uint8_t *dst = windowBuffer.bits;
int dstStride = windowBuffer.stride * 4; int dstStride = windowBuffer.stride * 4;
uint8_t *src = pFrameRGBA->data[0]; uint8_t *src = pFrameRGBA->data[0];
int srcStride = pFrameRGBA->linesize[0]; int srcStride = pFrameRGBA->linesize[0];
// 由于window的stride和帧的stride不同,因此需要逐行复制
int h; int h;
for (h = 0; h < pCodecCtx->height; h++) { for (h = 0; h < pCodecCtx->height; h++) {
memcpy(dst + h * dstStride, src + h * srcStride, (size_t) srcStride); memcpy(dst + h * dstStride, src + h * srcStride, (size_t) srcStride);
@ -404,11 +371,11 @@ VIDEO_PLAYER_FUNC(jint, filter, jstring filePath, jobject surface, jstring filte
} }
av_frame_unref(filter_frame); av_frame_unref(filter_frame);
} }
//延迟等待 //sleep to keep waiting
if (!playAudio) { if (!playAudio) {
usleep((unsigned long) (1000 * 40));//1000 * 40 usleep((unsigned long) (1000 * 40));//1000 * 40
} }
} else if(packet.stream_index == audio_stream_index){//音频帧 } else if (packet.stream_index == audio_stream_index) {//audio stream
if (playAudio) { if (playAudio) {
play_audio(env, &packet, pFrame); play_audio(env, &packet, pFrame);
} }
@ -417,7 +384,6 @@ VIDEO_PLAYER_FUNC(jint, filter, jstring filePath, jobject surface, jstring filte
} }
end: end:
is_playing = 0; is_playing = 0;
//释放内存以及关闭文件
av_free(buffer); av_free(buffer);
av_free(pFrameRGBA); av_free(pFrameRGBA);
av_free(filter_frame); av_free(filter_frame);

Loading…
Cancel
Save