From e96fa5608e430eeb14dfc44c6ecc0f260558b015 Mon Sep 17 00:00:00 2001 From: xufuji456 <839789740@qq.com> Date: Wed, 1 Sep 2021 09:18:25 +0800 Subject: [PATCH] Add: fft visualizer --- app/src/main/cpp/audio_player.cpp | 42 ++++++++++++++++--- app/src/main/cpp/visualizer/execute_fft.c | 2 +- app/src/main/cpp/visualizer/fft.h | 2 +- .../java/com/frank/ffmpeg/AudioPlayer.java | 17 ++++++++ .../ffmpeg/activity/EqualizerActivity.kt | 14 +++++++ .../main/res/layout/activity_equalizer.xml | 6 +++ 6 files changed, 76 insertions(+), 7 deletions(-) diff --git a/app/src/main/cpp/audio_player.cpp b/app/src/main/cpp/audio_player.cpp index 76c4a37..ba2f265 100644 --- a/app/src/main/cpp/audio_player.cpp +++ b/app/src/main/cpp/audio_player.cpp @@ -34,6 +34,10 @@ int filter_again = 0; int filter_release = 0; const char *filter_desc = "superequalizer=6b=4:8b=5:10b=5"; +JavaVM *java_vm; +jobject jni_object; +jmethodID fft_method; + static void fft_callback(void* arg); int init_volume_filter(AVFilterGraph **graph, AVFilterContext **src, AVFilterContext **sink, @@ -280,9 +284,19 @@ AUDIO_PLAYER_FUNC(void, play, jstring input_jstr, jstring filter_jstr) { ret = init_equalizer_filter(filter_desc, codecCtx, &audioFilterGraph, &audioSrcContext, &audioSinkContext); if (ret < 0) { LOGE(TAG, "Unable to init filter graph:%d", ret); - goto end; +// goto end; } + env->GetJavaVM(&java_vm); + jni_object = env->NewGlobalRef(thiz); + fft_method = env->GetMethodID(player_class, "fftCallbackFromJNI", "([I)V"); + + filter_sys_t *fft_filter = static_cast(malloc(sizeof(filter_sys_t))); + open_visualizer(fft_filter); + auto *block = static_cast(malloc(sizeof(block_t))); + block->i_nb_samples = 0; + block->fft_callback.callback = fft_callback; + //read audio frame while (av_read_frame(pFormatCtx, packet) >= 0 && !filter_release) { if (packet->stream_index != audio_stream_idx) { @@ -303,6 +317,19 @@ AUDIO_PLAYER_FUNC(void, play, jstring input_jstr, jstring filter_jstr) { break; } if (got_frame > 0) { + + if (block->i_nb_samples == 0 && frame->nb_samples > 100) { + auto *data = static_cast(malloc(frame->nb_samples * sizeof(uint8_t))); + block->p_buffer = data; + block->i_nb_samples = static_cast(frame->nb_samples); + LOGE(TAG, "frame->nb_samples=%d", frame->nb_samples); + } + if (block->i_nb_samples == frame->nb_samples) { + LOGE(TAG, "start frame->nb_samples=%d", frame->nb_samples); + memcpy(block->p_buffer, frame->data[0], static_cast(frame->nb_samples)); + filter_audio(fft_filter, block); + } + ret = av_buffersrc_add_frame(audioSrcContext, frame); if (ret < 0) { LOGE(TAG, "Error add the frame to the filter graph:%d", ret); @@ -376,10 +403,15 @@ AUDIO_PLAYER_FUNC(void, openFFT) { } static void fft_callback(void* arg) { - float* data = (float*) arg; - for (int i = 0; i < 3; ++i) { - LOGE(TAG, "fft data[0]=%f,data[1]=%f,data[2]=%f", data[0], data[1], data[2]); - } +// float* data = (float*) arg; +// LOGE(TAG, "fft data[0]=%f,data[1]=%f,data[2]=%f", data[0], data[1], data[2]); + + JNIEnv *jniEnv = nullptr; + java_vm->AttachCurrentThread(&jniEnv, nullptr); + jintArray dataArray = jniEnv->NewIntArray(/*20*/256); + jniEnv->SetIntArrayRegion(dataArray, 0, /*20*/256, (int*)arg); + jniEnv->CallVoidMethod(jni_object, fft_method, dataArray); + java_vm->DetachCurrentThread(); } static void *input(void *) { diff --git a/app/src/main/cpp/visualizer/execute_fft.c b/app/src/main/cpp/visualizer/execute_fft.c index 6a4f821..28ab61d 100644 --- a/app/src/main/cpp/visualizer/execute_fft.c +++ b/app/src/main/cpp/visualizer/execute_fft.c @@ -188,7 +188,7 @@ static void *fft_thread(void *p_data) // vlc_tick_wait(block->i_pts + (block->i_length / 2)); // vlc_gl_Swap(gl); usleep(10*1000 /*block->i_pts + (block->i_length / 2)*/); - block->fft_callback.callback(height); + block->fft_callback.callback(p_dest); release: window_close(&wind_ctx); diff --git a/app/src/main/cpp/visualizer/fft.h b/app/src/main/cpp/visualizer/fft.h index ac9605f..1456ac1 100644 --- a/app/src/main/cpp/visualizer/fft.h +++ b/app/src/main/cpp/visualizer/fft.h @@ -24,7 +24,7 @@ #ifndef VLC_VISUAL_FFT_H_ #define VLC_VISUAL_FFT_H_ -#define FFT_BUFFER_SIZE_LOG 9 +#define FFT_BUFFER_SIZE_LOG 8 #define FFT_BUFFER_SIZE (1 << FFT_BUFFER_SIZE_LOG) diff --git a/app/src/main/java/com/frank/ffmpeg/AudioPlayer.java b/app/src/main/java/com/frank/ffmpeg/AudioPlayer.java index be7fecf..8dbd6a6 100644 --- a/app/src/main/java/com/frank/ffmpeg/AudioPlayer.java +++ b/app/src/main/java/com/frank/ffmpeg/AudioPlayer.java @@ -59,4 +59,21 @@ public class AudioPlayer { mAudioTrack = null; } } + + private OnFFTCallback onFFTCallback; + + public interface OnFFTCallback { + void onFFT(int[] data); + } + + public void setOnFftCallback(OnFFTCallback onFFTCallback) { + this.onFFTCallback = onFFTCallback; + } + + public void fftCallbackFromJNI(int[] data) { + if (data != null && onFFTCallback != null) { + onFFTCallback.onFFT(data); + } + } + } diff --git a/app/src/main/java/com/frank/ffmpeg/activity/EqualizerActivity.kt b/app/src/main/java/com/frank/ffmpeg/activity/EqualizerActivity.kt index 5a3c151..fb6eb21 100644 --- a/app/src/main/java/com/frank/ffmpeg/activity/EqualizerActivity.kt +++ b/app/src/main/java/com/frank/ffmpeg/activity/EqualizerActivity.kt @@ -13,6 +13,7 @@ import com.frank.ffmpeg.AudioPlayer import com.frank.ffmpeg.R import com.frank.ffmpeg.adapter.EqualizerAdapter import com.frank.ffmpeg.listener.OnSeeBarListener +import com.frank.ffmpeg.view.VisualizerView import java.lang.StringBuilder import java.util.ArrayList @@ -33,8 +34,10 @@ class EqualizerActivity : BaseActivity(), OnSeeBarListener { private val selectBandList = IntArray(bandsList.size) private val minEQLevel = 0 + private val fftData = ByteArray(/*20*/256) private var filterThread: Thread? = null private var mAudioPlayer: AudioPlayer? = null + private var visualizerView: VisualizerView? = null private var equalizerAdapter: EqualizerAdapter? = null private var audioPath = Environment.getExternalStorageDirectory().path + "/know_play.mp3" @@ -50,6 +53,7 @@ class EqualizerActivity : BaseActivity(), OnSeeBarListener { } private fun initView() { + visualizerView = findViewById(R.id.visualizer_fft) val equalizerView = findViewById(R.id.list_equalizer) val layoutManager = LinearLayoutManager(this) layoutManager.orientation = RecyclerView.VERTICAL @@ -87,6 +91,16 @@ class EqualizerActivity : BaseActivity(), OnSeeBarListener { equalizerAdapter!!.setEqualizerList(equalizerList) } mAudioPlayer = AudioPlayer() + mAudioPlayer?.setOnFftCallback { + Log.e("Equalizer", "fft data[0]=" + it[0] + "--data[1]=" + it[1] + "--data[2]=" + it[2]) + if (visualizerView != null && it != null) { + for (i in it.indices) { + fftData[i] = (it[i]/*10/4.2f*/).toByte() + } + Log.e("Equalizer", "data[0]=" + fftData[0] + "--data[1]=" + fftData[1] + "--data[2]=" + fftData[2]) + visualizerView!!.post { visualizerView!!.setWaveData(fftData) } + } + } } private fun doEqualize() { diff --git a/app/src/main/res/layout/activity_equalizer.xml b/app/src/main/res/layout/activity_equalizer.xml index da09ca3..d61c203 100644 --- a/app/src/main/res/layout/activity_equalizer.xml +++ b/app/src/main/res/layout/activity_equalizer.xml @@ -55,6 +55,12 @@ + +