From 638ccffda09856115bdf0daddf8b705cfd121958 Mon Sep 17 00:00:00 2001 From: xufuji456 Date: Thu, 8 Sep 2022 14:47:14 +0800 Subject: [PATCH] Feature: add visualizer with fft data --- app/src/main/cpp/CMakeLists.txt | 1 - app/src/main/cpp/audio_player_jni.cpp | 13 ++++ app/src/main/cpp/ff_audio_player.cpp | 33 +++++++++ app/src/main/cpp/ff_audio_player.h | 12 ++++ app/src/main/cpp/visualizer/block_queue.c | 70 ------------------- app/src/main/cpp/visualizer/block_queue.h | 57 --------------- .../main/cpp/visualizer/frank_visualizer.cpp | 8 ++- .../main/cpp/visualizer/frank_visualizer.h | 2 + 8 files changed, 66 insertions(+), 130 deletions(-) delete mode 100644 app/src/main/cpp/visualizer/block_queue.c delete mode 100644 app/src/main/cpp/visualizer/block_queue.h diff --git a/app/src/main/cpp/CMakeLists.txt b/app/src/main/cpp/CMakeLists.txt index 30371d9..9966c87 100644 --- a/app/src/main/cpp/CMakeLists.txt +++ b/app/src/main/cpp/CMakeLists.txt @@ -33,7 +33,6 @@ add_library( # Sets the name of the library. video_cutting.cpp visualizer/fft.cpp visualizer/fixed_fft.cpp - visualizer/block_queue.c visualizer/frank_visualizer.cpp visualizer/frank_visualizer_jni.cpp visualizer/window.cpp diff --git a/app/src/main/cpp/audio_player_jni.cpp b/app/src/main/cpp/audio_player_jni.cpp index 0f2de9f..b0197da 100644 --- a/app/src/main/cpp/audio_player_jni.cpp +++ b/app/src/main/cpp/audio_player_jni.cpp @@ -10,6 +10,13 @@ FFAudioPlayer *audioPlayer; +void fftCallback(JNIEnv *env, jobject thiz, jmethodID fft_method, int8_t *data, int size) { + jbyteArray dataArray = env->NewByteArray(size); + env->SetByteArrayRegion(dataArray, 0, size, data); + env->CallVoidMethod(thiz, fft_method, dataArray); + env->DeleteLocalRef(dataArray); +} + AUDIO_PLAYER_FUNC(void, play, jstring path) { if (path == nullptr) return; @@ -32,6 +39,8 @@ AUDIO_PLAYER_FUNC(void, play, jstring path) { // method if of write jmethodID write_method = env->GetMethodID(audio_track_class, "write", "([BII)I"); + jmethodID fft_method = env->GetMethodID(audio_class, "fftCallbackFromJNI", "([B)V"); + // demux decode and play while (result >= 0) { result = audioPlayer->decodeAudio(); @@ -49,6 +58,10 @@ AUDIO_PLAYER_FUNC(void, play, jstring path) { env->CallIntMethod(audio_track, write_method, audio_array, 0, size); env->DeleteLocalRef(audio_array); + if (audioPlayer->enableVisualizer()) { + fftCallback(env, thiz, fft_method, audioPlayer->getFFTData(), audioPlayer->getFFTSize()); + } + // audio sync usleep(SLEEP_TIME); } diff --git a/app/src/main/cpp/ff_audio_player.cpp b/app/src/main/cpp/ff_audio_player.cpp index 845c1fa..e560d50 100644 --- a/app/src/main/cpp/ff_audio_player.cpp +++ b/app/src/main/cpp/ff_audio_player.cpp @@ -132,6 +132,9 @@ int FFAudioPlayer::open(const char *path) { filterFrame = av_frame_alloc(); initFilter(FILTER_DESC, codecContext, &audioFilterGraph, &audioSrcContext, &audioSinkContext); + // init visualizer + mVisualizer = new FrankVisualizer(); + mVisualizer->init_visualizer(); return 0; } @@ -172,6 +175,13 @@ int FFAudioPlayer::decodeAudio() { } } + // visualizer: do fft + int nb_samples = inputFrame->nb_samples < MAX_FFT_SIZE ? inputFrame->nb_samples : MAX_FFT_SIZE; + if (m_enableVisualizer && nb_samples >= MIN_FFT_SIZE) { + mVisualizer->fft_run(inputFrame->data[0], nb_samples); + } + + // change filter if (filterAgain) { filterAgain = false; avfilter_graph_free(&audioFilterGraph); @@ -216,6 +226,26 @@ uint8_t *FFAudioPlayer::getDecodeFrame() const { return out_buffer; } +void FFAudioPlayer::setEnableVisualizer(bool enable) { + m_enableVisualizer = enable; +} + +bool FFAudioPlayer::enableVisualizer() const { + return m_enableVisualizer; +} + +int8_t* FFAudioPlayer::getFFTData() const { + if (!mVisualizer) + return nullptr; + return mVisualizer->getFFTData(); +} + +int FFAudioPlayer::getFFTSize() const { + if (!mVisualizer) + return 0; + return mVisualizer->getOutputSample(); +} + void FFAudioPlayer::setFilterAgain(bool again) { filterAgain = again; } @@ -250,4 +280,7 @@ void FFAudioPlayer::close() { avfilter_graph_free(&audioFilterGraph); } delete[] out_buffer; + if (mVisualizer) { + mVisualizer->release_visualizer(); + } } \ No newline at end of file diff --git a/app/src/main/cpp/ff_audio_player.h b/app/src/main/cpp/ff_audio_player.h index 1f330cf..bbc4379 100644 --- a/app/src/main/cpp/ff_audio_player.h +++ b/app/src/main/cpp/ff_audio_player.h @@ -7,6 +7,7 @@ #include #include "ffmpeg_jni_define.h" +#include "visualizer/frank_visualizer.h" #ifdef __cplusplus extern "C" { @@ -46,6 +47,9 @@ private: AVFilterContext *audioSrcContext; AVFilterContext *audioSinkContext; + bool m_enableVisualizer = false; + FrankVisualizer *mVisualizer; + public: int open(const char* path); @@ -61,6 +65,14 @@ public: void setFilterDesc(const char *filterDescription); + void setEnableVisualizer(bool enable); + + bool enableVisualizer() const; + + int8_t* getFFTData() const; + + int getFFTSize() const; + void setExit(bool exit); void close(); diff --git a/app/src/main/cpp/visualizer/block_queue.c b/app/src/main/cpp/visualizer/block_queue.c deleted file mode 100644 index ef730db..0000000 --- a/app/src/main/cpp/visualizer/block_queue.c +++ /dev/null @@ -1,70 +0,0 @@ -// -// Created by frank on 2021/08/08. -// -#include "block_queue.h" -#include - -vlc_queue_t *vlc_queue_init(int size) { - vlc_queue_t *queue = (vlc_queue_t *)(malloc(sizeof(vlc_queue_t))); - queue->size = size; - queue->next_to_read = 0; - queue->next_to_write = 0; - int i; - queue->packets = (void **)(malloc(sizeof(*queue->packets) * size)); - for (i = 0; i < size; i++) { - block_t *packet = malloc(sizeof(block_t)); - packet->i_nb_samples = 0; - queue->packets[i] = packet; - } - pthread_mutex_init(&queue->mutex, NULL); - pthread_cond_init(&queue->cond, NULL); - return queue; -} - -void vlc_queue_free(vlc_queue_t *queue) { - int i; - for (i = 0; i < queue->size; i++) { - free(queue->packets[i]); - } - free(queue->packets); - free(queue); - pthread_cond_destroy(&queue->cond); - pthread_mutex_destroy(&queue->mutex); -} - -int vlc_queue_next(vlc_queue_t *queue, int current) { - return (current + 1) % queue->size; -} - -void *vlc_queue_push(vlc_queue_t *queue, void *data) { - pthread_mutex_lock(&queue->mutex);//lock - int current = queue->next_to_write; - int next_to_write; - for (;;) { - next_to_write = vlc_queue_next(queue, current); - if (next_to_write != queue->next_to_read) { - break; - } - pthread_cond_wait(&queue->cond, &queue->mutex); - } - queue->next_to_write = next_to_write; - queue->packets[current] = data;//TODO - pthread_cond_broadcast(&queue->cond); - pthread_mutex_unlock(&queue->mutex);//unlock - return queue->packets[current]; -} - -void *vlc_queue_pop(vlc_queue_t *queue) { - pthread_mutex_lock(&queue->mutex); - int current = queue->next_to_read; - for (;;) { - if (queue->next_to_write != queue->next_to_read) { - break; - } - pthread_cond_wait(&queue->cond, &queue->mutex); - } - queue->next_to_read = vlc_queue_next(queue, current); - pthread_cond_broadcast(&queue->cond); - pthread_mutex_unlock(&queue->mutex); - return queue->packets[current]; -} \ No newline at end of file diff --git a/app/src/main/cpp/visualizer/block_queue.h b/app/src/main/cpp/visualizer/block_queue.h deleted file mode 100644 index aaa98cf..0000000 --- a/app/src/main/cpp/visualizer/block_queue.h +++ /dev/null @@ -1,57 +0,0 @@ -// -// Created by frank on 2021/08/08. -// - -#ifndef BLOCK_QUEUE_H -#define BLOCK_QUEUE_H - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -typedef int64_t vlc_tick_t; - -typedef struct FFT_callback { - void (*callback)(void*); -} FFT_callback; - -typedef struct block_t { - uint8_t *p_buffer; /** Payload start */ - - unsigned i_nb_samples; /** Used for audio */ - - vlc_tick_t i_pts; - vlc_tick_t i_length; - - FFT_callback fft_callback; -} block_t; - -typedef struct vlc_queue { - //the size of queue - int size; - //packet array - void **packets; - //the packet next to write - int next_to_write; - //the packet next to read - int next_to_read; - - pthread_mutex_t mutex; - pthread_cond_t cond; -} vlc_queue_t; - -vlc_queue_t *vlc_queue_init(int size); - -void vlc_queue_free(vlc_queue_t *queue); - -void *vlc_queue_push(vlc_queue_t *queue, void *data); - -void *vlc_queue_pop(vlc_queue_t *queue); - -#endif //BLOCK_QUEUE_H - -#ifdef __cplusplus -} -#endif \ No newline at end of file diff --git a/app/src/main/cpp/visualizer/frank_visualizer.cpp b/app/src/main/cpp/visualizer/frank_visualizer.cpp index f2b432a..27fd26a 100644 --- a/app/src/main/cpp/visualizer/frank_visualizer.cpp +++ b/app/src/main/cpp/visualizer/frank_visualizer.cpp @@ -160,6 +160,8 @@ FrankVisualizer::~FrankVisualizer() { int8_t* FrankVisualizer::fft_run(uint8_t *input_buffer, int nb_samples) { mFftLock.lock(); + if (!fft_context) + return nullptr; fft_context->nb_samples = nb_samples; memcpy(fft_context->data, input_buffer, static_cast(nb_samples)); filter_sys_t *p_sys = fft_context; @@ -179,11 +181,13 @@ int FrankVisualizer::getOutputSample() { return 0; } +int8_t *FrankVisualizer::getFFTData() { + return fft_context ? fft_context->output : nullptr; +} + int FrankVisualizer::init_visualizer() { fft_context = new filter_sys_t(); filter_sys_t *p_filter = fft_context; - if (!p_filter) - return -1; p_filter->convert_to_float = false; p_filter->i_channels = 1; diff --git a/app/src/main/cpp/visualizer/frank_visualizer.h b/app/src/main/cpp/visualizer/frank_visualizer.h index bea980f..c14ec3f 100644 --- a/app/src/main/cpp/visualizer/frank_visualizer.h +++ b/app/src/main/cpp/visualizer/frank_visualizer.h @@ -47,6 +47,8 @@ public: int getOutputSample(); + int8_t* getFFTData(); + int8_t* fft_run(uint8_t *input_buffer, int nb_samples); int init_visualizer();