Audio Output plugin on Android

I am looking at how to create an Audio Output plugin on Android ,I refer to the implementation on windwos. file path :FMOD SoundSystem\FMOD Studio API Windows\api\lowlevel\examples\plugins\output_mp3.cpp

Exception in function OutputMP3_UpdateCallback debug Find exceptions in call
output_state->readfrommixer
FMOD version is: 09/08/18 1.10.08 - Studio API minor release (build 96768)
full code:

#include "fmod/fmod.hpp"
#include "fmod/fmod_errors.h"
#include "com_sencent_voicechange.h"
#include "lame_3.99.5_libmp3lame/lame.h"

#include <stdlib.h>
#include <unistd.h>
#include <android/log.h>
#include <jni.h>
#include <thread>

using namespace FMOD;
#define OPEN_MP3_PLUG  1

#define DEFAULT_SAMPLING_RATE 44100
#define DEFAULT_LAME_IN_CHANNEL 2
#define DEFAULT_SAMPLING_RATE 44100
#define DEFAULT_LAME_MP3_BIT_RATE 32
#define DEFAULT_LAME_MP3_QUALITY 7
#define DEFAULT_OUTPUTNAME  "fmodoutput.mp3"


extern "C" {

    bool bProcessing = false;
    bool bLoop = false;
    unsigned int nAudioLength = 1;
    unsigned int nCurrentPosition = 1;
    const int loopTime = 50;
    
    ////////////////////////FMOD plug
    typedef struct{
        FILE                *mFP;
        unsigned long      hbeStream;
        unsigned char*     pMP3Buffer;
        short*              pWAVBuffer;
        unsigned long      dwMP3Buffer;
        unsigned long      dwSamples;
    int                 dspbufferlength;
    lame_global_flags   *lame;
} outputmp3_state;

FMOD_OUTPUT_DESCRIPTION mp3output;
/*
//already OK
*/
FMOD_RESULT F_CALLBACK
OutputMP3_GetNumDriversCallback(FMOD_OUTPUT_STATE *output_state, int *numdrivers) {
    LOGE("%s", "OutputMP3_GetNumDriversCallback");
    *numdrivers = 1;
    return FMOD_OK;
}

/*
//already OK
*/
FMOD_RESULT F_CALLBACK
OutputMP3_GetDriverInfoCallback(FMOD_OUTPUT_STATE * /*output*/, int /*id*/, char *name, int namelen,
                                FMOD_GUID * /*guid*/, int * /*systemrate*/,
                                FMOD_SPEAKERMODE *speakermode, int *speakermodechannels) {
    LOGE("%s", "OutputMP3_GetDriverInfoCallback");
    //fmodoutput.mp3 output file
    strncpy(name, DEFAULT_OUTPUTNAME, namelen);
    *speakermode = FMOD_SPEAKERMODE_STEREO;
    *speakermodechannels = 2;
    return FMOD_OK;
}

/*
//already OK
*/
FMOD_RESULT F_CALLBACK
OutputMP3_InitCallback(FMOD_OUTPUT_STATE *output_state, int /*selecteddriver*/,
                    FMOD_INITFLAGS /*flags*/, int *outputrate, FMOD_SPEAKERMODE *speakermode,
                    int *speakermodechannels, FMOD_SOUND_FORMAT *outputformat,
                    int dspbufferlength, int /*dspnumbuffers*/, void *extradriverdata) {
    //LOGE("%s", "OutputMP3_InitCallback begin");
    outputmp3_state *state;
    char    filename[256]= {};
    /*
        Create a structure that we can attach to the plugin instance.
    */
    state = (outputmp3_state *)calloc(sizeof(outputmp3_state), 1);
    if (!state){
        return FMOD_ERR_MEMORY;
    }

    output_state->plugindata = state;

    *outputformat        = FMOD_SOUND_FORMAT_PCM16;
    *speakermode         = FMOD_SPEAKERMODE_STEREO;
    *speakermodechannels = 2;

    state->dspbufferlength = dspbufferlength;
    state->lame = lame_init();
    lame_set_in_samplerate(state->lame, DEFAULT_SAMPLING_RATE);
    lame_set_num_channels(state->lame, DEFAULT_LAME_IN_CHANNEL);
    lame_set_out_samplerate(state->lame, DEFAULT_SAMPLING_RATE);
    lame_set_brate(state->lame, DEFAULT_LAME_MP3_BIT_RATE);
    lame_set_quality(state->lame, DEFAULT_LAME_MP3_QUALITY);
    lame_init_params(state->lame);

    //setting default value
    state->dwMP3Buffer = 7200*2;
    state->dwSamples = DEFAULT_SAMPLING_RATE;
    // Allocate MP3 buffer
    state->pMP3Buffer = (unsigned char*)malloc(state->dwMP3Buffer);
    if(!state->pMP3Buffer){
        return FMOD_ERR_MEMORY;
    }

    // Allocate WAV buffer
    state->pWAVBuffer = (short*)malloc(state->dwSamples * sizeof(short));
    if (!state->pWAVBuffer){
        return FMOD_ERR_MEMORY;
    }
    if (!extradriverdata){
        strncpy(filename, DEFAULT_OUTPUTNAME, 256);
    }
    else{
        strncpy(filename, (char *)extradriverdata, 256);
    }

    state->mFP = fopen(filename, "wb");
    if (!state->mFP){
        return FMOD_ERR_FILE_NOTFOUND;
    }

    LOGE("%s", "OutputMP3_InitCallback ok");
    return FMOD_OK;
}

/*
//
*/
FMOD_RESULT F_CALLBACK OutputMP3_CloseCallback(FMOD_OUTPUT_STATE *output_state) {
    LOGE("%s", "OutputMP3_CloseCallback");
    outputmp3_state *state = (outputmp3_state *)output_state->plugindata;
    if (!state){
        return FMOD_OK;
    }

    int dwWrite =lame_encode_flush(state->lame,state->pMP3Buffer,state->dwMP3Buffer);
    if(dwWrite > 0 ){
        if (fwrite(state->pMP3Buffer, 1, dwWrite, state->mFP) != dwWrite){
            return FMOD_ERR_FILE_BAD;
        }
    }

    lame_close(state->lame);
    state->lame = NULL;

    if (state->pWAVBuffer){
        free(state->pWAVBuffer);
        state->pWAVBuffer = 0;
    }

    if (state->pMP3Buffer){
        free(state->pMP3Buffer);
        state->pMP3Buffer = 0;
    }

    if (state->mFP){
        fclose(state->mFP);
        state->mFP = 0;
    }

    if (state){
        free(state);
        output_state->plugindata = 0;
    }

    return FMOD_OK;
}

static int debug_count=0;
/*
//
*/
FMOD_RESULT F_CALLBACK OutputMP3_UpdateCallback(FMOD_OUTPUT_STATE *output_state) {
    LOGE("OutputMP3_UpdateCallback debug_count:%d", debug_count++);
    FMOD_RESULT result;
    outputmp3_state *state = (outputmp3_state *)output_state->plugindata;

    /*
        Update the mixer to the interleaved buffer.
    */
    int dwRead = state->dwSamples * sizeof(short);
    short* destptr = state->pWAVBuffer;
    /* /2 = stereo */;
    unsigned int len = state->dwSamples / 2;
    while (len)
    {
        unsigned int temp_len = len; // > state->dspbufferlength ? state->dspbufferlength : len;
        //readfrommixer exception
        result = output_state->readfrommixer(output_state, destptr, temp_len);
        if (result != FMOD_OK){
            return FMOD_OK;
        }

        len -= temp_len;
        destptr += (temp_len * 2); /* *2 = stereo. */
    }

    // Encode samples
    int dwWrite = lame_encode_buffer(state->lame, state->pWAVBuffer, state->pWAVBuffer,dwRead, state->pMP3Buffer, state->dwMP3Buffer);
    LOGE("OutputMP3_UpdateCallback dwWrite:%d", dwWrite);
    // write dwWrite bytes that are returned in tehe pMP3Buffer to disk
    if(dwWrite > 0 ){
        if (fwrite(state->pMP3Buffer, 1, dwWrite, state->mFP) != dwWrite){
        return FMOD_ERR_FILE_BAD;
        }
    }

    return FMOD_OK;
}

/*
//
*/
FMOD_RESULT F_CALLBACK OutputMP3_GetHandleCallback(FMOD_OUTPUT_STATE *output_state, void **handle) {
    LOGE("%s", "OutputMP3_GetHandleCallback");
    outputmp3_state *state = (outputmp3_state *)output_state->plugindata;
    *handle = state->mFP;
    return FMOD_OK;
}

/*
    FMODGetOutputDescription is mandantory for every fmod plugin.  This is the symbol the registerplugin function searches for.
    Must be declared with F_CALL to make it export as stdcall.
*/
F_EXPORT FMOD_OUTPUT_DESCRIPTION *F_CALL FMODGetOutputDescription() {
    memset(&mp3output, 0, sizeof(FMOD_OUTPUT_DESCRIPTION));
    mp3output.apiversion = FMOD_OUTPUT_PLUGIN_VERSION;
    mp3output.name = "Sencent MP3 Output";
    mp3output.version = 0x00010000;
    mp3output.polling = 0;                                      /* False = no thread is created.  Plugin must drive FMOD's mixer */
    mp3output.getnumdrivers = OutputMP3_GetNumDriversCallback;
    mp3output.getdriverinfo = OutputMP3_GetDriverInfoCallback;
    mp3output.init = OutputMP3_InitCallback;
    mp3output.close = OutputMP3_CloseCallback;
    mp3output.update = OutputMP3_UpdateCallback;
    mp3output.gethandle = OutputMP3_GetHandleCallback;
    mp3output.getposition = 0;                                    /* Not a polling output so getposition is never called */
    mp3output.lock = 0;                                           /* Not a polling output so lock is never called */
    mp3output.unlock = 0;                                         /* Not a polling output so unlock is never called */

    LOGE("%s", "mp3 plug");
    return &mp3output;
}
////////////////////////FMOD plug end

static void callJavaMethod(JNIEnv *env, jobject jthis, int what) {
    jclass voiceChangerClazz = env->FindClass("com/sencent/voicechange/FMODProcess");
    if (voiceChangerClazz == NULL) {
        LOGE("%s", "find class VoiceChanger error !");
        return;
    }

    jmethodID notifyID = env->GetStaticMethodID(voiceChangerClazz, "notify", "(I)V");
    if (notifyID == NULL) {
        LOGE("%s", "find method notify error !");
        return;
    }
    env->CallStaticVoidMethod(voiceChangerClazz, notifyID, what);
}

static void functionEnd(System *system, Sound *sound, Channel *channel) {
    bProcessing = false;
    bLoop = false;
    nAudioLength = 1;
    nCurrentPosition = 1;
    int numdsps = 0;
    channel->getNumDSPs(&numdsps);
    for (int i = 0; i < numdsps; i++) {
        DSP *dsp = NULL;
        channel->getDSP(i, &dsp);
        FMOD_DSP_TYPE type;
        dsp->getType(&type);
        channel->removeDSP(dsp);
        dsp->release();
    }

    channel->stop();
    sound->release();
    system->close();
    system->release();
}

JNIEXPORT jboolean JNICALL Java_com_sencent_voicechange_FMODProcess_isProcess
        (JNIEnv *env, jobject jthis) {
    return true;
}

JNIEXPORT jint JNICALL Java_com_sencent_voicechange_FMODProcess_getCurrentPosition
        (JNIEnv *env, jobject jthis) {
    return nCurrentPosition;
}

JNIEXPORT jint JNICALL Java_com_sencent_voicechange_FMODProcess_getDuration
        (JNIEnv *env, jobject jthis) {
    return nAudioLength;
}

JNIEXPORT jint JNICALL Java_com_sencent_voicechange_FMODProcess_getDelayTime
        (JNIEnv *env, jobject jthis) {
    return loopTime;
}

JNIEXPORT void JNICALL Java_com_sencent_voicechange_FMODProcess_nativeStop
        (JNIEnv *env, jobject jthis) {
    bLoop = false;
}

JNIEXPORT jboolean JNICALL Java_com_sencent_voicechange_FMODProcess_nativeProcessByte
        (JNIEnv *env, jobject jthis, jbyteArray byteData, jlong len, jstring out_path, jint type) {
    return true;
}

JNIEXPORT jboolean JNICALL Java_com_sencent_voicechange_FMODProcess_nativeProcess
        (JNIEnv *env, jobject jthis, jstring in_path, jstring out_path, jint type) {
    const char *inputPathChar = env->GetStringUTFChars(in_path, NULL);
    const char *outputPathChar = env->GetStringUTFChars(out_path, NULL);

    char input[256] = {};
    char output[256] = {};
    strcpy(input, inputPathChar);
    strcpy(output, outputPathChar);

    env->ReleaseStringUTFChars(in_path, inputPathChar);
    env->ReleaseStringUTFChars(out_path, outputPathChar);
    env->DeleteLocalRef(in_path);
    env->DeleteLocalRef(out_path);

    FMOD::System *system;
    FMOD::Sound *sound;
    FMOD::Channel *channel = 0;
    FMOD_RESULT result;
    unsigned int version;
    float frequency = 0;
    FMOD::DSP *dsp;

    result = System_Create(&system);
    if (result != FMOD_OK) {
        LOGE("System_Create failure %s", FMOD_ErrorString(result))
        callJavaMethod(env, jthis, ON_ERROR_PROCESS);
        return false;
    }

    result = system->getVersion(&version);
    if (result != FMOD_OK) {
        LOGE("getVersion failure %s", FMOD_ErrorString(result))
        callJavaMethod(env, jthis, ON_ERROR_PROCESS);
        return false;
    }

#if OPEN_MP3_PLUG
    unsigned int outputhandle=0;
    result = system->registerOutput(FMODGetOutputDescription(), &outputhandle);
    LOGE("registerOutput result %s", result)
    if (result != FMOD_OK) {
        LOGE("registerOutput failure %s", FMOD_ErrorString(result))
        callJavaMethod(env, jthis, ON_ERROR_PROCESS);
        return false;
    }
#endif

    unsigned int bufferlength = 512;
    int numbuffers = 4;
    result = system->getDSPBufferSize(&bufferlength, &numbuffers);
    if (result != FMOD_OK) {
        LOGE("getDSPBufferSize failure %s", FMOD_ErrorString(result))
        callJavaMethod(env, jthis, ON_ERROR_PROCESS);
        return false;
    }
    system->setDSPBufferSize(bufferlength * 4, numbuffers);

#if OPEN_MP3_PLUG
    result = system->setOutputByPlugin(outputhandle);
#else
    result = system->setOutput(FMOD_OUTPUTTYPE_WAVWRITER_NRT);
    if (result != FMOD_OK) {
        LOGE("setOutput failure %s", FMOD_ErrorString(result))
        callJavaMethod(env, jthis, ON_ERROR_PROCESS);
        return false;
    }
#endif

    result = system->init(16, FMOD_INIT_STREAM_FROM_UPDATE, output);
    if (result != FMOD_OK) {
        LOGE("init failure %s", FMOD_ErrorString(result))
        callJavaMethod(env, jthis, ON_ERROR_PROCESS);
        return false;
    }

    result = system->createSound(input, FMOD_DEFAULT, 0, &sound);
    if (result != FMOD_OK) {
        LOGE("createSound failure %s", FMOD_ErrorString(result))
        callJavaMethod(env, jthis, ON_ERROR_PROCESS);
        return false;
    }

    try {
        switch (type) {
            case MODE_NORMAL:
                system->playSound(sound, 0, false, &channel);
                break;
            case MODE_LUOLI:
                system->createDSPByType(FMOD_DSP_TYPE_PITCHSHIFT, &dsp);
                dsp->setParameterFloat(FMOD_DSP_PITCHSHIFT_PITCH, 2.5);
                system->playSound(sound, 0, false, &channel);
                channel->addDSP(0, dsp);
                break;
            case MODE_JINGSONG:
                system->createDSPByType(FMOD_DSP_TYPE_TREMOLO, &dsp);
                dsp->setParameterFloat(FMOD_DSP_TREMOLO_SKEW, 0.5);
                system->playSound(sound, 0, false, &channel);
                channel->addDSP(0, dsp);
                break;
            case MODE_DASHU:
                system->createDSPByType(FMOD_DSP_TYPE_PITCHSHIFT, &dsp);
                dsp->setParameterFloat(FMOD_DSP_PITCHSHIFT_PITCH, 0.8);
                system->playSound(sound, 0, false, &channel);
                channel->addDSP(0, dsp);
                break;
            case MODE_GAOGUAI:
                system->playSound(sound, 0, false, &channel);
                channel->getFrequency(&frequency);
                frequency = frequency * 1.6;
                channel->setFrequency(frequency);
                break;
            case MODE_KONGLING:
                system->createDSPByType(FMOD_DSP_TYPE_ECHO, &dsp);
                dsp->setParameterFloat(FMOD_DSP_ECHO_DELAY, 300);
                dsp->setParameterFloat(FMOD_DSP_ECHO_FEEDBACK, 20);
                system->playSound(sound, 0, false, &channel);
                channel->addDSP(0, dsp);
                break;
            default:
                break;
        }
    } catch (...) {
        LOGE("%s", "run error!!!");
        callJavaMethod(env, jthis, ON_ERROR_PROCESS);
        functionEnd(system, sound, channel);
        return false;
    }

    bProcessing = false;
    bLoop = true;
    sound->getLength(&nAudioLength, FMOD_TIMEUNIT_MS);
    callJavaMethod(env, jthis, ON_START_PROCESS);

    do {
        system->update();
        channel->isPlaying(&bProcessing);
        channel->getPosition(&nCurrentPosition, FMOD_TIMEUNIT_MS);
        usleep(10*loopTime*1000);
    } while (bProcessing && bLoop);

    if (bLoop)
        callJavaMethod(env, jthis, ON_SUCCESS_PROCESS);
    else
        callJavaMethod(env, jthis, ON_STOP_PROCESS);

    functionEnd(system, sound, channel);

    return true;
}

}

What is the possible cause of the problem?

Is that using a legitimate lame build that works on android? I’d be surprised if you can use windows code so easily on android.

I can only suggest debugging the contents of FMOD_OUTPUT_STATE and outputmp3_state
plugindata member a valid pointer in every callback that you got? You could put guards inside outputmp3_state to see if anything gets changed or overwritten.

Next i would verify the parameters are ok
ie, output_state, destptr, temp_len

Then I would strip out the mp3 stuff and just call readfrommixer and lets say, write it to a .pcm file, as you’re basing the output mode off a ‘no sound output’ method, ie the mp3 write to file output method.

1 Like

Hi luoyong6572? Did you manage to configure the output plugin on android?