Mux audio but bug exist.

utils/MediaBase/src/FfmpegDecoder.cpp

@@ -0,0 +1,220 @@
+/*
+ * Copyright (c) 2023 Fancy Code.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "FfmpegDecoder.h"
+#include "ILog.h"
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include <libavcodec/avcodec.h>
+#include <libavcodec/codec.h>
+#include <libavcodec/codec_id.h>
+#include <libavcodec/packet.h>
+#include <libavutil/avutil.h>
+#include <libavutil/channel_layout.h>
+#include <libavutil/error.h>
+#include <libavutil/frame.h>
+#include <libavutil/samplefmt.h>
+#ifdef __cplusplus
+}
+#endif
+#include <cstddef>
+#include <cstdlib>
+#include <errno.h>
+#include <functional>
+FfmpegDecoder::FfmpegDecoder(const enum AVCodecID &codecId)
+    : mCodecId(codecId), mCodec(nullptr), mCodecCtx(nullptr), mFrame(nullptr)
+{
+}
+bool FfmpegDecoder::Init(void)
+{
+    int ret = 0;
+    mCodec = (AVCodec *)avcodec_find_decoder(mCodecId);
+    // mCodec = (AVCodec *)avcodec_find_decoder_by_name("libfdk_aac");
+    if (!(mCodec)) {
+        LogError("Codec not found\n");
+        return false;
+    }
+    mCodecCtx = avcodec_alloc_context3((const AVCodec *)(mCodec));
+    if (!(mCodecCtx)) {
+        LogError("Could not allocate codec context\n");
+        return false;
+    }
+    if (AVMEDIA_TYPE_AUDIO == mCodec->type) {
+        LogInfo("Audio decoder.\n");
+        /* put sample parameters */
+        mCodecCtx->bit_rate = 352800;
+        mCodecCtx->sample_rate = 8000;
+
+        /* check that the encoder supports s16 pcm input */
+        mCodecCtx->sample_fmt = AV_SAMPLE_FMT_S16;
+        if (!check_sample_fmt(mCodec, mCodecCtx->sample_fmt)) {
+            LogError("Encoder does not support sample format %s", av_get_sample_fmt_name(mCodecCtx->sample_fmt));
+            return false;
+        }
+
+        /* select other audio parameters supported by the encoder */
+        mCodecCtx->sample_rate = select_sample_rate(mCodec);
+        ret = select_channel_layout(mCodec, &(mCodecCtx->ch_layout));
+        if (ret < 0) {
+            LogError("Could not set channel layout\n");
+            return false;
+        }
+    }
+    if ((ret = avcodec_open2(mCodecCtx, mCodec, nullptr)) < 0) {
+        char error_str[AV_ERROR_MAX_STRING_SIZE] = {0};
+        LogError("Could not open codec:%s\n", av_make_error_string(error_str, AV_ERROR_MAX_STRING_SIZE, ret));
+        return false;
+    }
+    mFrame = av_frame_alloc();
+    if (!mFrame) {
+        LogError("Could not allocate video frame\n");
+        return false;
+    }
+    return true;
+}
+bool FfmpegDecoder::UnInit(void)
+{
+    if (mFrame) {
+        av_frame_free(&mFrame);
+        mFrame = nullptr;
+    }
+    if (mCodecCtx) {
+        avcodec_free_context(&mCodecCtx);
+        mCodecCtx = nullptr;
+    }
+    return true;
+}
+void FfmpegDecoder::DecodeData(const void *data, const size_t &size, std::function<void(AVFrame *frame)> callback)
+{
+    AVPacket *packet = nullptr;
+    packet = av_packet_alloc();
+    packet->data = (unsigned char *)data;
+    packet->size = size;
+    int ret = avcodec_send_packet(mCodecCtx, packet);
+    if (ret < 0) {
+        LogInfo("Error sending a packet for decoding\n");
+        av_packet_unref(packet);
+        av_packet_free(&packet);
+        return;
+    }
+    while (ret >= 0) {
+        ret = avcodec_receive_frame(mCodecCtx, mFrame);
+        if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
+            break;
+        }
+        if (ret < 0) {
+            LogInfo("Error during decoding\n");
+            break;
+        }
+        if (callback) {
+            callback(mFrame);
+        }
+        // mFrame->pts = mAudioSt.next_pts;
+        // mAudioSt.next_pts += mFrame->nb_samples;
+        // ConvertAudioFrame(mFrame, mAudioSt.enc, &mAudioSt);
+        // write_frame(mOc, mAudioSt.enc, mAudioSt.st, mAudioSt.frame, mAudioSt.tmp_pkt);
+        break;
+    }
+    av_packet_unref(packet);
+    av_packet_free(&packet);
+}
+/* just pick the highest supported samplerate */
+int FfmpegDecoder::select_sample_rate(const AVCodec *codec)
+{
+    const int *p;
+    int best_samplerate = 0;
+
+    if (!codec->supported_samplerates)
+        return 44100;
+
+    p = codec->supported_samplerates;
+    while (*p) {
+        if (!best_samplerate || abs(44100 - *p) < abs(44100 - best_samplerate))
+            best_samplerate = *p;
+        p++;
+    }
+    return best_samplerate;
+}
+/* select layout with the highest channel count */
+int FfmpegDecoder::select_channel_layout(const AVCodec *codec, AVChannelLayout *dst)
+{
+    const AVChannelLayout *p, *best_ch_layout = nullptr;
+    int best_nb_channels = 0;
+    AVChannelLayout channelLayout = AV_CHANNEL_LAYOUT_STEREO;
+    if (!codec->ch_layouts)
+        return av_channel_layout_copy(dst, &channelLayout);
+
+    p = codec->ch_layouts;
+    while (p->nb_channels) {
+        int nb_channels = p->nb_channels;
+
+        if (nb_channels > best_nb_channels) {
+            best_ch_layout = p;
+            best_nb_channels = nb_channels;
+        }
+        p++;
+    }
+    return av_channel_layout_copy(dst, best_ch_layout);
+}
+/* check that a given sample format is supported by the encoder */
+int FfmpegDecoder::check_sample_fmt(const AVCodec *codec, enum AVSampleFormat sample_fmt)
+{
+    const enum AVSampleFormat *p = codec->sample_fmts;
+
+    while (*p != AV_SAMPLE_FMT_NONE) {
+        if (*p == sample_fmt)
+            return 1;
+        p++;
+    }
+    return 0;
+}
+// bool FfmpegDecoder::ConvertAudioFrame(AVFrame *decodeFrame, AVCodecContext *c, struct SwrContext *swr_ctx)
+// {
+//     if (nullptr == decodeFrame) {
+//         LogError("decodeFrame is null\n");
+//         return false;
+//     }
+//     int ret = 0;
+//     int dst_nb_samples = 0;
+//     /* convert samples from native format to destination codec format, using the resampler */
+//     /* compute destination number of samples */
+//     dst_nb_samples = av_rescale_rnd(
+//         swr_get_delay(swr_ctx, c->sample_rate) + decodeFrame->nb_samples, c->sample_rate, c->sample_rate,
+//         AV_ROUND_UP);
+//     av_assert0(dst_nb_samples == decodeFrame->nb_samples);
+
+//     /* when we pass a frame to the encoder, it may keep a reference to it
+//      * internally;
+//      * make sure we do not overwrite it here
+//      */
+//     ret = av_frame_make_writable(ost->frame);
+//     if (ret < 0) {
+//         LogError("av_frame_make_writable failed\n");
+//         return false;
+//     }
+
+//     /* convert to destination format */
+//     ret = swr_convert(
+//         swr_ctx, ost->frame->data, dst_nb_samples, (const uint8_t **)decodeFrame->data, decodeFrame->nb_samples);
+//     if (ret < 0) {
+//         LogError("Error while converting\n");
+//         return false;
+//     }
+//     decodeFrame = ost->frame;
+
+//     decodeFrame->pts = av_rescale_q(ost->samples_count, (AVRational){1, c->sample_rate}, c->time_base);
+//     ost->samples_count += dst_nb_samples;
+//     return true;
+// }

utils/MediaBase/src/FfmpegDecoder.h

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2023 Fancy Code.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef FFMPEG_DECODER_H
+#define FFMPEG_DECODER_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include <libavcodec/avcodec.h>
+#include <libavcodec/packet.h>
+#include <libavformat/avformat.h>
+#include <libavutil/avassert.h>
+#include <libavutil/avutil.h>
+#include <libavutil/channel_layout.h>
+#include <libavutil/imgutils.h>
+#include <libavutil/mathematics.h>
+#include <libavutil/opt.h>
+#include <libavutil/timestamp.h>
+#include <libswresample/swresample.h>
+#include <libswscale/swscale.h>
+#ifdef __cplusplus
+}
+#endif
+#include <functional>
+class FfmpegDecoder
+{
+public:
+    FfmpegDecoder(const enum AVCodecID &codecId);
+    virtual ~FfmpegDecoder() = default;
+    bool Init(void);
+    bool UnInit(void);
+    void DecodeData(const void *data, const size_t &size, std::function<void(AVFrame *frame)> callback);
+
+private:
+    static int select_sample_rate(const AVCodec *codec);
+    static int select_channel_layout(const AVCodec *codec, AVChannelLayout *dst);
+    static int check_sample_fmt(const AVCodec *codec, enum AVSampleFormat sample_fmt);
+    // static bool ConvertAudioFrame(AVFrame *decodeFrame, AVCodecContext *c, struct SwrContext *swr_ctx);
+
+private:
+    const enum AVCodecID mCodecId;
+    AVCodec *mCodec;
+    AVCodecContext *mCodecCtx;
+    AVFrame *mFrame;
+};
+#endif

utils/MediaBase/src/FfmpegEncoder.cpp

@@ -0,0 +1,320 @@
+/*
+ * Copyright (c) 2023 Fancy Code.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "FfmpegEncoder.h"
+#include "ILog.h"
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include <libavcodec/avcodec.h>
+#include <libavcodec/codec.h>
+#include <libavcodec/codec_id.h>
+#include <libavcodec/packet.h>
+#include <libavformat/avformat.h>
+#include <libavutil/avutil.h>
+#include <libavutil/channel_layout.h>
+#include <libavutil/dict.h>
+#include <libavutil/error.h>
+#include <libavutil/frame.h>
+#include <libavutil/opt.h>
+#include <libavutil/pixfmt.h>
+#include <libavutil/samplefmt.h>
+#include <libswresample/swresample.h>
+#ifdef __cplusplus
+}
+#endif
+#include <errno.h>
+#include <functional>
+#define STREAM_DURATION 10.0
+#define STREAM_FRAME_RATE 25              /* 25 images/s */
+#define STREAM_PIX_FMT AV_PIX_FMT_YUV420P /* default pix_fmt */
+FfmpegEncoder::FfmpegEncoder(const enum AVCodecID &codecId)
+    : mCodecId(codecId), mCodecCtx(nullptr), mCodec(nullptr), mFrame(nullptr), mTmpFrame(nullptr), mTmpPkt(nullptr)
+{
+}
+bool FfmpegEncoder::Init(int &outputFlags)
+{
+    mTmpPkt = av_packet_alloc();
+    if (!mTmpPkt) {
+        LogError("Could not allocate AVPacket\n");
+        return false;
+    }
+    int i = 0;
+    /* find the encoder */
+    mCodec = (AVCodec *)avcodec_find_encoder(mCodecId);
+    if (!mCodec) {
+        LogError("Could not find encoder for '%s'\n", avcodec_get_name(mCodecId));
+        return false;
+    }
+    mCodecCtx = avcodec_alloc_context3(mCodec);
+    if (!mCodecCtx) {
+        LogError("Could not alloc an encoding context\n");
+        return false;
+    }
+    const AVChannelLayout src = (AVChannelLayout)AV_CHANNEL_LAYOUT_STEREO;
+    switch (mCodec->type) {
+    case AVMEDIA_TYPE_AUDIO:
+        mCodecCtx->sample_fmt = mCodec->sample_fmts ? mCodec->sample_fmts[0] : AV_SAMPLE_FMT_FLTP;
+        mCodecCtx->bit_rate = 64000;
+        mCodecCtx->sample_rate = 44100;
+        if (mCodec->supported_samplerates) {
+            mCodecCtx->sample_rate = mCodec->supported_samplerates[0];
+            for (i = 0; mCodec->supported_samplerates[i]; i++) {
+                if (mCodec->supported_samplerates[i] == 44100)
+                    mCodecCtx->sample_rate = 44100;
+            }
+        }
+        av_channel_layout_copy(&mCodecCtx->ch_layout, &src);
+        // st->time_base = (AVRational){1, mCodecCtx->sample_rate};
+        break;
+
+    case AVMEDIA_TYPE_VIDEO:
+        mCodecCtx->codec_id = mCodecId;
+
+        mCodecCtx->bit_rate = 400000;
+        /* Resolution must be a multiple of two. */
+        mCodecCtx->width = 352;
+        mCodecCtx->height = 288;
+        /* timebase: This is the fundamental unit of time (in seconds) in terms
+         * of which frame timestamps are represented. For fixed-fps content,
+         * timebase should be 1/framerate and timestamp increments should be
+         * identical to 1. */
+        // st->time_base = (AVRational){1, STREAM_FRAME_RATE};
+        mCodecCtx->time_base = (AVRational){1, STREAM_FRAME_RATE};
+
+        mCodecCtx->gop_size = 12; /* emit one intra frame every twelve frames at most */
+        mCodecCtx->pix_fmt = STREAM_PIX_FMT;
+        if (mCodecCtx->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
+            /* just for testing, we also add B-frames */
+            mCodecCtx->max_b_frames = 2;
+        }
+        if (mCodecCtx->codec_id == AV_CODEC_ID_MPEG1VIDEO) {
+            /* Needed to avoid using macroblocks in which some coeffs overflow.
+             * This does not happen with normal video, it just happens here as
+             * the motion of the chroma plane does not match the luma plane. */
+            mCodecCtx->mb_decision = 2;
+        }
+        break;
+
+    default:
+        break;
+    }
+    /* Some formats want stream headers to be separate. */
+    if (outputFlags & AVFMT_GLOBALHEADER) {
+        mCodecCtx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
+    }
+    return true;
+}
+void FfmpegEncoder::UnInit(void)
+{
+    if (mFrame) {
+        av_frame_free(&mFrame);
+        mFrame = nullptr;
+    }
+    if (mTmpFrame) {
+        av_frame_free(&mTmpFrame);
+        mTmpFrame = nullptr;
+    }
+    if (mCodecCtx) {
+        avcodec_free_context(&mCodecCtx);
+        mCodecCtx = nullptr;
+    }
+    av_packet_free(&mTmpPkt);
+}
+AVRational FfmpegEncoder::GetTimeBase(void)
+{
+    switch (mCodec->type) {
+    case AVMEDIA_TYPE_AUDIO:
+        return (AVRational){1, mCodecCtx->sample_rate};
+
+    case AVMEDIA_TYPE_VIDEO:
+        return mCodecCtx->time_base;
+
+    default:
+        LogError("Unsupported media type.\n");
+        return (AVRational){0, -1};
+    }
+}
+bool FfmpegEncoder::OpenEncoder(AVDictionary *optArg, AVStream *stream, struct SwrContext *swr_ctx)
+{
+    switch (mCodec->type) {
+    case AVMEDIA_TYPE_AUDIO:
+        return OpenAudio(optArg, stream, swr_ctx);
+
+    case AVMEDIA_TYPE_VIDEO:
+        return OpenVideo(optArg, stream);
+
+    default:
+        LogError("Unsupported media type.\n");
+        return false;
+    }
+}
+int FfmpegEncoder::EncodeData(AVFrame *frame, AVStream *stream, std::function<void(AVPacket *pkt)> callback)
+{
+    int ret;
+    // send the frame to the encoder
+    ret = avcodec_send_frame(mCodecCtx, frame);
+    if (ret < 0) {
+        char error_str[AV_ERROR_MAX_STRING_SIZE] = {0};
+        LogInfo("Error sending a frame to the encoder: %s\n",
+                av_make_error_string(error_str, AV_ERROR_MAX_STRING_SIZE, ret));
+        return AVERROR_EXIT;
+    }
+
+    while (ret >= 0) {
+        ret = avcodec_receive_packet(mCodecCtx, mTmpPkt);
+        if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
+            break;
+        }
+        if (ret < 0) {
+            char error_str[AV_ERROR_MAX_STRING_SIZE] = {0};
+            LogInfo("Error encoding a frame: %s\n", av_make_error_string(error_str, AV_ERROR_MAX_STRING_SIZE, ret));
+            return AVERROR_EXIT;
+        }
+
+        /* rescale output packet timestamp values from codec to stream timebase */
+        av_packet_rescale_ts(mTmpPkt, mCodecCtx->time_base, stream->time_base);
+        mTmpPkt->stream_index = stream->index;
+
+        /* Write the compressed frame to the media file. */
+        // log_packet(fmt_ctx, pkt);
+        // ret = av_interleaved_write_frame(fmt_ctx, pkt);
+        if (callback) {
+            callback(mTmpPkt);
+        }
+        /* pkt is now blank (av_interleaved_write_frame() takes ownership of
+         * its contents and resets pkt), so that no unreferencing is necessary.
+         * This would be different if one used av_write_frame(). */
+        // if (ret < 0) {
+        //     fprintf(stderr, "Error while writing output packet: %s\n", av_err2str(ret));
+        //     return AVERROR_EXIT;
+        // }
+    }
+
+    return ret == AVERROR_EOF ? 1 : 0;
+}
+bool FfmpegEncoder::OpenVideo(AVDictionary *optArg, AVStream *stream)
+{
+    int ret = 0;
+    AVDictionary *opt = nullptr;
+    av_dict_copy(&opt, optArg, 0);
+    /* open the codec */
+    ret = avcodec_open2(mCodecCtx, mCodec, &opt);
+    av_dict_free(&opt);
+    if (ret < 0) {
+        char error_str[AV_ERROR_MAX_STRING_SIZE] = {0};
+        LogError("Could not open video codec: %s\n", av_make_error_string(error_str, AV_ERROR_MAX_STRING_SIZE, ret));
+        return false;
+    }
+    /* allocate and init a re-usable frame */
+    mFrame = alloc_frame(mCodecCtx->pix_fmt, mCodecCtx->width, mCodecCtx->height);
+    if (!mFrame) {
+        LogError("Could not allocate video frame\n");
+        return false;
+    }
+    if (mCodecCtx->pix_fmt != AV_PIX_FMT_YUV420P) {
+        mTmpFrame = alloc_frame(AV_PIX_FMT_YUV420P, mCodecCtx->width, mCodecCtx->height);
+        if (!mTmpFrame) {
+            LogError("Could not allocate temporary video frame\n");
+            return false;
+        }
+    }
+    /* copy the stream parameters to the muxer */
+    ret = avcodec_parameters_from_context(stream->codecpar, mCodecCtx);
+    if (ret < 0) {
+        LogError("Could not copy the stream parameters\n");
+        return false;
+    }
+    return true;
+}
+bool FfmpegEncoder::OpenAudio(AVDictionary *optArg, AVStream *stream, struct SwrContext *swr_ctx)
+{
+    int nb_samples = 0;
+    int ret = 0;
+    AVDictionary *opt = nullptr;
+    av_dict_copy(&opt, optArg, 0);
+    /* open it */
+    ret = avcodec_open2(mCodecCtx, mCodec, &opt);
+    av_dict_free(&opt);
+    if (ret < 0) {
+        char error_str[AV_ERROR_MAX_STRING_SIZE] = {0};
+        LogError("Could not open audio codec: %s\n", av_make_error_string(error_str, AV_ERROR_MAX_STRING_SIZE, ret));
+        return false;
+    }
+    if (mCodecCtx->codec->capabilities & AV_CODEC_CAP_VARIABLE_FRAME_SIZE)
+        nb_samples = 10000;
+    else
+        nb_samples = mCodecCtx->frame_size;
+    mFrame = alloc_audio_frame(mCodecCtx->sample_fmt, &mCodecCtx->ch_layout, mCodecCtx->sample_rate, nb_samples);
+    mTmpFrame = alloc_audio_frame(AV_SAMPLE_FMT_S16, &mCodecCtx->ch_layout, mCodecCtx->sample_rate, nb_samples);
+    /* copy the stream parameters to the muxer */
+    ret = avcodec_parameters_from_context(stream->codecpar, mCodecCtx);
+    if (ret < 0) {
+        LogError("Could not copy the stream parameters\n");
+        return false;
+    }
+    /* set options */
+    av_opt_set_chlayout(swr_ctx, "in_chlayout", &mCodecCtx->ch_layout, 0);
+    av_opt_set_int(swr_ctx, "in_sample_rate", mCodecCtx->sample_rate, 0);
+    av_opt_set_sample_fmt(swr_ctx, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
+    av_opt_set_chlayout(swr_ctx, "out_chlayout", &mCodecCtx->ch_layout, 0);
+    av_opt_set_int(swr_ctx, "out_sample_rate", mCodecCtx->sample_rate, 0);
+    av_opt_set_sample_fmt(swr_ctx, "out_sample_fmt", mCodecCtx->sample_fmt, 0);
+    return true;
+}
+AVFrame *FfmpegEncoder::alloc_frame(enum AVPixelFormat pix_fmt, int width, int height)
+{
+    AVFrame *frame;
+    int ret;
+
+    frame = av_frame_alloc();
+    if (!frame)
+        return nullptr;
+
+    frame->format = pix_fmt;
+    frame->width = width;
+    frame->height = height;
+
+    /* allocate the buffers for the frame data */
+    ret = av_frame_get_buffer(frame, 0);
+    if (ret < 0) {
+        LogInfo("Could not allocate frame data.\n");
+        return nullptr;
+    }
+
+    return frame;
+}
+AVFrame *FfmpegEncoder::alloc_audio_frame(enum AVSampleFormat sample_fmt, const AVChannelLayout *channel_layout,
+                                          int sample_rate, int nb_samples)
+{
+    AVFrame *frame = av_frame_alloc();
+    if (!frame) {
+        LogInfo("Error allocating an audio frame\n");
+        return nullptr;
+    }
+
+    frame->format = sample_fmt;
+    av_channel_layout_copy(&frame->ch_layout, channel_layout);
+    frame->sample_rate = sample_rate;
+    frame->nb_samples = nb_samples;
+
+    if (nb_samples) {
+        if (av_frame_get_buffer(frame, 0) < 0) {
+            LogInfo("Error allocating an audio buffer\n");
+            return nullptr;
+        }
+    }
+
+    return frame;
+}

utils/MediaBase/src/FfmpegEncoder.h

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2023 Fancy Code.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef FFMPEG_ENCODER_H
+#define FFMPEG_ENCODER_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include <libavcodec/avcodec.h>
+#include <libavcodec/packet.h>
+#include <libavformat/avformat.h>
+#include <libavutil/avassert.h>
+#include <libavutil/avutil.h>
+#include <libavutil/channel_layout.h>
+#include <libavutil/imgutils.h>
+#include <libavutil/mathematics.h>
+#include <libavutil/opt.h>
+#include <libavutil/timestamp.h>
+#include <libswresample/swresample.h>
+#include <libswscale/swscale.h>
+#ifdef __cplusplus
+}
+#endif
+#include <functional>
+class FfmpegEncoder
+{
+public:
+    FfmpegEncoder(const enum AVCodecID &codecId);
+    virtual ~FfmpegEncoder() = default;
+    bool Init(int &outputFlags);
+    void UnInit(void);
+    AVRational GetTimeBase(void);
+    bool OpenEncoder(AVDictionary *optArg, AVStream *stream, struct SwrContext *swr_ctx);
+    int EncodeData(AVFrame *frame, AVStream *stream, std::function<void(AVPacket *pkt)> callback);
+
+private:
+    bool OpenVideo(AVDictionary *optArg, AVStream *stream);
+    bool OpenAudio(AVDictionary *optArg, AVStream *stream, struct SwrContext *swr_ctx);
+
+private:
+    static AVFrame *alloc_frame(enum AVPixelFormat pix_fmt, int width, int height);
+    static AVFrame *alloc_audio_frame(enum AVSampleFormat sample_fmt, const AVChannelLayout *channel_layout,
+                                      int sample_rate, int nb_samples);
+
+private:
+    const enum AVCodecID mCodecId;
+    AVCodecContext *mCodecCtx;
+    AVCodec *mCodec;
+    AVFrame *mFrame;
+    AVFrame *mTmpFrame;
+    AVPacket *mTmpPkt;
+};
+#endif

utils/MediaBase/src/FfmpegMuxStream.cpp

@@ -18,6 +18,7 @@
 #include "StatusCode.h"
 #include <cstddef>
 #include <cstdlib>
+#include <stdint.h>
 #include <stdio.h>
 #include <string.h>
 #include <string>
@@ -30,11 +31,13 @@ extern "C" {
 #include <libavcodec/packet.h>
 #include <libavformat/avformat.h>
 #include <libavformat/avio.h>
+#include <libavutil/avassert.h>
 #include <libavutil/avutil.h>
 #include <libavutil/channel_layout.h>
 #include <libavutil/dict.h>
 #include <libavutil/error.h>
 #include <libavutil/frame.h>
+#include <libavutil/mathematics.h>
 #include <libavutil/opt.h>
 #include <libavutil/pixfmt.h>
 #include <libavutil/samplefmt.h>
@@ -124,7 +127,7 @@ StatusCode FfmpegMuxStream::OpenOutputFile(const std::string &fileName)
 }
 StatusCode FfmpegMuxStream::CloseOutputFile(void)
 {
-    if (mOc->pb) {
+    if (mOc && mOc->pb) {
         av_write_trailer(mOc);
     }
     if (mFrameVideo) {
@@ -143,6 +146,9 @@ StatusCode FfmpegMuxStream::CloseOutputFile(void)
         avcodec_free_context(&mCodecAudioContext);
         mCodecAudioContext = nullptr;
     }
+    if (nullptr == mOc) {
+        return CreateStatusCode(STATUS_CODE_OK);
+    }
     close_stream(mOc, &mVideoSt);
     close_stream(mOc, &mAudioSt);
     memset(&mVideoSt, 0, sizeof(mVideoSt));
@@ -191,12 +197,26 @@ void FfmpegMuxStream::GetVideoStream(const void *data, const size_t &size, const
     av_packet_unref(packet);
     av_packet_free(&packet);
 }
+void save_code_stream_file(const void *data, const unsigned int &size)
+{
+    FILE *file = NULL;
+    file = fopen("./test.g711a", "a+");
+
+    if (file) {
+        fwrite(data, 1, size, file);
+        fflush(file);
+    }
+
+    if (file)
+        fclose(file);
+}
 void FfmpegMuxStream::GetAudioStream(const void *data, const size_t &size, const StreamInfo &streamInfo)
 {
     AVPacket *packet = nullptr;
     packet = av_packet_alloc();
     packet->data = (unsigned char *)data;
     packet->size = size;
+    // save_code_stream_file(data, size);
     int ret = avcodec_send_packet(mCodecAudioContext, packet);
     if (ret < 0) {
         LogInfo("Error sending a packet for decoding\n");
@@ -215,7 +235,7 @@ void FfmpegMuxStream::GetAudioStream(const void *data, const size_t &size, const
         }
         mFrameAudio->pts = mAudioSt.next_pts;
         mAudioSt.next_pts += mFrameAudio->nb_samples;
-        ConvertAudioFrame(mFrameVideo, mAudioSt.enc, &mAudioSt);
+        ConvertAudioFrame(mFrameAudio, mAudioSt.enc, &mAudioSt);
         write_frame(mOc, mAudioSt.enc, mAudioSt.st, mAudioSt.frame, mAudioSt.tmp_pkt);
         break;
     }
@@ -226,7 +246,7 @@ bool FfmpegMuxStream::add_stream(OutputStream *ost, AVFormatContext *oc, const A
 {
     AVCodecContext *c;
     int i;
-
+    LogInfo("Encoder: %s\n", avcodec_get_name(codec_id));
     /* find the encoder */
     *codec = avcodec_find_encoder(codec_id);
     if (!(*codec)) {
@@ -266,6 +286,7 @@ bool FfmpegMuxStream::add_stream(OutputStream *ost, AVFormatContext *oc, const A
                     c->sample_rate = 44100;
             }
         }
+        c->sample_rate = 8000;
         av_channel_layout_copy(&c->ch_layout, &src);
         ost->st->time_base = (AVRational){1, c->sample_rate};
         break;
@@ -500,7 +521,7 @@ bool FfmpegMuxStream::InitCodecAudio(enum AVCodecID codecId, AVCodec **codec, AV
 {
     int ret = 0;
     *codec = (AVCodec *)avcodec_find_decoder(codecId);
-    // *codec = (AVCodec *)avcodec_find_encoder_by_name("libfdk_aac");
+    // *codec = (AVCodec *)avcodec_find_decoder_by_name("libfdk_aac");
     if (!(*codec)) {
         LogError("Codec not found\n");
         return false;
@@ -512,6 +533,8 @@ bool FfmpegMuxStream::InitCodecAudio(enum AVCodecID codecId, AVCodec **codec, AV
     }
     /* put sample parameters */
     (*codec_ctx)->bit_rate = 64000;
+    // (*codec_ctx)->bit_rate = 352800;
+    // (*codec_ctx)->sample_rate = 8000;
 
     /* check that the encoder supports s16 pcm input */
     (*codec_ctx)->sample_fmt = AV_SAMPLE_FMT_S16;
@@ -532,6 +555,8 @@ bool FfmpegMuxStream::InitCodecAudio(enum AVCodecID codecId, AVCodec **codec, AV
         LogError("Could not open codec:%s\n", av_make_error_string(error_str, AV_ERROR_MAX_STRING_SIZE, ret));
         return false;
     }
+    /* Set the packet timebase for the decoder. */
+    // (*codec_ctx)->pkt_timebase = {.num = 1, .den = 44100};
     *frame = av_frame_alloc();
     if (!(*frame)) {
         LogError("Could not allocate video frame\n");

utils/MediaBase/src/FfmpegMuxStreamV2.cpp

@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2023 Fancy Code.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "FfmpegMuxStreamV2.h"
+#include "FfmpegOutputStream.h"
+#include "ILog.h"
+#include "MediaBase.h"
+#include "StatusCode.h"
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include <libavcodec/codec_id.h>
+#include <libavcodec/packet.h>
+#include <libavformat/avformat.h>
+#include <libavformat/avio.h>
+#include <libavutil/dict.h>
+#include <libavutil/error.h>
+#ifdef __cplusplus
+}
+#endif
+#include <cstddef>
+#include <functional>
+#include <memory>
+#include <string>
+FfmpegMuxStreamV2::FfmpegMuxStreamV2() : mOutputFormat(nullptr), mOptions(nullptr)
+{
+}
+StatusCode FfmpegMuxStreamV2::OpenOutputFile(const std::string &fileName)
+{
+    return OpenMuxOutputFile(fileName);
+}
+StatusCode FfmpegMuxStreamV2::CloseOutputFile(void)
+{
+    if (mOutputFormat && mOutputFormat->pb) {
+        av_write_trailer(mOutputFormat);
+    }
+    mVideoStream->UnInit();
+    mAudioStream->UnInit();
+    if (nullptr == mOutputFormat) {
+        return CreateStatusCode(STATUS_CODE_OK);
+    }
+    if (!(mOutputFormat->oformat->flags & AVFMT_NOFILE)) {
+        /* Close the output file. */
+        avio_closep(&mOutputFormat->pb);
+    }
+    avformat_free_context(mOutputFormat);
+    return CreateStatusCode(STATUS_CODE_OK);
+}
+void FfmpegMuxStreamV2::GetStreamData(const void *data, const size_t &size, const StreamInfo &streamInfo)
+{
+    if (streamInfo.mType == STREAM_TYPE_VIDEO_H264) {
+        // GetVideoStream(data, size, streamInfo);
+        mVideoStream->WriteSourceData(data, size);
+    }
+    if (streamInfo.mType == STREAM_TYPE_AUDIO_G711A) {
+        // GetAudioStream(data, size, streamInfo);
+        mAudioStream->WriteSourceData(data, size);
+    }
+}
+StatusCode FfmpegMuxStreamV2::OpenMuxOutputFile(const std::string &fileName)
+{
+    AVDictionary *opt = nullptr;
+    int ret = 0;
+    /* allocate the output media context */
+    avformat_alloc_output_context2(&mOutputFormat, nullptr, "mp4", fileName.c_str());
+    if (!mOutputFormat) {
+        LogError("Could not deduce output format from file.\n");
+        return CreateStatusCode(STATUS_CODE_NOT_OK);
+    }
+    /* Add the audio and video streams using the default format codecs
+     * and initialize the codecs. */
+    if (mOutputFormat->oformat->video_codec != AV_CODEC_ID_NONE) {
+        mVideoStream = AddStream(mOutputFormat, mOutputFormat->oformat->video_codec);
+        mVideoStream->SetWriteSourceDataCallback(
+            std::bind(&FfmpegMuxStreamV2::GetAVPacketDataCallback, this, std::placeholders::_1));
+    }
+    if (mOutputFormat->oformat->audio_codec != AV_CODEC_ID_NONE) {
+        mAudioStream = AddStream(mOutputFormat, mOutputFormat->oformat->video_codec);
+        mAudioStream->SetWriteSourceDataCallback(
+            std::bind(&FfmpegMuxStreamV2::GetAVPacketDataCallback, this, std::placeholders::_1));
+    }
+    av_dump_format(mOutputFormat, 0, fileName.c_str(), 1);
+    /* open the output file, if needed */
+    if (!(mOutputFormat->oformat->flags & AVFMT_NOFILE)) {
+        ret = avio_open(&mOutputFormat->pb, fileName.c_str(), AVIO_FLAG_WRITE);
+        if (ret < 0) {
+            char error_str[AV_ERROR_MAX_STRING_SIZE] = {0};
+            LogError("Could not open '%s': %s\n",
+                     fileName.c_str(),
+                     av_make_error_string(error_str, AV_ERROR_MAX_STRING_SIZE, ret));
+        }
+    }
+    /* Write the stream header, if any. */
+    ret = avformat_write_header(mOutputFormat, &opt);
+    if (ret < 0) {
+        char error_str[AV_ERROR_MAX_STRING_SIZE] = {0};
+        LogError("Error occurred when opening output file: %s\n",
+                 av_make_error_string(error_str, AV_ERROR_MAX_STRING_SIZE, ret));
+        return CreateStatusCode(STATUS_CODE_NOT_OK);
+    }
+    return CreateStatusCode(STATUS_CODE_OK);
+}
+void FfmpegMuxStreamV2::GetAVPacketDataCallback(AVPacket *pkt)
+{
+    int ret = 0;
+    ret = av_interleaved_write_frame(mOutputFormat, pkt);
+    /* pkt is now blank (av_interleaved_write_frame() takes ownership of
+     * its contents and resets pkt), so that no unreferencing is necessary.
+     * This would be different if one used av_write_frame(). */
+    if (ret < 0) {
+        char error_str[AV_ERROR_MAX_STRING_SIZE] = {0};
+        LogInfo("Error while writing output packet: %s\n",
+                av_make_error_string(error_str, AV_ERROR_MAX_STRING_SIZE, ret));
+    }
+}
+std::shared_ptr<FfmpegOutputStream> FfmpegMuxStreamV2::AddStream(AVFormatContext *outputFormat, enum AVCodecID codecId)
+{
+    auto stream = std::make_shared<FfmpegOutputStream>(codecId);
+    stream->Init(outputFormat);
+    return stream;
+}

utils/MediaBase/src/FfmpegMuxStreamV2.h

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2023 Fancy Code.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef FFMPEG_MUX_STREAM_V2_H
+#define FFMPEG_MUX_STREAM_V2_H
+#include "FfmpegBase.h"
+#include "FfmpegOutputStream.h"
+#include "MediaBase.h"
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include <libavcodec/avcodec.h>
+#include <libavcodec/packet.h>
+#include <libavformat/avformat.h>
+#include <libavutil/avassert.h>
+#include <libavutil/avutil.h>
+#include <libavutil/channel_layout.h>
+#include <libavutil/imgutils.h>
+#include <libavutil/mathematics.h>
+#include <libavutil/opt.h>
+#include <libavutil/timestamp.h>
+#include <libswresample/swresample.h>
+#include <libswscale/swscale.h>
+#ifdef __cplusplus
+}
+#endif
+#include <memory>
+#include <string>
+class FfmpegMuxStreamV2 : virtual public FfmpegBase
+{
+public:
+    FfmpegMuxStreamV2();
+    virtual ~FfmpegMuxStreamV2() = default;
+
+public: // About combine file.
+    StatusCode OpenOutputFile(const std::string &fileName) override;
+    StatusCode CloseOutputFile(void) override;
+    void GetStreamData(const void *data, const size_t &size, const StreamInfo &streamInfo) override;
+
+private:
+    StatusCode OpenMuxOutputFile(const std::string &fileName);
+    void GetAVPacketDataCallback(AVPacket *pkt);
+
+private:
+    static std::shared_ptr<FfmpegOutputStream> AddStream(AVFormatContext *outputFormat, enum AVCodecID codecId);
+
+private:
+    AVFormatContext *mOutputFormat;
+    std::shared_ptr<FfmpegOutputStream> mVideoStream;
+    std::shared_ptr<FfmpegOutputStream> mAudioStream;
+    AVDictionary *mOptions;
+};
+#endif

utils/MediaBase/src/FfmpegOutputStream.cpp

@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2023 Fancy Code.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "FfmpegOutputStream.h"
+#include "FfmpegEncoder.h"
+#include "ILog.h"
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include <libavcodec/codec_id.h>
+#include <libavcodec/packet.h>
+#include <libavformat/avformat.h>
+#include <libavutil/frame.h>
+#include <libswresample/swresample.h>
+#ifdef __cplusplus
+}
+#endif
+#include <cstddef>
+#include <functional>
+#include <memory>
+FfmpegOutputStream::FfmpegOutputStream(const AVCodecID &codecId)
+    : mCodecId(codecId), mTmpPkt(nullptr), mStream(nullptr), swr_ctx(nullptr)
+{
+}
+bool FfmpegOutputStream::Init(AVFormatContext *outputFormat)
+{
+    mDecodeCallback = std::bind(&FfmpegOutputStream::GetDecodeDataCallback, this, std::placeholders::_1);
+    // mEncodeCallback = std::bind(&FfmpegOutputStream::GetEncodeDataCallback, this, std::placeholders::_1);
+    int ret = 0;
+    mTmpPkt = av_packet_alloc();
+    if (!mTmpPkt) {
+        LogError("Could not allocate AVPacket\n");
+        return false;
+    }
+    /* create resampler context */
+    swr_ctx = swr_alloc();
+    if (!swr_ctx) {
+        LogError("Could not allocate resampler context\n");
+        return false;
+    }
+    mStream = avformat_new_stream(outputFormat, nullptr);
+    if (!mStream) {
+        LogError("Could not allocate stream\n");
+        return false;
+    }
+    mStream->id = outputFormat->nb_streams - 1;
+    mEncoder = std::make_shared<FfmpegEncoder>(mCodecId);
+    mEncoder->Init(outputFormat->flags);
+    mEncoder->OpenEncoder(nullptr, mStream, swr_ctx);
+    /* initialize the resampling context */
+    if ((ret = swr_init(swr_ctx)) < 0) {
+        LogError("Failed to initialize the resampling context\n");
+        return false;
+    }
+    // mDecoder = std::make_shared<FfmpegDecoder>();
+    return true;
+}
+void FfmpegOutputStream::UnInit(void)
+{
+    mEncoder->UnInit();
+    swr_free(&swr_ctx);
+    av_packet_free(&mTmpPkt);
+}
+void FfmpegOutputStream::WriteSourceData(const void *data, const size_t &size)
+{
+    mDecoder->DecodeData(data, size, mDecodeCallback);
+}
+void FfmpegOutputStream::SetWriteSourceDataCallback(std::function<void(AVPacket *pkt)> callback)
+{
+    mEncodeCallback = callback;
+}
+void FfmpegOutputStream::GetDecodeDataCallback(AVFrame *frame)
+{
+    mEncoder->EncodeData(frame, mStream, mEncodeCallback);
+}
+void FfmpegOutputStream::GetEncodeDataCallback(AVPacket *pkt)
+{
+}

utils/MediaBase/src/FfmpegOutputStream.h

@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2023 Fancy Code.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef FFMPEG_OUTPUT_STREAM_H
+#define FFMPEG_OUTPUT_STREAM_H
+#include "FfmpegDecoder.h"
+#include "FfmpegEncoder.h"
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include <libavcodec/avcodec.h>
+#include <libavcodec/packet.h>
+#include <libavformat/avformat.h>
+#include <libavutil/avassert.h>
+#include <libavutil/avutil.h>
+#include <libavutil/channel_layout.h>
+#include <libavutil/imgutils.h>
+#include <libavutil/mathematics.h>
+#include <libavutil/opt.h>
+#include <libavutil/timestamp.h>
+#include <libswresample/swresample.h>
+#include <libswscale/swscale.h>
+#ifdef __cplusplus
+}
+#endif
+#include <functional>
+#include <memory>
+class FfmpegOutputStream
+{
+public:
+    FfmpegOutputStream(const AVCodecID &codecId);
+    virtual ~FfmpegOutputStream() = default;
+    bool Init(AVFormatContext *outputFormat);
+    void UnInit(void);
+    void WriteSourceData(const void *data, const size_t &size);
+    void SetWriteSourceDataCallback(std::function<void(AVPacket *pkt)> callback);
+
+private:
+    void GetDecodeDataCallback(AVFrame *frame);
+    void GetEncodeDataCallback(AVPacket *pkt);
+
+private:
+    const AVCodecID mCodecId;
+    AVPacket *mTmpPkt;
+    std::shared_ptr<FfmpegEncoder> mEncoder;
+    std::shared_ptr<FfmpegDecoder> mDecoder;
+    AVStream *mStream;
+    struct SwrContext *swr_ctx;
+    std::function<void(AVFrame *)> mDecodeCallback;
+    std::function<void(AVPacket *)> mEncodeCallback;
+};
+#endif

utils/MediaBase/src/FfmpegReadFile.cpp

@@ -123,6 +123,10 @@ void FfmpegReadFile::ReadFileThread(AVFormatContext *pFormatCtx, int mediaStream
                          pFormatCtx->streams[mediaStreamIndex]->time_base.den;
             // LogInfo("Frame data address: %p, length: %zu\n", packet.data, packet.size);
             // LogInfo("Play time ms:%d\n", playTimeMs);
+            // LogInfo("time base: num = %d, den = %d\n",
+            //         pFormatCtx->streams[mediaStreamIndex]->time_base.num,
+            //         pFormatCtx->streams[mediaStreamIndex]->time_base.den);
+            // LogInfo("pFormatCtx->bit_rate = %ld\n", pFormatCtx->bit_rate);
             ReadFrame(&packet);
             std::this_thread::sleep_for(std::chrono::milliseconds(playTimeMs));
         }