first commit

nodes/ffio/ff_common.cpp

@@ -0,0 +1,55 @@
+
+#ifdef VP_WITH_FFMPEG
+#include "ff_common.h"
+
+namespace vp_nodes {
+    ff_scaler::ff_scaler(int src_width, 
+            int src_height, 
+            AVPixelFormat src_fmt, 
+            int dst_width, 
+            int dst_height,
+            AVPixelFormat dst_fmt):
+            m_src_width(src_width),
+            m_src_height(src_height),
+            m_src_fmt(src_fmt),
+            m_dst_width(dst_width),
+            m_dst_height(dst_height),
+            m_dst_fmt(dst_fmt) {
+        sws_ctx = sws_getContext(src_width, src_height, src_fmt, dst_width,dst_height, dst_fmt, SWS_FAST_BILINEAR, NULL, NULL, NULL);
+    }
+
+    ff_scaler::~ff_scaler() {
+        if (!sws_ctx) {
+            sws_freeContext(sws_ctx);
+        }
+    }
+
+    bool ff_scaler::scale(ff_av_frame_ptr src, ff_av_frame_ptr& dst) {
+        if (!sws_ctx || !src || !dst) {
+            return false;
+        }
+
+        if (src->format != m_src_fmt || src->width != m_src_width || src->height != m_src_height ||
+            dst->format != m_dst_fmt || dst->width != m_dst_width || dst->height != m_dst_height) {
+            return false;
+        }
+
+        if (dst->format == AV_PIX_FMT_BGR24) {
+            dst->linesize[0] = m_dst_width * 3;
+        }
+
+        /* allocate buffer if need for dst */
+        if (dst->data[0] == NULL) {
+            auto ret = av_frame_get_buffer(dst.get(), 0);
+            if (ret < 0) {
+                return false;
+            }
+        }
+        
+        auto ret = sws_scale(sws_ctx, src->data, src->linesize, 0, src->height, dst->data, dst->linesize);
+        dst->pts = src->pts;
+        dst->pkt_dts = src->pkt_dts;
+        return ret >= 0;
+    }
+}
+#endif

nodes/ffio/ff_common.h

@@ -0,0 +1,64 @@
+#pragma once
+#ifdef VP_WITH_FFMPEG
+#include <memory>
+#include <thread>
+#include <condition_variable>
+#include <mutex>
+#include <sstream>
+#include <unordered_map>
+#include <queue>
+extern "C" {
+#include <libavformat/avformat.h>
+#include <libavcodec/avcodec.h>
+#include <libswscale/swscale.h>
+#include <libavutil/avutil.h>
+#include <libavutil/pixdesc.h>
+#include <libavutil/imgutils.h>
+#include <libavdevice/avdevice.h>
+}
+
+#define ff_av_packet_ptr std::shared_ptr<AVPacket>
+#define ff_av_frame_ptr std::shared_ptr<AVFrame>
+#define ff_src_ptr std::shared_ptr<vp_nodes::ff_src>
+#define ff_des_ptr std::shared_ptr<vp_nodes::ff_des>
+#define ff_scaler_ptr std::shared_ptr<vp_nodes::ff_scaler>
+
+#define alloc_ff_src(channel_index) std::make_shared<vp_nodes::ff_src>(channel_index)
+#define alloc_ff_des(channel_index) std::make_shared<vp_nodes::ff_des>(channel_index)
+#define alloc_ff_scaler(src_width, src_height, src_fmt, dst_width, dst_height, dst_fmt)               \
+        std::make_shared<vp_nodes::ff_scaler>(src_width, src_height, src_fmt, dst_width, dst_height, dst_fmt)
+        
+#define alloc_ff_av_frame()                                                                           \
+        std::shared_ptr<AVFrame>(av_frame_alloc(), [](AVFrame *frame) { av_frame_free(&frame); })
+#define alloc_ff_av_packet()                                                                          \
+        std::shared_ptr<AVPacket>(av_packet_alloc(), [](AVPacket *pkt) { av_packet_free(&pkt); })
+    
+namespace vp_nodes {
+    /**
+     * tools for color conversion & image resize using FFmpeg on CPUs.
+     * 
+     */
+    class ff_scaler {
+    private:
+        SwsContext* sws_ctx = NULL;
+        int m_src_width;
+        int m_src_height;
+        AVPixelFormat m_src_fmt;
+        int m_dst_width;
+        int m_dst_height;
+        AVPixelFormat m_dst_fmt;
+    public:
+        /* disable copy and assignment operations */
+        ff_scaler(const ff_scaler&) = delete;
+        ff_scaler& operator=(const ff_scaler&) = delete;
+        ff_scaler(int src_width, 
+                 int src_height, 
+                 AVPixelFormat src_fmt, 
+                 int dst_width, 
+                 int dst_height,
+                 AVPixelFormat dst_fmt);
+        ~ff_scaler();
+        bool scale(ff_av_frame_ptr src, ff_av_frame_ptr& dst);
+    };
+}
+#endif

nodes/ffio/ff_des.cpp

@@ -0,0 +1,482 @@
+#ifdef VP_WITH_FFMPEG
+#include <iostream>
+#include <sstream>
+#include "ff_des.h"
+
+namespace vp_nodes {
+    ff_des::ff_des(int channel_index): m_channel_index(channel_index) {
+    }
+
+    ff_des::~ff_des() {
+        inner_close();
+    }
+
+    void ff_des::enmux_run() {
+        auto ret = 0;
+        while (m_enmux_running) {
+            ff_av_packet_ptr ff_packet = nullptr;
+            m_enmux_semaphore.wait();
+            {
+                std::lock_guard<std::mutex> g(m_enmux_packets_m);
+                ff_packet = m_enmux_packets_q.front();
+                m_enmux_packets_q.pop();
+            }
+
+            if (!ff_packet) {
+                VP_INFO(vp_utils::string_format("[ffio/ff_des][%d][enmux_run] get exit flag.", 
+                                                get_channel_index()));
+                break;
+            } else {
+                /* set parameters for packets */
+                av_packet_rescale_ts(ff_packet.get(), m_enc_ctx->time_base, m_ofmt_ctx->streams[m_inner_stream_index]->time_base);
+                ff_packet->stream_index = m_inner_stream_index;
+                ret = av_interleaved_write_frame(m_ofmt_ctx, ff_packet.get());
+            }
+
+            if (ret < 0) {
+                VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][enmux_run] av_interleaved_write_frame failed. ret: %d", 
+                                                 get_channel_index(), ret));
+                break;
+            }
+        }
+        /* write the trailer for output stream */
+        ret = av_write_trailer(m_ofmt_ctx);
+        if (ret != 0) {
+            VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][enmux_run] av_write_trailer failed. ret: %d", 
+                                             get_channel_index(), ret));
+        }
+        
+        /* set enmux running flag to false in case of abnormal exit */
+        m_enmux_running = false;
+        VP_INFO(vp_utils::string_format("[ffio/ff_des][%d][enmux_run] enmux thread exits.", 
+                                        get_channel_index()));
+    }
+
+    void ff_des::encode_run() {
+        while (m_encode_running) {
+            ff_av_frame_ptr ff_frame;
+            m_encode_semaphore.wait();
+            {
+                std::lock_guard<std::mutex> g(m_encode_frames_m);
+                ff_frame = m_encode_frames_q.front();
+                m_encode_frames_q.pop();
+            }
+
+            auto ret = 0;
+            if (!ff_frame) {
+                VP_INFO(vp_utils::string_format("[ffio/ff_des][%d][encode_run] get exit flag.", 
+                                                get_channel_index()));
+                break;
+            }
+            ff_frame->pts = m_enc_ctx->frame_number;
+            ret = avcodec_send_frame(m_enc_ctx, ff_frame.get());
+            if (ret < 0) {
+                VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][encode_run] avcodec_send_frame failed. ret: %d", 
+                                                 get_channel_index(), ret));
+                break;
+            }
+            
+            while (ret >= 0) {
+                auto ff_packet = alloc_ff_av_packet();
+                ret = avcodec_receive_packet(m_enc_ctx, ff_packet.get());
+                if (ret == AVERROR(EAGAIN)) {
+                    break;
+                } else if (ret < 0) {
+                    VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][encode_run] avcodec_receive_packet failed. ret: %d", 
+                                                     get_channel_index(), ret));
+                    ff_frame = nullptr;
+                    break;
+                }
+
+                bool notify = true;
+                {
+                    std::lock_guard<std::mutex> g(m_enmux_packets_m);
+                    m_enmux_packets_q.push(ff_packet);
+                    if (m_enmux_packets_q.size() > m_enmux_packets_q_max_size) {
+                        m_enmux_packets_q.pop();
+                        notify = false;
+                        VP_WARN(vp_utils::string_format("[ffio/ff_des][%d][encode_run] exceed m_enmux_packets_q_max_size(%d), discard the front in queue.", 
+                                                        get_channel_index(), m_enmux_packets_q_max_size));
+                    }
+                }
+                if (notify) {
+                    m_enmux_semaphore.signal();
+                }
+            }
+            
+            if (!ff_frame) {
+                break;
+            }  
+        }
+        /* set encode running flag to false in case of abnormal exit */
+        m_encode_running = false;
+        {
+            // send exit flag to notify enmux thread
+            std::lock_guard<std::mutex> g(m_enmux_packets_m);
+            m_enmux_packets_q.push(nullptr);
+            VP_INFO(vp_utils::string_format("[ffio/ff_des][%d][encode_run] send exit flag to enmux thread.", 
+                                            get_channel_index()));
+        }
+        m_enmux_semaphore.signal();
+        VP_INFO(vp_utils::string_format("[ffio/ff_des][%d][encode_run] encode thread exits.", 
+                                        get_channel_index()));
+    }
+
+    bool ff_des::
+    open(const std::string& uri,
+         int width, int height, int fps, int bitrate, int max_b_frames,
+         const std::string& encoder_name, 
+         AVPixelFormat sw_pix_fmt,
+         AVHWDeviceType hw_type,
+         AVPixelFormat hw_pix_fmt) {
+        inner_close();
+
+        auto uri_valid = false;
+        if (!uri_valid) {
+            auto uri_parts = vp_utils::string_split(uri, '.');  // file
+            if (std::find(m_supported_files.begin(), m_supported_files.end(), uri_parts[uri_parts.size() - 1]) != m_supported_files.end()) {
+                uri_valid = true;
+                m_live_stream = false;
+            }
+        }
+        if (!uri_valid) {
+            auto uri_parts = vp_utils::string_split(uri, ':');  // live stream
+            if (std::find(m_supported_protocols.begin(), m_supported_protocols.end(), uri_parts[0]) != m_supported_protocols.end()) {
+                uri_valid = true;
+                m_live_stream = true;
+            }
+        }
+        if (!uri_valid) {
+            VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][open] uri invalid! uri MUST start with `rtsp/rtmp/..`(network streams) or end with `mp4/mkv/..`(file streams).", 
+                                             get_channel_index()));
+            return false;
+        }
+
+        return inner_open(uri, width, height, fps, bitrate, max_b_frames, encoder_name, sw_pix_fmt, hw_type, hw_pix_fmt);
+    }
+
+    void ff_des::close() {
+        inner_close();
+    }
+
+    bool ff_des::is_opened() const {
+        return m_encode_running && m_enmux_running;
+    }
+
+    void ff_des::inner_close() {
+        /* set running flags to false */
+        m_encode_running = false;
+        m_enmux_running = false;
+
+        inner_exit_signal();
+        /* waiting for threads exist */
+        if (m_encode_th != nullptr && m_encode_th->joinable()) {
+            m_encode_th->join();
+        }
+        if (m_enmux_th != nullptr && m_enmux_th->joinable()) {
+            m_enmux_th->join();
+        }
+
+        /* free format & codec context */
+        if (m_ofmt_ctx) {
+            if (m_ofmt_ctx->pb) {
+                avio_closep(&m_ofmt_ctx->pb);
+            }
+            avformat_free_context(m_ofmt_ctx);
+        }
+        if (m_enc_ctx) {
+            avcodec_free_context(&m_enc_ctx);
+        }
+
+        /* clear queue */
+        {
+            // clear enmux_packets_q
+            std::lock_guard<std::mutex> g(m_enmux_packets_m);
+            m_enmux_packets_q = {};
+            m_enmux_semaphore.reset();
+        }
+        {
+            // clear encode_frames_q
+            std::lock_guard<std::mutex> g(m_encode_frames_m);
+            m_encode_frames_q = {};
+            m_encode_semaphore.reset();
+        }    
+        VP_INFO(vp_utils::string_format("[ffio/ff_des][%d][inner_close] close successfully.", 
+                                        get_channel_index()));
+    }
+
+    std::string ff_des::print_summary() {
+        return "";
+    }
+
+    bool ff_des::
+    inner_open(const std::string& uri,
+               int width, int height, int fps, int bitrate, int max_b_frames,
+               const std::string& encoder_name, 
+               AVPixelFormat sw_pix_fmt,
+               AVHWDeviceType hw_type,
+               AVPixelFormat hw_pix_fmt) {
+        auto ret = 0;
+        /* allocate the output media context */
+        ret = avformat_alloc_output_context2(&m_ofmt_ctx, NULL, NULL, uri.c_str());
+        if (ret < 0) {
+            VP_WARN(vp_utils::string_format("[ffio/ff_des][%d][inner_open] could not deduce output format from uri, try to use FLV.", 
+                                             get_channel_index()));
+            ret = avformat_alloc_output_context2(&m_ofmt_ctx, NULL, "flv", uri.c_str());
+        }
+        if (ret < 0) {
+            VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][inner_open] could not deduce output format from uri(used FLV).", 
+                                             get_channel_index()));
+            return false;
+        }
+        auto ofmt = m_ofmt_ctx->oformat;   
+        
+        /* find encoder for the output stream */
+        const AVCodec* enc = nullptr;
+        if (encoder_name.empty()) {
+            enc = avcodec_find_encoder(ofmt->video_codec);             // get default encoder by AVCodecID
+        } else {
+            enc = avcodec_find_encoder_by_name(encoder_name.c_str());  // get by encoder name
+        }
+        if (!enc) {
+            VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][inner_open] could not find proper encoder for output stream.", 
+                                             get_channel_index()));
+            avformat_free_context(m_ofmt_ctx);
+            m_ofmt_ctx = NULL;
+            return false;
+        }
+        /* allocate a codec context for the encoder */
+        m_enc_ctx = avcodec_alloc_context3(enc);
+        if (!m_enc_ctx) {
+            VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][inner_open] could not allocate context for encoder(%s).", 
+                                             get_channel_index(), enc->name));
+            avformat_free_context(m_ofmt_ctx);
+            m_ofmt_ctx = NULL;
+            return false;
+        }
+        /* set parameters for encoder */
+        m_enc_ctx->framerate = (AVRational) {fps, 1};
+        m_enc_ctx->time_base = av_inv_q(m_enc_ctx->framerate);
+        m_enc_ctx->pix_fmt   = sw_pix_fmt;
+        m_enc_ctx->width     = width;
+        m_enc_ctx->height    = height;
+        m_enc_ctx->bit_rate  = bitrate * 1024;
+        m_enc_ctx->max_b_frames = max_b_frames;
+        /* check if need init hwaccels context for encoder */
+        if (hw_type != AVHWDeviceType::AV_HWDEVICE_TYPE_NONE) {
+            m_enc_ctx->pix_fmt   = hw_pix_fmt;
+            if ((ret = hw_encoder_init(m_enc_ctx, hw_type, sw_pix_fmt)) < 0) {
+                VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][inner_open] hw_encoder_init failed. ret: %d", 
+                                                 get_channel_index(), ret));
+                avcodec_free_context(&m_enc_ctx);
+                avformat_free_context(m_ofmt_ctx);
+                m_ofmt_ctx = NULL;
+                return false;
+            }
+        }
+        /* open the encoder */
+        if ((ret = avcodec_open2(m_enc_ctx, enc, NULL)) < 0) {
+            VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][inner_open] could not open encoder. ret: %d", 
+                                             get_channel_index(), ret));
+            avcodec_free_context(&m_enc_ctx);
+            avformat_free_context(m_ofmt_ctx);
+            m_ofmt_ctx = NULL;
+            return false;
+        }
+        /* create new stream for output */
+        auto out_stream = avformat_new_stream(m_ofmt_ctx, NULL);
+        if (!out_stream) {
+            VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][inner_open] could not create new stream for output uri.", 
+                                             get_channel_index()));
+            avcodec_free_context(&m_enc_ctx);
+            avformat_free_context(m_ofmt_ctx);
+            m_ofmt_ctx = NULL;
+            return false;
+        }
+        m_inner_stream_index = out_stream->index;
+        out_stream->time_base = m_enc_ctx->time_base;
+        ret = avcodec_parameters_from_context(out_stream->codecpar, m_enc_ctx);
+        if (ret < 0) {
+            VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][inner_open] failed to copy codec parameters from encode context. ret: %d", 
+                                             get_channel_index(), ret));
+            avcodec_free_context(&m_enc_ctx);
+            avformat_free_context(m_ofmt_ctx);
+            m_ofmt_ctx = NULL;
+            return false;
+        }
+        if (!(m_ofmt_ctx->flags & AVFMT_NOFILE)) {
+            ret = avio_open(&m_ofmt_ctx->pb, uri.c_str(), AVIO_FLAG_WRITE);
+            m_ofmt_ctx->flush_packets = 1;
+            m_ofmt_ctx->flags |= AVFMT_FLAG_FLUSH_PACKETS;
+            if (ret < 0) {
+                VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][inner_open] avio_open failed on output uri(%s). ret: %d", 
+                                                 get_channel_index(), uri.c_str(), ret));
+                avcodec_free_context(&m_enc_ctx);
+                avformat_free_context(m_ofmt_ctx);
+                m_ofmt_ctx = NULL;
+                return false;
+            }
+        }
+
+        /* write the stream header */
+        if ((ret = avformat_write_header(m_ofmt_ctx, NULL)) < 0) {
+            VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][inner_open] avformat_write_header failed. ret: %d", 
+                                             get_channel_index(), ret));
+            avcodec_free_context(&m_enc_ctx);
+            avformat_free_context(m_ofmt_ctx);
+            m_ofmt_ctx = NULL;
+            return false;
+        }
+        
+        /* initialize video & ff_des properties */
+        m_hw_type_name = hw_type != AV_HWDEVICE_TYPE_NONE ? std::string(av_hwdevice_get_type_name(hw_type)) : "none";
+        m_encoder_name = encoder_name.empty() ? std::string(m_enc_ctx->codec->name) : encoder_name;
+        m_uri = uri;
+        m_fps = fps;
+        m_width = width;
+        m_height = height;
+        m_bitrate = bitrate;
+        m_codec_name = std::string(avcodec_get_name(out_stream->codecpar->codec_id));;
+        m_pixel_format = std::string(av_get_pix_fmt_name(static_cast<AVPixelFormat>(out_stream->codecpar->format)));
+        m_max_b_frames = max_b_frames;
+
+        /* collect summary notify caller */
+        auto summary = print_summary();
+
+        /* go! */
+        m_enmux_running = true;
+        m_encode_running = true;
+        m_enmux_th = std::make_shared<std::thread>(&ff_des::enmux_run, this);
+        m_encode_th = std::make_shared<std::thread>(&ff_des::encode_run, this);
+        VP_INFO(vp_utils::string_format("[ffio/ff_des][%d][inner_open] open successfully.", 
+                                        get_channel_index()));
+        return true;   
+    }
+
+    bool ff_des::write(const ff_av_frame_ptr& frame) {
+        if (!is_opened()) {
+            return false;
+        }
+        
+        bool notify = true;
+        {
+            std::lock_guard<std::mutex> g(m_encode_frames_m);
+            m_encode_frames_q.push(frame);
+            if (m_encode_frames_q.size() > m_encode_frames_q_max_size) {
+                m_encode_frames_q.pop();
+                notify = false;
+                VP_WARN(vp_utils::string_format("[ffio/ff_des][%d][write] exceed m_encode_frames_q_max_size(%d), discard the front in queue.", 
+                                                get_channel_index(), m_encode_frames_q_max_size));
+            }
+        }
+        if (notify) {
+            m_encode_semaphore.signal();
+        }
+        return true;
+    }
+
+    ff_des& ff_des::operator<<(const ff_av_frame_ptr& frame) {
+        write(frame);
+        return *this;
+    }
+
+    void ff_des::inner_exit_signal() {
+        {
+            // send exit flag to notify encode thread
+            std::lock_guard<std::mutex> g(m_encode_frames_m);
+            m_encode_frames_q.push(nullptr);
+        }
+        m_encode_semaphore.signal();
+    }
+
+    int ff_des::get_video_fps() const {
+        return m_fps;
+    }
+
+    int ff_des::get_video_width() const {
+        return m_width;
+    }
+
+    int ff_des::get_video_height() const {
+        return m_height;    
+    }
+
+    long ff_des::get_video_bitrate() const {
+        return m_bitrate;
+    }
+
+    std::string ff_des::get_video_codec_name() const {
+        return m_codec_name;
+    }
+
+    std::string ff_des::get_video_pixel_format_name() const {
+        return m_pixel_format;
+    }
+
+    bool ff_des::is_live_stream() const {
+        return m_live_stream;
+    }
+
+    std::string ff_des::get_hw_type_name() const {
+        return m_hw_type_name;
+    }
+
+    std::string ff_des::get_encoder_name() const {
+        return m_encoder_name;
+    }
+
+    std::string ff_des::get_uri() const {
+        return m_uri;
+    }
+
+    int ff_des::get_channel_index() const {
+        return m_channel_index;   
+    }
+
+    const AVCodecContext* ff_des::get_encode_ctx() const {
+        return m_enc_ctx;
+    }
+
+    int ff_des::
+    hw_encoder_init(AVCodecContext* enc_ctx, 
+                    const enum AVHWDeviceType hw_type, 
+                    const enum AVPixelFormat sw_pix_fmt) {
+        AVBufferRef* hw_device_ctx = nullptr;
+        AVBufferRef* hw_frames_ref = nullptr;
+        AVHWFramesContext* frames_ctx = nullptr;
+        int err = 0;
+        if ((err = av_hwdevice_ctx_create(&hw_device_ctx, hw_type,
+                                        NULL, NULL, 0)) < 0) {
+            VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][hw_encoder_init] failed to create specified(%s) HW device context for encoder.", 
+                                             get_channel_index(), av_hwdevice_get_type_name(hw_type)));
+            return err;
+        }
+
+        if (!(hw_frames_ref = av_hwframe_ctx_alloc(hw_device_ctx))) {
+            VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][hw_encoder_init] failed to create specified(%s) HW frame context for encoder.", 
+                                             get_channel_index(), av_hwdevice_get_type_name(hw_type)));
+            return err;
+        }
+        frames_ctx = (AVHWFramesContext* )(hw_frames_ref->data);
+        frames_ctx->format    = enc_ctx->pix_fmt;
+        frames_ctx->sw_format = sw_pix_fmt;
+        frames_ctx->width     = enc_ctx->width;
+        frames_ctx->height    = enc_ctx->height;
+        frames_ctx->initial_pool_size = 20;
+        if ((err = av_hwframe_ctx_init(hw_frames_ref)) < 0) {
+            VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][hw_encoder_init] failed to initialize specified(%s) HW frame context for encoder.", 
+                                             get_channel_index(), av_hwdevice_get_type_name(hw_type)));
+            av_buffer_unref(&hw_frames_ref);
+            av_buffer_unref(&hw_device_ctx);
+            return err;
+        }
+        enc_ctx->hw_frames_ctx = av_buffer_ref(hw_frames_ref);
+        if (!enc_ctx->hw_frames_ctx)
+            err = AVERROR(ENOMEM);
+
+        av_buffer_unref(&hw_frames_ref);
+        av_buffer_unref(&hw_device_ctx);
+        return err;
+    }
+}
+#endif

nodes/ffio/ff_des.h

@@ -0,0 +1,241 @@
+#pragma once
+#ifdef VP_WITH_FFMPEG
+#include <string>
+#include <queue>
+#include <vector>
+#include <mutex>
+#include <algorithm>
+#include <functional>
+#include "ff_common.h"
+#include "../../utils/vp_semaphore.h"
+#include "../../utils/vp_utils.h"
+#include "../../utils/logger/vp_logger.h"
+
+namespace vp_nodes {
+    /**
+     * encode and enmux using FFmpeg.
+     * used to encode & enmux network streams or file streams.
+     */
+    class ff_des: public std::enable_shared_from_this<ff_des> {
+    private:
+        /* core members */
+        const std::vector<std::string> m_supported_files = {"mp4", "mkv", "flv", "h265", "h264"};
+        const std::vector<std::string> m_supported_protocols = {"rtsp", "rtmp", "udp", "rtp"};
+        AVFormatContext* m_ofmt_ctx = nullptr;
+        AVCodecContext* m_enc_ctx = nullptr;
+        std::queue<ff_av_packet_ptr> m_enmux_packets_q;
+        std::queue<ff_av_frame_ptr> m_encode_frames_q;
+        std::mutex m_enmux_packets_m;
+        std::mutex m_encode_frames_m;
+        int m_enmux_packets_q_max_size = 25;
+        int m_encode_frames_q_max_size = 25;
+        std::shared_ptr<std::thread> m_enmux_th = nullptr;
+        std::shared_ptr<std::thread> m_encode_th = nullptr;
+        vp_utils::vp_semaphore m_enmux_semaphore;
+        vp_utils::vp_semaphore m_encode_semaphore;
+
+        /**
+         * live stream or not for output.
+         */
+        bool m_live_stream = false;
+
+        /**
+         * fps of output stream.
+         */
+        int m_fps = 0;
+        
+        /**
+         * width of output stream in pixels.
+         */
+        int m_width = 0;
+
+        /**
+         * height of output stream in pixels.
+         */
+        int m_height = 0;
+
+        /**
+         * bitrate of output stream (kbit/s).
+         */
+        long m_bitrate = 0;
+
+        /**
+         * codec type name of output stream (strings like `h264/hevc/vp8/...`).
+         */
+        std::string m_codec_name = "";
+
+        /**
+         * pixel format of output stream (strings like `YUV420P/NV12/...`). 
+         */
+        std::string m_pixel_format = "";
+
+        /**
+         * max B frames for encode.
+         */
+        int m_max_b_frames = 0;
+
+        /**
+         * channel index of output.
+         */
+        int m_channel_index = -1;
+
+        /**
+         * type name of hardware used for encode (`none` if no hardware used).
+         */
+        std::string m_hw_type_name = "";
+
+        /**
+         * encoder name used for encoding.
+         */
+        std::string m_encoder_name = "";
+
+        /**
+         * uri for output (rtmp/file/...).
+         */
+        std::string m_uri = "";
+
+        /* inner flags */
+        int m_inner_stream_index = -1;
+        bool m_enmux_running = false;
+        bool m_encode_running = false;
+
+        /* inner methods */
+        void enmux_run();
+        void encode_run();
+        void inner_close();
+        void inner_exit_signal();
+        std::string print_summary();
+        bool inner_open(const std::string& uri,
+                        int width, int height, int fps, int bitrate, int max_b_frames,
+                        const std::string& encoder_name = "", 
+                        AVPixelFormat sw_pix_fmt = AVPixelFormat::AV_PIX_FMT_YUV420P,
+                        AVHWDeviceType hw_type = AVHWDeviceType::AV_HWDEVICE_TYPE_NONE,
+                        AVPixelFormat hw_pix_fmt = AVPixelFormat::AV_PIX_FMT_NONE);
+        int hw_encoder_init(AVCodecContext* enc_ctx, const enum AVHWDeviceType hw_type, const enum AVPixelFormat sw_pix_fmt);
+
+    public:
+        /**
+         * create ff_des instance using initial parameters.
+         * 
+         * @param channel_index specify the channel index for output stream.
+         */
+        ff_des(int channel_index);
+        ~ff_des();
+
+        /* disable copy and assignment operations */
+        ff_des(const ff_des&) = delete;
+        ff_des& operator=(const ff_des&) = delete;
+
+        /**
+         * try to open ff_des, not thread-safe.
+         * 
+         * @param 
+         */
+        bool open(const std::string& uri,
+                  int width, int height, int fps, int bitrate, int max_b_frames = 0,
+                  const std::string& encoder_name = "", 
+                  AVPixelFormat sw_pix_fmt = AVPixelFormat::AV_PIX_FMT_YUV420P,
+                  AVHWDeviceType hw_type = AVHWDeviceType::AV_HWDEVICE_TYPE_NONE,
+                  AVPixelFormat hw_pix_fmt = AVPixelFormat::AV_PIX_FMT_NONE);
+        
+        /**
+         * close ff_des.
+         */
+        void close();
+
+        /**
+         * if working or not.
+         * 
+         * @return
+         * true if it is working.
+         */
+        bool is_opened() const;
+
+        /**
+         * write the next frame to ff_des, keep as same as cv::VideoWriter.
+         * 
+         * @param frame frame to be written.
+         * 
+         * @return
+         * true if write successfully.
+         * 
+         * @note
+         * return false means:
+         * 1. write too quickly no space prepared, please try to write again.
+         * 2. ff_des not opened yet or not working.
+         */
+        bool write(const ff_av_frame_ptr& frame);
+
+        /**
+         * write the next frame to ff_des, support for `<<` operator.
+         * 
+         * @param frame frame to be written.
+         * 
+         * @return
+         * reference for ff_des.
+         */
+        ff_des& operator<<(const ff_av_frame_ptr& frame);
+
+        /**
+         * get fps of output stream in pixels.
+         */
+        int get_video_fps() const;
+
+        /**
+         * get width of output stream in pixels.
+         */
+        int get_video_width() const;
+
+        /**
+         * get height of output stream in pixels.
+         */
+        int get_video_height() const;
+
+        /**
+         * get bitrate of output stream (kbit/s).
+         */
+        long get_video_bitrate() const;
+
+        /**
+         * get codec type name of output stream (strings like `h264/hevc/vp8/...`).
+         */
+        std::string get_video_codec_name() const;
+
+        /**
+         * get pixel format of output stream (strings like `YUV420P/NV12/...`). 
+         */
+        std::string get_video_pixel_format_name() const;
+
+        /**
+         * check is live stream or not.
+         */
+        bool is_live_stream() const;
+
+        /**
+         * get type name of hardware used for encoding (strings like 'cuda/vaapi', 'none' if no hardware used).
+         */
+        std::string get_hw_type_name() const;
+
+        /**
+         * get encoder name used for encoding.
+         */
+        std::string get_encoder_name() const;
+
+        /**
+         * get uri of output.
+         */
+        std::string get_uri() const;
+
+        /**
+         * get channel index of output.
+         */
+        int get_channel_index() const;
+
+        /**
+         * get pointer of encode context from FFmpeg.
+         * used to allocate hardware AVFrame outside of ff_des.
+         */
+        const AVCodecContext* get_encode_ctx() const;
+    };
+}
+#endif

nodes/ffio/ff_src.cpp

@@ -0,0 +1,475 @@
+#ifdef VP_WITH_FFMPEG
+#include <iostream>
+#include <sstream>
+#include "ff_src.h"
+
+namespace vp_nodes {
+    ff_src::ff_src(int channel_index): m_channel_index(channel_index) {
+    }
+    
+    ff_src::~ff_src() {
+        inner_close();
+    }
+
+    void ff_src::demux_run() {
+        auto start_time = std::chrono::system_clock::now();
+        auto total_bytes = 0;
+        while (m_demux_running) {
+            auto demux_start_t = std::chrono::system_clock::now();
+            auto ff_packet = alloc_ff_av_packet();
+            auto ret = 0;
+
+            if ((ret = av_read_frame(m_ifmt_ctx, ff_packet.get())) < 0) {
+                VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][demux_run] av_read_frame failed. ret: %d", 
+                                                 get_channel_index(), 
+                                                 ret));
+                break;
+            }
+
+            // only demux the right video stream
+            if (ff_packet->stream_index != m_inner_stream_index) {
+                continue;
+            }
+/*
+            // analyse
+            vp_media::stream_analyser analyser(ff_packet->data, ff_packet->size, true);
+            std::vector<vp_media::nal_unit> nal_units;
+            analyser.analyse(nal_units);
+            std::ostringstream oss;
+            for (auto& nal: nal_units) {
+                oss << std::setw(8) << std::setfill(' ') << nal.index 
+                    << std::setw(16) << std::setfill(' ') << nal.offset
+                    << std::setw(8) << std::setfill(' ') << nal.nal_length 
+                    << std::setw(16) << std::setfill(' ') << vp_media::stream_analyser::to_hex(nal.start_bytes) << vp_media::stream_analyser::to_hex(nal.head_bytes, false) 
+                    << std::setw(4) << std::setfill(' ') <<  nal.nal_type
+                    << std::setw(24) << std::setfill(' ') << nal.nal_type_name << std::endl;
+            }
+            std::cout << oss.str();
+*/
+            // need calculate bitrate manually
+            if (m_bitrate <= 0) {
+                total_bytes += ff_packet->size;
+                auto current_time = std::chrono::system_clock::now();
+                auto elapsed_seconds = std::chrono::duration_cast<std::chrono::seconds>(current_time - start_time);
+                if (elapsed_seconds.count() > 5.0) {
+                    m_bitrate = (total_bytes * 8) / elapsed_seconds.count() / 1024;   // convert to kbit/s 
+                    total_bytes = 0;
+                    start_time = std::chrono::system_clock::now();
+                }
+            }
+            
+            bool notify = true;
+            {
+                std::lock_guard<std::mutex> g(m_demux_packets_m);
+                m_demux_packets_q.push(ff_packet);
+                if (m_demux_packets_q.size() > m_demux_packets_q_max_size) {
+                    m_demux_packets_q.pop();
+                    notify = false;
+                    VP_WARN(vp_utils::string_format("[ffio/ff_src][%d][demux_run] exceed m_demux_packets_q_max_size(%d), discard the front in queue.", 
+                                                    get_channel_index(), 
+                                                    m_demux_packets_q_max_size));
+                }
+            }
+            if (notify) {
+                m_demux_semaphore.signal();
+            }
+
+            /* 1. wait for a while for video file
+             * 2. demux as soon as possible for live stream
+            */
+            if (!m_live_stream) {
+                auto cost_t = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - demux_start_t);
+                auto duration_t = std::chrono::milliseconds(int(1000.0 / m_fps));
+                if (cost_t < duration_t) {
+                    std::this_thread::sleep_for(duration_t - cost_t);
+                }
+            }
+        }
+
+        /* set demux running flag to false in case of abnormal exit */
+        m_demux_running = false;
+        {
+            // send exit flag to notify decode thread
+            std::lock_guard<std::mutex> g(m_demux_packets_m);
+            m_demux_packets_q.push(nullptr);
+            VP_INFO(vp_utils::string_format("[ffio/ff_src][%d][demux_run] send exit flag to decode thread.", 
+                                            get_channel_index()));
+        }
+        m_demux_semaphore.signal();
+        VP_INFO(vp_utils::string_format("[ffio/ff_src][%d][demux_run] demux thread exits.", 
+                                        get_channel_index()));
+    }
+
+    void ff_src::decode_run() {
+        while (m_decode_running) {
+            m_demux_semaphore.wait();
+            ff_av_packet_ptr ff_packet = nullptr;
+            {
+                std::lock_guard<std::mutex> g(m_demux_packets_m);
+                ff_packet = m_demux_packets_q.front();
+                m_demux_packets_q.pop();
+            }
+
+            auto ret = 0;
+            /* get exit flag */
+            if (!ff_packet) {
+                VP_INFO(vp_utils::string_format("[ffio/ff_src][%d][decode_run] get exit flag, go to flush decoder.", 
+                                                get_channel_index()));
+                ret = avcodec_send_packet(m_dec_ctx, NULL);  // flush decoder
+            } else {
+                ret = avcodec_send_packet(m_dec_ctx, ff_packet.get());
+            }
+
+            if (ret < 0) {
+                VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][decode_run] avcodec_send_packet failed. ret: %d", 
+                                                 get_channel_index(), ret));
+                break;
+            }
+            
+            while (ret >= 0) {
+                auto ff_frame = alloc_ff_av_frame();
+                ret = avcodec_receive_frame(m_dec_ctx, ff_frame.get());
+                if (ret == AVERROR(EAGAIN)) {
+                    break;
+                } else if (ret < 0) {
+                    VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][decode_run] avcodec_receive_frame failed. ret: %d", 
+                                                     get_channel_index(), 
+                                                     ret));
+                    ff_packet = nullptr;
+                    break;
+                }
+                {
+                    std::lock_guard<std::mutex> g(m_decode_frames_m);
+                    m_decode_frames_q.push(ff_frame);
+                    if (m_decode_frames_q.size() > m_decode_frames_q_max_size) {
+                        m_decode_frames_q.pop();
+                        VP_WARN(vp_utils::string_format("[ffio/ff_src][%d][decode_run] exceed m_decode_frames_q_max_size(%d), discard the front in queue.", 
+                                                        get_channel_index(), 
+                                                        m_decode_frames_q_max_size));
+                    }
+                }
+            }
+
+            if (!ff_packet) {
+                break;
+            }
+        }
+
+        /* set decode running flag to false in case of abnormal exit */
+        m_decode_running = false;
+        VP_INFO(vp_utils::string_format("[ffio/ff_src][%d][decode_run] decode thread exits.", 
+                                        get_channel_index()));
+    }   
+
+    bool ff_src::open(const std::string& uri, const std::string& decoder_name, AVHWDeviceType hw_type) {
+        inner_close();
+        auto uri_valid = false;
+        if (!uri_valid) {
+            auto uri_parts = vp_utils::string_split(uri, '.');  // file
+            if (std::find(m_supported_files.begin(), m_supported_files.end(), uri_parts[uri_parts.size() - 1]) != m_supported_files.end()) {
+                uri_valid = true;
+                m_live_stream = false;
+            }
+        }
+        if (!uri_valid) {
+            auto uri_parts = vp_utils::string_split(uri, ':');  // live stream
+            if (std::find(m_supported_protocols.begin(), m_supported_protocols.end(), uri_parts[0]) != m_supported_protocols.end()) {
+                uri_valid = true;
+                m_live_stream = true;
+            }
+        }
+        if (!uri_valid) {
+            VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][open] uri invalid! uri MUST start with `rtsp/rtmp/..`(network streams) or end with `mp4/mkv/..`(file streams).", 
+                                             get_channel_index()));
+            return false;
+        }
+        
+        return inner_open(uri, decoder_name, hw_type);
+    }
+
+    bool ff_src::inner_open(const std::string& uri, const std::string& decoder_name, AVHWDeviceType hw_type) {
+        auto ret = 0;
+        AVDictionary *options = NULL;
+        // Set options
+        av_dict_set(&options, "max_delay", "100000", 0);       // 设置最大延迟为100ms
+        av_dict_set(&options, "buffer_size", "2000000", 0);    // 设置缓冲区大小为2MB
+        av_dict_set(&options, "timeout", "3000000", 0);        // 设置超时时间为3秒
+        av_dict_set(&options, "rtsp_transport", "tcp", 0);     // 使用TCP传输
+        av_dict_set(&options, "max_interleave_delta", "1000000", 0);  // 设置最大间隔时间为1秒
+
+        /* open input uri(file/stream), and allocate format context */
+        if (avformat_open_input(&m_ifmt_ctx, uri.c_str(), NULL, NULL) < 0) {
+            VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][inner_open] could not open input uri.", 
+                                             get_channel_index()));
+            return false;
+        }
+        /* retrieve stream information */
+        if (avformat_find_stream_info(m_ifmt_ctx, NULL) < 0) {
+            VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][inner_open] could not find stream information.", 
+                                             get_channel_index()));
+            avformat_close_input(&m_ifmt_ctx);
+            return false;
+        }
+        /* find video stream */
+        if ((ret = av_find_best_stream(m_ifmt_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, NULL, 0)) < 0) {
+            VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][inner_open] could not find proper VIDEO stream.", 
+                                             get_channel_index()));
+            avformat_close_input(&m_ifmt_ctx);
+            return false;
+        }
+        m_inner_stream_index = ret;
+        auto in_stream = m_ifmt_ctx->streams[m_inner_stream_index];
+        
+        /* initialize video properties */
+        m_fps = int(av_q2d(in_stream->r_frame_rate));
+        m_width = in_stream->codecpar->width;
+        m_height = in_stream->codecpar->height;
+        m_bitrate = in_stream->codecpar->bit_rate / 1024;   // kb/s
+        m_codec_name = std::string(avcodec_get_name(in_stream->codecpar->codec_id));
+        m_duration = av_q2d(in_stream->time_base) * in_stream->duration;
+        m_duration = m_duration < 0 ? 0 : m_duration;
+        m_pixel_format = std::string(av_get_pix_fmt_name(static_cast<AVPixelFormat>(in_stream->codecpar->format)));
+        
+        /* find decoder for the input video stream */
+        const AVCodec* dec = nullptr;
+        if (decoder_name.empty()) {
+            dec = avcodec_find_decoder(in_stream->codecpar->codec_id);   // get default decoder by AVCodecID
+        } else {
+            dec = avcodec_find_decoder_by_name(decoder_name.c_str());    // get by decoder name
+        }
+        if (!dec) {
+            VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][inner_open] could not find the proper decoder for input stream.", 
+                                             get_channel_index()));
+            avformat_close_input(&m_ifmt_ctx);
+            return false;
+        }
+        /* allocate codec context for the decoder */
+        m_dec_ctx = avcodec_alloc_context3(dec);
+        if (!m_dec_ctx) {
+            VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][inner_open] could not allocate context for decoder(%s).", 
+                                             get_channel_index(), dec->name));
+            avformat_close_input(&m_ifmt_ctx);
+            return false;
+        }
+        /* copy codec parameters from input stream to codec context */
+        if ((ret = avcodec_parameters_to_context(m_dec_ctx, in_stream->codecpar)) < 0) {
+            VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][inner_open] could not copy codec parameters to decode context. ret: %d", 
+                                             get_channel_index(), ret));
+            avcodec_free_context(&m_dec_ctx);
+            avformat_close_input(&m_ifmt_ctx);
+            return false;
+        }
+        /* check if need init hwaccels context for decoder */
+        if (hw_type != AVHWDeviceType::AV_HWDEVICE_TYPE_NONE 
+            && hw_decoder_init(m_dec_ctx, hw_type) < 0) {
+            VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][inner_open] failed to create specified(%s) HW device context for decoder.", 
+                                             get_channel_index(), av_hwdevice_get_type_name(hw_type)));
+            avcodec_free_context(&m_dec_ctx);
+            avformat_close_input(&m_ifmt_ctx);
+            return false;
+        }
+        /* open the decoder */
+        if ((ret = avcodec_open2(m_dec_ctx, dec, NULL)) < 0) {
+            VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][inner_open] could not open decoder. ret: %d", 
+                                             get_channel_index(), ret));
+            avcodec_free_context(&m_dec_ctx);
+            avformat_close_input(&m_ifmt_ctx);
+            return false;
+        }
+        /* initialize node properties */
+        m_decoder_name = decoder_name.empty() ? std::string(m_dec_ctx->codec->name) : decoder_name;
+        m_uri = uri;
+        m_hw_type_name = hw_type != AV_HWDEVICE_TYPE_NONE ? std::string(av_hwdevice_get_type_name(hw_type)) : "none";
+
+        /* collect summary notify caller */
+        auto summary = print_summary();
+        if (m_src_opened_hooker) {
+            m_src_opened_hooker(shared_from_this(), summary);
+        }
+        
+        /* go! */
+        m_demux_running = true;
+        m_decode_running = true;
+        m_demux_th = std::make_shared<std::thread>(&ff_src::demux_run, this);
+        m_decode_th = std::make_shared<std::thread>(&ff_src::decode_run, this);
+        VP_INFO(vp_utils::string_format("[ffio/ff_src][%d][inner_open] open successfully.", 
+                                         get_channel_index()));
+        return true;   
+    }
+
+    void ff_src::close() {
+        inner_close();
+    }
+
+    void ff_src::inner_close() {
+        /* set running flags to false */
+        m_demux_running = false;
+        m_decode_running = false;
+
+        /* waiting for threads exist */
+        if (m_demux_th != nullptr && m_demux_th->joinable()) {
+            m_demux_th->join();
+        }
+        if (m_decode_th != nullptr && m_decode_th->joinable()) {
+            m_decode_th->join();
+        }
+
+        /* free format & codec context */
+        if (m_ifmt_ctx) {
+            avformat_close_input(&m_ifmt_ctx);
+        }
+        if (m_dec_ctx) {
+            avcodec_free_context(&m_dec_ctx);
+        }
+
+        /* clear queue */
+        {
+            // clear m_demux_packets_q
+            std::lock_guard<std::mutex> g(m_demux_packets_m);
+            m_demux_packets_q = {};
+            m_demux_semaphore.reset();
+        } 
+        {
+            // clear m_decode_frames_q
+            std::lock_guard<std::mutex> g(m_decode_frames_m);
+            m_decode_frames_q = {};
+        }
+        VP_INFO(vp_utils::string_format("[ffio/ff_src][%d][inner_close] close successfully.", 
+                                         get_channel_index()));
+    }
+
+    bool ff_src::is_opened() const {
+        return m_demux_running && m_decode_running; 
+    }
+
+    int ff_src::get_video_fps() const {
+        // no check is_opened or not
+        return m_fps;
+    }
+
+    int ff_src::get_video_width() const {
+        // no check is_opened or not
+        return m_width;
+    }
+
+    int ff_src::get_video_height() const {
+        // no check is opened or not
+        return m_height;
+    }
+
+    long ff_src::get_video_bitrate() const {
+        // no check is opened or not
+        return m_bitrate;
+    }
+
+    std::string ff_src::get_video_codec_name() const {
+        // no check is_opened or not
+        return m_codec_name;
+    }
+
+    double ff_src::get_video_duration() const {
+        // no check is opened or not
+        return m_duration;    
+    }
+
+    bool ff_src::is_live_stream() const {
+        // no check is opened or not
+        return m_live_stream;
+    }
+
+    std::string ff_src::get_hw_type_name() const {
+        // no check is opened or not
+        return m_hw_type_name;
+    }
+
+    std::string ff_src::get_decoder_name() const {
+        // no check is opened or not
+        return m_decoder_name;
+    }
+
+    std::string ff_src::get_uri() const {
+        // no check is opened or not
+        return m_uri;
+    }
+
+    int ff_src::get_channel_index() const {
+        // no check is opened or not
+        return m_channel_index;
+    }
+
+    std::string ff_src::get_video_pixel_format_name() const {
+        // no check is opened or not
+        return m_pixel_format;
+    }
+
+    std::string ff_src::print_summary() {
+        std::ostringstream s_stream;
+        s_stream << std::endl;
+        s_stream << "################# summary for [ff_src] #################" << std::endl;
+        s_stream << "|channel_index     |: " << get_channel_index() << std::endl;
+        s_stream << "|uri               |: " << get_uri() << std::endl;
+        s_stream << "|is_live_stream    |: " << (is_live_stream() ? std::string("YES") : std::string("NO")) << std::endl;
+        s_stream << "|decoder_name      |: " << get_decoder_name() << std::endl;
+        s_stream << "|hw_type_name      |: " << get_hw_type_name() << std::endl;
+        s_stream << "|------------------|  " << std::endl;
+        s_stream << "|video_codec       |: " << get_video_codec_name() << std::endl;
+        s_stream << "|video_pic_format  |: " << get_video_pixel_format_name() << std::endl;
+        s_stream << "|video_fps         |: " << get_video_fps() << std::endl;
+        s_stream << "|video_width       |: " << get_video_width() << std::endl;
+        s_stream << "|video_height      |: " << get_video_height() << std::endl;
+        s_stream << "|video_bitrate     |: " << get_video_bitrate() << " [kbit/s]" << std::endl;
+        s_stream << "|video_duration    |: " << get_video_duration() << " [seconds]" << std::endl;
+        s_stream << "|------------------|  " << std::endl;
+        s_stream << "################# summary for [ff_src] #################" << std::endl;
+
+        auto summary = s_stream.str();
+        VP_INFO(summary);
+        return summary;
+    }
+
+    bool ff_src::read(ff_av_frame_ptr& frame) {
+        auto ret = false;
+        {
+            std::lock_guard<std::mutex> g(m_decode_frames_m);
+            if (!m_decode_frames_q.empty()) {
+                frame = m_decode_frames_q.front();
+                m_decode_frames_q.pop();
+                ret = true;
+            } else {
+                frame = nullptr;
+            }
+        }
+        if (!ret) {
+            // switch control of CPUs if no frame returned, avoid of occupying CPUs for a long time by caller outside
+            //std::this_thread::sleep_for(std::chrono::milliseconds(1));
+            //VP_DEBUG(vp_utils::string_format("[ffio/ff_src][%d][read] read frame failed, sleep for 1 millisecond.", 
+            //                                 get_channel_index()));
+        }
+
+        /* false means no frame returned (not opened or read too quickly so data not prepared) */
+        return ret;
+    }
+
+    ff_src& ff_src::operator>>(ff_av_frame_ptr& frame) {
+        read(frame);
+        return *this;
+    }
+
+    int ff_src::hw_decoder_init(AVCodecContext* dec_ctx, const enum AVHWDeviceType type) {
+        int err = 0;
+        AVBufferRef* hw_device_ctx = nullptr;
+        if ((err = av_hwdevice_ctx_create(&hw_device_ctx, type,
+                                        NULL, NULL, 0)) < 0) {
+            return err;
+        }
+        // update using AVHWDeviceContext*(hw_device_ctx->data)
+        dec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
+        av_buffer_unref(&hw_device_ctx);
+        return err;
+    }
+
+    void ff_src::set_src_opened_hooker(ff_src_opened_hooker src_opened_hooker) {
+        m_src_opened_hooker = src_opened_hooker;
+    }
+}
+#endif

nodes/ffio/ff_src.h

@@ -0,0 +1,246 @@
+#pragma once
+#ifdef VP_WITH_FFMPEG
+#include <string>
+#include <queue>
+#include <vector>
+#include <mutex>
+#include <algorithm>
+#include <functional>
+#include "ff_common.h"
+#include "../../utils/vp_semaphore.h"
+#include "../../utils/vp_utils.h"
+#include "../../utils/logger/vp_logger.h"
+
+namespace vp_nodes {
+    class ff_src;
+    typedef std::function<void(ff_src_ptr, const std::string&)> ff_src_opened_hooker;
+
+    /**
+     * demux and decode using FFmpeg.
+     * used to demux & decode network streams or file streams.
+     */
+    class ff_src: public std::enable_shared_from_this<ff_src> {
+    private:
+        /* core members */
+        const std::vector<std::string> m_supported_files = {"mp4", "mkv", "flv", "avi", "h264"};
+        const std::vector<std::string> m_supported_protocols = {"rtsp", "rtmp", "http", "rtp"};
+        AVFormatContext* m_ifmt_ctx = nullptr;
+        AVCodecContext* m_dec_ctx = nullptr;
+        std::queue<ff_av_packet_ptr> m_demux_packets_q;
+        std::queue<ff_av_frame_ptr> m_decode_frames_q;
+        std::mutex m_demux_packets_m;
+        std::mutex m_decode_frames_m;        
+        int m_demux_packets_q_max_size = 25;
+        int m_decode_frames_q_max_size = 25;
+        std::shared_ptr<std::thread> m_demux_th = nullptr;
+        std::shared_ptr<std::thread> m_decode_th = nullptr;
+        vp_utils::vp_semaphore m_demux_semaphore;
+
+        /**
+         * live stream or not for input.
+         */
+        bool m_live_stream = false;
+        
+        /**
+         * fps of input stream.
+         */
+        int m_fps = 0;
+        
+        /**
+         * width of input stream in pixels.
+         */
+        int m_width = 0;
+
+        /**
+         * height of input stream in pixels.
+         */
+        int m_height = 0;
+
+        /**
+         * bitrate of input stream (kbit/s).
+         */
+        long m_bitrate = 0;
+
+        /**
+         * codec type name of input stream (strings like `h264/hevc/vp8/...`).
+         */
+        std::string m_codec_name = "";
+
+        /**
+         * pixel format of input stream (strings like `YUV420P/NV12/...`). 
+         */
+        std::string m_pixel_format = "";
+
+        /**
+         * duration of input stream (seconds), only for file stream.
+         */
+        double m_duration = 0.0;
+
+        /**
+         * channel index of input.
+         */
+        int m_channel_index = -1;
+
+        /**
+         * type name of hardware used for decoding (`none` if no hardware used).
+         */
+        std::string m_hw_type_name = "";
+
+        /**
+         * decoder name used for decoding.
+         */
+        std::string m_decoder_name = "";
+
+        /**
+         * uri of input (rtsp/file/...).
+         */
+        std::string m_uri = "";
+
+        /* inner flags */
+        int m_inner_stream_index = -1;
+        bool m_demux_running = false;
+        bool m_decode_running = false;
+
+        /* inner methods */
+        void demux_run();
+        void decode_run();
+        void inner_close();
+        std::string print_summary();
+        bool inner_open(const std::string& uri, const std::string& decoder_name, AVHWDeviceType hw_type);
+        int hw_decoder_init(AVCodecContext* dec_ctx, const enum AVHWDeviceType type);
+
+        /* callbacks */
+        ff_src_opened_hooker m_src_opened_hooker;
+    public:
+        /**
+         * create ff_src instance using initial parameters.
+         * 
+         * @param channel_index specify the channel index for input stream.
+         */
+        ff_src(int channel_index);
+        ~ff_src();
+
+        /* disable copy and assignment operations */
+        ff_src(const ff_src&) = delete;
+        ff_src& operator=(const ff_src&) = delete;
+
+        /**
+         * try to open ff_src, not thread-safe.
+         * 
+         * @param uri uri to open, url for network streams or file path for file streams.
+         * @param decoder_name specify the decoder name used for decoding, MUST supported by FFmpeg.
+         * @param hw_type type of hardware for decoding, MUST supported by FFmpeg.
+         * 
+         * @note
+         * if `decoder_name` not specified, ff_src will choose the default decoder accordding to the codec type of input stream.
+         */
+        bool open(const std::string& uri, 
+                  const std::string& decoder_name = "", 
+                  AVHWDeviceType hw_type = AVHWDeviceType::AV_HWDEVICE_TYPE_NONE);
+        /**
+         * close ff_src.
+         */
+        void close();
+
+        /**
+         * if working or not.
+         * 
+         * @return
+         * true if it is working.
+         */
+        bool is_opened() const;
+
+        /**
+         * read the next frame from ff_src, keep as same as cv::VideoCapture.
+         * 
+         * @param frame frame to be returned (return nullptr if read failed).
+         * 
+         * @return
+         * true if read successfully.
+         * 
+         * @note
+         * return false means:
+         * 1. read too quickly no data prepared, please try to read again.
+         * 2. ff_src not opened yet or not working.
+         */
+        bool read(ff_av_frame_ptr& frame);
+
+        /**
+         * read the next frame from ff_src, support for `>>` operator.
+         * 
+         * @param frame frame to be returned (return nullptr if read failed).
+         * 
+         * @return
+         * reference for ff_src.
+         */
+        ff_src& operator>>(ff_av_frame_ptr& frame);
+
+        /**
+         * get fps of input stream.
+         */
+        int get_video_fps() const;
+
+        /**
+         * get width of input stream in pixels.
+         */
+        int get_video_width() const;
+
+        /**
+         * get height of input stream in pixels.
+         */
+        int get_video_height() const;
+
+        /**
+         * get bitrate of input stream (kbit/s).
+         */
+        long get_video_bitrate() const;
+
+        /**
+         * get codec type name of input stream (strings like `h264/hevc/vp8/...`).
+         */
+        std::string get_video_codec_name() const;
+        
+        /**
+         * get pixel format of input stream (strings like `YUV420P/NV12/...`). 
+         */
+        std::string get_video_pixel_format_name() const;
+        
+        /**
+         * get duration of input stream (seconds), only for file stream.
+         */
+        double get_video_duration() const;
+        
+        /**
+         * check is live stream or not.
+         */
+        bool is_live_stream() const;
+        
+        /**
+         * get type name of hardware used for decoding (strings like 'cuda/vaapi', 'none' if no hardware used).
+         */
+        std::string get_hw_type_name() const;
+        
+        /**
+         * get decoder name used for decoding.
+         */
+        std::string get_decoder_name() const;
+        
+        /**
+         * get uri of input.
+         */
+        std::string get_uri() const;
+        
+        /**
+         * get channel index of input.
+         */
+        int get_channel_index() const;
+
+        /**
+         * set callback for opened event. would be activated every time ff_src opened.
+         * 
+         * @param src_opened_hooker callback activated when ff_src opened.
+         */
+        void set_src_opened_hooker(ff_src_opened_hooker src_opened_hooker);
+    };
+}
+#endif

nodes/ffio/vp_ff_des_node.cpp

@@ -0,0 +1,97 @@
+#ifdef VP_WITH_FFMPEG
+#include "vp_ff_des_node.h"
+
+namespace vp_nodes {
+    vp_ff_des_node::
+    vp_ff_des_node(const std::string& node_name,
+                          int channel_index,
+                          const std::string& out_uri,
+                          vp_objects::vp_size resolution_w_h, 
+                          int bitrate,
+                          bool osd,
+                          std::string encoder_name):
+                          vp_des_node(node_name, channel_index),
+                          m_out_uri(out_uri),
+                          m_resolution_w_h(resolution_w_h),
+                          m_out_bitrate(bitrate),
+                          m_use_osd(osd),
+                          m_encoder_name(encoder_name) {
+        m_ff_des = alloc_ff_des(channel_index);
+        VP_INFO(vp_utils::string_format("[%s] [%s]", node_name.c_str(), out_uri.c_str()));
+        this->initialized();
+    }
+
+    vp_ff_des_node::~vp_ff_des_node() {
+        deinitialized();
+        if (sws_ctx) {
+            sws_freeContext(sws_ctx);
+        }
+    }
+
+    std::shared_ptr<vp_objects::vp_meta> vp_ff_des_node::handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) {
+        VP_DEBUG(vp_utils::string_format("[%s] received frame meta, channel_index=>%d, frame_index=>%d", node_name.c_str(), meta->channel_index, meta->frame_index));
+        
+        cv::Mat resize_frame = (m_use_osd && !meta->osd_frame.empty()) ? meta->osd_frame : meta->frame;
+        auto out_width = resize_frame.cols;
+        auto out_height = resize_frame.rows;
+        if (m_resolution_w_h.width != 0 && m_resolution_w_h.height != 0) {
+            out_width = m_resolution_w_h.width;
+            out_height = m_resolution_w_h.height;
+        }
+              
+        /* try to open ff_dec. */
+        if (!m_ff_des->is_opened()) {
+            if(!m_ff_des->open(m_out_uri, 
+                                out_width, 
+                                out_height, 
+                                meta->fps, 
+                                m_out_bitrate, 
+                                0, 
+                                m_encoder_name, 
+                                AV_PIX_FMT_YUV420P)) {
+                VP_WARN(vp_utils::string_format("[%s] could not open ff_des.", node_name.c_str()));
+                /* general works in vp_des_node. */
+                return vp_des_node::handle_frame_meta(meta);
+            }
+        }
+
+        /* initialize sws_ctx. */
+        if (!sws_ctx) {
+            sws_ctx = sws_getContext(resize_frame.cols, 
+                                     resize_frame.rows, 
+                                     AV_PIX_FMT_BGR24, 
+                                     out_width, 
+                                     out_height, 
+                                     AV_PIX_FMT_YUV420P, 
+                                     0, NULL, NULL, NULL);
+        }
+        if (!sws_ctx) {
+            VP_WARN(vp_utils::string_format("[%s] could not initialize sws_ctx.", node_name.c_str()));
+            /* general works in vp_des_node. */
+            return vp_des_node::handle_frame_meta(meta);
+        }
+        
+        /* cv::Mat -> AVFrame. */
+        /* resize and convert to AV_PIX_FMT_YUV420P. */
+        auto dst_frame = alloc_ff_av_frame();
+        dst_frame->width = out_width;
+        dst_frame->height = out_height;
+        dst_frame->format= AV_PIX_FMT_YUV420P;
+        av_frame_get_buffer(dst_frame.get(), 0);
+        
+        auto p = resize_frame.data;
+        int linesize[1] = {resize_frame.cols * 3};
+        sws_scale(sws_ctx, &p, linesize, 0, resize_frame.rows, dst_frame->data, dst_frame->linesize);
+
+        /* write to ff_des. */
+        m_ff_des->write(dst_frame);
+
+        /* general works in vp_des_node. */
+        return vp_des_node::handle_frame_meta(meta);
+    }
+
+    std::string vp_ff_des_node::to_string() {
+        return m_out_uri;
+    }
+}
+#endif

nodes/ffio/vp_ff_des_node.h

@@ -0,0 +1,70 @@
+#pragma once
+#ifdef VP_WITH_FFMPEG
+#include "ff_des.h"
+#include "../vp_des_node.h"
+
+namespace vp_nodes {
+    /**
+     * universal DES node using FFmpeg.
+     * 
+     * support output uri:
+     * 1. path of file streams like `./vp_data/out_vp_test.mp4`.
+     * 2. url of network streams like `rtmp://192.168.77.68/live/stream`.
+     */
+    class vp_ff_des_node final: public vp_des_node {
+    private:
+        /* inner members. */
+        std::string m_out_uri = "";
+        bool m_use_osd = true;
+        int m_out_bitrate = 1024;
+        std::string m_encoder_name = "";
+        vp_objects::vp_size m_resolution_w_h;
+
+        /**
+         * encode & enmux.
+         */
+        ff_des_ptr m_ff_des = nullptr;
+        /**
+         * SwsContext used fot scale by FFmpeg.
+         */
+        SwsContext* sws_ctx = NULL;
+    protected:
+        // re-implementation, return nullptr.
+        virtual std::shared_ptr<vp_objects::vp_meta> handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) override; 
+    public:
+        /**
+         * create vp_ff_des_node instance using initial parameters.
+         * 
+         * @param node_name specify the name of DES node.
+         * @param channel_index specify the channel index of DES node.
+         * @param out_uri specify the uri to be written.
+         * @param use_osd specify use osd frame as output or not.
+         * @param out_width specify the final width of output, 0 means use the width of frame flowing in pipeline.
+         * @param out_height specify the final height of output, 0 means use the height of frame flowing in pipeline.
+         * @param out_fps specify the fps of output, 0 means use the fps of original stream in pipeline.
+         * @param out_bitrate specify the bitrate of output (kbit/s).
+         * @param out_max_b_frames specify the max B frames in a GOP for encoding.
+         * @param encoder_name specify the encoder name (`libx264`/`libx265`/`h264_nvenc`/`hevc_nvenc`) used for encoding in FFmpeg.
+         * @param out_sw_pix_fmt specify the pixel format of output.
+         * 
+         * @note
+         * the encoder specified by `encoder_name` MUST be supported already in FFmpeg, 
+         * we can run `ffmpeg -encoders` to show list of encoders supported in FFmpeg.
+         * if the encoder not found, please reconfigure & rebuild your FFmpeg.
+         */
+        vp_ff_des_node(const std::string& node_name,
+                              int channel_index,
+                              const std::string& out_uri,
+                              vp_objects::vp_size resolution_w_h = {}, 
+                              int bitrate = 1024,
+                              bool osd = true,
+                              std::string encoder_name = "libx264");
+        ~vp_ff_des_node();
+
+        /**
+         * return out uri of DES node.
+         */
+        virtual std::string to_string() override;
+    };
+}
+#endif

nodes/ffio/vp_ff_src_node.cpp

@@ -0,0 +1,149 @@
+#ifdef VP_WITH_FFMPEG
+#include "vp_ff_src_node.h"
+
+namespace vp_nodes {
+    
+    vp_ff_src_node::
+    vp_ff_src_node(const std::string& node_name, 
+                          int channel_index,
+                          const std::string& uri,
+                          const std::string& decoder_name,
+                          float resize_ratio,
+                          int skip_interval):
+                          vp_src_node(node_name, channel_index, resize_ratio),
+                          m_uri(uri),
+                          m_decoder_name(decoder_name),
+                          m_skip_interval(skip_interval) {
+        assert(skip_interval >= 0 && skip_interval <= 9);
+        m_ff_src = alloc_ff_src(channel_index);
+        VP_INFO(vp_utils::string_format("[%s] [%s]", node_name.c_str(), uri.c_str()));
+        this->initialized();
+    }
+    
+    vp_ff_src_node::~vp_ff_src_node() {
+        deinitialized();
+    }
+
+    void vp_ff_src_node::handle_run() {
+        SwsContext* sws_ctx = NULL;
+        auto free_sws_ctx = [&]() {
+            /* free swsContext. */
+            if (sws_ctx) {
+                sws_freeContext(sws_ctx);
+                sws_ctx = NULL;
+            }
+        };
+        auto reopen_wait = 10;
+        auto reopen_times = 0;
+        ff_av_frame_ptr src_frame;
+        int video_width = 0;
+        int video_height = 0;
+        int fps = 0;
+        int skip = 0;
+        while (alive) {
+            /* wait for data coming. */
+            gate.knock();
+            if (!m_ff_src->is_opened()) {
+                if(!m_ff_src->open(m_uri, m_decoder_name)) {
+                    reopen_times++;
+                    if (reopen_times < 5) {
+                        VP_WARN(vp_utils::string_format("[%s] open uri failed, try again right now...", node_name.c_str()));
+                    }
+                    else {
+                        VP_WARN(vp_utils::string_format("[%s] open uri failed too many times, wait for %d seconds then try again...", node_name.c_str(), reopen_wait));
+                        std::this_thread::sleep_for(std::chrono::milliseconds(1000 * reopen_wait));
+                        reopen_times = 0;
+                    }
+                    continue;
+                }
+                free_sws_ctx();
+            }
+
+            // video properties
+            if (video_width == 0 || video_height == 0 || fps == 0) {
+                video_width = m_ff_src->get_video_width();
+                video_height = m_ff_src->get_video_height();
+                fps = m_ff_src->get_video_fps();
+                
+                original_fps = fps;
+                original_width = video_width;
+                original_height = video_height;
+
+                // set true fps because skip some frames
+                fps = fps / (m_skip_interval + 1);
+            }
+            // stream_info_hooker activated if need
+            vp_stream_info stream_info {channel_index, original_fps, original_width, original_height, to_string()};
+            invoke_stream_info_hooker(node_name, stream_info);
+
+            /* try to read next frame from ff_src. */
+            if (!m_ff_src->read(src_frame)) {
+                //VP_WARN(vp_utils::string_format("[%s] reading frame failed, total frame==>%d", node_name.c_str(), frame_index));
+                continue;
+            }
+
+            // need skip
+            if (skip < m_skip_interval) {
+                skip++;
+                continue;
+            }
+            skip = 0;
+
+            /* AVFrame -> cv::Mat. */
+            /* resize and convert to BGR24. */
+            auto n_width = int(src_frame->width * resize_ratio);
+            auto n_height = int(src_frame->height * resize_ratio);
+            if (!sws_ctx) {
+                sws_ctx = sws_getContext(src_frame->width, 
+                                         src_frame->height, 
+                                         AVPixelFormat(src_frame->format), 
+                                         n_width, n_height, 
+                                         AV_PIX_FMT_BGR24, 
+                                         0, NULL, NULL, NULL);
+            }
+            if (!sws_ctx) {
+                VP_WARN(vp_utils::string_format("[%s] could not initialize sws_ctx.", node_name.c_str()));
+                continue;
+            }
+            auto buffer_size = n_width * n_height * 3;
+            uchar* bgr24 = new uchar[buffer_size];
+            int linesize[1] = {n_width * 3};
+            sws_scale(sws_ctx, src_frame->data, src_frame->linesize, 0, src_frame->height, &bgr24, linesize);
+
+            cv::Mat frame(n_height, n_width, CV_8UC3, bgr24);
+            auto c_frame = frame.clone();
+            delete[] bgr24;
+           // set true size because resize
+            video_width = c_frame.cols;
+            video_height = c_frame.rows;
+            
+            this->frame_index++;
+            // create frame meta
+            auto out_meta = 
+                std::make_shared<vp_objects::vp_frame_meta>(c_frame, this->frame_index, this->channel_index, video_width, video_height, fps);
+
+            if (out_meta != nullptr) {
+                this->out_queue.push(out_meta);
+                
+                // handled hooker activated if need
+                if (this->meta_handled_hooker) {
+                    meta_handled_hooker(node_name, out_queue.size(), out_meta);
+                }
+
+                // important! notify consumer of out_queue in case it is waiting.
+                this->out_queue_semaphore.signal();
+                VP_DEBUG(vp_utils::string_format("[%s] after handling meta, out_queue.size()==>%d", node_name.c_str(), out_queue.size()));
+            } 
+        }
+        
+        free_sws_ctx();
+        // send dead flag for dispatch_thread
+        this->out_queue.push(nullptr);
+        this->out_queue_semaphore.signal();    
+    }
+
+    std::string vp_ff_src_node::to_string() {
+        return m_uri;
+    }
+}
+#endif

nodes/ffio/vp_ff_src_node.h

@@ -0,0 +1,61 @@
+
+#pragma once
+#ifdef VP_WITH_FFMPEG
+#include "ff_src.h"
+#include "../vp_src_node.h"
+
+namespace vp_nodes {
+    /**
+     * universal SRC node using FFmpeg.
+     * 
+     * support uri:
+     * 1. path of file streams like `./vp_data/vp_test.mp4`.
+     * 2. url of network streams like `rtsp://192.168.77.68/main_stream`.
+     */
+    class vp_ff_src_node final: public vp_src_node {
+    private:
+        /* inner members. */
+        std::string m_decoder_name = "";
+        std::string m_uri = "";
+        // 0 means no skip
+        int m_skip_interval = 0;
+
+        /**
+         * demux & decode.
+         */
+        ff_src_ptr m_ff_src = nullptr;
+    protected:
+        /**
+         * get frames using FFmpeg.
+         */
+        virtual void handle_run() override;
+    public:
+        /**
+         * create vp_ff_src_node instance using initial parameters.
+         * 
+         * @param node_name specify the name of SRC node.
+         * @param channel_index specify the channel index of SRC node.
+         * @param uri specify the uri to be opened by SRC node.
+         * @param decoder_name specify the decoder name (`h264`/`hevc`/`h264_cuvid`/`hevc_cuvid`) used for decoding in FFmpeg.
+         * @param resize_ratio specify the resize ratio applied to frames.
+         * 
+         * @note
+         * the decoder specified by `decoder_name` MUST be supported already in FFmpeg, 
+         * we can run `ffmpeg -decoders` to show list of decoders supported in FFmpeg.
+         * if the decoder not found, please reconfigure & rebuild your FFmpeg.
+         */
+        vp_ff_src_node(const std::string& node_name, 
+                              int channel_index,
+                              const std::string& uri,
+                              const std::string& decoder_name = "h264",
+                              float resize_ratio = 1.0,
+                              int skip_interval = 0);
+        ~vp_ff_src_node();
+
+        /**
+         * return uri of SRC node.
+         */
+        virtual std::string to_string() override;
+    };
+}
+#endif