first commit

.gitignore

@@ -0,0 +1,160 @@
+
+# binary files
+*
+!*/
+!*.*
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# model and video
+*.cfg
+*.onnx
+*.caffemodel
+*.prototxt
+*.mp4
+*.weights
+*.h264
+*.trt
+models/
+test_video/
+log/
+debug/
+build/
+
+# doc
+*.pptx
+
+# Distribution / packaging
+.Python
+develop-eggs/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+scripts/include/
+scripts/libs/
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+.DS_Store
+.idea/
+cmake-build-debug-remote/
+cmake-build-debug/
+build/*
+!build/build.sh

.vscode/c_cpp_properties.json

@@ -0,0 +1,23 @@
+{
+    "configurations": [
+        {
+            "name": "Win32",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "/usr/local/include/**",
+                "/usr/include/**"
+            ],
+            "defines": [
+                "_DEBUG",
+                "UNICODE",
+                "_UNICODE"
+            ],
+            "windowsSdkVersion": "8.1",
+            "compilerPath": "/usr/bin/g++",
+            "cStandard": "c17",
+            "cppStandard": "c++17",
+            "intelliSenseMode": "linux-gcc-x64"
+        }
+    ],
+    "version": 4
+}

.vscode/launch.json

@@ -0,0 +1,92 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "debug samples for videopipe",  // debug for samples in videopipe
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "${workspaceFolder}/build/bin/${fileBasenameNoExtension}",
+            "args": [""],
+            "stopAtEntry": false,
+            "cwd": "${workspaceFolder}/..",  // change this value to the path of your vp_data
+            "environment": [],
+            "externalConsole": false,
+            "MIMode": "gdb",
+            "miDebuggerPath": "/usr/bin/gdb",
+            "setupCommands": [
+              {
+                "description": "Enable pretty-printing for gdb",
+                "text": "-enable-pretty-printing",
+                "ignoreFailures": true
+              }
+            ],
+            "preLaunchTask": ""
+        },
+        {
+          "name": "debug complex samples for videopipe",  // debug for complex samples in videopipe
+          "type": "cppdbg",
+          "request": "launch",
+          "program": "${fileDirname}/${fileBasenameNoExtension}",
+          "args": ["../../.."],  // change this value to the path of your vp_data
+          "stopAtEntry": false,
+          "cwd": "${fileDirname}",
+          "environment": [],
+          "externalConsole": false,
+          "MIMode": "gdb",
+          "miDebuggerPath": "/usr/bin/gdb",
+          "setupCommands": [
+            {
+              "description": "Enable pretty-printing for gdb",
+              "text": "-enable-pretty-printing",
+              "ignoreFailures": true
+            }
+          ],
+          "preLaunchTask": ""
+        },
+        {
+          "name": "debug trt_yolov8 samples",  // debug for trt samples at third_party/trt_yolov8
+          "type": "cppdbg",
+          "request": "launch",
+          "program": "${workspaceFolder}/third_party/trt_yolov8/build/samples/${fileBasenameNoExtension}",
+          "args": [""],
+          "stopAtEntry": false,
+          "cwd": "${workspaceFolder}/..",
+          "environment": [],
+          "externalConsole": false,
+          "MIMode": "gdb",
+          "miDebuggerPath": "/usr/bin/gdb",
+          "setupCommands": [
+            {
+              "description": "Enable pretty-printing for gdb",
+              "text": "-enable-pretty-printing",
+              "ignoreFailures": true
+            }
+          ],
+          "preLaunchTask": ""
+        },
+        {
+          "name": "debug trt_vehicle samples",  // debug for trt samples at third_party/trt_vehicle
+          "type": "cppdbg",
+          "request": "launch",
+          "program": "${workspaceFolder}/third_party/trt_vehicle/build/samples/${fileBasenameNoExtension}",
+          "args": [""],
+          "stopAtEntry": false,
+          "cwd": "${workspaceFolder}/..",
+          "environment": [],
+          "externalConsole": false,
+          "MIMode": "gdb",
+          "miDebuggerPath": "/usr/bin/gdb",
+          "setupCommands": [
+            {
+              "description": "Enable pretty-printing for gdb",
+              "text": "-enable-pretty-printing",
+              "ignoreFailures": true
+            }
+          ],
+          "preLaunchTask": ""
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,94 @@
+{
+    "files.associations": {
+        "memory": "cpp",
+        "cmath": "cpp",
+        "cstddef": "cpp",
+        "cstdint": "cpp",
+        "cstdio": "cpp",
+        "cstdlib": "cpp",
+        "cstring": "cpp",
+        "cwchar": "cpp",
+        "exception": "cpp",
+        "initializer_list": "cpp",
+        "iosfwd": "cpp",
+        "limits": "cpp",
+        "new": "cpp",
+        "stdexcept": "cpp",
+        "type_traits": "cpp",
+        "typeinfo": "cpp",
+        "utility": "cpp",
+        "vector": "cpp",
+        "xmemory": "cpp",
+        "xmemory0": "cpp",
+        "xstddef": "cpp",
+        "xstring": "cpp",
+        "xtr1common": "cpp",
+        "xutility": "cpp",
+        "queue": "cpp",
+        "thread": "cpp",
+        "mutex": "cpp",
+        "algorithm": "cpp",
+        "chrono": "cpp",
+        "deque": "cpp",
+        "functional": "cpp",
+        "ios": "cpp",
+        "istream": "cpp",
+        "ostream": "cpp",
+        "ratio": "cpp",
+        "streambuf": "cpp",
+        "string": "cpp",
+        "system_error": "cpp",
+        "xthread": "cpp",
+        "tuple": "cpp",
+        "xfacet": "cpp",
+        "xfunctional": "cpp",
+        "xiosbase": "cpp",
+        "xlocale": "cpp",
+        "xlocinfo": "cpp",
+        "xlocnum": "cpp",
+        "iostream": "cpp",
+        "any": "cpp",
+        "array": "cpp",
+        "atomic": "cpp",
+        "bit": "cpp",
+        "*.tcc": "cpp",
+        "cctype": "cpp",
+        "clocale": "cpp",
+        "compare": "cpp",
+        "concepts": "cpp",
+        "condition_variable": "cpp",
+        "cstdarg": "cpp",
+        "ctime": "cpp",
+        "cwctype": "cpp",
+        "map": "cpp",
+        "unordered_map": "cpp",
+        "iterator": "cpp",
+        "memory_resource": "cpp",
+        "numeric": "cpp",
+        "random": "cpp",
+        "string_view": "cpp",
+        "numbers": "cpp",
+        "semaphore": "cpp",
+        "stop_token": "cpp",
+        "complex": "cpp",
+        "sstream": "cpp",
+        "list": "cpp",
+        "set": "cpp",
+        "fstream": "cpp",
+        "iomanip": "cpp",
+        "optional": "cpp",
+        "hash_map": "cpp",
+        "hash_set": "cpp",
+        "bitset": "cpp",
+        "codecvt": "cpp",
+        "unordered_set": "cpp",
+        "filesystem": "cpp",
+        "future": "cpp",
+        "cfenv": "cpp",
+        "cinttypes": "cpp",
+        "__nullptr": "cpp",
+        "__locale": "cpp",
+        "regex": "cpp"
+    },
+    "C_Cpp.errorSquiggles": "Disabled"
+}

CMakeLists.txt

@@ -0,0 +1,171 @@
+#[[
+    prepare in advance：
+    1. OpenCV >= 4.6
+    2. Gstreamer >= 1.14.5
+    3. CUDA, TensorRT, PADDLE and others for optional
+]]
+
+cmake_minimum_required(VERSION 3.10)
+project(video_pipe VERSION 1.0)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_BUILD_TYPE Debug)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fPIC -w -fdiagnostics-color=always -pthread")
+# save all libs(including third_party's) to 'libs'
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/libs)
+
+# optional for build, modify values when configure the project using 'cmake -DVP_WITH_CUDA=OFF ..'
+option(VP_WITH_CUDA "prepared CUDA or not?" OFF)
+option(VP_WITH_TRT "prepared TensorRT or not?" OFF)
+option(VP_WITH_PADDLE "prepared PaddlePaddle or not?" OFF)
+option(VP_WITH_KAFKA "prepared Kafka or not?" OFF)
+option(VP_WITH_LLM "prepared LLM or not?" OFF)
+option(VP_WITH_FFMPEG "prepared FFMPEG or not?" OFF)
+option(VP_BUILD_COMPLEX_SAMPLES "build complex samples or not? (maybe source code not provided)" OFF)
+
+# OpenCV required
+find_package(OpenCV REQUIRED)
+message(STATUS "OpenCV library status:")
+message(STATUS "    version: ${OpenCV_VERSION}")
+message(STATUS "    libraries: ${OpenCV_LIBS}")
+message(STATUS "    include path: ${OpenCV_INCLUDE_DIRS}")
+include_directories(${OpenCV_INCLUDE_DIRS})
+
+# Gstreamer required
+include(FindPkgConfig)
+pkg_check_modules(GST   REQUIRED gstreamer-1.0)
+pkg_check_modules(GSTAPP REQUIRED gstreamer-app-1.0)
+pkg_check_modules(GST_RTSP REQUIRED gstreamer-rtsp-server-1.0)
+message(STATUS "GStreamer library status:")
+message(STATUS "    version: ${GST_VERSION}")
+message(STATUS "    libraries: ${GST_LIBRARIES} ${GSTAPP_LIBRARIES} ${GST_RTSP_LIBRARIES}")
+message(STATUS "    include path: ${GST_INCLUDE_DIRS}")
+include_directories(${GST_INCLUDE_DIRS})
+set (GST_DEPEND_LIBS ${GST_LIBRARIES} ${GSTAPP_LIBRARIES} ${GST_RTSP_LIBRARIES})
+
+# collect dependent libs for videopipe
+list(APPEND VP_DEPEND_LIBS ${OpenCV_LIBS} ${GST_DEPEND_LIBS} stdc++fs)
+
+# optional for CUDA
+if(VP_WITH_CUDA) # CUDA enabled
+    add_definitions(-DVP_WITH_CUDA)
+endif()
+
+# optional for TensorRT
+if(VP_WITH_TRT)  # TensorRT enabled
+    add_definitions(-DVP_WITH_TRT)
+    set(VP_BUILD_FROM ON)  # set flag for sub project
+    set(VP_TRT_LIB_PATH "")
+    set(VP_TRT_INC_PATH "")
+    # trt_vehicle
+    message("-------------start build trt_vehicle--------------")
+    add_subdirectory(third_party/trt_vehicle)
+    list(APPEND VP_DEPEND_LIBS trt_vehicle)
+    link_directories(${VP_TRT_LIB_PATH})
+    message(STATUS "TensorRT library status:")
+    message(STATUS "    include path: ${VP_TRT_INC_PATH}")
+    message(STATUS "    library path: ${VP_TRT_LIB_PATH}")
+    message("--------------end build trt_vehicle---------------")
+    # trt_yolov8    
+    message("-------------start build trt_yolov8--------------")
+    add_subdirectory(third_party/trt_yolov8)
+    list(APPEND VP_DEPEND_LIBS trt_yolov8)
+    message("--------------end build trt_yolov8---------------")
+endif()
+
+# optional for PaddlePaddle
+if(VP_WITH_PADDLE)  # PaddlePaddle enabled
+    add_definitions(-DVP_WITH_PADDLE)
+    set(VP_BUILD_FROM ON)  # set flag for sub project
+    set(VP_PADDLE_LIB_PATH "")
+    set(VP_PADDLE_INC_PATH "")
+    message("-------------start build paddle_ocr--------------")
+    add_subdirectory(third_party/paddle_ocr) # build paddle_ocr
+    list(APPEND VP_DEPEND_LIBS paddle_ocr)
+    link_directories(${VP_PADDLE_LIB_PATH})
+    message(STATUS "PaddlePaddle library status:")
+    message(STATUS "    include path: ${VP_PADDLE_INC_PATH}")
+    message(STATUS "    library path: ${VP_PADDLE_LIB_PATH}")
+    message("--------------end build paddle_ocr---------------")
+endif()
+
+# optional for Kafka
+if(VP_WITH_KAFKA)
+    add_definitions(-DVP_WITH_KAFKA)
+    list(APPEND VP_DEPEND_LIBS rdkafka++)
+endif()
+
+# optional for LLM
+if(VP_WITH_LLM)
+    find_package(OpenSSL REQUIRED)
+    message(STATUS "OpenSSL library status:")
+    message(STATUS "    version: ${OPENSSL_VERSION}")
+    message(STATUS "    libraries: ${OPENSSL_LIBRARIES}")
+    message(STATUS "    include path: ${OPENSSL_INCLUDE_DIR}")
+    include_directories(${OPENSSL_INCLUDE_DIR})
+    list(APPEND VP_DEPEND_LIBS ${OPENSSL_LIBRARIES})
+    add_definitions(-DVP_WITH_LLM)
+endif()
+
+# optional for FFmpeg
+if(VP_WITH_FFMPEG)
+    add_definitions(-DVP_WITH_FFMPEG)
+    list(APPEND VP_DEPEND_LIBS avcodec avformat avdevice swscale swresample avutil)
+endif()
+
+# tinyexpr
+message("-------------start build tinyexpr--------------")
+add_subdirectory(third_party/tinyexpr)
+list(APPEND VP_DEPEND_LIBS tinyexpr)
+message("--------------end build tinyexpr---------------")
+
+message("-------------collect version info--------------")
+string(TIMESTAMP BUILD_TIME "%Y%m%d-%H%M%S")
+find_package(Git QUIET)
+if(GIT_FOUND)
+    execute_process(
+        COMMAND ${GIT_EXECUTABLE} rev-parse --short HEAD
+        WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+        OUTPUT_VARIABLE GIT_COMMIT_HASH
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+    execute_process(
+        COMMAND ${GIT_EXECUTABLE} diff --quiet
+        WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+        RESULT_VARIABLE GIT_DIFF_RESULT
+    )
+    if(GIT_DIFF_RESULT EQUAL 0)
+        set(GIT_DIRTY "")
+    else()
+        set(GIT_DIRTY "-dirty")
+    endif()
+else()
+    message(WARNING "Git not found! Using default version info.")
+    set(GIT_COMMIT_HASH "nogit")
+    set(GIT_DIRTY "")
+endif()
+set(FINAL_GIT_VERSION "${GIT_COMMIT_HASH}${GIT_DIRTY}")
+configure_file(
+    ${CMAKE_SOURCE_DIR}/utils/vp_version.h.in
+    ${CMAKE_BINARY_DIR}/vp_version.h
+    @ONLY
+)
+include_directories(${CMAKE_BINARY_DIR})
+message(STATUS "version info:")
+message(STATUS "    build_time: ${BUILD_TIME}")
+message(STATUS "    commit_hash: ${FINAL_GIT_VERSION}")
+message("-------------collect version info--------------")
+
+# collect source files for videopipe
+file(GLOB_RECURSE NODES "nodes/*.cpp")
+file(GLOB_RECURSE OBJECTS "objects/*.cpp")
+file(GLOB_RECURSE UTILS "utils/*.cpp")
+#...#
+list(APPEND VP_CPPS_SOURCES ${NODES} ${OBJECTS} ${UTILS})
+
+# build for videopipe
+add_library(${PROJECT_NAME} SHARED ${VP_CPPS_SOURCES})
+target_link_libraries(${PROJECT_NAME} ${VP_DEPEND_LIBS})
+
+# build for samples
+add_subdirectory(samples)

README.md

@@ -0,0 +1,223 @@
+<p style="" align="center">
+  <img src="./doc/logo.png" alt="Logo" width="75%">
+</p>
+<p style="margin:0px" align="center">
+  <a href='./README_CN.md'>中文README</a> | <a href='http://www.videopipe.cool'>VideoPipe Website </a> | <a href='http://www.videopipe.cool/index.php/2024/09/11/videopipetutorials/'>VideoPipe tutorials(视频教程) </a>
+</p>
+<p style="margin:0px" align="center">
+  <a href='https://github.com/sherlockchou86/one-yolo'>🚀one-yolo, make all in one for Yolo integration. All Tasks, All Versions, All Runtimes. 🚀</a>
+</p>
+
+---
+
+## Introduction
+
+`VideoPipe` is a framework for video analysis and structuring, written in C++. It has minimal dependencies and is easy to use. It operates like a pipeline, where each node is independent and can be combined in various ways. `VideoPipe` can be used to build different types of video analysis applications, suitable for scenarios such as video structuring, image search, face recognition, and behavior analysis in traffic/security fields (such as traffic incident detection).
+
+![](./doc/g1.gif)
+
+## Advantages and Features
+
+`VideoPipe` is similar to NVIDIA's DeepStream and Huawei's mxVision frameworks, but it is easier to use and more portable.
+
+Here is a comparison table:
+
+| **Name**      | **Open Source** | **Learning Curve** | **Supported Platforms** | **Performance** | **Third-Party Dependencies** |
+|---------------|-----------------|---------------------|--------------------------|-----------------|-------------------------------|
+| DeepStream    | No              | High                | NVIDIA only              | High            | Many                          |
+| mxVision      | No              | High                | Huawei only              | High            | Many                          |
+| VideoPipe     | Yes             | Low                 | Any platform             | Medium          | Few                           |
+
+`VideoPipe` uses a plugin-oriented coding style that allows for flexible configuration based on different needs. We can use independent plugins (referred to as `Node` types within the framework) to build various types of video analysis applications. You only need to prepare the model and understand how to parse its output. Inference can be implemented using different backends, such as OpenCV::DNN (default), TensorRT, PaddleInference, ONNXRuntime, or any other backend you prefer.
+
+![](./doc/p1.png)
+
+## Demonstration
+
+https://github.com/sherlockchou86/video_pipe_c/assets/13251045/b1289faa-e2c7-4d38-871e-879ae36f6d50
+
+To watch in fullscreen, use the button in the bottom right corner of the player，[more video demos](./SAMPLES.md)
+
+## Functions
+
+`VideoPipe` is a framework that simplifies the integration of computer vision algorithm models. It is important to note that it is not a deep learning framework like TensorFlow or TensorRT. The main features of `VideoPipe` are as follows:
+
+- **Stream Reading**: Supports mainstream video stream protocols such as UDP, RTSP, RTMP, file, and application. It also supports image reading.
+- **Video Decoding**: Supports video and image decoding based on OpenCV/GStreamer (with hardware acceleration).
+- **Algorithm Inference**: Supports multi-level inference based on deep learning algorithms, such as object detection, image classification, feature extraction, and image generation. It also supports the integration of traditional image algorithms. **Support mLLM(Multimodal Large Language Model) integration now (update 2025/8/12)**
+- **Object Tracking**: Supports object tracking, such as IOU and SORT tracking algorithms.
+- **Behavior Analysis (BA)**: Supports behavior analysis based on tracking, such as traffic behavior detection like line-crossing, parking, and violations.
+- **Business Logic**: Allows integration of any custom business logic, which can be closely related to specific business requirements.
+- **Data Proxy**: Supports pushing structured data (in JSON, XML, or custom formats) to the cloud, files, or other third-party platforms via methods like Kafka or Socket.
+- **Recording**: Supports video recording for specific time periods and capturing screenshots of specific frames, with the ability to save them as files.
+- **On-Screen Display (OSD)**: Supports overlaying structured data and business logic processing results onto frames.
+- **Video Encoding**: Supports video and image encoding based on OpenCV/GStreamer (with hardware acceleration).
+- **Stream Pushing**: Supports mainstream video stream protocols such as UDP, RTSP, RTMP, file, and application. It also supports image streaming.
+
+## Getting Started Quickly
+
+### Dependencies
+
+Platforms
+- Ubuntu 18.04 x86_64 NVIDIA rtx/tesla GPUs
+- Ubuntu 18.04 aarch64 NVIDIA jetson serials device，tx2 tested
+- Ubuntu 22.04 x86_64 by VMware virtual machine on Windows 10, pure CPUs
+- Ubuntu 18.04 x86_64 Cambrian MLU serials device, MLU 370 tested (code not provided)
+- Ubuntu 18.04 aarch64 Rockchip RK35** serials device, RK3588 tested (code not provided)
+- Ubuntu 22.04 aarch64 Ascend 310/910 serials device, Atlas 300I-Pro tested (code not provided)
+- Wait for your test
+
+Basics
+- C++ 17
+- OpenCV >= 4.6
+- GStreamer 1.14.5 (Required by OpenCV)
+- GCC >= 7.5
+
+Optional, if you need to implement your own inference backend or use a backend other than `opencv::dnn`.
+- CUDA
+- TensorRT
+- Paddle Inference
+- ONNX Runtime
+- mLLM（Ollama/vLLM/OpenAI-compatible API Services）
+- Anything you like
+
+[how to install CUDA and TensorRT](./third_party/trt_vehicle/README.md)
+
+[how to install Paddle_Inference](./third_party/paddle_ocr/README.md)
+
+### Compilation and Debugging
+
+1. run `git clone https://github.com/sherlockchou86/VideoPipe.git`
+2. run `cd VideoPipe`
+3. run `mkdir build && cd build`
+4. run `cmake ..`
+5. run `make -j8`
+
+After compilation, all library files are stored in `build/libs`, and all sample executables are located in `build/bin`. During Step 4, you can add some compilation options:
+
+- `-DVP_WITH_CUDA=ON` (Compile CUDA-related features; default is OFF)
+- `-DVP_WITH_TRT=ON` (Compile TensorRT-related features and samples; default is OFF)
+- `-DVP_WITH_PADDLE=ON` (Compile PaddlePaddle-related features and samples; default is OFF)
+- `-DVP_WITH_KAFKA=ON` (Compile Kafka-related features and samples; default is OFF)
+- `-DVP_WITH_LLM=ON` (Compile LLM-related features and samples; default is OFF)
+- `-DVP_BUILD_COMPLEX_SAMPLES=ON` (Compile advanced samples; default is OFF)
+
+For example, to enable CUDA and TensorRT modules, you can run:
+```bash
+cmake -DVP_WITH_CUDA=ON -DVP_WITH_TRT=ON ..
+```
+If you run just:
+```bash
+cmake ..
+```
+all code will be executed on the CPU.
+
+To run the compiled samples, first download the model files and test data:
+
+1. [Google Drive - Download test files and models](https://drive.google.com/drive/folders/1v9dVcR6xttUTB-WPsH3mZ_ZZMzD4wG-v?usp=sharing)
+2. [Baidu Drive - Download test files and models](https://pan.baidu.com/s/1jr2nBnEDmuNaM5DiMjbC0g?pwd=nf53)
+
+Place the downloaded directory (named `vp_data`) in any location (e.g., `/root/abc`). Then, run the sample in the same directory where `vp_data` is located. For example, execute the command:
+
+```bash
+[path to VideoPipe]/build/bin/1-1-1_sample
+```
+at `/root/abc`.
+
+**Note**: The `./third_party/` directory contains independent projects. Some are header-only libraries directly referenced by VideoPipe, while others include CPP files that can be compiled or run independently. VideoPipe depends on these libraries, and they will be automatically compiled during the VideoPipe build process. These libraries also contain their own samples; for specific usage instructions, refer to the README files in the corresponding subdirectories.
+
+### How to use
+
+Here’s a guide on how to build and run a sample pipeline with `VideoPipe`. You can either compile `VideoPipe` as a library and link it, or directly include the source code and compile the entire application.
+
+Below is a sample code demonstrating how to construct a pipeline and run it. Please make sure to update the file paths in the code accordingly:
+
+```c++
+#include "../nodes/vp_file_src_node.h"
+#include "../nodes/infers/vp_yunet_face_detector_node.h"
+#include "../nodes/infers/vp_sface_feature_encoder_node.h"
+#include "../nodes/osd/vp_face_osd_node_v2.h"
+#include "../nodes/vp_screen_des_node.h"
+#include "../nodes/vp_rtmp_des_node.h"
+#include "../utils/analysis_board/vp_analysis_board.h"
+
+/*
+* Name: 1-1-N Sample
+* Complete code located at: samples/1-1-N_sample.cpp
+* Functionality: 1 video input, 1 video analysis task (face detection and recognition), 2 outputs (screen display/RTMP stream)
+*/
+
+int main() {
+    VP_SET_LOG_INCLUDE_CODE_LOCATION(false);
+    VP_SET_LOG_INCLUDE_THREAD_ID(false);
+    VP_LOGGER_INIT();
+
+    // 1. Create nodes
+    // Video Source Node
+    auto file_src_0 = std::make_shared<vp_nodes::vp_file_src_node>("file_src_0", 0, "./test_video/10.mp4", 0.6);
+    
+    // 2. Model Inference Nodes
+    // First-level inference: Face detection
+    auto yunet_face_detector_0 = std::make_shared<vp_nodes::vp_yunet_face_detector_node>("yunet_face_detector_0", "./models/face/face_detection_yunet_2022mar.onnx");
+    // Second-level inference: Face recognition
+    auto sface_face_encoder_0 = std::make_shared<vp_nodes::vp_sface_feature_encoder_node>("sface_face_encoder_0", "./models/face/face_recognition_sface_2021dec.onnx");
+    
+    // 3. OSD Node
+    // Draw results on frames
+    auto osd_0 = std::make_shared<vp_nodes::vp_face_osd_node_v2>("osd_0");
+    
+    // Screen Display Node
+    auto screen_des_0 = std::make_shared<vp_nodes::vp_screen_des_node>("screen_des_0", 0);
+    // RTMP Stream Node
+    auto rtmp_des_0 = std::make_shared<vp_nodes::vp_rtmp_des_node>("rtmp_des_0", 0, "rtmp://192.168.77.60/live/10000");
+
+    // Build the pipeline by linking the nodes
+    yunet_face_detector_0->attach_to({file_src_0});
+    sface_face_encoder_0->attach_to({yunet_face_detector_0});
+    osd_0->attach_to({sface_face_encoder_0});
+
+    // Split the pipeline automatically to display results on screen and stream via RTMP
+    screen_des_0->attach_to({osd_0});
+    rtmp_des_0->attach_to({osd_0});
+
+    // Start the pipeline
+    file_src_0->start();
+
+    // Visualize the pipeline
+    vp_utils::vp_analysis_board board({file_src_0});
+    board.display();
+}
+```
+
+**Note**: Running this code will show three displays:
+1. **Pipeline Status**: A live update of the pipeline’s status.
+2. **Screen Output**: The GUI display showing results.
+3. **RTMP Output**: The streaming output available at the specified RTMP URL.
+
+![](./doc/g3.png)
+
+
+### Prototype Examples
+|ID|Sample|Screenshot|
+|--|--|--|
+|1|face_tracking_sample|![](./doc/p18.png)|
+|2|vehicle_tracking_sample|![](./doc/p22.png)|
+|3|mask_rcnn_sample|![](./doc/p30.png)|
+|4|openpose_sample|![](./doc/p31.png)|
+|5|face_swap_sample|![](./doc/p57.png)|
+|6|mllm_analyse_sample|![](./doc/p69.png)|
+
+A total of over 40 prototype examples are available. [Click here](./SAMPLES.md) to view more.
+
+## Read More
+- [Sample Code](./samples)
+- [Node Table](./nodes/README.md)
+- [How VideoPipe Works](./doc/about.md)
+- [Development Environment For Reference](./doc/env.md)
+
+## WeChat Discussion Group
+![](./doc/vx.png)
+
+## Thanks
+
+<a href="https://hellogithub.com/repository/4284d29e778642a4a51a471ab1eae6f0" target="_blank"><img src="https://api.hellogithub.com/v1/widgets/recommend.svg?rid=4284d29e778642a4a51a471ab1eae6f0&claim_uid=hBZTk0E2RAzJKyq" alt="Featured｜HelloGitHub" style="width: 250px; height: 54px;" width="250" height="54" /></a>

README_CN.md

@@ -0,0 +1,216 @@
+<p style="" align="center">
+  <img src="./doc/logo.png" alt="Logo" width="75%">
+</p>
+<p style="margin:0px" align="center">
+  <a href='./README.md'>英文README</a> | <a href='http://www.videopipe.cool'>VideoPipe网站</a> | <a href='http://www.videopipe.cool/index.php/2024/09/11/videopipetutorials/'>VideoPipe视频教程</a>
+</p>
+<p style="margin:0px" align="center">
+  <a href='https://github.com/sherlockchou86/one-yolo'>🚀one-yolo, make all in one for Yolo integration. All Tasks, All Versions, All Runtimes. 🚀</a>
+</p>
+
+---
+
+## 一、介绍
+
+`VideoPipe` 是一个用于视频分析和结构化的框架，采用 C++ 编写、依赖少、易上手。它像管道一样，其中每个节点相互独立并可自行搭配，`VideoPipe` 可用来构建不同类型的视频分析应用，适用于视频结构化、图片搜索、人脸识别、交通/安防领域的行为分析（如交通事件检测）等场景。
+
+![](./doc/g1.gif)
+
+## 二、优势和特点
+
+`VideoPipe` 类似于英伟达的 DeepStream 和华为的 mxVision 框架，但它更易于使用、更具备可移植性。
+
+|名称|是否开源|学习门槛|适用平台|性能|三方依赖|
+|--|--|--|--|--|--|
+|DeepStream|否|高|仅限英伟达|高|多|
+|mxVision|否|高|仅限华为|高|多|
+|VideoPipe|是|低|不限平台|中|少|
+
+`VideoPipe` 采用面向插件的编码风格，可以根据不同的需求按需搭配，我们可以使用独立的插件（即框架中的 `Node` 类型），来构建不同类型的视频分析应用。你只需准备好模型并了解如何解析其输出即可，推理可以基于不同的后端实现，如 OpenCV::DNN（默认）、TensorRT、PaddleInference、ONNXRuntime 等，任何你喜欢的都可以。
+
+![](./doc/p1-1.png)
+
+## 三、演示
+
+https://github.com/sherlockchou86/video_pipe_c/assets/13251045/b1289faa-e2c7-4d38-871e-879ae36f6d50
+
+播放器右下角全屏观看，[更多视频演示](./SAMPLES.md)
+
+## 四、功能
+
+VideoPipe 是一个让计算机视觉算法模型集成更加简单的框架，注意它不是像 TensorFlow、TensorRT 类似的深度学习框架。VideoPipe主要功能如下：
+
+- 流读取：⽀持主流的视频流协议，如 udp、rtsp、rtmp、file、application。同时支持图片读取。
+- 视频解码：⽀持基于 OpenCV/GStreamer 的视频和图片解码（⽀持硬件加速）。
+- 算法推理：⽀持基于深度学习算法的多级推理，例如⽬标检测、图像分类、特征提取、图像生成等相关网络集成。同时支持传统图像算法集成。**支持多模态大模型（mLLM）集成（2025/8/12更新）**
+- ⽬标跟踪：⽀持⽬标追踪，例如 IOU、SORT 跟踪算法等。
+- ⾏为分析（BA）：⽀持基于跟踪的⾏为分析，例如越线、停⻋、违章等交通行为判断。
+- 业务逻辑：支持任意自定义业务逻辑的集成，可以与业务强相关。
+- 数据代理：⽀持将结构化数据（json/xml/⾃定义格式）以 kafka/Sokcet 等⽅式推送到云端、文件或其他第三⽅平台。
+- 录制：⽀持特定时间段的视频录制，特定帧的截图，并存文件。
+- 屏幕显⽰（OSD）：支持将结构化数据、业务逻辑处理结果绘制到帧上。
+- 视频编码：⽀持基于 OpenCV/GStreamer 的视频和图片编码（⽀持硬件加速）。
+- 流推送：⽀持主流的视频流协议，如 udp、rtsp、rtmp、file、application。同时支持图片推送。
+
+## 五、快速上手
+
+### 5.1 依赖
+
+平台
+- Ubuntu 18.04 x86_64 NVIDIA rtx/tesla GPUs
+- Ubuntu 18.04 aarch64 NVIDIA jetson serials device，tx2 tested
+- Ubuntu 22.04 x86_64 by VMware virtual machine on Windows 10, pure CPUs
+- Ubuntu 18.04 x86_64 Cambrian MLU serials device, MLU 370 tested (code not provided)
+- Ubuntu 18.04 aarch64 Rockchip RK35** serials device, RK3588 tested (code not provided)
+- Ubuntu 22.04 aarch64 Ascend 310/910 serials device, Atlas 300I-Pro tested (code not provided)
+- Wait for your test
+
+基础
+- C++ 17
+- OpenCV >= 4.6
+- GStreamer 1.14.5 (Required by OpenCV)
+- GCC >= 7.5
+
+可选，如果你需要实现自己的推理后端，或者使用除 `opencv::dnn` 之外的其他推理后端.
+- CUDA
+- TensorRT
+- Paddle Inference
+- ONNX Runtime
+- mLLM（Ollama/vLLM/OpenAI-compatible API Services）
+- Anything you like
+
+[如何安装CUDA和TensorRT](./third_party/trt_vehicle/README.md)
+
+[如何安装Paddle_Inference](./third_party/paddle_ocr/README.md)
+
+### 5.2 编译和调试
+
+1. 运行 `git clone https://github.com/sherlockchou86/VideoPipe.git`
+2. 运行 `cd VideoPipe`
+3. 运行 `mkdir build && cd build`
+4. 运行 `cmake ..`
+5. 运行 `make -j8`
+
+编译完成后，所有的库文件存放在 `build/libs` 中，所有的 Sample 运行文件存放在 `build/bin` 中。在执行第 4 步的时候，可以添加一些编译选项：
+- -DVP_WITH_CUDA=ON （编译 CUDA 相关功能，默认为 OFF）
+- -DVP_WITH_TRT=ON （编译 TensorRT 相关功能和 Samples，默认为 OFF）
+- -DVP_WITH_PADDLE=ON （编译 PaddlePaddle 相关功能和 Samples，默认为 OFF）
+- -DVP_WITH_KAFKA=ON （编译 Kafka 相关功能和 Samples，默认为 OFF）
+- -DVP_WITH_LLM=ON （编译 LLM 相关功能和 Samples，默认为 OFF）
+- -DVP_BUILD_COMPLEX_SAMPLES=ON （编译高级 Samples，默认为 OFF）
+
+比如需要开启CUDA和TensorRT相关的模块，可以运行 `cmake -DVP_WITH_CUDA=ON -DVP_WITH_TRT=ON ..`。如果只运行 `cmake ..`，那么所有代码运行在 CPU 上。
+
+```
+# 开启全部
+cmake -DVP_WITH_CUDA=ON \
+-DVP_WITH_TRT=ON \
+-DVP_WITH_PADDLE=ON \
+-DVP_WITH_KAFKA=ON \
+-DVP_BUILD_COMPLEX_SAMPLES=ON ..
+
+# 关闭全部（默认）
+cmake ..
+```
+
+如果要运行编译生成的 Samples，先下载模型文件和测试数据：
+
+1. [谷歌网盘下载测试文件和模型](https://drive.google.com/drive/folders/1v9dVcR6xttUTB-WPsH3mZ_ZZMzD4wG-v?usp=sharing)
+2. [百度网盘下载测试文件和模型](https://pan.baidu.com/s/1jr2nBnEDmuNaM5DiMjbC0g?pwd=nf53)
+
+将下载好的目录（名称为 vp_data）放在任何位置（比如放在 `/root/abc` 下面），然后在 `同一目录` 下运行 Sample，比如在 `/root/abc` 下面执行命令：`[path to VideoPipe]/build/bin/1-1-1_sample` 即可运行 1-1-1_sample。
+
+**注意**：`./third_party/` 下面都是独立的项目，有的是 header-only 库，被 VideoPipe 直接引用；有的包含有 cpp 文件，可以独立编译或运行，VideoPipe 依赖这些库，在编译 VideoPipe 的过程中会自动编译这些库。这些库也包含自己的 Samples，具体使用方法可参见对应子目录下的 README 文件.
+
+### 5.3 如何使用
+
+1. 先将 VideoPipe 编译成库，然后引用它.
+2. 或者直接引用源代码，然后编译整个Application.
+
+下面是一个如何构建 Pipeline 然后运行的 Sample(请先修改代码中的相关文件路径)：
+
+```c++
+#include "../nodes/vp_file_src_node.h"
+#include "../nodes/infers/vp_yunet_face_detector_node.h"
+#include "../nodes/infers/vp_sface_feature_encoder_node.h"
+#include "../nodes/osd/vp_face_osd_node_v2.h"
+#include "../nodes/vp_screen_des_node.h"
+#include "../nodes/vp_rtmp_des_node.h"
+#include "../utils/analysis_board/vp_analysis_board.h"
+
+/*
+* 名称：1-1-N sample
+* 完整代码位于：samples/1-1-N_sample.cpp
+* 功能说明：1个视频输入，1个视频分析任务（人脸检测和识别），2个输出（屏幕输出/RTMP推流输出）
+*/
+
+int main() {
+    VP_SET_LOG_INCLUDE_CODE_LOCATION(false);
+    VP_SET_LOG_INCLUDE_THREAD_ID(false);
+    VP_LOGGER_INIT();
+
+    // 1、创建节点
+    // 视频获取 Node
+    auto file_src_0 = std::make_shared<vp_nodes::vp_file_src_node>("file_src_0", 0, "./test_video/10.mp4", 0.6);
+    // 2、模型推理 Node
+    // 一级推理：人脸检测
+    auto yunet_face_detector_0 = std::make_shared<vp_nodes::vp_yunet_face_detector_node>("yunet_face_detector_0", "./models/face/face_detection_yunet_2022mar.onnx");
+    // 二级推理：人脸识别
+    auto sface_face_encoder_0 = std::make_shared<vp_nodes::vp_sface_feature_encoder_node>("sface_face_encoder_0", "./models/face/face_recognition_sface_2021dec.onnx");
+    // 3、OSD Node
+    // 处理结果绘制到帧上
+    auto osd_0 = std::make_shared<vp_nodes::vp_face_osd_node_v2>("osd_0");
+    // 屏幕展示
+    auto screen_des_0 = std::make_shared<vp_nodes::vp_screen_des_node>("screen_des_0", 0);
+    // 推流展示
+    auto rtmp_des_0 = std::make_shared<vp_nodes::vp_rtmp_des_node>("rtmp_des_0", 0, "rtmp://192.168.77.60/live/10000");
+
+    // 构建管道，将节点的处理结果关联起来
+    yunet_face_detector_0->attach_to({file_src_0});
+    sface_face_encoder_0->attach_to({yunet_face_detector_0});
+    osd_0->attach_to({sface_face_encoder_0});
+
+    // 管道自动拆分，通过屏幕/推流输出结果
+    screen_des_0->attach_to({osd_0});
+    rtmp_des_0->attach_to({osd_0});
+
+    // 启动管道
+    file_src_0->start();
+
+    // 可视化管道
+    vp_utils::vp_analysis_board board({file_src_0});
+    board.display();
+}
+```
+上面代码运行后，会出现 3 个画面:
+1. 管道的运行状态图，状态自动刷新
+2. 屏幕显示结果（GUI）
+3. 播放器显示结果（RTMP）
+
+![](./doc/g3.png)
+
+
+### 5.4 案例原型
+|ID|Sample|截图|
+|--|--|--|
+|1|face_tracking_sample|![](./doc/p18.png)|
+|2|vehicle_tracking_sample|![](./doc/p22.png)|
+|3|mask_rcnn_sample|![](./doc/p30.png)|
+|4|openpose_sample|![](./doc/p31.png)|
+|5|face_swap_sample|![](./doc/p57.png)|
+|6|mllm_analyse_sample|![](./doc/p69.png)|
+
+共计 40 多个原型案例，[点击](./SAMPLES.md)查看更多。
+
+## 六、更多资料
+- [Sample Code](./samples)
+- [Node Table](./nodes/README.md)
+- [How VideoPipe Works](./doc/about.md)
+- [Development Environment For Reference](./doc/env.md)
+
+## 扫码入群交流
+![](./doc/vx.png)
+
+## 鸣谢
+
+<a href="https://hellogithub.com/repository/4284d29e778642a4a51a471ab1eae6f0" target="_blank"><img src="https://api.hellogithub.com/v1/widgets/recommend.svg?rid=4284d29e778642a4a51a471ab1eae6f0&claim_uid=hBZTk0E2RAzJKyq" alt="Featured｜HelloGitHub" style="width: 250px; height: 54px;" width="250" height="54" /></a>

SAMPLES.md

@@ -0,0 +1,93 @@
+## Samples Video（视频演示） ##
+### Face Swap（AI换脸） ###
+https://github.com/sherlockchou86/video_pipe_c/assets/13251045/aa1162f3-2f61-4ac7-8add-12d9f8b7ab23
+
+### Vehicle CLustering（车辆分类聚合） ###
+https://github.com/sherlockchou86/video_pipe_c/assets/13251045/867b981b-5d8c-4ee3-9831-9c755b520ad6
+
+### Search Vehicle by Image（以图搜车） ###
+https://github.com/sherlockchou86/video_pipe_c/assets/13251045/a0ae5422-904b-4ad1-8201-3e1a07477882
+
+### Stop Detecting（停车检测） ###
+https://github.com/sherlockchou86/video_pipe_c/assets/13251045/dd518798-6345-4121-a4d5-ad5bde18e3f3
+
+### Vehicle Behavior Analysis（车辆行为分析） ###
+https://github.com/sherlockchou86/video_pipe_c/assets/13251045/3c115d3b-45e4-4df9-8459-bfadca974e4c
+
+### Jam Detecting（拥堵检测） ###
+https://github.com/sherlockchou86/video_pipe_c/assets/13251045/b31612ae-6b44-4cbd-8439-610bd71834d5
+
+### Face Recognition（人脸识别） ###
+https://github.com/sherlockchou86/video_pipe_c/assets/13251045/7958c5e3-4e6e-453b-bde9-663779d960cb
+
+### Dynamical Pipeline（动态管道） ###
+https://github.com/sherlockchou86/video_pipe_c/assets/13251045/7fe3e45d-e528-4315-ba4f-21c3f11163e3
+
+### License Plate Recognition（车牌识别相机） ###
+https://github.com/sherlockchou86/video_pipe_c/assets/13251045/f680dccf-92c3-41eb-a472-7f0e27c28257
+
+### Parallel Task & Synchronization（并行任务、数据同步） ###
+https://github.com/sherlockchou86/video_pipe_c/assets/13251045/28a71383-8864-4a25-8da3-27922f0b5baf
+
+### Math Review（口算检查） ###
+https://github.com/sherlockchou86/video_pipe_c/assets/13251045/9c3c1a87-d9f7-4630-a541-96610d564f13
+
+
+## Samples Screenshot（案例原型） ##
+
+|id|sample|screenshot|
+|--|--|--|
+|1|1-1-1_sample|![](./doc//p10.png)|
+|2|1-1-N_sample|![](./doc//p11.png)|
+|3|1-N-N_sample|![](./doc//p12.png)|
+|4|N-1-N_sample|![](./doc//p13.png)|
+|5|N-N_sample|![](./doc//p14.png)|
+|6|paddle_infer_sample|![](./doc//p15.png)|
+|7|src_des_sample|![](./doc//p16.png)|
+|8|trt_infer_sample|![](./doc//p17.png)|
+|9|vp_logger_sample|-|
+|10|face_tracking_sample|![](./doc//p18.png)|
+|11|vehicle_tracking_sample|![](./doc//p22.png)|
+|12|interaction with pipe sample|--|
+|13|record_sample|--|
+|14|message_broker_sample & message_broker_sample2|![](./doc//p21.png)|
+|15|mask_rcnn_sample|![](./doc//p30.png)|
+|16|openpose_sample|![](./doc//p31.png)|
+|17|enet_seg_sample|![](./doc//p32.png)|
+|18|multi detectors and classifiers sample|![](./doc//p33.png)|
+|19|image_des_sample|![](./doc//p34.png)|
+|20|image_src_sample|![](./doc//p35.png)|
+|21|rtsp_des_sample|![](./doc//p36.png)|
+|22|ba_crossline_sample|![](./doc//p37.png)|
+|23|plate_recognize_sample|![](./doc//p38.png)|
+|24|vehicle body scan sample|![](./doc/p40.png)|
+|25|body scan and plate detect sample|![](./doc/p39.png)|
+|26|app_src_sample|![](./doc/p41.png)|
+|27|vehicle cluster based on classify encoding sample|![](./doc/p42.png)|
+|28|ba_stop_sample|![](./doc/p49.png)|
+|29|behaviour analysis|![](./doc/p48.png)|
+|30|similiarity search|![](./doc/p44.png)![](./doc/p43.png)![](./doc/p45.png)|
+|31|property and similiarity search|![](./doc/p46.png)![](./doc/p47.png)|
+|32|ba_jam_sample|![](./doc/p50.png)|
+|33|face recognize|![](./doc/p51.png)|
+|34|license plate recognize(LPR) camera|![](./doc/p52.png)|
+|35|math expression check|![](./doc/p53.png)|
+|36|skip_sample|![](./doc/p54.png)|
+|37|obstacle_detect_sample|![](./doc/p55.png)|
+|38|firesmoke_detect_sample|![](./doc/p56.png)|
+|39|face_swap_sample|![](./doc/p57.png)|
+|40|video_restoration_sample|![](./doc/p58.png)|
+|41|app_des_sample|![](./doc/p59.png)|
+|42|app_src_des_sample|![](./doc/p60.png)|
+|43|lane_detect_sample|![](./doc/p61.png)|
+|44|frame_fusion_sample|![](./doc/p62.png)|
+|45|1-N-1_sample|![](./doc/p65.png)|
+|46|1-N-1_sample2|![](./doc/p64.png)|
+|47|1-N-1_sample3|![](./doc/p63.png)|
+|48|message_broker_kafka_sample|![](./doc/p66.png)|
+|49|trt_yolov8_sample|![](./doc/p67.png)|
+|50|trt_yolov8_sample2|![](./doc/p68.png)|
+|51|mllm_analyse_sample|![](./doc/p69.png)|
+|52|mllm_analyse_sample_openai|![](./doc/p70.png)|
+|53|rtmp_src_sample|![](./doc/p71.png)|
+|54|yolov5_seg_sample|![](./doc/p72.png)|

doc/about.md

@@ -0,0 +1,101 @@
+
+here are important instructions for VideoPipe if you want to figure out how it works!
+
+## Core parts in video structured application ## 
+`video structured` is a process which converts unstructure data (video here) into structured data. unstructure data:
+- video
+- image
+- audio
+- nature text
+
+and structure data mainly includes something like json, xml or data table in DataBase which can by processed directly by machine (program).
+
+Specifically in terms of video, the process of structured mainly involves these core parts:
+1. `read stream`. capture video stream from network or local machine.
+2. `decode`. decode byte stream to frames, algorithm/procedure can only act on images.
+3. `inference`. deep learning work on images, detect, classification or feature extraction.
+4. `track`. track on objects in video.
+5. `behaviour analysis` (optional). analysis on objects' tracks.
+6. `osd`. on screen display, show results on images for debug purpose or intuitive effects.
+7. `message broker`. push structured data to external.
+8. `encode`. encode frames which contains results to byte stream for transfer/serialization purpose.
+9. `push stream`. push byte stream to external or save it directly.
+
+<p align="center">
+<img src="./p23.png">
+</p>
+<div align="center">figure 1. core parts in video structured application. </div>
+
+each core part in `video structured` corresponding to one type of plugin in `VideoPipe`, namely **`Node`** in code.
+
+
+## Node in VideoPipe ##
+one `Node` in VideoPipe responsible for single task such as decoding or inference. we put many nodes together to construct a pipe, and let video data flow through the whole pipeline. every `Node` has 2 queues inside, one is for caching data from upstream nodes and another one is for caching data waiting for being pushed to downstream nodes. we can write logic code between the 2 queues, they are typical `producer-consumer` pattern.
+
+<p align="center">
+<img src="./p24.png">
+</p>
+<div align="center">figure 2. what Node looks like? </div>
+
+by default, producer and consumer work with single thread inside node, we need write async code when deal with complex tasks (for example, pushing data is a time-consuming operation in `vp_message_broker_node`) to avoid blocking the pipeline.
+
+<p align="center">
+<img src="./p25.png">
+</p>
+<div align="center">figure 3. async task inside Node. </div>
+
+there are 3 types of `Node` in VideoPipe, namely:
+- `SRC Node`. source node where data was created (only 1 queue inside used for caching data being pushed to downstream nodes).
+- `MID Node`. middle node where data would be handled.
+- `DES Node`. destination node where data disappears (only 1 queue inside used for caching data from upstream nodes).
+
+each `Node` `itself` has the ability to merge multi upstream nodes, and split into multi downstream nodes as well. note that `Node` use shallow-copy and copy Equally when data transfered from one node to other nodes by default, if you need deep-copy or want to transfer data by channel index (just hope data unconfused), add a `vp_split_node` at the point of spliting which would get different behaviour.
+<p align="center">
+<img src="./p28.png">
+</p>
+<div align="center">figure 4. merge & split in VideoPipe. </div>
+
+## Data flow in VideoPipe ##
+video (frame here) is a type of heavyweight data, so deep copying frequently would decrease the performance of pipeline. actually data transfered between 2 nodes in VideoPipe use `smart pointers` by default, once data was created by source nodes, the data content would NOT be copyed later at most time in the whole pipeline (but we can specify deep-copy if we need, using `vp_split_node` for instance).
+
+<p align="center">
+<img src="./p26.png">
+</p>
+<div align="center">figure 5. how data flows in VideoPipe. </div>
+
+video consist of continuous frames, VideoPipe handle these frames One by One, so the `frame index` in frame meta would increase continuously as well.
+
+
+## Hooks in VideoPipe ##
+hook is a mechanism which let host notify listeners when something happens, VideoPipe support hooks as well. pipeline invokes callback functions (via `std::function` object) to communicate with external code, such as export `fps`, `latency` and other status of pipeline itself. we should NOT block the callback functions when writing custom code inside it.
+
+<p align="center">
+<img src="./p27.png">
+</p>
+<div align="center">figure 6. hooks in VideoPipe. </div>
+
+hooks help to debug with our application and quickly find the bottleneck in whole pipe, visualization tool `vp_analysis_board` works depend on hooks.
+
+
+## Implement new Node type in VideoPipe ##
+`vp_node` is the base class for all nodes in VideoPipe. we can define a new node class derived from `vp_node` and override some virtual functions like `handle_frame_meta` and `handle_control_meta`.
+- `handle_frame_meta`. handle frame data flowing current node.
+- `handle_control_meta`. handle control data flowing current node.
+
+<p align="center">
+<img src="./p29.png">
+</p>
+<div align="center">figure 7. override virtual functions in custom Node. </div>
+
+frame data means `vp_frame_meta` in VideoPipe, contains data related to frame such as `frame index`, `data buffer`, `original width`. control data means `vp_control_meta` in VideoPipe, contains data related command such as `record video`, `record image`.
+
+note, NOT all data flowing current node should be handled using new logic, they just pass through if no operations work on them. we just need handle what we are interested in. 
+
+## Hardware Acceleration in VideoPipe ##
+some operations in `video structured` applications can benefit from hardware such as GPUs/NPUs. for example, video encoding on GPUs have higher speed/performace than CPUs. VideoPipe support hardware acceleration for these parts:
+
+- decode/encode. based on HARD decode/encode gstreamer plugins, [look more](https://github.com/sherlockchou86/video_pipe_c/blob/master/doc/env.md#about-hardware-acceleration).
+- inference. no doubt about it.
+- osd. need implement based on hardware acceleration SDKs by yourself.
+
+it is important to note that, although VideoPipe support hardware acceleration for above logic, `they could NOT share memory between each others`. it means that data will be copyed from GPU to CPU or CPU to GPU over and over again, which is the biggest disadvantage compared to other similar SDKs such as DeepStream.

doc/env.md

@@ -0,0 +1,101 @@
+
+
+## Personal Development Environment ##
+
+- VS Code for Windows 11
+- Ubuntu 18.04 x86_64 / C++17  / GCC 7.5 / GTX 1080 GPU
+- GStreamer 1.14.5 / OpenCV 4.6
+---------
+
+Install GStreamer (1.14.5 for Ubuntu 18.04 by default):
+```
+apt-get install libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libgstreamer-plugins-bad1.0-dev gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly gstreamer1.0-libav gstreamer1.0-tools gstreamer1.0-x gstreamer1.0-alsa gstreamer1.0-gl gstreamer1.0-gtk3 gstreamer1.0-qt5 gstreamer1.0-pulseaudio libgstrtspserver-1.0-dev gstreamer1.0-rtsp
+```
+
+Install OpenCV from source with `gstreamer` ON (CUDA optional). download source code of OpenCV 4.6.0 (with extra contrib modules) from github first, put them at the same directory then run `cmake` and `make` command:
+
+```
+step 1:
+cd `the path of opencv 4.6.0`
+mkdir build && cd build
+```
+
+```
+step 2:
+cmake -D CMAKE_BUILD_TYPE=RELEASE \
+-D CMAKE_INSTALL_PREFIX=/usr/local \
+-D WITH_TBB=ON \
+-D ENABLE_FAST_MATH=1 \
+-D CUDA_FAST_MATH=1 \
+-D WITH_CUBLAS=1 \
+-D WITH_CUDA=ON \
+-D BUILD_opencv_cudacodec=OFF \
+-D WITH_CUDNN=ON \
+-D OPENCV_DNN_CUDA=ON \
+-D CUDA_ARCH_BIN=6.1 \
+-D WITH_V4L=ON \
+-D WITH_QT=OFF \
+-D WITH_OPENGL=ON \
+-D WITH_GSTREAMER=ON \
+-D OPENCV_GENERATE_PKGCONFIG=ON \
+-D OPENCV_PC_FILE_NAME=opencv.pc \
+-D OPENCV_ENABLE_NONFREE=ON \
+-D OPENCV_EXTRA_MODULES_PATH=../../opencv_contrib-4.6.0/modules \
+-D INSTALL_PYTHON_EXAMPLES=OFF \
+-D INSTALL_C_EXAMPLES=OFF \
+-D BUILD_EXAMPLES=OFF ..
+```
+
+```
+step 3:
+make -j8
+```
+
+---------
+`VcXsrv` for screen display from remote machine to local desktop in case of using SSH terminal.
+
+- first install PC client from: https://sourceforge.net/p/vcxsrv/wiki/Home/
+- then run `export DISPLAY=local_ip:0.0` (or add it to ~/.bashrc) on remote machine (linux server or embedded board)
+
+---------
+Maybe you need install nginx with `http-rtmp-module` as rtmp server for debug purpose (other tools such as `ZLMediaKit` works fine **which was highly recommended**). Also, maybe you need a rtsp server from which we can receive rtsp stream for debug purpose.
+
+- [how to install ZLMeidaKit](https://github.com/ZLMediaKit/ZLMediaKit/wiki/vcpkg%E6%96%B9%E5%BC%8F%E5%AE%89%E8%A3%85zlmediakit)
+- [how to push stream to ZLMediaKit](https://github.com/ZLMediaKit/ZLMediaKit/wiki/ZLMediaKit%E6%8E%A8%E6%B5%81%E6%B5%8B%E8%AF%95), a few samples which used `vp_rtmp_des_node` in VidepPipe would push rtmp stream to server.
+- [how to pull/play stream from ZLMediaKit](https://github.com/ZLMediaKit/ZLMediaKit/wiki/%E6%92%AD%E6%94%BEurl%E8%A7%84%E5%88%99), VLC player or ffplay was recommended.
+
+---------
+Please prepare kafka server AND install `librdkafka` client sdk if you want to enable kafka-related codes.
+- [how to install kafka server](https://kafka.apache.org/quickstart)
+- how to install `librdkafka`? run `sudo apt-get install librdkafka-dev`
+
+
+## tips ##
+- Use shared_ptr/make_shared in whole project, do not use new/delete.
+- The pipeline is driven by stream data, if your app is not responding, maybe no stream input.
+
+
+## about Hardware Acceleration ##
+Since decode & encode in VideoPipe depend on gstreamer (encapsulated inside opencv), if you want to use your GPUs/NPUs to accelerate decoding and encoding performace, you need get/install HARD decode or HARD encode `gstreamer plugins` correctly first and modify gst launch string (take `vp_file_des_node` for example):
+```cpp
+appsrc ! videoconvert ! x264enc bitrate=%d ! mp4mux ! filesink location=%s
+```
+to
+```
+appsrc ! videoconvert ! nvv4l2h264enc bitrate=%d ! mp4mux ! filesink location=%s
+```
+the plugin `x264enc` use CPUs to encode video stream, but `nvv4l2h264enc`(comes from DeepStream SDK) use GPUs instread. if you use other platforms other than NVIDIA, you need Corresponding Hardware Acceleration plugins.
+
+**soft/hard decode example**
+```
+gst-launch-1.0 filesrc location=./face.mp4 ! qtdemux ! h264parse ! avdec_h264 ! videoconvert ! autovideosink    // decode by avdec_h264 use CPUs
+gst-launch-1.0 filesrc location=./face.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! videoconvert ! autovideosink // decode by nvv4l2decoder use NVIDIA GPUs
+```
+
+**soft/hard encode example**
+```
+gst-launch-1.0 filesrc location=./face.mp4 ! qtdemux ! h264parse ! avdec_h264 ! x264enc ! h264parse ! flvmux ! filesink location=./new_face.flv    // encode by x264enc use CPUs
+gst-launch-1.0 filesrc location=./face.mp4 ! qtdemux ! h264parse ! avdec_h264 ! nvv4l2h264enc ! h264parse ! flvmux ! filesink location=./new_face.flv  // encode by nvv4l2h264enc use NVIDIA GPUs
+```
+[source code of hard decode/encode gstreamer plugins for NVIDIA](https://gitlab.freedesktop.org/gstreamer/gstreamer/-/tree/main/subprojects/gst-plugins-bad/sys/nvcodec).(developed by community, open source), we could also use decode/encode plugins from [DeepStream SDK](https://docs.nvidia.com/metropolis/deepstream/6.0/dev-guide/text/DS_Quickstart.html) which maintained by NVIDIA but closed source.
+

excepts/vp_invalid_argument_error.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <stdexcept>
+
+namespace vp_excepts {
+
+    class vp_invalid_argument_error: public std::runtime_error {
+    private:
+        /* data */
+    public:
+        vp_invalid_argument_error(const std::string& what_arg);
+        ~vp_invalid_argument_error();
+    };
+    
+    inline vp_invalid_argument_error::vp_invalid_argument_error(const std::string& what_arg): std::runtime_error(what_arg) {
+    }
+    
+    inline vp_invalid_argument_error::~vp_invalid_argument_error() {
+    }
+}

excepts/vp_invalid_calling_error.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <stdexcept>
+
+namespace vp_excepts {
+
+    class vp_invalid_calling_error: public std::runtime_error {
+    private:
+        /* data */
+    public:
+        vp_invalid_calling_error(const std::string& what_arg);
+        ~vp_invalid_calling_error();
+    };
+    
+    inline vp_invalid_calling_error::vp_invalid_calling_error(const std::string& what_arg): std::runtime_error(what_arg) {
+    }
+    
+    inline vp_invalid_calling_error::~vp_invalid_calling_error() {
+    }
+
+}

excepts/vp_invalid_pipeline_error.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <stdexcept>
+
+namespace vp_excepts {
+
+    class vp_invalid_pipeline_error: public std::runtime_error {
+    private:
+        /* data */
+    public:
+        vp_invalid_pipeline_error(const std::string& what_arg);
+        ~vp_invalid_pipeline_error();
+    };
+    
+    inline vp_invalid_pipeline_error::vp_invalid_pipeline_error(const std::string& what_arg): std::runtime_error(what_arg) {
+    }
+    
+    inline vp_invalid_pipeline_error::~vp_invalid_pipeline_error() {
+    }
+
+}