commit ba76cfae28e807e63105ae3b09f92c4f483d4f9e Author: 陈赣 Date: Wed Jun 3 12:43:14 2026 +0800 first commit diff --git a/.gitignore b/.gitignore new file mode 100755 index 0000000..0fb3108 --- /dev/null +++ b/.gitignore @@ -0,0 +1,160 @@ + +# binary files +* +!*/ +!*.* + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# model and video +*.cfg +*.onnx +*.caffemodel +*.prototxt +*.mp4 +*.weights +*.h264 +*.trt +models/ +test_video/ +log/ +debug/ +build/ + +# doc +*.pptx + +# Distribution / packaging +.Python +develop-eggs/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST +scripts/include/ +scripts/libs/ + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +.DS_Store +.idea/ +cmake-build-debug-remote/ +cmake-build-debug/ +build/* +!build/build.sh diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json new file mode 100755 index 0000000..d7913d6 --- /dev/null +++ b/.vscode/c_cpp_properties.json @@ -0,0 +1,23 @@ +{ + "configurations": [ + { + "name": "Win32", + "includePath": [ + "${workspaceFolder}/**", + "/usr/local/include/**", + "/usr/include/**" + ], + "defines": [ + "_DEBUG", + "UNICODE", + "_UNICODE" + ], + "windowsSdkVersion": "8.1", + "compilerPath": "/usr/bin/g++", + "cStandard": "c17", + "cppStandard": "c++17", + "intelliSenseMode": "linux-gcc-x64" + } + ], + "version": 4 +} \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..a1b30da --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,92 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "debug samples for videopipe", // debug for samples in videopipe + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/build/bin/${fileBasenameNoExtension}", + "args": [""], + "stopAtEntry": false, + "cwd": "${workspaceFolder}/..", // change this value to the path of your vp_data + "environment": [], + "externalConsole": false, + "MIMode": "gdb", + "miDebuggerPath": "/usr/bin/gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ], + "preLaunchTask": "" + }, + { + "name": "debug complex samples for videopipe", // debug for complex samples in videopipe + "type": "cppdbg", + "request": "launch", + "program": "${fileDirname}/${fileBasenameNoExtension}", + "args": ["../../.."], // change this value to the path of your vp_data + "stopAtEntry": false, + "cwd": "${fileDirname}", + "environment": [], + "externalConsole": false, + "MIMode": "gdb", + "miDebuggerPath": "/usr/bin/gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ], + "preLaunchTask": "" + }, + { + "name": "debug trt_yolov8 samples", // debug for trt samples at third_party/trt_yolov8 + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/third_party/trt_yolov8/build/samples/${fileBasenameNoExtension}", + "args": [""], + "stopAtEntry": false, + "cwd": "${workspaceFolder}/..", + "environment": [], + "externalConsole": false, + "MIMode": "gdb", + "miDebuggerPath": "/usr/bin/gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ], + "preLaunchTask": "" + }, + { + "name": "debug trt_vehicle samples", // debug for trt samples at third_party/trt_vehicle + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/third_party/trt_vehicle/build/samples/${fileBasenameNoExtension}", + "args": [""], + "stopAtEntry": false, + "cwd": "${workspaceFolder}/..", + "environment": [], + "externalConsole": false, + "MIMode": "gdb", + "miDebuggerPath": "/usr/bin/gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ], + "preLaunchTask": "" + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100755 index 0000000..6d9d6d7 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,94 @@ +{ + "files.associations": { + "memory": "cpp", + "cmath": "cpp", + "cstddef": "cpp", + "cstdint": "cpp", + "cstdio": "cpp", + "cstdlib": "cpp", + "cstring": "cpp", + "cwchar": "cpp", + "exception": "cpp", + "initializer_list": "cpp", + "iosfwd": "cpp", + "limits": "cpp", + "new": "cpp", + "stdexcept": "cpp", + "type_traits": "cpp", + "typeinfo": "cpp", + "utility": "cpp", + "vector": "cpp", + "xmemory": "cpp", + "xmemory0": "cpp", + "xstddef": "cpp", + "xstring": "cpp", + "xtr1common": "cpp", + "xutility": "cpp", + "queue": "cpp", + "thread": "cpp", + "mutex": "cpp", + "algorithm": "cpp", + "chrono": "cpp", + "deque": "cpp", + "functional": "cpp", + "ios": "cpp", + "istream": "cpp", + "ostream": "cpp", + "ratio": "cpp", + "streambuf": "cpp", + "string": "cpp", + "system_error": "cpp", + "xthread": "cpp", + "tuple": "cpp", + "xfacet": "cpp", + "xfunctional": "cpp", + "xiosbase": "cpp", + "xlocale": "cpp", + "xlocinfo": "cpp", + "xlocnum": "cpp", + "iostream": "cpp", + "any": "cpp", + "array": "cpp", + "atomic": "cpp", + "bit": "cpp", + "*.tcc": "cpp", + "cctype": "cpp", + "clocale": "cpp", + "compare": "cpp", + "concepts": "cpp", + "condition_variable": "cpp", + "cstdarg": "cpp", + "ctime": "cpp", + "cwctype": "cpp", + "map": "cpp", + "unordered_map": "cpp", + "iterator": "cpp", + "memory_resource": "cpp", + "numeric": "cpp", + "random": "cpp", + "string_view": "cpp", + "numbers": "cpp", + "semaphore": "cpp", + "stop_token": "cpp", + "complex": "cpp", + "sstream": "cpp", + "list": "cpp", + "set": "cpp", + "fstream": "cpp", + "iomanip": "cpp", + "optional": "cpp", + "hash_map": "cpp", + "hash_set": "cpp", + "bitset": "cpp", + "codecvt": "cpp", + "unordered_set": "cpp", + "filesystem": "cpp", + "future": "cpp", + "cfenv": "cpp", + "cinttypes": "cpp", + "__nullptr": "cpp", + "__locale": "cpp", + "regex": "cpp" + }, + "C_Cpp.errorSquiggles": "Disabled" +} \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..6df9d74 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,171 @@ +#[[ + prepare in advance： + 1. OpenCV >= 4.6 + 2. Gstreamer >= 1.14.5 + 3. CUDA, TensorRT, PADDLE and others for optional +]] + +cmake_minimum_required(VERSION 3.10) +project(video_pipe VERSION 1.0) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_BUILD_TYPE Debug) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fPIC -w -fdiagnostics-color=always -pthread") +# save all libs(including third_party's) to 'libs' +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/libs) + +# optional for build, modify values when configure the project using 'cmake -DVP_WITH_CUDA=OFF ..' +option(VP_WITH_CUDA "prepared CUDA or not?" OFF) +option(VP_WITH_TRT "prepared TensorRT or not?" OFF) +option(VP_WITH_PADDLE "prepared PaddlePaddle or not?" OFF) +option(VP_WITH_KAFKA "prepared Kafka or not?" OFF) +option(VP_WITH_LLM "prepared LLM or not?" OFF) +option(VP_WITH_FFMPEG "prepared FFMPEG or not?" OFF) +option(VP_BUILD_COMPLEX_SAMPLES "build complex samples or not? (maybe source code not provided)" OFF) + +# OpenCV required +find_package(OpenCV REQUIRED) +message(STATUS "OpenCV library status:") +message(STATUS " version: ${OpenCV_VERSION}") +message(STATUS " libraries: ${OpenCV_LIBS}") +message(STATUS " include path: ${OpenCV_INCLUDE_DIRS}") +include_directories(${OpenCV_INCLUDE_DIRS}) + +# Gstreamer required +include(FindPkgConfig) +pkg_check_modules(GST REQUIRED gstreamer-1.0) +pkg_check_modules(GSTAPP REQUIRED gstreamer-app-1.0) +pkg_check_modules(GST_RTSP REQUIRED gstreamer-rtsp-server-1.0) +message(STATUS "GStreamer library status:") +message(STATUS " version: ${GST_VERSION}") +message(STATUS " libraries: ${GST_LIBRARIES} ${GSTAPP_LIBRARIES} ${GST_RTSP_LIBRARIES}") +message(STATUS " include path: ${GST_INCLUDE_DIRS}") +include_directories(${GST_INCLUDE_DIRS}) +set (GST_DEPEND_LIBS ${GST_LIBRARIES} ${GSTAPP_LIBRARIES} ${GST_RTSP_LIBRARIES}) + +# collect dependent libs for videopipe +list(APPEND VP_DEPEND_LIBS ${OpenCV_LIBS} ${GST_DEPEND_LIBS} stdc++fs) + +# optional for CUDA +if(VP_WITH_CUDA) # CUDA enabled + add_definitions(-DVP_WITH_CUDA) +endif() + +# optional for TensorRT +if(VP_WITH_TRT) # TensorRT enabled + add_definitions(-DVP_WITH_TRT) + set(VP_BUILD_FROM ON) # set flag for sub project + set(VP_TRT_LIB_PATH "") + set(VP_TRT_INC_PATH "") + # trt_vehicle + message("-------------start build trt_vehicle--------------") + add_subdirectory(third_party/trt_vehicle) + list(APPEND VP_DEPEND_LIBS trt_vehicle) + link_directories(${VP_TRT_LIB_PATH}) + message(STATUS "TensorRT library status:") + message(STATUS " include path: ${VP_TRT_INC_PATH}") + message(STATUS " library path: ${VP_TRT_LIB_PATH}") + message("--------------end build trt_vehicle---------------") + # trt_yolov8 + message("-------------start build trt_yolov8--------------") + add_subdirectory(third_party/trt_yolov8) + list(APPEND VP_DEPEND_LIBS trt_yolov8) + message("--------------end build trt_yolov8---------------") +endif() + +# optional for PaddlePaddle +if(VP_WITH_PADDLE) # PaddlePaddle enabled + add_definitions(-DVP_WITH_PADDLE) + set(VP_BUILD_FROM ON) # set flag for sub project + set(VP_PADDLE_LIB_PATH "") + set(VP_PADDLE_INC_PATH "") + message("-------------start build paddle_ocr--------------") + add_subdirectory(third_party/paddle_ocr) # build paddle_ocr + list(APPEND VP_DEPEND_LIBS paddle_ocr) + link_directories(${VP_PADDLE_LIB_PATH}) + message(STATUS "PaddlePaddle library status:") + message(STATUS " include path: ${VP_PADDLE_INC_PATH}") + message(STATUS " library path: ${VP_PADDLE_LIB_PATH}") + message("--------------end build paddle_ocr---------------") +endif() + +# optional for Kafka +if(VP_WITH_KAFKA) + add_definitions(-DVP_WITH_KAFKA) + list(APPEND VP_DEPEND_LIBS rdkafka++) +endif() + +# optional for LLM +if(VP_WITH_LLM) + find_package(OpenSSL REQUIRED) + message(STATUS "OpenSSL library status:") + message(STATUS " version: ${OPENSSL_VERSION}") + message(STATUS " libraries: ${OPENSSL_LIBRARIES}") + message(STATUS " include path: ${OPENSSL_INCLUDE_DIR}") + include_directories(${OPENSSL_INCLUDE_DIR}) + list(APPEND VP_DEPEND_LIBS ${OPENSSL_LIBRARIES}) + add_definitions(-DVP_WITH_LLM) +endif() + +# optional for FFmpeg +if(VP_WITH_FFMPEG) + add_definitions(-DVP_WITH_FFMPEG) + list(APPEND VP_DEPEND_LIBS avcodec avformat avdevice swscale swresample avutil) +endif() + +# tinyexpr +message("-------------start build tinyexpr--------------") +add_subdirectory(third_party/tinyexpr) +list(APPEND VP_DEPEND_LIBS tinyexpr) +message("--------------end build tinyexpr---------------") + +message("-------------collect version info--------------") +string(TIMESTAMP BUILD_TIME "%Y%m%d-%H%M%S") +find_package(Git QUIET) +if(GIT_FOUND) + execute_process( + COMMAND ${GIT_EXECUTABLE} rev-parse --short HEAD + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + OUTPUT_VARIABLE GIT_COMMIT_HASH + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + execute_process( + COMMAND ${GIT_EXECUTABLE} diff --quiet + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + RESULT_VARIABLE GIT_DIFF_RESULT + ) + if(GIT_DIFF_RESULT EQUAL 0) + set(GIT_DIRTY "") + else() + set(GIT_DIRTY "-dirty") + endif() +else() + message(WARNING "Git not found! Using default version info.") + set(GIT_COMMIT_HASH "nogit") + set(GIT_DIRTY "") +endif() +set(FINAL_GIT_VERSION "${GIT_COMMIT_HASH}${GIT_DIRTY}") +configure_file( + ${CMAKE_SOURCE_DIR}/utils/vp_version.h.in + ${CMAKE_BINARY_DIR}/vp_version.h + @ONLY +) +include_directories(${CMAKE_BINARY_DIR}) +message(STATUS "version info:") +message(STATUS " build_time: ${BUILD_TIME}") +message(STATUS " commit_hash: ${FINAL_GIT_VERSION}") +message("-------------collect version info--------------") + +# collect source files for videopipe +file(GLOB_RECURSE NODES "nodes/*.cpp") +file(GLOB_RECURSE OBJECTS "objects/*.cpp") +file(GLOB_RECURSE UTILS "utils/*.cpp") +#...# +list(APPEND VP_CPPS_SOURCES ${NODES} ${OBJECTS} ${UTILS}) + +# build for videopipe +add_library(${PROJECT_NAME} SHARED ${VP_CPPS_SOURCES}) +target_link_libraries(${PROJECT_NAME} ${VP_DEPEND_LIBS}) + +# build for samples +add_subdirectory(samples) \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..a05888e --- /dev/null +++ b/README.md @@ -0,0 +1,223 @@ +

+ Logo +

+

+ 中文README | VideoPipe Website | VideoPipe tutorials(视频教程) +

+

+ 🚀one-yolo, make all in one for Yolo integration. All Tasks, All Versions, All Runtimes. 🚀 +

+ +--- + +## Introduction + +`VideoPipe` is a framework for video analysis and structuring, written in C++. It has minimal dependencies and is easy to use. It operates like a pipeline, where each node is independent and can be combined in various ways. `VideoPipe` can be used to build different types of video analysis applications, suitable for scenarios such as video structuring, image search, face recognition, and behavior analysis in traffic/security fields (such as traffic incident detection). + +![](./doc/g1.gif) + +## Advantages and Features + +`VideoPipe` is similar to NVIDIA's DeepStream and Huawei's mxVision frameworks, but it is easier to use and more portable. + +Here is a comparison table: + +| **Name** | **Open Source** | **Learning Curve** | **Supported Platforms** | **Performance** | **Third-Party Dependencies** | +|---------------|-----------------|---------------------|--------------------------|-----------------|-------------------------------| +| DeepStream | No | High | NVIDIA only | High | Many | +| mxVision | No | High | Huawei only | High | Many | +| VideoPipe | Yes | Low | Any platform | Medium | Few | + +`VideoPipe` uses a plugin-oriented coding style that allows for flexible configuration based on different needs. We can use independent plugins (referred to as `Node` types within the framework) to build various types of video analysis applications. You only need to prepare the model and understand how to parse its output. Inference can be implemented using different backends, such as OpenCV::DNN (default), TensorRT, PaddleInference, ONNXRuntime, or any other backend you prefer. + +![](./doc/p1.png) + +## Demonstration + +https://github.com/sherlockchou86/video_pipe_c/assets/13251045/b1289faa-e2c7-4d38-871e-879ae36f6d50 + +To watch in fullscreen, use the button in the bottom right corner of the player，[more video demos](./SAMPLES.md) + +## Functions + +`VideoPipe` is a framework that simplifies the integration of computer vision algorithm models. It is important to note that it is not a deep learning framework like TensorFlow or TensorRT. The main features of `VideoPipe` are as follows: + +- **Stream Reading**: Supports mainstream video stream protocols such as UDP, RTSP, RTMP, file, and application. It also supports image reading. +- **Video Decoding**: Supports video and image decoding based on OpenCV/GStreamer (with hardware acceleration). +- **Algorithm Inference**: Supports multi-level inference based on deep learning algorithms, such as object detection, image classification, feature extraction, and image generation. It also supports the integration of traditional image algorithms. **Support mLLM(Multimodal Large Language Model) integration now (update 2025/8/12)** +- **Object Tracking**: Supports object tracking, such as IOU and SORT tracking algorithms. +- **Behavior Analysis (BA)**: Supports behavior analysis based on tracking, such as traffic behavior detection like line-crossing, parking, and violations. +- **Business Logic**: Allows integration of any custom business logic, which can be closely related to specific business requirements. +- **Data Proxy**: Supports pushing structured data (in JSON, XML, or custom formats) to the cloud, files, or other third-party platforms via methods like Kafka or Socket. +- **Recording**: Supports video recording for specific time periods and capturing screenshots of specific frames, with the ability to save them as files. +- **On-Screen Display (OSD)**: Supports overlaying structured data and business logic processing results onto frames. +- **Video Encoding**: Supports video and image encoding based on OpenCV/GStreamer (with hardware acceleration). +- **Stream Pushing**: Supports mainstream video stream protocols such as UDP, RTSP, RTMP, file, and application. It also supports image streaming. + +## Getting Started Quickly + +### Dependencies + +Platforms +- Ubuntu 18.04 x86_64 NVIDIA rtx/tesla GPUs +- Ubuntu 18.04 aarch64 NVIDIA jetson serials device，tx2 tested +- Ubuntu 22.04 x86_64 by VMware virtual machine on Windows 10, pure CPUs +- Ubuntu 18.04 x86_64 Cambrian MLU serials device, MLU 370 tested (code not provided) +- Ubuntu 18.04 aarch64 Rockchip RK35** serials device, RK3588 tested (code not provided) +- Ubuntu 22.04 aarch64 Ascend 310/910 serials device, Atlas 300I-Pro tested (code not provided) +- Wait for your test + +Basics +- C++ 17 +- OpenCV >= 4.6 +- GStreamer 1.14.5 (Required by OpenCV) +- GCC >= 7.5 + +Optional, if you need to implement your own inference backend or use a backend other than `opencv::dnn`. +- CUDA +- TensorRT +- Paddle Inference +- ONNX Runtime +- mLLM（Ollama/vLLM/OpenAI-compatible API Services） +- Anything you like + +[how to install CUDA and TensorRT](./third_party/trt_vehicle/README.md) + +[how to install Paddle_Inference](./third_party/paddle_ocr/README.md) + +### Compilation and Debugging + +1. run `git clone https://github.com/sherlockchou86/VideoPipe.git` +2. run `cd VideoPipe` +3. run `mkdir build && cd build` +4. run `cmake ..` +5. run `make -j8` + +After compilation, all library files are stored in `build/libs`, and all sample executables are located in `build/bin`. During Step 4, you can add some compilation options: + +- `-DVP_WITH_CUDA=ON` (Compile CUDA-related features; default is OFF) +- `-DVP_WITH_TRT=ON` (Compile TensorRT-related features and samples; default is OFF) +- `-DVP_WITH_PADDLE=ON` (Compile PaddlePaddle-related features and samples; default is OFF) +- `-DVP_WITH_KAFKA=ON` (Compile Kafka-related features and samples; default is OFF) +- `-DVP_WITH_LLM=ON` (Compile LLM-related features and samples; default is OFF) +- `-DVP_BUILD_COMPLEX_SAMPLES=ON` (Compile advanced samples; default is OFF) + +For example, to enable CUDA and TensorRT modules, you can run: +```bash +cmake -DVP_WITH_CUDA=ON -DVP_WITH_TRT=ON .. +``` +If you run just: +```bash +cmake .. +``` +all code will be executed on the CPU. + +To run the compiled samples, first download the model files and test data: + +1. [Google Drive - Download test files and models](https://drive.google.com/drive/folders/1v9dVcR6xttUTB-WPsH3mZ_ZZMzD4wG-v?usp=sharing) +2. [Baidu Drive - Download test files and models](https://pan.baidu.com/s/1jr2nBnEDmuNaM5DiMjbC0g?pwd=nf53) + +Place the downloaded directory (named `vp_data`) in any location (e.g., `/root/abc`). Then, run the sample in the same directory where `vp_data` is located. For example, execute the command: + +```bash +[path to VideoPipe]/build/bin/1-1-1_sample +``` +at `/root/abc`. + +**Note**: The `./third_party/` directory contains independent projects. Some are header-only libraries directly referenced by VideoPipe, while others include CPP files that can be compiled or run independently. VideoPipe depends on these libraries, and they will be automatically compiled during the VideoPipe build process. These libraries also contain their own samples; for specific usage instructions, refer to the README files in the corresponding subdirectories. + +### How to use + +Here’s a guide on how to build and run a sample pipeline with `VideoPipe`. You can either compile `VideoPipe` as a library and link it, or directly include the source code and compile the entire application. + +Below is a sample code demonstrating how to construct a pipeline and run it. Please make sure to update the file paths in the code accordingly: + +```c++ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yunet_face_detector_node.h" +#include "../nodes/infers/vp_sface_feature_encoder_node.h" +#include "../nodes/osd/vp_face_osd_node_v2.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtmp_des_node.h" +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* Name: 1-1-N Sample +* Complete code located at: samples/1-1-N_sample.cpp +* Functionality: 1 video input, 1 video analysis task (face detection and recognition), 2 outputs (screen display/RTMP stream) +*/ + +int main() { + VP_SET_LOG_INCLUDE_CODE_LOCATION(false); + VP_SET_LOG_INCLUDE_THREAD_ID(false); + VP_LOGGER_INIT(); + + // 1. Create nodes + // Video Source Node + auto file_src_0 = std::make_shared("file_src_0", 0, "./test_video/10.mp4", 0.6); + + // 2. Model Inference Nodes + // First-level inference: Face detection + auto yunet_face_detector_0 = std::make_shared("yunet_face_detector_0", "./models/face/face_detection_yunet_2022mar.onnx"); + // Second-level inference: Face recognition + auto sface_face_encoder_0 = std::make_shared("sface_face_encoder_0", "./models/face/face_recognition_sface_2021dec.onnx"); + + // 3. OSD Node + // Draw results on frames + auto osd_0 = std::make_shared("osd_0"); + + // Screen Display Node + auto screen_des_0 = std::make_shared("screen_des_0", 0); + // RTMP Stream Node + auto rtmp_des_0 = std::make_shared("rtmp_des_0", 0, "rtmp://192.168.77.60/live/10000"); + + // Build the pipeline by linking the nodes + yunet_face_detector_0->attach_to({file_src_0}); + sface_face_encoder_0->attach_to({yunet_face_detector_0}); + osd_0->attach_to({sface_face_encoder_0}); + + // Split the pipeline automatically to display results on screen and stream via RTMP + screen_des_0->attach_to({osd_0}); + rtmp_des_0->attach_to({osd_0}); + + // Start the pipeline + file_src_0->start(); + + // Visualize the pipeline + vp_utils::vp_analysis_board board({file_src_0}); + board.display(); +} +``` + +**Note**: Running this code will show three displays: +1. **Pipeline Status**: A live update of the pipeline’s status. +2. **Screen Output**: The GUI display showing results. +3. **RTMP Output**: The streaming output available at the specified RTMP URL. + +![](./doc/g3.png) + + +### Prototype Examples +|ID|Sample|Screenshot| +|--|--|--| +|1|face_tracking_sample|![](./doc/p18.png)| +|2|vehicle_tracking_sample|![](./doc/p22.png)| +|3|mask_rcnn_sample|![](./doc/p30.png)| +|4|openpose_sample|![](./doc/p31.png)| +|5|face_swap_sample|![](./doc/p57.png)| +|6|mllm_analyse_sample|![](./doc/p69.png)| + +A total of over 40 prototype examples are available. [Click here](./SAMPLES.md) to view more. + +## Read More +- [Sample Code](./samples) +- [Node Table](./nodes/README.md) +- [How VideoPipe Works](./doc/about.md) +- [Development Environment For Reference](./doc/env.md) + +## WeChat Discussion Group +![](./doc/vx.png) + +## Thanks + +Featured｜HelloGitHub diff --git a/README_CN.md b/README_CN.md new file mode 100644 index 0000000..c36d2d0 --- /dev/null +++ b/README_CN.md @@ -0,0 +1,216 @@ +

+ Logo +

+

+ 英文README | VideoPipe网站 | VideoPipe视频教程 +

+

+ 🚀one-yolo, make all in one for Yolo integration. All Tasks, All Versions, All Runtimes. 🚀 +

+ +--- + +## 一、介绍 + +`VideoPipe` 是一个用于视频分析和结构化的框架，采用 C++ 编写、依赖少、易上手。它像管道一样，其中每个节点相互独立并可自行搭配，`VideoPipe` 可用来构建不同类型的视频分析应用，适用于视频结构化、图片搜索、人脸识别、交通/安防领域的行为分析（如交通事件检测）等场景。 + +![](./doc/g1.gif) + +## 二、优势和特点 + +`VideoPipe` 类似于英伟达的 DeepStream 和华为的 mxVision 框架，但它更易于使用、更具备可移植性。 + +|名称|是否开源|学习门槛|适用平台|性能|三方依赖| +|--|--|--|--|--|--| +|DeepStream|否|高|仅限英伟达|高|多| +|mxVision|否|高|仅限华为|高|多| +|VideoPipe|是|低|不限平台|中|少| + +`VideoPipe` 采用面向插件的编码风格，可以根据不同的需求按需搭配，我们可以使用独立的插件（即框架中的 `Node` 类型），来构建不同类型的视频分析应用。你只需准备好模型并了解如何解析其输出即可，推理可以基于不同的后端实现，如 OpenCV::DNN（默认）、TensorRT、PaddleInference、ONNXRuntime 等，任何你喜欢的都可以。 + +![](./doc/p1-1.png) + +## 三、演示 + +https://github.com/sherlockchou86/video_pipe_c/assets/13251045/b1289faa-e2c7-4d38-871e-879ae36f6d50 + +播放器右下角全屏观看，[更多视频演示](./SAMPLES.md) + +## 四、功能 + +VideoPipe 是一个让计算机视觉算法模型集成更加简单的框架，注意它不是像 TensorFlow、TensorRT 类似的深度学习框架。VideoPipe主要功能如下： + +- 流读取：⽀持主流的视频流协议，如 udp、rtsp、rtmp、file、application。同时支持图片读取。 +- 视频解码：⽀持基于 OpenCV/GStreamer 的视频和图片解码（⽀持硬件加速）。 +- 算法推理：⽀持基于深度学习算法的多级推理，例如⽬标检测、图像分类、特征提取、图像生成等相关网络集成。同时支持传统图像算法集成。**支持多模态大模型（mLLM）集成（2025/8/12更新）** +- ⽬标跟踪：⽀持⽬标追踪，例如 IOU、SORT 跟踪算法等。 +- ⾏为分析（BA）：⽀持基于跟踪的⾏为分析，例如越线、停⻋、违章等交通行为判断。 +- 业务逻辑：支持任意自定义业务逻辑的集成，可以与业务强相关。 +- 数据代理：⽀持将结构化数据（json/xml/⾃定义格式）以 kafka/Sokcet 等⽅式推送到云端、文件或其他第三⽅平台。 +- 录制：⽀持特定时间段的视频录制，特定帧的截图，并存文件。 +- 屏幕显⽰（OSD）：支持将结构化数据、业务逻辑处理结果绘制到帧上。 +- 视频编码：⽀持基于 OpenCV/GStreamer 的视频和图片编码（⽀持硬件加速）。 +- 流推送：⽀持主流的视频流协议，如 udp、rtsp、rtmp、file、application。同时支持图片推送。 + +## 五、快速上手 + +### 5.1 依赖 + +平台 +- Ubuntu 18.04 x86_64 NVIDIA rtx/tesla GPUs +- Ubuntu 18.04 aarch64 NVIDIA jetson serials device，tx2 tested +- Ubuntu 22.04 x86_64 by VMware virtual machine on Windows 10, pure CPUs +- Ubuntu 18.04 x86_64 Cambrian MLU serials device, MLU 370 tested (code not provided) +- Ubuntu 18.04 aarch64 Rockchip RK35** serials device, RK3588 tested (code not provided) +- Ubuntu 22.04 aarch64 Ascend 310/910 serials device, Atlas 300I-Pro tested (code not provided) +- Wait for your test + +基础 +- C++ 17 +- OpenCV >= 4.6 +- GStreamer 1.14.5 (Required by OpenCV) +- GCC >= 7.5 + +可选，如果你需要实现自己的推理后端，或者使用除 `opencv::dnn` 之外的其他推理后端. +- CUDA +- TensorRT +- Paddle Inference +- ONNX Runtime +- mLLM（Ollama/vLLM/OpenAI-compatible API Services） +- Anything you like + +[如何安装CUDA和TensorRT](./third_party/trt_vehicle/README.md) + +[如何安装Paddle_Inference](./third_party/paddle_ocr/README.md) + +### 5.2 编译和调试 + +1. 运行 `git clone https://github.com/sherlockchou86/VideoPipe.git` +2. 运行 `cd VideoPipe` +3. 运行 `mkdir build && cd build` +4. 运行 `cmake ..` +5. 运行 `make -j8` + +编译完成后，所有的库文件存放在 `build/libs` 中，所有的 Sample 运行文件存放在 `build/bin` 中。在执行第 4 步的时候，可以添加一些编译选项： +- -DVP_WITH_CUDA=ON （编译 CUDA 相关功能，默认为 OFF） +- -DVP_WITH_TRT=ON （编译 TensorRT 相关功能和 Samples，默认为 OFF） +- -DVP_WITH_PADDLE=ON （编译 PaddlePaddle 相关功能和 Samples，默认为 OFF） +- -DVP_WITH_KAFKA=ON （编译 Kafka 相关功能和 Samples，默认为 OFF） +- -DVP_WITH_LLM=ON （编译 LLM 相关功能和 Samples，默认为 OFF） +- -DVP_BUILD_COMPLEX_SAMPLES=ON （编译高级 Samples，默认为 OFF） + +比如需要开启CUDA和TensorRT相关的模块，可以运行 `cmake -DVP_WITH_CUDA=ON -DVP_WITH_TRT=ON ..`。如果只运行 `cmake ..`，那么所有代码运行在 CPU 上。 + +``` +# 开启全部 +cmake -DVP_WITH_CUDA=ON \ +-DVP_WITH_TRT=ON \ +-DVP_WITH_PADDLE=ON \ +-DVP_WITH_KAFKA=ON \ +-DVP_BUILD_COMPLEX_SAMPLES=ON .. + +# 关闭全部（默认） +cmake .. +``` + +如果要运行编译生成的 Samples，先下载模型文件和测试数据： + +1. [谷歌网盘下载测试文件和模型](https://drive.google.com/drive/folders/1v9dVcR6xttUTB-WPsH3mZ_ZZMzD4wG-v?usp=sharing) +2. [百度网盘下载测试文件和模型](https://pan.baidu.com/s/1jr2nBnEDmuNaM5DiMjbC0g?pwd=nf53) + +将下载好的目录（名称为 vp_data）放在任何位置（比如放在 `/root/abc` 下面），然后在 `同一目录` 下运行 Sample，比如在 `/root/abc` 下面执行命令：`[path to VideoPipe]/build/bin/1-1-1_sample` 即可运行 1-1-1_sample。 + +**注意**：`./third_party/` 下面都是独立的项目，有的是 header-only 库，被 VideoPipe 直接引用；有的包含有 cpp 文件，可以独立编译或运行，VideoPipe 依赖这些库，在编译 VideoPipe 的过程中会自动编译这些库。这些库也包含自己的 Samples，具体使用方法可参见对应子目录下的 README 文件. + +### 5.3 如何使用 + +1. 先将 VideoPipe 编译成库，然后引用它. +2. 或者直接引用源代码，然后编译整个Application. + +下面是一个如何构建 Pipeline 然后运行的 Sample(请先修改代码中的相关文件路径)： + +```c++ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yunet_face_detector_node.h" +#include "../nodes/infers/vp_sface_feature_encoder_node.h" +#include "../nodes/osd/vp_face_osd_node_v2.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtmp_des_node.h" +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* 名称：1-1-N sample +* 完整代码位于：samples/1-1-N_sample.cpp +* 功能说明：1个视频输入，1个视频分析任务（人脸检测和识别），2个输出（屏幕输出/RTMP推流输出） +*/ + +int main() { + VP_SET_LOG_INCLUDE_CODE_LOCATION(false); + VP_SET_LOG_INCLUDE_THREAD_ID(false); + VP_LOGGER_INIT(); + + // 1、创建节点 + // 视频获取 Node + auto file_src_0 = std::make_shared("file_src_0", 0, "./test_video/10.mp4", 0.6); + // 2、模型推理 Node + // 一级推理：人脸检测 + auto yunet_face_detector_0 = std::make_shared("yunet_face_detector_0", "./models/face/face_detection_yunet_2022mar.onnx"); + // 二级推理：人脸识别 + auto sface_face_encoder_0 = std::make_shared("sface_face_encoder_0", "./models/face/face_recognition_sface_2021dec.onnx"); + // 3、OSD Node + // 处理结果绘制到帧上 + auto osd_0 = std::make_shared("osd_0"); + // 屏幕展示 + auto screen_des_0 = std::make_shared("screen_des_0", 0); + // 推流展示 + auto rtmp_des_0 = std::make_shared("rtmp_des_0", 0, "rtmp://192.168.77.60/live/10000"); + + // 构建管道，将节点的处理结果关联起来 + yunet_face_detector_0->attach_to({file_src_0}); + sface_face_encoder_0->attach_to({yunet_face_detector_0}); + osd_0->attach_to({sface_face_encoder_0}); + + // 管道自动拆分，通过屏幕/推流输出结果 + screen_des_0->attach_to({osd_0}); + rtmp_des_0->attach_to({osd_0}); + + // 启动管道 + file_src_0->start(); + + // 可视化管道 + vp_utils::vp_analysis_board board({file_src_0}); + board.display(); +} +``` +上面代码运行后，会出现 3 个画面: +1. 管道的运行状态图，状态自动刷新 +2. 屏幕显示结果（GUI） +3. 播放器显示结果（RTMP） + +![](./doc/g3.png) + + +### 5.4 案例原型 +|ID|Sample|截图| +|--|--|--| +|1|face_tracking_sample|![](./doc/p18.png)| +|2|vehicle_tracking_sample|![](./doc/p22.png)| +|3|mask_rcnn_sample|![](./doc/p30.png)| +|4|openpose_sample|![](./doc/p31.png)| +|5|face_swap_sample|![](./doc/p57.png)| +|6|mllm_analyse_sample|![](./doc/p69.png)| + +共计 40 多个原型案例，[点击](./SAMPLES.md)查看更多。 + +## 六、更多资料 +- [Sample Code](./samples) +- [Node Table](./nodes/README.md) +- [How VideoPipe Works](./doc/about.md) +- [Development Environment For Reference](./doc/env.md) + +## 扫码入群交流 +![](./doc/vx.png) + +## 鸣谢 + +Featured｜HelloGitHub diff --git a/SAMPLES.md b/SAMPLES.md new file mode 100644 index 0000000..9180ced --- /dev/null +++ b/SAMPLES.md @@ -0,0 +1,93 @@ +## Samples Video（视频演示） ## +### Face Swap（AI换脸） ### +https://github.com/sherlockchou86/video_pipe_c/assets/13251045/aa1162f3-2f61-4ac7-8add-12d9f8b7ab23 + +### Vehicle CLustering（车辆分类聚合） ### +https://github.com/sherlockchou86/video_pipe_c/assets/13251045/867b981b-5d8c-4ee3-9831-9c755b520ad6 + +### Search Vehicle by Image（以图搜车） ### +https://github.com/sherlockchou86/video_pipe_c/assets/13251045/a0ae5422-904b-4ad1-8201-3e1a07477882 + +### Stop Detecting（停车检测） ### +https://github.com/sherlockchou86/video_pipe_c/assets/13251045/dd518798-6345-4121-a4d5-ad5bde18e3f3 + +### Vehicle Behavior Analysis（车辆行为分析） ### +https://github.com/sherlockchou86/video_pipe_c/assets/13251045/3c115d3b-45e4-4df9-8459-bfadca974e4c + +### Jam Detecting（拥堵检测） ### +https://github.com/sherlockchou86/video_pipe_c/assets/13251045/b31612ae-6b44-4cbd-8439-610bd71834d5 + +### Face Recognition（人脸识别） ### +https://github.com/sherlockchou86/video_pipe_c/assets/13251045/7958c5e3-4e6e-453b-bde9-663779d960cb + +### Dynamical Pipeline（动态管道） ### +https://github.com/sherlockchou86/video_pipe_c/assets/13251045/7fe3e45d-e528-4315-ba4f-21c3f11163e3 + +### License Plate Recognition（车牌识别相机） ### +https://github.com/sherlockchou86/video_pipe_c/assets/13251045/f680dccf-92c3-41eb-a472-7f0e27c28257 + +### Parallel Task & Synchronization（并行任务、数据同步） ### +https://github.com/sherlockchou86/video_pipe_c/assets/13251045/28a71383-8864-4a25-8da3-27922f0b5baf + +### Math Review（口算检查） ### +https://github.com/sherlockchou86/video_pipe_c/assets/13251045/9c3c1a87-d9f7-4630-a541-96610d564f13 + + +## Samples Screenshot（案例原型） ## + +|id|sample|screenshot| +|--|--|--| +|1|1-1-1_sample|![](./doc//p10.png)| +|2|1-1-N_sample|![](./doc//p11.png)| +|3|1-N-N_sample|![](./doc//p12.png)| +|4|N-1-N_sample|![](./doc//p13.png)| +|5|N-N_sample|![](./doc//p14.png)| +|6|paddle_infer_sample|![](./doc//p15.png)| +|7|src_des_sample|![](./doc//p16.png)| +|8|trt_infer_sample|![](./doc//p17.png)| +|9|vp_logger_sample|-| +|10|face_tracking_sample|![](./doc//p18.png)| +|11|vehicle_tracking_sample|![](./doc//p22.png)| +|12|interaction with pipe sample|--| +|13|record_sample|--| +|14|message_broker_sample & message_broker_sample2|![](./doc//p21.png)| +|15|mask_rcnn_sample|![](./doc//p30.png)| +|16|openpose_sample|![](./doc//p31.png)| +|17|enet_seg_sample|![](./doc//p32.png)| +|18|multi detectors and classifiers sample|![](./doc//p33.png)| +|19|image_des_sample|![](./doc//p34.png)| +|20|image_src_sample|![](./doc//p35.png)| +|21|rtsp_des_sample|![](./doc//p36.png)| +|22|ba_crossline_sample|![](./doc//p37.png)| +|23|plate_recognize_sample|![](./doc//p38.png)| +|24|vehicle body scan sample|![](./doc/p40.png)| +|25|body scan and plate detect sample|![](./doc/p39.png)| +|26|app_src_sample|![](./doc/p41.png)| +|27|vehicle cluster based on classify encoding sample|![](./doc/p42.png)| +|28|ba_stop_sample|![](./doc/p49.png)| +|29|behaviour analysis|![](./doc/p48.png)| +|30|similiarity search|![](./doc/p44.png)![](./doc/p43.png)![](./doc/p45.png)| +|31|property and similiarity search|![](./doc/p46.png)![](./doc/p47.png)| +|32|ba_jam_sample|![](./doc/p50.png)| +|33|face recognize|![](./doc/p51.png)| +|34|license plate recognize(LPR) camera|![](./doc/p52.png)| +|35|math expression check|![](./doc/p53.png)| +|36|skip_sample|![](./doc/p54.png)| +|37|obstacle_detect_sample|![](./doc/p55.png)| +|38|firesmoke_detect_sample|![](./doc/p56.png)| +|39|face_swap_sample|![](./doc/p57.png)| +|40|video_restoration_sample|![](./doc/p58.png)| +|41|app_des_sample|![](./doc/p59.png)| +|42|app_src_des_sample|![](./doc/p60.png)| +|43|lane_detect_sample|![](./doc/p61.png)| +|44|frame_fusion_sample|![](./doc/p62.png)| +|45|1-N-1_sample|![](./doc/p65.png)| +|46|1-N-1_sample2|![](./doc/p64.png)| +|47|1-N-1_sample3|![](./doc/p63.png)| +|48|message_broker_kafka_sample|![](./doc/p66.png)| +|49|trt_yolov8_sample|![](./doc/p67.png)| +|50|trt_yolov8_sample2|![](./doc/p68.png)| +|51|mllm_analyse_sample|![](./doc/p69.png)| +|52|mllm_analyse_sample_openai|![](./doc/p70.png)| +|53|rtmp_src_sample|![](./doc/p71.png)| +|54|yolov5_seg_sample|![](./doc/p72.png)| \ No newline at end of file diff --git a/doc/3rdparty/1.png b/doc/3rdparty/1.png new file mode 100755 index 0000000..e73b711 Binary files /dev/null and b/doc/3rdparty/1.png differ diff --git a/doc/3rdparty/10.png b/doc/3rdparty/10.png new file mode 100755 index 0000000..de20498 Binary files /dev/null and b/doc/3rdparty/10.png differ diff --git a/doc/3rdparty/11.png b/doc/3rdparty/11.png new file mode 100755 index 0000000..2acaabc Binary files /dev/null and b/doc/3rdparty/11.png differ diff --git a/doc/3rdparty/12.png b/doc/3rdparty/12.png new file mode 100755 index 0000000..cae5893 Binary files /dev/null and b/doc/3rdparty/12.png differ diff --git a/doc/3rdparty/2.png b/doc/3rdparty/2.png new file mode 100755 index 0000000..11b261c Binary files /dev/null and b/doc/3rdparty/2.png differ diff --git a/doc/3rdparty/3.png b/doc/3rdparty/3.png new file mode 100755 index 0000000..9240432 Binary files /dev/null and b/doc/3rdparty/3.png differ diff --git a/doc/3rdparty/4.png b/doc/3rdparty/4.png new file mode 100755 index 0000000..d0a82a0 Binary files /dev/null and b/doc/3rdparty/4.png differ diff --git a/doc/3rdparty/5.png b/doc/3rdparty/5.png new file mode 100755 index 0000000..911b347 Binary files /dev/null and b/doc/3rdparty/5.png differ diff --git a/doc/3rdparty/6.png b/doc/3rdparty/6.png new file mode 100755 index 0000000..ae386f8 Binary files /dev/null and b/doc/3rdparty/6.png differ diff --git a/doc/3rdparty/7.png b/doc/3rdparty/7.png new file mode 100755 index 0000000..128d96b Binary files /dev/null and b/doc/3rdparty/7.png differ diff --git a/doc/3rdparty/8.png b/doc/3rdparty/8.png new file mode 100755 index 0000000..ee1b027 Binary files /dev/null and b/doc/3rdparty/8.png differ diff --git a/doc/3rdparty/9.png b/doc/3rdparty/9.png new file mode 100755 index 0000000..1a94bb5 Binary files /dev/null and b/doc/3rdparty/9.png differ diff --git a/doc/about.md b/doc/about.md new file mode 100644 index 0000000..48b2ee1 --- /dev/null +++ b/doc/about.md @@ -0,0 +1,101 @@ + +here are important instructions for VideoPipe if you want to figure out how it works! + +## Core parts in video structured application ## +`video structured` is a process which converts unstructure data (video here) into structured data. unstructure data: +- video +- image +- audio +- nature text + +and structure data mainly includes something like json, xml or data table in DataBase which can by processed directly by machine (program). + +Specifically in terms of video, the process of structured mainly involves these core parts: +1. `read stream`. capture video stream from network or local machine. +2. `decode`. decode byte stream to frames, algorithm/procedure can only act on images. +3. `inference`. deep learning work on images, detect, classification or feature extraction. +4. `track`. track on objects in video. +5. `behaviour analysis` (optional). analysis on objects' tracks. +6. `osd`. on screen display, show results on images for debug purpose or intuitive effects. +7. `message broker`. push structured data to external. +8. `encode`. encode frames which contains results to byte stream for transfer/serialization purpose. +9. `push stream`. push byte stream to external or save it directly. + +

+ +

+
figure 1. core parts in video structured application.
+ +each core part in `video structured` corresponding to one type of plugin in `VideoPipe`, namely **`Node`** in code. + + +## Node in VideoPipe ## +one `Node` in VideoPipe responsible for single task such as decoding or inference. we put many nodes together to construct a pipe, and let video data flow through the whole pipeline. every `Node` has 2 queues inside, one is for caching data from upstream nodes and another one is for caching data waiting for being pushed to downstream nodes. we can write logic code between the 2 queues, they are typical `producer-consumer` pattern. + +

+ +

+
figure 2. what Node looks like?
+ +by default, producer and consumer work with single thread inside node, we need write async code when deal with complex tasks (for example, pushing data is a time-consuming operation in `vp_message_broker_node`) to avoid blocking the pipeline. + +

+ +

+
figure 3. async task inside Node.
+ +there are 3 types of `Node` in VideoPipe, namely: +- `SRC Node`. source node where data was created (only 1 queue inside used for caching data being pushed to downstream nodes). +- `MID Node`. middle node where data would be handled. +- `DES Node`. destination node where data disappears (only 1 queue inside used for caching data from upstream nodes). + +each `Node` `itself` has the ability to merge multi upstream nodes, and split into multi downstream nodes as well. note that `Node` use shallow-copy and copy Equally when data transfered from one node to other nodes by default, if you need deep-copy or want to transfer data by channel index (just hope data unconfused), add a `vp_split_node` at the point of spliting which would get different behaviour. +

+ +

+
figure 4. merge & split in VideoPipe.
+ +## Data flow in VideoPipe ## +video (frame here) is a type of heavyweight data, so deep copying frequently would decrease the performance of pipeline. actually data transfered between 2 nodes in VideoPipe use `smart pointers` by default, once data was created by source nodes, the data content would NOT be copyed later at most time in the whole pipeline (but we can specify deep-copy if we need, using `vp_split_node` for instance). + +

+ +

+
figure 5. how data flows in VideoPipe.
+ +video consist of continuous frames, VideoPipe handle these frames One by One, so the `frame index` in frame meta would increase continuously as well. + + +## Hooks in VideoPipe ## +hook is a mechanism which let host notify listeners when something happens, VideoPipe support hooks as well. pipeline invokes callback functions (via `std::function` object) to communicate with external code, such as export `fps`, `latency` and other status of pipeline itself. we should NOT block the callback functions when writing custom code inside it. + +

+ +

+
figure 6. hooks in VideoPipe.
+ +hooks help to debug with our application and quickly find the bottleneck in whole pipe, visualization tool `vp_analysis_board` works depend on hooks. + + +## Implement new Node type in VideoPipe ## +`vp_node` is the base class for all nodes in VideoPipe. we can define a new node class derived from `vp_node` and override some virtual functions like `handle_frame_meta` and `handle_control_meta`. +- `handle_frame_meta`. handle frame data flowing current node. +- `handle_control_meta`. handle control data flowing current node. + +

+ +

+
figure 7. override virtual functions in custom Node.
+ +frame data means `vp_frame_meta` in VideoPipe, contains data related to frame such as `frame index`, `data buffer`, `original width`. control data means `vp_control_meta` in VideoPipe, contains data related command such as `record video`, `record image`. + +note, NOT all data flowing current node should be handled using new logic, they just pass through if no operations work on them. we just need handle what we are interested in. + +## Hardware Acceleration in VideoPipe ## +some operations in `video structured` applications can benefit from hardware such as GPUs/NPUs. for example, video encoding on GPUs have higher speed/performace than CPUs. VideoPipe support hardware acceleration for these parts: + +- decode/encode. based on HARD decode/encode gstreamer plugins, [look more](https://github.com/sherlockchou86/video_pipe_c/blob/master/doc/env.md#about-hardware-acceleration). +- inference. no doubt about it. +- osd. need implement based on hardware acceleration SDKs by yourself. + +it is important to note that, although VideoPipe support hardware acceleration for above logic, `they could NOT share memory between each others`. it means that data will be copyed from GPU to CPU or CPU to GPU over and over again, which is the biggest disadvantage compared to other similar SDKs such as DeepStream. diff --git a/doc/env.md b/doc/env.md new file mode 100755 index 0000000..9eda30e --- /dev/null +++ b/doc/env.md @@ -0,0 +1,101 @@ + + +## Personal Development Environment ## + +- VS Code for Windows 11 +- Ubuntu 18.04 x86_64 / C++17 / GCC 7.5 / GTX 1080 GPU +- GStreamer 1.14.5 / OpenCV 4.6 +--------- + +Install GStreamer (1.14.5 for Ubuntu 18.04 by default): +``` +apt-get install libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libgstreamer-plugins-bad1.0-dev gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly gstreamer1.0-libav gstreamer1.0-tools gstreamer1.0-x gstreamer1.0-alsa gstreamer1.0-gl gstreamer1.0-gtk3 gstreamer1.0-qt5 gstreamer1.0-pulseaudio libgstrtspserver-1.0-dev gstreamer1.0-rtsp +``` + +Install OpenCV from source with `gstreamer` ON (CUDA optional). download source code of OpenCV 4.6.0 (with extra contrib modules) from github first, put them at the same directory then run `cmake` and `make` command: + +``` +step 1: +cd `the path of opencv 4.6.0` +mkdir build && cd build +``` + +``` +step 2: +cmake -D CMAKE_BUILD_TYPE=RELEASE \ +-D CMAKE_INSTALL_PREFIX=/usr/local \ +-D WITH_TBB=ON \ +-D ENABLE_FAST_MATH=1 \ +-D CUDA_FAST_MATH=1 \ +-D WITH_CUBLAS=1 \ +-D WITH_CUDA=ON \ +-D BUILD_opencv_cudacodec=OFF \ +-D WITH_CUDNN=ON \ +-D OPENCV_DNN_CUDA=ON \ +-D CUDA_ARCH_BIN=6.1 \ +-D WITH_V4L=ON \ +-D WITH_QT=OFF \ +-D WITH_OPENGL=ON \ +-D WITH_GSTREAMER=ON \ +-D OPENCV_GENERATE_PKGCONFIG=ON \ +-D OPENCV_PC_FILE_NAME=opencv.pc \ +-D OPENCV_ENABLE_NONFREE=ON \ +-D OPENCV_EXTRA_MODULES_PATH=../../opencv_contrib-4.6.0/modules \ +-D INSTALL_PYTHON_EXAMPLES=OFF \ +-D INSTALL_C_EXAMPLES=OFF \ +-D BUILD_EXAMPLES=OFF .. +``` + +``` +step 3: +make -j8 +``` + +--------- +`VcXsrv` for screen display from remote machine to local desktop in case of using SSH terminal. + +- first install PC client from: https://sourceforge.net/p/vcxsrv/wiki/Home/ +- then run `export DISPLAY=local_ip:0.0` (or add it to ~/.bashrc) on remote machine (linux server or embedded board) + +--------- +Maybe you need install nginx with `http-rtmp-module` as rtmp server for debug purpose (other tools such as `ZLMediaKit` works fine **which was highly recommended**). Also, maybe you need a rtsp server from which we can receive rtsp stream for debug purpose. + +- [how to install ZLMeidaKit](https://github.com/ZLMediaKit/ZLMediaKit/wiki/vcpkg%E6%96%B9%E5%BC%8F%E5%AE%89%E8%A3%85zlmediakit) +- [how to push stream to ZLMediaKit](https://github.com/ZLMediaKit/ZLMediaKit/wiki/ZLMediaKit%E6%8E%A8%E6%B5%81%E6%B5%8B%E8%AF%95), a few samples which used `vp_rtmp_des_node` in VidepPipe would push rtmp stream to server. +- [how to pull/play stream from ZLMediaKit](https://github.com/ZLMediaKit/ZLMediaKit/wiki/%E6%92%AD%E6%94%BEurl%E8%A7%84%E5%88%99), VLC player or ffplay was recommended. + +--------- +Please prepare kafka server AND install `librdkafka` client sdk if you want to enable kafka-related codes. +- [how to install kafka server](https://kafka.apache.org/quickstart) +- how to install `librdkafka`? run `sudo apt-get install librdkafka-dev` + + +## tips ## +- Use shared_ptr/make_shared in whole project, do not use new/delete. +- The pipeline is driven by stream data, if your app is not responding, maybe no stream input. + + +## about Hardware Acceleration ## +Since decode & encode in VideoPipe depend on gstreamer (encapsulated inside opencv), if you want to use your GPUs/NPUs to accelerate decoding and encoding performace, you need get/install HARD decode or HARD encode `gstreamer plugins` correctly first and modify gst launch string (take `vp_file_des_node` for example): +```cpp +appsrc ! videoconvert ! x264enc bitrate=%d ! mp4mux ! filesink location=%s +``` +to +``` +appsrc ! videoconvert ! nvv4l2h264enc bitrate=%d ! mp4mux ! filesink location=%s +``` +the plugin `x264enc` use CPUs to encode video stream, but `nvv4l2h264enc`(comes from DeepStream SDK) use GPUs instread. if you use other platforms other than NVIDIA, you need Corresponding Hardware Acceleration plugins. + +**soft/hard decode example** +``` +gst-launch-1.0 filesrc location=./face.mp4 ! qtdemux ! h264parse ! avdec_h264 ! videoconvert ! autovideosink // decode by avdec_h264 use CPUs +gst-launch-1.0 filesrc location=./face.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! videoconvert ! autovideosink // decode by nvv4l2decoder use NVIDIA GPUs +``` + +**soft/hard encode example** +``` +gst-launch-1.0 filesrc location=./face.mp4 ! qtdemux ! h264parse ! avdec_h264 ! x264enc ! h264parse ! flvmux ! filesink location=./new_face.flv // encode by x264enc use CPUs +gst-launch-1.0 filesrc location=./face.mp4 ! qtdemux ! h264parse ! avdec_h264 ! nvv4l2h264enc ! h264parse ! flvmux ! filesink location=./new_face.flv // encode by nvv4l2h264enc use NVIDIA GPUs +``` +[source code of hard decode/encode gstreamer plugins for NVIDIA](https://gitlab.freedesktop.org/gstreamer/gstreamer/-/tree/main/subprojects/gst-plugins-bad/sys/nvcodec).(developed by community, open source), we could also use decode/encode plugins from [DeepStream SDK](https://docs.nvidia.com/metropolis/deepstream/6.0/dev-guide/text/DS_Quickstart.html) which maintained by NVIDIA but closed source. + diff --git a/doc/g1.gif b/doc/g1.gif new file mode 100755 index 0000000..ca863c6 Binary files /dev/null and b/doc/g1.gif differ diff --git a/doc/g3.png b/doc/g3.png new file mode 100644 index 0000000..9b2b5c8 Binary files /dev/null and b/doc/g3.png differ diff --git a/doc/logo.png b/doc/logo.png new file mode 100644 index 0000000..aaaa369 Binary files /dev/null and b/doc/logo.png differ diff --git a/doc/p1-1.png b/doc/p1-1.png new file mode 100644 index 0000000..9b074c9 Binary files /dev/null and b/doc/p1-1.png differ diff --git a/doc/p1.png b/doc/p1.png new file mode 100644 index 0000000..3af5229 Binary files /dev/null and b/doc/p1.png differ diff --git a/doc/p10.png b/doc/p10.png new file mode 100755 index 0000000..64cd991 Binary files /dev/null and b/doc/p10.png differ diff --git a/doc/p11.png b/doc/p11.png new file mode 100755 index 0000000..9410775 Binary files /dev/null and b/doc/p11.png differ diff --git a/doc/p12.png b/doc/p12.png new file mode 100755 index 0000000..1402b6a Binary files /dev/null and b/doc/p12.png differ diff --git a/doc/p13.png b/doc/p13.png new file mode 100755 index 0000000..6a76c75 Binary files /dev/null and b/doc/p13.png differ diff --git a/doc/p14.png b/doc/p14.png new file mode 100755 index 0000000..fd91641 Binary files /dev/null and b/doc/p14.png differ diff --git a/doc/p15.png b/doc/p15.png new file mode 100755 index 0000000..565548a Binary files /dev/null and b/doc/p15.png differ diff --git a/doc/p16.png b/doc/p16.png new file mode 100755 index 0000000..5911760 Binary files /dev/null and b/doc/p16.png differ diff --git a/doc/p17.png b/doc/p17.png new file mode 100755 index 0000000..72eba46 Binary files /dev/null and b/doc/p17.png differ diff --git a/doc/p18.png b/doc/p18.png new file mode 100755 index 0000000..8c47aff Binary files /dev/null and b/doc/p18.png differ diff --git a/doc/p19.png b/doc/p19.png new file mode 100755 index 0000000..e66c117 Binary files /dev/null and b/doc/p19.png differ diff --git a/doc/p2.png b/doc/p2.png new file mode 100755 index 0000000..9eae3f1 Binary files /dev/null and b/doc/p2.png differ diff --git a/doc/p20.png b/doc/p20.png new file mode 100755 index 0000000..9f751a3 Binary files /dev/null and b/doc/p20.png differ diff --git a/doc/p21.png b/doc/p21.png new file mode 100755 index 0000000..fb5ab69 Binary files /dev/null and b/doc/p21.png differ diff --git a/doc/p22.png b/doc/p22.png new file mode 100755 index 0000000..efd4b5e Binary files /dev/null and b/doc/p22.png differ diff --git a/doc/p23.png b/doc/p23.png new file mode 100755 index 0000000..2cdc058 Binary files /dev/null and b/doc/p23.png differ diff --git a/doc/p24.png b/doc/p24.png new file mode 100755 index 0000000..28f6dd8 Binary files /dev/null and b/doc/p24.png differ diff --git a/doc/p25.png b/doc/p25.png new file mode 100755 index 0000000..d280df2 Binary files /dev/null and b/doc/p25.png differ diff --git a/doc/p26.png b/doc/p26.png new file mode 100755 index 0000000..bffa9f6 Binary files /dev/null and b/doc/p26.png differ diff --git a/doc/p27.png b/doc/p27.png new file mode 100755 index 0000000..b649837 Binary files /dev/null and b/doc/p27.png differ diff --git a/doc/p28.png b/doc/p28.png new file mode 100755 index 0000000..e04111a Binary files /dev/null and b/doc/p28.png differ diff --git a/doc/p29.png b/doc/p29.png new file mode 100755 index 0000000..c0c6365 Binary files /dev/null and b/doc/p29.png differ diff --git a/doc/p3.png b/doc/p3.png new file mode 100755 index 0000000..e24ef8a Binary files /dev/null and b/doc/p3.png differ diff --git a/doc/p30.png b/doc/p30.png new file mode 100755 index 0000000..6b20e34 Binary files /dev/null and b/doc/p30.png differ diff --git a/doc/p31.png b/doc/p31.png new file mode 100755 index 0000000..6a85620 Binary files /dev/null and b/doc/p31.png differ diff --git a/doc/p32.png b/doc/p32.png new file mode 100755 index 0000000..4bcc4a0 Binary files /dev/null and b/doc/p32.png differ diff --git a/doc/p33.png b/doc/p33.png new file mode 100755 index 0000000..86b5516 Binary files /dev/null and b/doc/p33.png differ diff --git a/doc/p34.png b/doc/p34.png new file mode 100755 index 0000000..d4cbcbd Binary files /dev/null and b/doc/p34.png differ diff --git a/doc/p35.png b/doc/p35.png new file mode 100755 index 0000000..6f4e422 Binary files /dev/null and b/doc/p35.png differ diff --git a/doc/p36.png b/doc/p36.png new file mode 100755 index 0000000..c53ab06 Binary files /dev/null and b/doc/p36.png differ diff --git a/doc/p37.png b/doc/p37.png new file mode 100755 index 0000000..a99bef0 Binary files /dev/null and b/doc/p37.png differ diff --git a/doc/p38.png b/doc/p38.png new file mode 100755 index 0000000..329eb1d Binary files /dev/null and b/doc/p38.png differ diff --git a/doc/p39.png b/doc/p39.png new file mode 100755 index 0000000..75a5e5f Binary files /dev/null and b/doc/p39.png differ diff --git a/doc/p4.png b/doc/p4.png new file mode 100755 index 0000000..dd8ccca Binary files /dev/null and b/doc/p4.png differ diff --git a/doc/p40.png b/doc/p40.png new file mode 100755 index 0000000..b878e12 Binary files /dev/null and b/doc/p40.png differ diff --git a/doc/p41.png b/doc/p41.png new file mode 100755 index 0000000..982ab0b Binary files /dev/null and b/doc/p41.png differ diff --git a/doc/p42.png b/doc/p42.png new file mode 100755 index 0000000..0485865 Binary files /dev/null and b/doc/p42.png differ diff --git a/doc/p43.png b/doc/p43.png new file mode 100755 index 0000000..02fad76 Binary files /dev/null and b/doc/p43.png differ diff --git a/doc/p44.png b/doc/p44.png new file mode 100755 index 0000000..34ef7d7 Binary files /dev/null and b/doc/p44.png differ diff --git a/doc/p45.png b/doc/p45.png new file mode 100755 index 0000000..980303d Binary files /dev/null and b/doc/p45.png differ diff --git a/doc/p46.png b/doc/p46.png new file mode 100755 index 0000000..25ea58c Binary files /dev/null and b/doc/p46.png differ diff --git a/doc/p47.png b/doc/p47.png new file mode 100755 index 0000000..2f93ca5 Binary files /dev/null and b/doc/p47.png differ diff --git a/doc/p48.png b/doc/p48.png new file mode 100755 index 0000000..a26af9e Binary files /dev/null and b/doc/p48.png differ diff --git a/doc/p49.png b/doc/p49.png new file mode 100755 index 0000000..519ffc9 Binary files /dev/null and b/doc/p49.png differ diff --git a/doc/p5.png b/doc/p5.png new file mode 100755 index 0000000..2fc8025 Binary files /dev/null and b/doc/p5.png differ diff --git a/doc/p50.png b/doc/p50.png new file mode 100755 index 0000000..e034db2 Binary files /dev/null and b/doc/p50.png differ diff --git a/doc/p51.png b/doc/p51.png new file mode 100755 index 0000000..36f1012 Binary files /dev/null and b/doc/p51.png differ diff --git a/doc/p52.png b/doc/p52.png new file mode 100755 index 0000000..84e47b6 Binary files /dev/null and b/doc/p52.png differ diff --git a/doc/p53.png b/doc/p53.png new file mode 100755 index 0000000..28aa453 Binary files /dev/null and b/doc/p53.png differ diff --git a/doc/p54.png b/doc/p54.png new file mode 100755 index 0000000..facee25 Binary files /dev/null and b/doc/p54.png differ diff --git a/doc/p55.png b/doc/p55.png new file mode 100755 index 0000000..96b0f71 Binary files /dev/null and b/doc/p55.png differ diff --git a/doc/p56.png b/doc/p56.png new file mode 100755 index 0000000..b40a1e4 Binary files /dev/null and b/doc/p56.png differ diff --git a/doc/p57.png b/doc/p57.png new file mode 100755 index 0000000..15858a4 Binary files /dev/null and b/doc/p57.png differ diff --git a/doc/p58.png b/doc/p58.png new file mode 100755 index 0000000..c255c7c Binary files /dev/null and b/doc/p58.png differ diff --git a/doc/p59.png b/doc/p59.png new file mode 100755 index 0000000..40e81a1 Binary files /dev/null and b/doc/p59.png differ diff --git a/doc/p6.png b/doc/p6.png new file mode 100755 index 0000000..7c8d014 Binary files /dev/null and b/doc/p6.png differ diff --git a/doc/p60.png b/doc/p60.png new file mode 100755 index 0000000..ac23840 Binary files /dev/null and b/doc/p60.png differ diff --git a/doc/p61.png b/doc/p61.png new file mode 100755 index 0000000..b47666e Binary files /dev/null and b/doc/p61.png differ diff --git a/doc/p62.png b/doc/p62.png new file mode 100755 index 0000000..6aa4031 Binary files /dev/null and b/doc/p62.png differ diff --git a/doc/p63.png b/doc/p63.png new file mode 100755 index 0000000..0eeae46 Binary files /dev/null and b/doc/p63.png differ diff --git a/doc/p64.png b/doc/p64.png new file mode 100755 index 0000000..df61ecc Binary files /dev/null and b/doc/p64.png differ diff --git a/doc/p65.png b/doc/p65.png new file mode 100755 index 0000000..f7495aa Binary files /dev/null and b/doc/p65.png differ diff --git a/doc/p66.png b/doc/p66.png new file mode 100755 index 0000000..5a1de18 Binary files /dev/null and b/doc/p66.png differ diff --git a/doc/p67.png b/doc/p67.png new file mode 100755 index 0000000..c84501b Binary files /dev/null and b/doc/p67.png differ diff --git a/doc/p68.png b/doc/p68.png new file mode 100755 index 0000000..cd8474b Binary files /dev/null and b/doc/p68.png differ diff --git a/doc/p69.png b/doc/p69.png new file mode 100644 index 0000000..c20242f Binary files /dev/null and b/doc/p69.png differ diff --git a/doc/p7.png b/doc/p7.png new file mode 100755 index 0000000..96b012e Binary files /dev/null and b/doc/p7.png differ diff --git a/doc/p70.png b/doc/p70.png new file mode 100644 index 0000000..f42f424 Binary files /dev/null and b/doc/p70.png differ diff --git a/doc/p71.png b/doc/p71.png new file mode 100644 index 0000000..2c15676 Binary files /dev/null and b/doc/p71.png differ diff --git a/doc/p72.png b/doc/p72.png new file mode 100644 index 0000000..31592b2 Binary files /dev/null and b/doc/p72.png differ diff --git a/doc/vx.png b/doc/vx.png new file mode 100755 index 0000000..aa1e50c Binary files /dev/null and b/doc/vx.png differ diff --git a/excepts/vp_invalid_argument_error.h b/excepts/vp_invalid_argument_error.h new file mode 100755 index 0000000..616fe59 --- /dev/null +++ b/excepts/vp_invalid_argument_error.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +namespace vp_excepts { + + class vp_invalid_argument_error: public std::runtime_error { + private: + /* data */ + public: + vp_invalid_argument_error(const std::string& what_arg); + ~vp_invalid_argument_error(); + }; + + inline vp_invalid_argument_error::vp_invalid_argument_error(const std::string& what_arg): std::runtime_error(what_arg) { + } + + inline vp_invalid_argument_error::~vp_invalid_argument_error() { + } +} \ No newline at end of file diff --git a/excepts/vp_invalid_calling_error.h b/excepts/vp_invalid_calling_error.h new file mode 100755 index 0000000..0844309 --- /dev/null +++ b/excepts/vp_invalid_calling_error.h @@ -0,0 +1,21 @@ +#pragma once + +#include + +namespace vp_excepts { + + class vp_invalid_calling_error: public std::runtime_error { + private: + /* data */ + public: + vp_invalid_calling_error(const std::string& what_arg); + ~vp_invalid_calling_error(); + }; + + inline vp_invalid_calling_error::vp_invalid_calling_error(const std::string& what_arg): std::runtime_error(what_arg) { + } + + inline vp_invalid_calling_error::~vp_invalid_calling_error() { + } + +} \ No newline at end of file diff --git a/excepts/vp_invalid_pipeline_error.h b/excepts/vp_invalid_pipeline_error.h new file mode 100755 index 0000000..dccf52c --- /dev/null +++ b/excepts/vp_invalid_pipeline_error.h @@ -0,0 +1,21 @@ +#pragma once + +#include + +namespace vp_excepts { + + class vp_invalid_pipeline_error: public std::runtime_error { + private: + /* data */ + public: + vp_invalid_pipeline_error(const std::string& what_arg); + ~vp_invalid_pipeline_error(); + }; + + inline vp_invalid_pipeline_error::vp_invalid_pipeline_error(const std::string& what_arg): std::runtime_error(what_arg) { + } + + inline vp_invalid_pipeline_error::~vp_invalid_pipeline_error() { + } + +} \ No newline at end of file diff --git a/excepts/vp_not_implemented_error.h b/excepts/vp_not_implemented_error.h new file mode 100755 index 0000000..a34dfe2 --- /dev/null +++ b/excepts/vp_not_implemented_error.h @@ -0,0 +1,22 @@ + +#pragma once + +#include + +namespace vp_excepts { + // not implemented error + class vp_not_implemented_error: public std::runtime_error { + private: + /* data */ + public: + vp_not_implemented_error(const std::string& what_arg); + ~vp_not_implemented_error(); + }; + + inline vp_not_implemented_error::vp_not_implemented_error(const std::string& what_arg): std::runtime_error(what_arg) { + } + + inline vp_not_implemented_error::~vp_not_implemented_error() { + } + +} \ No newline at end of file diff --git a/nodes/README.md b/nodes/README.md new file mode 100644 index 0000000..67c2cfd --- /dev/null +++ b/nodes/README.md @@ -0,0 +1,145 @@ + +## important tips when using node ## +Most nodes support multi-channel, which means that multiple channels of data can be used as its input. In addition, some nodes do not support multi-channel by default, that is, there can only be one channel of data as its input (the input channel index cannot be changed once it is determined), otherwise an error will occur. The following are common examples: + +*support multi-channel* +- all infer nodes, broker nodes, because they work independently of the channel index. +- split node, it works independently of channel index too. it could be used to split pipeline into multi branches accordding to different channel index. +- some nodes which has been designed to support multi-channel, such as record nodes, track nodes, ba nodes, which are able to distinguish different channels inside logic code (`such as using std::map to maintain different data of channels`). + +*do NOT support multi-channel* +- all src node and all des node, you MUST specify channel index when initializing instances of them. +- part of osd nodes, because they work dependently of channel index, you can do some work to let them support multi-channel, for example just use some data structure like `std::map` to maintain different data of channels inside code of nodes. + + +Note, although some nodes support multi-channel, you'd better be careful because using single instance of node to deal with multiple channels of data would have lower performance(process data serially +). On the contrary, one instance dealing with only one channel of data (multiple channels use multiple instances of the SAME node) have higher performance(process data in parallel). below is an example of 2 methods to create pipeline: +``` +single instance of track/ba node works on 2 channels： +file_src_0 --> osd_0 --> screen_des_0 + --> detector --> multi-classifiers --> tracker --> ba_crossline --> split(by channel index) +file_src_1 --> osd_1 --> screen_des_1 + +2 instances of track/ba node work on 2 channels: +file_src_0 --> tracker_0 --> ba_crossline_0 --> osd_0 --> screen_des_0 + --> detector --> multi-classifiers --> split(by channel index) +file_src_1 --> tracker_0 --> ba_crossline_0 --> osd_1 --> screen_des_1 +``` + + +## 节点目录 ## + +
+ ba + + - vp_ba_crossline_node：跨线判断 + - vp_ba_jam_node：拥堵判断 + - vp_ba_stop_node：停止判断 +
+ +
+ broker + + - vp_ba_socket_broker_node：使用udp转发行为分析结果 + - vp_embeddings_properties_socket_broker_node：使用udp转发目标特征、属性结果 + - vp_embeddings_socket_broker_node：使用udp转发目标特征结果 + - vp_expr_socket_broker_node：使用udp转发数学表达式检查结果 + - vp_json_console_broker_node：以json格式将结构化数据输出到控制台 + - vp_json_kafka_broker_node：以json格式将结构化数据通过kafka发送给第三方 + - vp_msg_broker_node：数据代理基类节点 + - vp_plate_socket_broker_node：使用udp转发车牌识别结果 + - vp_xml_file_broker_node：以xml格式将结构化数据存储到文件 + - vp_xml_socket_broker_node：以xml格式将结构化数据通过udp发送给第三方 +
+ +
+ infers + + - vp_classifier_node：基于resnet系列的图像分类节点（opencv::dnn） + - vp_enet_seg_node：基于ENet网络的图像分割节点（opencv::dnn） + - vp_face_swap_node：基于insightface的人脸替换节点（opencv::dnn） + - vp_feature_encoder_node：基于resnet系列的目标特征提取节点（opencv::dnn） + - vp_lane_detector_node：基于CenterNet的车道线检测节点（opencv::dnn） + - vp_mask_rcnn_detector_node：基于maskrcnn的目标检测节点（opencv::dnn） + - vp_openpose_detector_node：基于openpose的肢体检测节点（opencv::dnn） + - vp_ppocr_text_detector_node：基于paddleocr的文字检测节点（paddleinference） + - vp_restoration_node：基于real-esrgan的图像增强修复节点（opencv::dnn） + - vp_sface_feature_encoder_node：基于sface网络的人脸特征提取节点（opencv::dnn） + - vp_trt_vehicle_color_classifier：基于resnet18的车辆颜色分类节点（tensorrt） + - vp_trt_vehicle_detector：基于yolov5s的车辆检测节点（tensorrt） + - vp_trt_vehicle_feature_encoder：基于fastreid的车辆特征提取节点（tensorrt） + - vp_trt_vehicle_plate_detector_v2：基于yolov5s的车牌检测识别节点（一级推理）（tensorrt） + - vp_trt_vehicle_plate_detector：基于yolov5s的车牌检测识别节点（二级推理）（tensorrt） + - vp_trt_vehicle_scanner：基于yolov5s的车身扫描节点（tensorrt） + - vp_trt_vehicle_type_classifier：基于resnet18的车辆车型分类节点（tensorrt） + - vp_yolo_detector_node：基于yolov3（含tiny）的目标检测节点（opencv::dnn） + - yolo_yunet_face_detector_node：基于yunet网络的人脸检测节点（opencv::dnn） + +
+ +
+ osd + + - vp_ba_crossline_osd_node：跨线判断结果绘制节点 + - vp_ba_jam_osd_node：拥堵判断结果绘制节点 + - vp_ba_stop_osd_node：停止判断结果绘制节点 + - vp_cluster_node：目标聚类结果绘制节点 + - vp_expr_osd_node：数学表达式检查结果绘制节点 + - vp_face_osd_node_v2：人脸检测结果绘制节点（含相似度显示） + - vp_face_osd_node：人脸检测结果绘制节点 + - vp_lane_osd_node：车道线检测结果绘制节点 + - vp_osd_node_v2：目标绘制节点（含子目标） + - vp_osd_node_v3：目标绘制节点（含目标mask） + - vp_osd_node：目标绘制节点 + - vp_plate_osd_node：车牌检测识别结果绘制节点 + - vp_pose_osd_node：肢体检测结果绘制节点 + - vp_seg_osd_node：图像分割结果绘制节点 + - vp_text_osd_node：文字检测识别结果绘制节点 +
+ +
+ proc + + - vp_expr_check_node：数学等式准确性判断节点 + - vp_frame_fusion_node：视频帧按像素比融合节点（支持2个通道） +
+ +
+ record + + - vp_record_node：视频/图片录制节点 +
+ +
+ track + + - vp_dsort_track_node：基于deepsort的跟踪节点 + - vp_sort_track_node：基于sort的跟踪节点 +
+ +
+ common + + - vp_app_des_node：将图片数据推送给application的目标节点 + - vp_app_src_node：从application接收图片数据的原始节点 + - vp_des_node：所有目标节点基类 + - vp_fake_des_node：虚拟目标节点（不做任何事） + - vp_file_des_node：将视频数据存入文件的目标节点 + - vp_file_src_node：从文件读取视频数据的原始节点 + - vp_image_des_node：将数据以图片的形式发送给socket或者file的目标节点 + - vp_image_src_node：从file或者socket读取图片数据的原始节点 + - vp_infer_node：所有推理节点基类 + - vp_message_broker_node：所有数据代理节点基类 + - vp_node：所有节点基类 + - vp_placeholder_node：虚拟中间节点（不做任何事） + - vp_primary_infer_node：所有一级推理节点基类 + - vp_rtmp_des_node：将视频数据以rtmp格式推送到rtmp服务器的目标节点 + - vp_rtsp_des_node：将视频数据以rtsp格式推送（无需rtsp服务器）的目标节点 + - vp_rtsp_src_node：以rtsp格式读取网络流的原始节点 + - vp_screen_des_node：将视频/图片显示到屏幕的目标节点 + - vp_secondary_infer_node：所有二级推理节点基类 + - vp_split_node：管道拆分节点 + - vp_src_node：所有原始节点基类 + - vp_sync_node：管道分支同步节点 + - vp_udp_src_node：以udp格式读取网络流的原始节点 +
\ No newline at end of file diff --git a/nodes/ba/README.md b/nodes/ba/README.md new file mode 100644 index 0000000..be3b260 --- /dev/null +++ b/nodes/ba/README.md @@ -0,0 +1,5 @@ +## tips for BA node ## +- behaviour analysis（short as BA） works dependently on tracking, all BA nodes Must attached after track node in pipeline. +- all types of BA nodes support multi-channel, single instance of BA node supports multi channels as input. +- all BA nodes can be divided into 2 categories, one is instantaneous(like crossline) and another one is continuous(like stop, jam). `vp_ba_type` contains pair member for continuous BA such as `STOP` and `UNSTOP`, which means target enter BA status and leave BA status. +- there is no flag of BA inside `vp_frame_target` type, all nodes in pipeline need maintain it by themself if they want to know BA status for some task(like counter for crossline). \ No newline at end of file diff --git a/nodes/ba/vp_ba_crossline_node.cpp b/nodes/ba/vp_ba_crossline_node.cpp new file mode 100644 index 0000000..1fcb0f6 --- /dev/null +++ b/nodes/ba/vp_ba_crossline_node.cpp @@ -0,0 +1,124 @@ + + +#include "vp_ba_crossline_node.h" + +namespace vp_nodes { + + vp_ba_crossline_node::vp_ba_crossline_node(std::string node_name, + std::map lines, + bool need_record_image, + bool need_record_video): + vp_node(node_name), all_lines(lines), need_record_image(need_record_image), need_record_video(need_record_video) { + VP_INFO(vp_utils::string_format("[%s] %s", node_name.c_str(), to_string().c_str())); + this->initialized(); + } + + vp_ba_crossline_node::~vp_ba_crossline_node() { + deinitialized(); + } + + std::string vp_ba_crossline_node::to_string() { + /* + * return 2 points of all lines + * [channel 0: x1,y1 x2,y2][channel 1: x1,y1 x2,y2]... + */ + std::stringstream ss; + for(auto& p: all_lines) { + ss << "[channel" << p.first << ": " << p.second.start.x << "," << p.second.start.y << " " << p.second.end.x << "," << p.second.end.y << "]"; + } + return ss.str(); + } + + std::shared_ptr vp_ba_crossline_node::handle_frame_meta(std::shared_ptr meta) { + // if need applied on current channel or not + if (all_lines.count(meta->channel_index) == 0) { + return meta; + } + + // for current channel + auto& total_crossline = all_total_crossline[meta->channel_index]; + auto& line = all_lines[meta->channel_index]; + + // for vp_frame_target only + for (auto& target : meta->targets) { + auto len = target->tracks.size(); + std::vector involve_targets; + if (len > 1 && target->track_id >= 0) { + // check the last 2 points in tracks + auto p1 = target->tracks[len - 1].track_point(); + auto p2 = target->tracks[len - 2].track_point(); + + auto check1 = at_1_side_of_line(p1, line); + auto check2 = at_1_side_of_line(p2, line); + + // `true and false` or `false and true` + // means target passed the line in current frame + if (check1 ^ check2) { + total_crossline++; + involve_targets.push_back(target->track_id); + + // not empty need fill ba result back to frame meta + if (involve_targets.size() > 0) { + // send record image and record video signal, recording actions would occur if record nodes exist in pipeline + std::string image_file_name_without_ext = ""; // empty means no recording image + std::string video_file_name_without_ext = ""; // empty means no recording video + + // send image record control meta + if (need_record_image) { + image_file_name_without_ext = vp_utils::time_format(NOW, "crossline_image__"); + auto image_record_control_meta = std::make_shared(meta->channel_index, image_file_name_without_ext, true); + pendding_meta(image_record_control_meta); + } + // send video record control meta + if (need_record_video) { + video_file_name_without_ext = vp_utils::time_format(NOW, "crossline_video__"); + auto video_record_control_meta = std::make_shared(meta->channel_index, video_file_name_without_ext); + pendding_meta(video_record_control_meta); + } + + std::vector involve_region {line.start, line.end}; + auto ba_result = std::make_shared(vp_objects::vp_ba_type::CROSSLINE, + meta->channel_index, + meta->frame_index, + involve_targets, + involve_region, + "cross line", // meaningful label + image_file_name_without_ext, + video_file_name_without_ext); + // fill back to frame meta + meta->ba_results.push_back(ba_result); + // info log + VP_INFO(vp_utils::string_format("[%s] [channel %d] has found target cross line, total number of crossline: [%d]", node_name.c_str(), meta->channel_index, total_crossline)); + if (need_record_image || need_record_video) { + VP_INFO(vp_utils::string_format("[%s] [channel %d] image & video record file names are: [%s & %s]", node_name.c_str(), meta->channel_index, image_file_name_without_ext.c_str(), video_file_name_without_ext.c_str())); + } + } + } + } + } + + return meta; + } + + bool vp_ba_crossline_node::at_1_side_of_line(vp_objects::vp_point p, vp_objects::vp_line line) { + auto p1 = line.start; + auto p2 = line.end; + + if (p1.x == p2.x) { + return p.x < p1.x; + } + + if (p1.y == p2.y) { + return p.y < p1.y; + } + + if (p2.x < p1.x) { + auto tmp = p2; + p2 = p1; + p1 = tmp; + } + + int ret = (p2.y - p.y) * (p2.x - p1.x) - (p2.y - p1.y) * (p2.x - p.x); + return ret < 0; + } +} \ No newline at end of file diff --git a/nodes/ba/vp_ba_crossline_node.h b/nodes/ba/vp_ba_crossline_node.h new file mode 100644 index 0000000..1e625de --- /dev/null +++ b/nodes/ba/vp_ba_crossline_node.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include "../vp_node.h" +#include "../../objects/shapes/vp_point.h" +#include "../../objects/shapes/vp_line.h" +#include "../../objects/vp_image_record_control_meta.h" +#include "../../objects/vp_video_record_control_meta.h" + +namespace vp_nodes { + // behaviour analysis node for crossline (support multi channels) + class vp_ba_crossline_node: public vp_node + { + private: + // channel -> counter + std::map all_total_crossline; + + // channel -> line, 1 channel supports only 1 line at most (can be 0, which means no crossline check on this channel) + std::map all_lines; + + // record params + bool need_record_image; + bool need_record_video; + + // check if point at one side of line + bool at_1_side_of_line(vp_objects::vp_point p, vp_objects::vp_line line); + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + public: + vp_ba_crossline_node(std::string node_name, + std::map lines, + bool need_record_image = true, + bool need_record_video = false); + ~vp_ba_crossline_node(); + + std::string to_string() override; + }; +} \ No newline at end of file diff --git a/nodes/ba/vp_ba_jam_node.cpp b/nodes/ba/vp_ba_jam_node.cpp new file mode 100644 index 0000000..9180eec --- /dev/null +++ b/nodes/ba/vp_ba_jam_node.cpp @@ -0,0 +1,180 @@ + + +#include "vp_ba_jam_node.h" + +namespace vp_nodes { + + vp_ba_jam_node::vp_ba_jam_node(std::string node_name, + std::map> jam_regions, + bool need_record_image, + bool need_record_video): + vp_node(node_name), all_jam_regions(jam_regions), need_record_image(need_record_image), need_record_video(need_record_video) { + VP_INFO(vp_utils::string_format("[%s] %s", node_name.c_str(), to_string().c_str())); + this->initialized(); + } + + vp_ba_jam_node::~vp_ba_jam_node() { + deinitialized(); + } + + std::string vp_ba_jam_node::to_string() { + /* + * return vertexs of all jam regions + * [channel0: x1,y1 x2,y2 ...][channel1: x1,y1 x2,y2 ...]... + */ + std::stringstream ss; + for(auto& r: all_jam_regions) { + ss << "[" << r.first << ":"; + for(auto& p: r.second) { + ss << " " << p.x << "," << p.y; + } + ss << "]"; + } + return ss.str(); + } + + bool vp_ba_jam_node::point_in_poly(vp_objects::vp_point p, std::vector region) { + int i, j, c = 0; + int nvert = region.size(); + + for (i = 0, j = nvert-1; i < nvert; j = i++) { + if (((region[i].y > p.y) != (region[j].y > p.y)) && + (p.x < (region[j].x - region[i].x) * (p.y - region[i].y) + / (region[j].y - region[i].y) + region[i].x)) + c = !c; + } + return c; + } + + std::shared_ptr vp_ba_jam_node::handle_frame_meta(std::shared_ptr meta) { + // if need applied on current channel or not + if (all_jam_regions.count(meta->channel_index) == 0) { + return meta; + } + + // for current channel + auto& jam_region = all_jam_regions[meta->channel_index]; + auto& stop_checking_status = all_stop_checking_status[meta->channel_index]; + auto& jam_result = all_jam_results[meta->channel_index]; + auto& last_notify = all_last_notifys[meta->channel_index]; + + // for vp_frame_target only + std::vector hit_traget_ids; + for (auto& target : meta->targets) { + auto len = target->tracks.size(); + auto loc = target->get_rect().track_point(); + + // target has been tracked AND tracked enough frames + if (len < check_interval_frames || target->track_id < 0) { + continue; + } + // if target inside of stop region or not + if (!point_in_poly(loc, jam_region)) { + continue; + } + + auto pre_loc = target->tracks[len - check_interval_frames].track_point(); + if (pre_loc.distance_with(loc) <= check_max_distance) { + stop_checking_status[target->track_id]++; + hit_traget_ids.push_back(target->track_id); + } + } + + // count for stop targets + auto stops = 0; + std::vector involve_targets; + for (auto i = stop_checking_status.begin(); i != stop_checking_status.end();) { + if (std::find(hit_traget_ids.begin(), hit_traget_ids.end(), i->first) == hit_traget_ids.end()) { + // remove since it not satisfy stop condition + i = stop_checking_status.erase(i); + continue; + } + if (i->second >= check_min_hit_frames) { + involve_targets.push_back(i->first); + stops++; + } + i++; + } + + // enter jam status + if (stops >= check_min_stops && !jam_result && (meta->frame_index - last_notify) > (check_notify_interval * meta->fps)) { + jam_result = true; + last_notify = meta->frame_index; + + // send record image and record video signal, recording actions would occur if record nodes exist in pipeline + std::string image_file_name_without_ext = ""; // empty means no recording image + std::string video_file_name_without_ext = ""; // empty means no recording video + + // send image record control meta + if (need_record_image) { + image_file_name_without_ext = vp_utils::time_format(NOW, "jam_image__"); + auto image_record_control_meta = std::make_shared(meta->channel_index, image_file_name_without_ext, true); + pendding_meta(image_record_control_meta); + } + // send video record control meta + if (need_record_video) { + video_file_name_without_ext = vp_utils::time_format(NOW, "jam_video__"); + auto video_record_control_meta = std::make_shared(meta->channel_index, video_file_name_without_ext); + pendding_meta(video_record_control_meta); + } + + std::vector involve_region = jam_region; + auto ba_result = std::make_shared(vp_objects::vp_ba_type::JAM, + meta->channel_index, + meta->frame_index, + involve_targets, + involve_region, + "jam", // meaningful label + image_file_name_without_ext, + video_file_name_without_ext); + // fill back to frame meta + meta->ba_results.push_back(ba_result); + // info log + VP_INFO(vp_utils::string_format("[%s] [channel %d] has found jam", node_name.c_str(), meta->channel_index)); + if (need_record_image || need_record_video) { + VP_INFO(vp_utils::string_format("[%s] [channel %d] image & video record file names are: [%s & %s]", node_name.c_str(), meta->channel_index, image_file_name_without_ext.c_str(), video_file_name_without_ext.c_str())); + } + } + + // leave jam status + if (stops < check_min_stops && jam_result) { + jam_result = false; + last_notify = meta->frame_index; + + // send record image and record video signal, recording actions would occur if record nodes exist in pipeline + std::string image_file_name_without_ext = ""; // empty means no recording image + std::string video_file_name_without_ext = ""; // empty means no recording video + + // send image record control meta + if (need_record_image) { + image_file_name_without_ext = vp_utils::time_format(NOW, "unjam_image__"); + auto image_record_control_meta = std::make_shared(meta->channel_index, image_file_name_without_ext, true); + pendding_meta(image_record_control_meta); + } + // send video record control meta + if (need_record_video) { + video_file_name_without_ext = vp_utils::time_format(NOW, "unjam_video__"); + auto video_record_control_meta = std::make_shared(meta->channel_index, video_file_name_without_ext); + pendding_meta(video_record_control_meta); + } + + std::vector involve_region = jam_region; + auto ba_result = std::make_shared(vp_objects::vp_ba_type::UNJAM, + meta->channel_index, + meta->frame_index, + involve_targets, + involve_region, + "unjam", // meaningful label + image_file_name_without_ext, + video_file_name_without_ext); + // fill back to frame meta + meta->ba_results.push_back(ba_result); + // info log + VP_INFO(vp_utils::string_format("[%s] [channel %d] has found unjam", node_name.c_str(), meta->channel_index)); + if (need_record_image || need_record_video) { + VP_INFO(vp_utils::string_format("[%s] [channel %d] image & video record file names are: [%s & %s]", node_name.c_str(), meta->channel_index, image_file_name_without_ext.c_str(), video_file_name_without_ext.c_str())); + } + } + return meta; + } +} \ No newline at end of file diff --git a/nodes/ba/vp_ba_jam_node.h b/nodes/ba/vp_ba_jam_node.h new file mode 100644 index 0000000..7911136 --- /dev/null +++ b/nodes/ba/vp_ba_jam_node.h @@ -0,0 +1,50 @@ +#pragma once + +#include +#include "../vp_node.h" +#include "../../objects/shapes/vp_point.h" +#include "../../objects/shapes/vp_line.h" +#include "../../objects/vp_image_record_control_meta.h" +#include "../../objects/vp_video_record_control_meta.h" + +namespace vp_nodes { + // behaviour analysis node for stop (support multi channels) + class vp_ba_jam_node: public vp_node + { + private: + // channel -> vertexs of region, 1 channel supports only 1 region at most (can be 0, which means no jam check on this channel) + std::map> all_jam_regions; + + // channel -> status of targets (id -> num of hit frames) + std::map> all_stop_checking_status; + + // channel -> jam status of channel (jam or not) + std::map all_jam_results; + + // channel -> last frame index to notify jam + std::map all_last_notifys; + + // record params + bool need_record_image; + bool need_record_video; + + // check if point inside of polygon + bool point_in_poly(vp_objects::vp_point p, std::vector region); + + // jam checking logic parameters which may be configed by constructor passed in by user + const int check_interval_frames = 20; + const int check_min_hit_frames = 25 * 2; // 25 fps * 2 seconds + const int check_max_distance = 8; + const int check_min_stops = 8; + const int check_notify_interval = 10; // interval time (seconds) to notify (ensure not frequently) + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + public: + vp_ba_jam_node(std::string node_name, + std::map> jam_regions, + bool need_record_image = true, + bool need_record_video = true); + ~vp_ba_jam_node(); + std::string to_string() override; + }; +} \ No newline at end of file diff --git a/nodes/ba/vp_ba_stop_node.cpp b/nodes/ba/vp_ba_stop_node.cpp new file mode 100644 index 0000000..669f544 --- /dev/null +++ b/nodes/ba/vp_ba_stop_node.cpp @@ -0,0 +1,172 @@ + + +#include "vp_ba_stop_node.h" + +namespace vp_nodes { + + vp_ba_stop_node::vp_ba_stop_node(std::string node_name, + std::map> stop_regions, + bool need_record_image, + bool need_record_video): + vp_node(node_name), all_stop_regions(stop_regions), need_record_image(need_record_image), need_record_video(need_record_video) { + VP_INFO(vp_utils::string_format("[%s] %s", node_name.c_str(), to_string().c_str())); + this->initialized(); + } + + vp_ba_stop_node::~vp_ba_stop_node() { + deinitialized(); + } + + std::string vp_ba_stop_node::to_string() { + /* + * return vertexs of all stop regions + * [channel0: x1,y1 x2,y2 ...][channel1: x1,y1 x2,y2 ...]... + */ + std::stringstream ss; + for(auto& r: all_stop_regions) { + ss << "[channel" << r.first << ":"; + for(auto& p: r.second) { + ss << " " << p.x << "," << p.y; + } + ss << "]"; + } + return ss.str(); + } + + bool vp_ba_stop_node::point_in_poly(vp_objects::vp_point p, std::vector region) { + int i, j, c = 0; + int nvert = region.size(); + + for (i = 0, j = nvert-1; i < nvert; j = i++) { + if (((region[i].y > p.y) != (region[j].y > p.y)) && + (p.x < (region[j].x - region[i].x) * (p.y - region[i].y) + / (region[j].y - region[i].y) + region[i].x)) + c = !c; + } + return c; + } + + std::shared_ptr vp_ba_stop_node::handle_frame_meta(std::shared_ptr meta) { + // if need applied on current channel or not + if (all_stop_regions.count(meta->channel_index) == 0) { + return meta; + } + + // for current channel + auto& stop_region = all_stop_regions[meta->channel_index]; + auto& stop_checking_status = all_stop_checking_status[meta->channel_index]; + + // for vp_frame_target only + std::vector hit_traget_ids; + for (auto& target : meta->targets) { + auto len = target->tracks.size(); + auto loc = target->get_rect().track_point(); + + // target has been tracked AND tracked enough frames + if (len < check_interval_frames || target->track_id < 0) { + continue; + } + // if target inside of stop region or not + if (!point_in_poly(loc, stop_region)) { + continue; + } + + auto pre_loc = target->tracks[len - check_interval_frames].track_point(); + if (pre_loc.distance_with(loc) <= check_max_distance) { + stop_checking_status[target->track_id]++; + hit_traget_ids.push_back(target->track_id); + } + } + + for (auto i = stop_checking_status.begin(); i != stop_checking_status.end();) { + if (std::find(hit_traget_ids.begin(), hit_traget_ids.end(), i->first) == hit_traget_ids.end()) { + // statisfy unstop condition + if (i->second >= check_min_hit_frames) { + std::vector involve_targets; + involve_targets.push_back(i->first); + + // send record image and record video signal, recording actions would occur if record nodes exist in pipeline + std::string image_file_name_without_ext = ""; // empty means no recording image + std::string video_file_name_without_ext = ""; // empty means no recording video + + // send image record control meta + if (need_record_image) { + image_file_name_without_ext = vp_utils::time_format(NOW, "unstop_image__"); + auto image_record_control_meta = std::make_shared(meta->channel_index, image_file_name_without_ext, true); + pendding_meta(image_record_control_meta); + } + // send video record control meta + if (need_record_video) { + video_file_name_without_ext = vp_utils::time_format(NOW, "unstop_video__"); + auto video_record_control_meta = std::make_shared(meta->channel_index, video_file_name_without_ext); + pendding_meta(video_record_control_meta); + } + + std::vector involve_region = stop_region; + auto ba_result = std::make_shared(vp_objects::vp_ba_type::UNSTOP, + meta->channel_index, + meta->frame_index, + involve_targets, + involve_region, + "unstop", // meaningful label + image_file_name_without_ext, + video_file_name_without_ext); + // fill back to frame meta + meta->ba_results.push_back(ba_result); + // info log + VP_INFO(vp_utils::string_format("[%s] [channel %d] has found target unstop", node_name.c_str(), meta->channel_index)); + if (need_record_image || need_record_video) { + VP_INFO(vp_utils::string_format("[%s] [channel %d] image & video record file names are: [%s & %s]", node_name.c_str(), meta->channel_index, image_file_name_without_ext.c_str(), video_file_name_without_ext.c_str())); + } + } + + // remove since it not satisfy stop condition + i = stop_checking_status.erase(i); + continue; + } + + // equal means first time to satisfy stop condition + if (i->second == check_min_hit_frames) { + std::vector involve_targets; + involve_targets.push_back(i->first); + + // send record image and record video signal, recording actions would occur if record nodes exist in pipeline + std::string image_file_name_without_ext = ""; // empty means no recording image + std::string video_file_name_without_ext = ""; // empty means no recording video + + // send image record control meta + if (need_record_image) { + image_file_name_without_ext = vp_utils::time_format(NOW, "stop_image__"); + auto image_record_control_meta = std::make_shared(meta->channel_index, image_file_name_without_ext, true); + pendding_meta(image_record_control_meta); + } + // send video record control meta + if (need_record_video) { + video_file_name_without_ext = vp_utils::time_format(NOW, "stop_video__"); + auto video_record_control_meta = std::make_shared(meta->channel_index, video_file_name_without_ext); + pendding_meta(video_record_control_meta); + } + + std::vector involve_region = stop_region; + auto ba_result = std::make_shared(vp_objects::vp_ba_type::STOP, + meta->channel_index, + meta->frame_index, + involve_targets, + involve_region, + "stop", // meaningful label + image_file_name_without_ext, + video_file_name_without_ext); + // fill back to frame meta + meta->ba_results.push_back(ba_result); + // info log + VP_INFO(vp_utils::string_format("[%s] [channel %d] has found target stop", node_name.c_str(), meta->channel_index)); + if (need_record_image || need_record_video) { + VP_INFO(vp_utils::string_format("[%s] [channel %d] image & video record file names are: [%s & %s]", node_name.c_str(), meta->channel_index, image_file_name_without_ext.c_str(), video_file_name_without_ext.c_str())); + } + } + i++; + } + + return meta; + } +} \ No newline at end of file diff --git a/nodes/ba/vp_ba_stop_node.h b/nodes/ba/vp_ba_stop_node.h new file mode 100644 index 0000000..9575c5c --- /dev/null +++ b/nodes/ba/vp_ba_stop_node.h @@ -0,0 +1,42 @@ +#pragma once + +#include +#include "../vp_node.h" +#include "../../objects/shapes/vp_point.h" +#include "../../objects/shapes/vp_line.h" +#include "../../objects/vp_image_record_control_meta.h" +#include "../../objects/vp_video_record_control_meta.h" + +namespace vp_nodes { + // behaviour analysis node for stop (support multi channels) + class vp_ba_stop_node: public vp_node + { + private: + // channel -> vertexs of region, 1 channel supports only 1 region at most (can be 0, which means no stop check on this channel) + std::map> all_stop_regions; + + // channel -> status of targets (id -> num of hit frames) + std::map> all_stop_checking_status; + + // record params + bool need_record_image; + bool need_record_video; + + // check if point inside of polygon + bool point_in_poly(vp_objects::vp_point p, std::vector region); + + // stop checking logic parameters which may be configed by constructor passed in by user + const int check_interval_frames = 20; + const int check_min_hit_frames = 25 * 2; // 25 fps * 2 seconds + const int check_max_distance = 5; + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + public: + vp_ba_stop_node(std::string node_name, + std::map> stop_regions, + bool need_record_image = true, + bool need_record_video = true); + ~vp_ba_stop_node(); + std::string to_string() override; + }; +} \ No newline at end of file diff --git a/nodes/broker/cereal_archive/vp_objects_cereal_archive.h b/nodes/broker/cereal_archive/vp_objects_cereal_archive.h new file mode 100644 index 0000000..ca89719 --- /dev/null +++ b/nodes/broker/cereal_archive/vp_objects_cereal_archive.h @@ -0,0 +1,100 @@ +#pragma once + +/* +* define EXTERNAL archive functions for objects which need to be serialized by cereal library in VideoPipe. +* refer to `https://uscilab.github.io/cereal/serialization_functions.html` for more details. +*/ + +// object types +#include "../../../objects/vp_frame_target.h" +#include "../../../objects/vp_frame_face_target.h" +#include "../../../objects/vp_frame_text_target.h" +#include "../../../objects/vp_frame_pose_target.h" +#include "../../../objects/vp_sub_target.h" +/* extend for more types of objects in VideoPipe. */ + +// headers from cereal +#include "../../../third_party/cereal/cereal.hpp" +#include "../../../third_party/cereal/types/vector.hpp" +#include "../../../third_party/cereal/types/memory.hpp" +#include "../../../third_party/cereal/types/string.hpp" +#include "../../../third_party/cereal/types/utility.hpp" +#include "../../../third_party/cereal/archives/json.hpp" +#include "../../../third_party/cereal/archives/xml.hpp" + +/* same namespace as object types */ +namespace vp_objects { + /* vp_frame_target */ + template + void serialize(Archive& archive, vp_frame_target& target) { + // define the form of structured data for vp_frame_target + archive(cereal::make_nvp("x", target.x), + cereal::make_nvp("y", target.y), + cereal::make_nvp("width", target.width), + cereal::make_nvp("height", target.height), + cereal::make_nvp("primary_class_id", target.primary_class_id), + cereal::make_nvp("primary_score", target.primary_score), + cereal::make_nvp("primary_label", target.primary_label), + cereal::make_nvp("channel_index", target.channel_index), + cereal::make_nvp("frame_index", target.frame_index), + cereal::make_nvp("track_id", target.track_id), + cereal::make_nvp("secondary_class_ids", target.secondary_class_ids), + cereal::make_nvp("secondary_scores", target.secondary_scores), + cereal::make_nvp("secondary_labels", target.secondary_labels), + cereal::make_nvp("sub_targets", target.sub_targets), + cereal::make_nvp("embeddings", target.embeddings)); + } + + template + void serialize(Archive& archive, vp_sub_target& target) { + // define the form of structured data for vp_sub_target + archive(cereal::make_nvp("x", target.x), + cereal::make_nvp("y", target.y), + cereal::make_nvp("width", target.width), + cereal::make_nvp("height", target.height), + cereal::make_nvp("class_id", target.class_id), + cereal::make_nvp("score", target.score), + cereal::make_nvp("label", target.label), + cereal::make_nvp("frame_index", target.frame_index), + cereal::make_nvp("channel_index", target.channel_index), + cereal::make_nvp("attachments", target.attachments)); + } + /* END OF vp_frame_target */ + + + /* vp_frame_face_target */ + template + void serialize(Archive& archive, vp_frame_face_target& target) { + // define the form of structured data for vp_frame_face_target + archive(cereal::make_nvp("x", target.x), + cereal::make_nvp("y", target.y), + cereal::make_nvp("width", target.width), + cereal::make_nvp("height", target.height), + cereal::make_nvp("score", target.score), + cereal::make_nvp("embeddings", target.embeddings), + cereal::make_nvp("key_points", target.key_points), + cereal::make_nvp("track_id", target.track_id)); + } + /* END OF vp_frame_face_target */ + + + /* vp_frame_text_target */ + template + void serialize(Archive& archive, vp_frame_text_target& target) { + // define the form of structured data for vp_frame_text_target + archive(cereal::make_nvp("text", target.text), + cereal::make_nvp("score", target.score), + cereal::make_nvp("region", target.region_vertexes), + cereal::make_nvp("flags", target.flags)); + } + /* END OF vp_frame_text_target */ + + + /* vp_frame_pose_target */ + template + void serialize(Archive& archive, vp_frame_pose_target& target) { + // define the form of structured data for vp_frame_pose_target + + } + /* END OF vp_frame_pose_target */ +} \ No newline at end of file diff --git a/nodes/broker/kafka_utils/KafkaProducer.cpp b/nodes/broker/kafka_utils/KafkaProducer.cpp new file mode 100644 index 0000000..790659e --- /dev/null +++ b/nodes/broker/kafka_utils/KafkaProducer.cpp @@ -0,0 +1,167 @@ + +#ifdef VP_WITH_KAFKA +#include "KafkaProducer.h" + +// callbacks +class ProducerDeliveryReportCb : public RdKafka::DeliveryReportCb { +public: + void dr_cb(RdKafka::Message &message) { + if (message.err()) + std::cerr << "Message delivery failed: " << message.errstr() << std::endl; + else { + // Message delivered to topic test [0] at offset 135000 + /* + std::cerr << "Message delivered to topic " << message.topic_name() + << " [" << message.partition() << "] at offset " + << message.offset() << std::endl;*/ + } + } +}; + +class ProducerEventCb : public RdKafka::EventCb { +public: + void event_cb(RdKafka::Event &event) { + switch (event.type()) { + case RdKafka::Event::EVENT_ERROR: + std::cout << "RdKafka::Event::EVENT_ERROR: " << RdKafka::err2str(event.err()) << std::endl; + break; + case RdKafka::Event::EVENT_STATS: + //std::cout << "RdKafka::Event::EVENT_STATS: " << event.str() << std::endl; + break; + case RdKafka::Event::EVENT_LOG: + //std::cout << "RdKafka::Event::EVENT_LOG " << event.fac() << std::endl; + break; + case RdKafka::Event::EVENT_THROTTLE: + //std::cout << "RdKafka::Event::EVENT_THROTTLE " << event.broker_name() << std::endl; + break; + } + } +}; + + +class HashPartitionerCb : public RdKafka::PartitionerCb { +public: + int32_t partitioner_cb(const RdKafka::Topic *topic, const std::string *key, + int32_t partition_cnt, void *msg_opaque) { + char msg[128] = { 0 }; + int32_t partition_id = generate_hash(key->c_str(), key->size()) % partition_cnt; + // [topic][key][partition_cnt][partition_id] + // :[test][6419][2][1] + /*sprintf(msg, "HashPartitionerCb:topic:[%s], key:[%s]partition_cnt:[%d], partition_id:[%d]", topic->name().c_str(), + key->c_str(), partition_cnt, partition_id); + std::cout << msg << std::endl;*/ + return partition_id; + } +private: + + static inline unsigned int generate_hash(const char *str, size_t len) { + unsigned int hash = 5381; + for (size_t i = 0; i < len; i++) + hash = ((hash << 5) + hash) + str[i]; + return hash; + } +}; + + +KafkaProducer::KafkaProducer(const std::string& brokers, const std::string& topic, int partition) { + m_brokers = brokers; + m_topicStr = topic; + m_partition = partition; + + m_config = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL); + if(m_config==NULL) + std::cout << "Create RdKafka Conf failed." << std::endl; + + m_topicConfig = RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC); + if (m_topicConfig == NULL) + std::cout << "Create RdKafka Topic Conf failed." << std::endl; + + RdKafka::Conf::ConfResult errCode; + std::string errorStr; + m_dr_cb = new ProducerDeliveryReportCb; + + errCode = m_config->set("dr_cb", m_dr_cb, errorStr); + if (errCode != RdKafka::Conf::CONF_OK) + { + std::cout << "Conf set failed:" << errorStr << std::endl; + } + + m_event_cb = new ProducerEventCb; + errCode = m_config->set("event_cb", m_event_cb, errorStr); + if (errCode != RdKafka::Conf::CONF_OK) + { + std::cout << "Conf set failed:" << errorStr << std::endl; + } + + m_partitioner_cb = new HashPartitionerCb; + errCode = m_topicConfig->set("partitioner_cb", m_partitioner_cb, errorStr); + if (errCode != RdKafka::Conf::CONF_OK) + { + std::cout << "Conf set failed:" << errorStr << std::endl; + } + + errCode = m_config->set("statistics.interval.ms", "10000", errorStr); + if (errCode != RdKafka::Conf::CONF_OK) + { + std::cout << "Conf set failed:" << errorStr << std::endl; + } + errCode = m_config->set("message.max.bytes", "10240000", errorStr); + if (errCode != RdKafka::Conf::CONF_OK) + { + std::cout << "Conf set failed:" << errorStr << std::endl; + } + errCode = m_config->set("bootstrap.servers", m_brokers, errorStr); + if (errCode != RdKafka::Conf::CONF_OK) + { + std::cout << "Conf set failed:" << errorStr << std::endl; + } + + m_producer = RdKafka::Producer::create(m_config, errorStr); + if (m_producer == NULL) + { + std::cout << "Create Producer failed:" << errorStr << std::endl; + } + + m_topic = RdKafka::Topic::create(m_producer, m_topicStr, m_topicConfig, errorStr); + if (m_topic == NULL) + { + std::cout << "Create Topic failed:" << errorStr << std::endl; + } +} + +KafkaProducer::~KafkaProducer() { + while (m_producer->outq_len() > 0) { + std::cerr << "Waiting for " << m_producer->outq_len() << std::endl; + m_producer->flush(5000); + } + delete m_config; + delete m_topicConfig; + delete m_topic; + delete m_producer; + delete m_dr_cb; + delete m_event_cb; + delete m_partitioner_cb; +} + + +void KafkaProducer::pushMessage(const std::string& str) { + int32_t len = str.length(); + void* payload = const_cast(static_cast(str.data())); + RdKafka::ErrorCode errorCode = m_producer->produce( + m_topic, + RdKafka::Topic::PARTITION_UA, + RdKafka::Producer::RK_MSG_COPY, + payload, + len, + NULL, + NULL); + m_producer->poll(0); + if (errorCode != RdKafka::ERR_NO_ERROR) { + std::cerr << "Produce failed: " << RdKafka::err2str(errorCode) << std::endl; + if (errorCode == RdKafka::ERR__QUEUE_FULL) { + m_producer->poll(100); + } + } +} + +#endif \ No newline at end of file diff --git a/nodes/broker/kafka_utils/KafkaProducer.h b/nodes/broker/kafka_utils/KafkaProducer.h new file mode 100644 index 0000000..69d8e45 --- /dev/null +++ b/nodes/broker/kafka_utils/KafkaProducer.h @@ -0,0 +1,35 @@ +#pragma once + +#ifdef VP_WITH_KAFKA +#include +#include + +// compile tips: +// run `apt-get install librdkafka-dev`, v0.11.3 for ubuntu 18.04 by default. +#include + +// wrapper class for Producer in librdkafka +class KafkaProducer +{ +public: + explicit KafkaProducer(const std::string& brokers, const std::string& topic, int partition); + + void pushMessage(const std::string& str); + ~KafkaProducer(); + + +private: + std::string m_brokers; + std::string m_topicStr; + int m_partition; + + RdKafka::Conf* m_config; + RdKafka::Conf* m_topicConfig; + RdKafka::Topic* m_topic; + RdKafka::Producer* m_producer; + + RdKafka::DeliveryReportCb* m_dr_cb; + RdKafka::EventCb* m_event_cb; + RdKafka::PartitionerCb* m_partitioner_cb; +}; +#endif \ No newline at end of file diff --git a/nodes/broker/vp_ba_socket_broker_node.cpp b/nodes/broker/vp_ba_socket_broker_node.cpp new file mode 100644 index 0000000..101d5fd --- /dev/null +++ b/nodes/broker/vp_ba_socket_broker_node.cpp @@ -0,0 +1,130 @@ + + +#include "vp_ba_socket_broker_node.h" + + +namespace vp_nodes { + + vp_ba_socket_broker_node::vp_ba_socket_broker_node(std::string node_name, + std::string des_ip, + int des_port, + vp_broke_for broke_for, + int broking_cache_warn_threshold, + int broking_cache_ignore_threshold): + vp_msg_broker_node(node_name, broke_for, broking_cache_warn_threshold, broking_cache_ignore_threshold), + des_ip(des_ip), + des_port(des_port) { + // only for vp_frame_target since BA logic ONLY works on vp_frame_target + assert(broke_for == vp_broke_for::NORMAL); + udp_writer = kissnet::udp_socket(kissnet::endpoint(des_ip, des_port)); + VP_INFO(vp_utils::string_format("[%s] [message broker] set des_ip as `%s` and des_port as [%d]", node_name.c_str(), des_ip.c_str(), des_port)); + this->initialized(); + } + + vp_ba_socket_broker_node::~vp_ba_socket_broker_node() { + deinitialized(); + stop_broking(); + } + + void vp_ba_socket_broker_node::format_msg(const std::shared_ptr& meta, std::string& msg) { + /* format: + line 0, <-- + line 1, time + line 2, channel_index, frame_index + line 3, ba_type, ba_label + line 4, ids of involve targets (target_id_1, target_id_2,...) + line 5, vertexs of involve region (x1,y1,x2,y2,...) + line 6, property labels split by '|' and split by ',' between different involve targets + line 7, record image path + line 8, record video path + line 9, --> + line 10, <-- + line 11, time + line 12, channel_index, frame_index + line 13, ba_type, ba_label + line 14, ids of involve targets (target_id_1, target_id_2,...) + line 15, vertexs of involve region (x1,y1,x2,y2,...) + line 16, property labels split by '|' and split by ',' between different involve targets + line 17, record image path + linr 18, record video path + line 19, --> + line 20, ... + */ + + std::stringstream msg_stream; + auto format_basic_info = [&](int channel_index, int frame_index) { + msg_stream << vp_utils::time_format(NOW) << std::endl; // line1 + msg_stream << channel_index << "," << frame_index << std::endl; // line2 + }; + auto format_ba_info = [&](vp_objects::vp_ba_type ba_type, + std::string ba_label, + const std::vector& involve_target_ids, + const std::vector& involve_region_vertexs) { + msg_stream << int(ba_type) << "," << ba_label << std::endl; // line3 + for (int i = 0; i < involve_target_ids.size(); i++) { + msg_stream << involve_target_ids[i]; // line 4 + if (i != involve_target_ids.size() - 1) { + msg_stream << ","; + } + } + msg_stream << std::endl; + + for (int i = 0; i < involve_region_vertexs.size(); i++) { + msg_stream << involve_region_vertexs[i].x << "," << involve_region_vertexs[i].y; // line 5 + if (i != involve_region_vertexs.size() - 1) { + msg_stream << ","; + } + } + msg_stream << std::endl; + + auto targets = meta->get_targets_by_ids(involve_target_ids); + for (int i = 0; i < targets.size(); i++) { + auto t = targets[i]; + for (int j = 0; j < t->secondary_labels.size(); j++) { + msg_stream << t->secondary_labels[j]; // line 6 + if (j != t->secondary_labels.size() - 1) { + msg_stream << "|"; + } + } + if (i != targets.size() - 1) { + msg_stream << ","; + } + } + msg_stream << std::endl; + }; + auto format_record_info = [&](const std::string& record_image_name, const std::string& record_video_name) { + msg_stream << record_image_name << std::endl; // line7 + msg_stream << record_video_name << std::endl; // line8 + }; + + if (broke_for == vp_broke_for::NORMAL) { + for (int i = 0; i < meta->ba_results.size(); i++) { + auto& ba = meta->ba_results[i]; + + // start flag + msg_stream << "<--" << std::endl; + // basic info + format_basic_info(meta->channel_index, meta->frame_index); + //ba info + format_ba_info(ba->type, ba->ba_label, ba->involve_target_ids_in_frame, ba->involve_region_in_frame); + // record info + format_record_info(ba->record_image_name, ba->record_video_name); + // end flag + msg_stream << "-->"; + + if (i != meta->ba_results.size() - 1) { + msg_stream << std::endl; // not the last one + } + } + } + + msg = msg_stream.str(); + } + + void vp_ba_socket_broker_node::broke_msg(const std::string& msg) { + // broke msg to socket by udp + auto bytes_2_send = reinterpret_cast(msg.c_str()); + auto bytes_2_send_len = msg.size(); + udp_writer.send(bytes_2_send, bytes_2_send_len); + } +} \ No newline at end of file diff --git a/nodes/broker/vp_ba_socket_broker_node.h b/nodes/broker/vp_ba_socket_broker_node.h new file mode 100644 index 0000000..1c1cff0 --- /dev/null +++ b/nodes/broker/vp_ba_socket_broker_node.h @@ -0,0 +1,37 @@ +#pragma once + +#include "vp_msg_broker_node.h" +#include "../../objects/ba/vp_ba_result.h" +#include "cereal_archive/vp_objects_cereal_archive.h" + +// light weight socket support +#include "../../third_party/kissnet/kissnet.hpp" + +namespace vp_nodes { + // message broker node, broke BA results (ONLY for vp_frame_target) to socket via udp. + // BA results could be used for archive. + class vp_ba_socket_broker_node: public vp_msg_broker_node + { + private: + // host the data sent to via udp + std::string des_ip = ""; + // port the data sent to via udp + int des_port = 0; + + // udp socket writer + kissnet::udp_socket udp_writer; + protected: + // to custom format + virtual void format_msg(const std::shared_ptr& meta, std::string& msg) override; + // to socket via udp + virtual void broke_msg(const std::string& msg) override; + public: + vp_ba_socket_broker_node(std::string node_name, + std::string des_ip = "", + int des_port = 0, + vp_broke_for broke_for = vp_broke_for::NORMAL, + int broking_cache_warn_threshold = 50, + int broking_cache_ignore_threshold = 200); + ~vp_ba_socket_broker_node(); + }; +} \ No newline at end of file diff --git a/nodes/broker/vp_embeddings_properties_socket_broker_node.cpp b/nodes/broker/vp_embeddings_properties_socket_broker_node.cpp new file mode 100644 index 0000000..244dd74 --- /dev/null +++ b/nodes/broker/vp_embeddings_properties_socket_broker_node.cpp @@ -0,0 +1,137 @@ + + +#include "vp_embeddings_properties_socket_broker_node.h" + + +namespace vp_nodes { + + vp_embeddings_properties_socket_broker_node::vp_embeddings_properties_socket_broker_node(std::string node_name, + std::string des_ip, + int des_port, + std::string cropped_dir, + int min_crop_width, + int min_crop_height, + vp_broke_for broke_for, + bool only_for_tracked, + int broking_cache_warn_threshold, + int broking_cache_ignore_threshold): + vp_msg_broker_node(node_name, broke_for, broking_cache_warn_threshold, broking_cache_ignore_threshold), + des_ip(des_ip), + des_port(des_port), + cropped_dir(cropped_dir), + min_crop_width(min_crop_width), + min_crop_height(min_crop_height), + only_for_tracked(only_for_tracked) { + // only for vp_frame_target + assert(broke_for == vp_broke_for::NORMAL); + udp_writer = kissnet::udp_socket(kissnet::endpoint(des_ip, des_port)); + VP_INFO(vp_utils::string_format("[%s] [message broker] set des_ip as `%s` and des_port as [%d]", node_name.c_str(), des_ip.c_str(), des_port)); + this->initialized(); + } + + vp_embeddings_properties_socket_broker_node::~vp_embeddings_properties_socket_broker_node() { + deinitialized(); + stop_broking(); + } + + void vp_embeddings_properties_socket_broker_node::format_msg(const std::shared_ptr& meta, std::string& msg) { + /* format: + line 0, <-- + line 1, 1st cropped image's path + line 2, 1st properties, multi property labels splited by ',' + line 3, 1st sub target info, empty line if sub target detected (sub target may be vehicle plate) + line 4, 1st embeddings + line 5, --> + line 6, <-- + line 7, 2nd cropped image's path + line 8, 2nd properties, multi property labels splited by ',' + line 9, 2nd sub target info, empty line if sub target detected (sub target may be vehicle plate) + line 10, 2nd embeddings + line 11, --> + line 12, ... + */ + auto& broked = all_broked[meta->channel_index]; + // remove 50 elements every 100 ids + if (broked.size() > 100) { + broked.erase(broked.begin(), broked.begin() + 50); + } + + std::stringstream msg_stream; + auto format_embeddings = [&](const std::vector& embeddings) { + for (int i = 0; i < embeddings.size(); i++) { + msg_stream << embeddings[i]; + if (i != embeddings.size() - 1) { + msg_stream << ","; + } + } + msg_stream << std::endl; + }; + auto format_sub_target = [&](const std::string& sub_target_info) { + msg_stream << sub_target_info << std::endl; + }; + auto format_properties = [&](const std::vector& secondary_labels) { + for (int i = 0; i < secondary_labels.size(); i++) { + msg_stream << secondary_labels[i]; + if (i != secondary_labels.size() - 1) { + msg_stream << ","; + } + } + msg_stream << std::endl; + }; + auto save_cropped_image = [&](cv::Mat& frame, cv::Rect rect, std::string name) { + auto cropped = frame(rect); + cv::imwrite(name, cropped); + msg_stream << name << std::endl; + }; + if (broke_for == vp_broke_for::NORMAL) { + for (int i = 0; i < meta->targets.size(); i++) { + auto& t = meta->targets[i]; + // only broke for tracked targets and have enough frames + if ((only_for_tracked && t->track_id < 0) || (only_for_tracked && t->tracks.size() < min_tracked_frames)) { + continue; + } + + // only broke 1 time for specific track id if it has been tracked, or broke many times + if (t->track_id >= 0 && + std::find(broked.begin(), broked.end(), t->track_id) != broked.end()) { + continue; + } + + // size filter + if (t->width < min_crop_width || t->height < min_crop_height) { + continue; + } + + if (t->track_id >= 0) { + broked.push_back(t->track_id); + } + auto name = cropped_dir + "/" + std::to_string(t->channel_index) + "_" + std::to_string(t->frame_index) + "_" + std::to_string(t->track_id >= 0 ? t->track_id : i) + ".jpg"; + // start flag + msg_stream << "<--" << std::endl; + // save small cropped image + save_cropped_image(meta->frame, cv::Rect(t->x, t->y, t->width, t->height), name); + // format properties + format_properties(t->secondary_labels); + // format sub target (vehicle plate here), just using the first sub target's label is enough. + format_sub_target(t->sub_targets.size() > 0 ? t->sub_targets[0]->label : ""); + // format embeddings + format_embeddings(t->embeddings); + // end flag + msg_stream << "-->"; + + if (i != meta->targets.size() - 1) { + msg_stream << std::endl; // not the last one + } + } + } + + msg = msg_stream.str(); + } + + void vp_embeddings_properties_socket_broker_node::broke_msg(const std::string& msg) { + // broke msg to socket by udp + auto bytes_2_send = reinterpret_cast(msg.c_str()); + auto bytes_2_send_len = msg.size(); + udp_writer.send(bytes_2_send, bytes_2_send_len); + } +} \ No newline at end of file diff --git a/nodes/broker/vp_embeddings_properties_socket_broker_node.h b/nodes/broker/vp_embeddings_properties_socket_broker_node.h new file mode 100644 index 0000000..a00811d --- /dev/null +++ b/nodes/broker/vp_embeddings_properties_socket_broker_node.h @@ -0,0 +1,55 @@ +#pragma once + +#include "vp_msg_broker_node.h" +#include "cereal_archive/vp_objects_cereal_archive.h" + +// light weight socket support +#include "../../third_party/kissnet/kissnet.hpp" + +namespace vp_nodes { + // message broker node, broke embeddings, AND properties (ONLY for vp_frame_target) to socket via udp. + // embeddings and properties could be used for similiarity search and property search later. + class vp_embeddings_properties_socket_broker_node: public vp_msg_broker_node + { + private: + // save dir for cropped images, which would be used for embeddings similiarity search or property search later + std::string cropped_dir = "cropped_images"; + // min width to crop (embedding/property will be ignored if target's width is smaller than this value) + int min_crop_width = 50; + // min height to crop (embedding/property will be ignored if target's height is smaller than this value) + int min_crop_height = 50; + // only broke for tracked targets (track_id is not -1) + bool only_for_tracked = false; + + // min tracked frames if only_for_tracked is true + int min_tracked_frames = 25; + + // host the data sent to via udp + std::string des_ip = ""; + // port the data sent to via udp + int des_port = 0; + + // udp socket writer + kissnet::udp_socket udp_writer; + + // support multi-channel + std::map> all_broked; // channel -> target ids which have been broked + protected: + // to custom format + virtual void format_msg(const std::shared_ptr& meta, std::string& msg) override; + // to socket via udp + virtual void broke_msg(const std::string& msg) override; + public: + vp_embeddings_properties_socket_broker_node(std::string node_name, + std::string des_ip = "", + int des_port = 0, + std::string cropped_dir = "cropped_images", + int min_crop_width = 50, + int min_crop_height = 50, + vp_broke_for broke_for = vp_broke_for::NORMAL, + bool only_for_tracked = false, + int broking_cache_warn_threshold = 50, + int broking_cache_ignore_threshold = 200); + ~vp_embeddings_properties_socket_broker_node(); + }; +} \ No newline at end of file diff --git a/nodes/broker/vp_embeddings_socket_broker_node.cpp b/nodes/broker/vp_embeddings_socket_broker_node.cpp new file mode 100644 index 0000000..4123a3e --- /dev/null +++ b/nodes/broker/vp_embeddings_socket_broker_node.cpp @@ -0,0 +1,156 @@ + + +#include "vp_embeddings_socket_broker_node.h" + + +namespace vp_nodes { + + vp_embeddings_socket_broker_node::vp_embeddings_socket_broker_node(std::string node_name, + std::string des_ip, + int des_port, + std::string cropped_dir, + int min_crop_width, + int min_crop_height, + vp_broke_for broke_for, + bool only_for_tracked, + int broking_cache_warn_threshold, + int broking_cache_ignore_threshold): + vp_msg_broker_node(node_name, broke_for, broking_cache_warn_threshold, broking_cache_ignore_threshold), + des_ip(des_ip), + des_port(des_port), + cropped_dir(cropped_dir), + min_crop_width(min_crop_width), + min_crop_height(min_crop_height), + only_for_tracked(only_for_tracked) { + // only for vp_frame_target or vp_frame_face_target + assert(broke_for == vp_broke_for::NORMAL || broke_for == vp_broke_for::FACE); + udp_writer = kissnet::udp_socket(kissnet::endpoint(des_ip, des_port)); + VP_INFO(vp_utils::string_format("[%s] [message broker] set des_ip as `%s` and des_port as [%d]", node_name.c_str(), des_ip.c_str(), des_port)); + this->initialized(); + } + + vp_embeddings_socket_broker_node::~vp_embeddings_socket_broker_node() { + deinitialized(); + stop_broking(); + } + + void vp_embeddings_socket_broker_node::format_msg(const std::shared_ptr& meta, std::string& msg) { + /* format: + line 0, <-- + line 1, 1st cropped image's path + line 2, 1st embeddings + line 3, --> + line 4, <-- + line 5, 2nd cropped image's path + line 6, 2nd embeddings + line 7, --> + line 8, ... + */ + auto& broked = all_broked[meta->channel_index]; + // remove 50 elements every 100 ids + if (broked.size() > 100) { + broked.erase(broked.begin(), broked.begin() + 50); + } + + std::stringstream msg_stream; + auto format_embeddings = [&](const std::vector& embeddings) { + for (int i = 0; i < embeddings.size(); i++) { + msg_stream << embeddings[i]; + if (i != embeddings.size() - 1) { + msg_stream << ","; + } + } + msg_stream << std::endl; + }; + auto save_cropped_image = [&](cv::Mat& frame, cv::Rect rect, std::string name) { + auto cropped = frame(rect); + cv::imwrite(name, cropped); + msg_stream << name << std::endl; + }; + + if (broke_for == vp_broke_for::NORMAL) { + for (int i = 0; i < meta->targets.size(); i++) { + auto& t = meta->targets[i]; + // only broke for tracked targets and have enough frames + if ((only_for_tracked && t->track_id < 0) || (only_for_tracked && t->tracks.size() < min_tracked_frames)) { + continue; + } + + // only broke 1 time for specific track id if it has been tracked, or broke many times + if (t->track_id >= 0 && + std::find(broked.begin(), broked.end(), t->track_id) != broked.end()) { + continue; + } + + // size filter + if (t->width < min_crop_width || t->height < min_crop_height) { + continue; + } + + if (t->track_id >= 0) { + broked.push_back(t->track_id); + } + auto name = cropped_dir + "/" + std::to_string(t->channel_index) + "_" + std::to_string(t->frame_index) + "_" + std::to_string(t->track_id >= 0 ? t->track_id : i) + ".jpg"; + // start flag + msg_stream << "<--" << std::endl; + // save small cropped image + save_cropped_image(meta->frame, cv::Rect(t->x, t->y, t->width, t->height), name); + // format embeddings + format_embeddings(t->embeddings); + // end flag + msg_stream << "-->"; + + if (i != meta->targets.size() - 1) { + msg_stream << std::endl; // not the last one + } + } + } + + if (broke_for == vp_broke_for::FACE) { + for (int i = 0; i < meta->face_targets.size(); i++) { + auto& t = meta->face_targets[i]; + // only broke for tracked targets and have enough frames + if ((only_for_tracked && t->track_id < 0) || (only_for_tracked && t->tracks.size() < min_tracked_frames)) { + continue; + } + + // only broke 1 time for specific track id if it has been tracked, or broke many times + if (t->track_id >= 0 && + std::find(broked.begin(), broked.end(), t->track_id) != broked.end()) { + continue; + } + + // size filter + if (t->width < min_crop_width || t->height < min_crop_height) { + continue; + } + + if (t->track_id >= 0) { + broked.push_back(t->track_id); + } + auto name = cropped_dir + "/" + std::to_string(meta->channel_index) + "_" + std::to_string(meta->frame_index) + "_" + std::to_string(t->track_id >= 0 ? t->track_id : i) + ".jpg"; + // start flag + msg_stream << "<--" << std::endl; + // save small cropped image + save_cropped_image(meta->frame, cv::Rect(t->x, t->y, t->width, t->height), name); + // format embeddings + format_embeddings(t->embeddings); + // end flag + msg_stream << "-->"; + + if (i != meta->face_targets.size() - 1) { + msg_stream << std::endl; // not the last one + } + } + } + + msg = msg_stream.str(); + } + + void vp_embeddings_socket_broker_node::broke_msg(const std::string& msg) { + // broke msg to socket by udp + auto bytes_2_send = reinterpret_cast(msg.c_str()); + auto bytes_2_send_len = msg.size(); + udp_writer.send(bytes_2_send, bytes_2_send_len); + } +} \ No newline at end of file diff --git a/nodes/broker/vp_embeddings_socket_broker_node.h b/nodes/broker/vp_embeddings_socket_broker_node.h new file mode 100644 index 0000000..a47fd52 --- /dev/null +++ b/nodes/broker/vp_embeddings_socket_broker_node.h @@ -0,0 +1,55 @@ +#pragma once + +#include "vp_msg_broker_node.h" +#include "cereal_archive/vp_objects_cereal_archive.h" + +// light weight socket support +#include "../../third_party/kissnet/kissnet.hpp" + +namespace vp_nodes { + // message broker node, broke ONLY embeddings (for vp_frame_target or vp_frame_face_target) to socket via udp. + // embeddings could be used for similiarity search later. + class vp_embeddings_socket_broker_node: public vp_msg_broker_node + { + private: + // save dir for cropped images, which would be used for embeddings similiarity search later + std::string cropped_dir = "cropped_images"; + // min width to crop (embedding will be ignored if target's width is smaller than this value) + int min_crop_width = 50; + // min height to crop (embedding will be ignored if target's height is smaller than this value) + int min_crop_height = 50; + // only broke for tracked targets (track_id is not -1) + bool only_for_tracked = false; + + // min tracked frames if only_for_tracked is true + int min_tracked_frames = 25; + + // host the data sent to via udp + std::string des_ip = ""; + // port the data sent to via udp + int des_port = 0; + + // udp socket writer + kissnet::udp_socket udp_writer; + + // support multi-channel + std::map> all_broked; // channel -> target ids which have been broked + protected: + // to custom format + virtual void format_msg(const std::shared_ptr& meta, std::string& msg) override; + // to socket via udp + virtual void broke_msg(const std::string& msg) override; + public: + vp_embeddings_socket_broker_node(std::string node_name, + std::string des_ip = "", + int des_port = 0, + std::string cropped_dir = "cropped_images", + int min_crop_width = 50, + int min_crop_height = 50, + vp_broke_for broke_for = vp_broke_for::NORMAL, + bool only_for_tracked = false, + int broking_cache_warn_threshold = 50, + int broking_cache_ignore_threshold = 200); + ~vp_embeddings_socket_broker_node(); + }; +} \ No newline at end of file diff --git a/nodes/broker/vp_expr_socket_broker_node.cpp b/nodes/broker/vp_expr_socket_broker_node.cpp new file mode 100644 index 0000000..f7e8a88 --- /dev/null +++ b/nodes/broker/vp_expr_socket_broker_node.cpp @@ -0,0 +1,95 @@ + + +#include "vp_expr_socket_broker_node.h" + + +namespace vp_nodes { + + vp_expr_socket_broker_node::vp_expr_socket_broker_node(std::string node_name, + std::string des_ip, + int des_port, + std::string screenshot_dir, + vp_broke_for broke_for, + int broking_cache_warn_threshold, + int broking_cache_ignore_threshold): + vp_msg_broker_node(node_name, broke_for, broking_cache_warn_threshold, broking_cache_ignore_threshold), + des_ip(des_ip), + des_port(des_port), + screenshot_dir(screenshot_dir) { + // only for vp_frame_text_target + assert(broke_for == vp_broke_for::TEXT); + udp_writer = kissnet::udp_socket(kissnet::endpoint(des_ip, des_port)); + VP_INFO(vp_utils::string_format("[%s] [message broker] set des_ip as `%s` and des_port as [%d]", node_name.c_str(), des_ip.c_str(), des_port)); + this->initialized(); + } + + vp_expr_socket_broker_node::~vp_expr_socket_broker_node() { + deinitialized(); + stop_broking(); + } + + void vp_expr_socket_broker_node::format_msg(const std::shared_ptr& meta, std::string& msg) { + /* format: + line 0, <-- + line 1, time + line 2, channel_index, frame_index + line 3, total num, num of yes, num of no, number of invalid + line 4, 1st expression string, 2nd expression string, 3rd expression string, ... + line 5, screenshot path + line 6, --> + */ + + std::stringstream msg_stream; + auto format_basic_info = [&](int channel_index, int frame_index) { + msg_stream << vp_utils::time_format(NOW) << std::endl; // line1 + msg_stream << channel_index << "," << frame_index << std::endl; // line2 + }; + auto format_expr_info = [&](const std::vector>& expr_targets) { + auto num_yes = 0, num_no = 0, num_invalid = 0; + auto expr_strs = std::string(); + for (int i = 0; i < expr_targets.size(); i++) { + expr_strs += expr_targets[i]->text; + if (i != expr_targets.size() - 1) { + expr_strs += ","; + } + + if (expr_targets[i]->flags.find("yes") != std::string::npos) { + num_yes++; + } + if (expr_targets[i]->flags.find("no") != std::string::npos) { + num_no++; + } + if (expr_targets[i]->flags.find("invalid") != std::string::npos) { + num_invalid++; + } + } + msg_stream << expr_targets.size() << "," << num_yes << "," << num_no << "," << num_invalid << std::endl; // line3 + msg_stream << expr_strs << std::endl; // line4 + }; + auto format_screenshot = [&](cv::Mat& screenshot, const std::string& name) { + cv::imwrite(name, screenshot); + msg_stream << name << std::endl; // line5 + }; + + // at most 1 record for each frame + if (broke_for == vp_broke_for::TEXT) { + // start flag + msg_stream << "<--" << std::endl; + format_basic_info(meta->channel_index, meta->frame_index); + format_expr_info(meta->text_targets); + auto screenshot_name = screenshot_dir + "/" + std::to_string(meta->channel_index) + "_" + std::to_string(meta->frame_index) + ".jpg"; + format_screenshot(meta->osd_frame.empty() ? meta->frame : meta->osd_frame, screenshot_name); + // end flag + msg_stream << "-->" << std::endl; + } + + msg = msg_stream.str(); + } + + void vp_expr_socket_broker_node::broke_msg(const std::string& msg) { + // broke msg to socket by udp + auto bytes_2_send = reinterpret_cast(msg.c_str()); + auto bytes_2_send_len = msg.size(); + udp_writer.send(bytes_2_send, bytes_2_send_len); + } +} \ No newline at end of file diff --git a/nodes/broker/vp_expr_socket_broker_node.h b/nodes/broker/vp_expr_socket_broker_node.h new file mode 100644 index 0000000..68ac336 --- /dev/null +++ b/nodes/broker/vp_expr_socket_broker_node.h @@ -0,0 +1,40 @@ +#pragma once + +#include "vp_msg_broker_node.h" +#include "../../objects/ba/vp_ba_result.h" +#include "cereal_archive/vp_objects_cereal_archive.h" + +// light weight socket support +#include "../../third_party/kissnet/kissnet.hpp" + +namespace vp_nodes { + // message broker node, broke math expression checking results (ONLY for vp_frame_text_target) to socket via udp. + // math expression checking results could be used for archive. + class vp_expr_socket_broker_node: public vp_msg_broker_node + { + private: + // save dir for screenshot images, which would be used for displaying in web client + std::string screenshot_dir = "screenshot_images"; + // host the data sent to via udp + std::string des_ip = ""; + // port the data sent to via udp + int des_port = 0; + + // udp socket writer + kissnet::udp_socket udp_writer; + protected: + // to custom format + virtual void format_msg(const std::shared_ptr& meta, std::string& msg) override; + // to socket via udp + virtual void broke_msg(const std::string& msg) override; + public: + vp_expr_socket_broker_node(std::string node_name, + std::string des_ip = "", + int des_port = 0, + std::string screenshot_dir = "screenshot_images", + vp_broke_for broke_for = vp_broke_for::TEXT, + int broking_cache_warn_threshold = 50, + int broking_cache_ignore_threshold = 200); + ~vp_expr_socket_broker_node(); + }; +} \ No newline at end of file diff --git a/nodes/broker/vp_json_console_broker_node.cpp b/nodes/broker/vp_json_console_broker_node.cpp new file mode 100644 index 0000000..5709902 --- /dev/null +++ b/nodes/broker/vp_json_console_broker_node.cpp @@ -0,0 +1,58 @@ + +#include "vp_json_console_broker_node.h" + +namespace vp_nodes { + + vp_json_console_broker_node::vp_json_console_broker_node(std::string node_name, + vp_broke_for broke_for, + int broking_cache_warn_threshold, + int broking_cache_ignore_threshold): + vp_msg_broker_node(node_name, broke_for, broking_cache_warn_threshold, broking_cache_ignore_threshold) { + this->initialized(); + } + + vp_json_console_broker_node::~vp_json_console_broker_node() { + deinitialized(); + stop_broking(); + } + + void vp_json_console_broker_node::format_msg(const std::shared_ptr& meta, std::string& msg) { + // serialize objects to json by cereal + std::stringstream msg_stream; + { + cereal::JSONOutputArchive json_archive(msg_stream); + + // global values + json_archive(cereal::make_nvp("channel_index", meta->channel_index), + cereal::make_nvp("frame_index", meta->frame_index), + cereal::make_nvp("width", meta->frame.cols), + cereal::make_nvp("height", meta->frame.rows), + cereal::make_nvp("fps", meta->fps), + cereal::make_nvp("broke_for", broke_fors.at(broke_for))); + + // serialize values according to broke_for + if (broke_for == vp_broke_for::NORMAL) { + json_archive(cereal::make_nvp("target_size", meta->targets.size()), + cereal::make_nvp("targets", meta->targets)); + } + else if (broke_for == vp_broke_for::FACE) { + json_archive(cereal::make_nvp("face_target_size", meta->face_targets.size()), + cereal::make_nvp("face_targets", meta->face_targets)); + } + else if (broke_for == vp_broke_for::TEXT) { + json_archive(cereal::make_nvp("text_target_size", meta->text_targets.size()), + cereal::make_nvp("text_targets", meta->text_targets)); + } + else { + throw "invalid broke_for!"; + } + } // flush + + msg = msg_stream.str(); + } + + void vp_json_console_broker_node::broke_msg(const std::string& msg) { + // broke msg to console by std::cout + std::cout << msg << std::endl; + } +} \ No newline at end of file diff --git a/nodes/broker/vp_json_console_broker_node.h b/nodes/broker/vp_json_console_broker_node.h new file mode 100644 index 0000000..c29ea19 --- /dev/null +++ b/nodes/broker/vp_json_console_broker_node.h @@ -0,0 +1,26 @@ +#pragma once + +#include + +#include "vp_msg_broker_node.h" +#include "cereal_archive/vp_objects_cereal_archive.h" + +namespace vp_nodes { + // message broker node (for debug purpose), broke json data to console. + class vp_json_console_broker_node: public vp_msg_broker_node + { + private: + /* data */ + protected: + // to json + virtual void format_msg(const std::shared_ptr& meta, std::string& msg) override; + // to console + virtual void broke_msg(const std::string& msg) override; + public: + vp_json_console_broker_node(std::string node_name, + vp_broke_for broke_for = vp_broke_for::NORMAL, + int broking_cache_warn_threshold = 50, + int broking_cache_ignore_threshold = 200); + ~vp_json_console_broker_node(); + }; +} \ No newline at end of file diff --git a/nodes/broker/vp_json_kafka_broker_node.cpp b/nodes/broker/vp_json_kafka_broker_node.cpp new file mode 100644 index 0000000..6a20408 --- /dev/null +++ b/nodes/broker/vp_json_kafka_broker_node.cpp @@ -0,0 +1,65 @@ +#ifdef VP_WITH_KAFKA +#include "vp_json_kafka_broker_node.h" + +namespace vp_nodes { + + vp_json_kafka_broker_node::vp_json_kafka_broker_node(std::string node_name, + std::string kafka_servers, + std::string topic_name, + vp_broke_for broke_for, + int broking_cache_warn_threshold, + int broking_cache_ignore_threshold): + vp_msg_broker_node(node_name, broke_for, broking_cache_warn_threshold, broking_cache_ignore_threshold) { + VP_INFO(vp_utils::string_format("[%s] kafka_servers:[%s] topic_name:[%s]", node_name.c_str(), kafka_servers.c_str(), topic_name.c_str())); + kafka_producer = std::make_shared(kafka_servers, topic_name, 0); + this->initialized(); + } + + vp_json_kafka_broker_node::~vp_json_kafka_broker_node() { + deinitialized(); + stop_broking(); + } + + void vp_json_kafka_broker_node::format_msg(const std::shared_ptr& meta, std::string& msg) { + // serialize objects to json by cereal + std::stringstream msg_stream; + { + cereal::JSONOutputArchive json_archive(msg_stream); + + // global values + json_archive(cereal::make_nvp("channel_index", meta->channel_index), + cereal::make_nvp("frame_index", meta->frame_index), + cereal::make_nvp("width", meta->frame.cols), + cereal::make_nvp("height", meta->frame.rows), + cereal::make_nvp("fps", meta->fps), + cereal::make_nvp("broke_for", broke_fors.at(broke_for))); + + // serialize values according to broke_for + if (broke_for == vp_broke_for::NORMAL) { + json_archive(cereal::make_nvp("target_size", meta->targets.size()), + cereal::make_nvp("targets", meta->targets)); + } + else if (broke_for == vp_broke_for::FACE) { + json_archive(cereal::make_nvp("face_target_size", meta->face_targets.size()), + cereal::make_nvp("face_targets", meta->face_targets)); + } + else if (broke_for == vp_broke_for::TEXT) { + json_archive(cereal::make_nvp("text_target_size", meta->text_targets.size()), + cereal::make_nvp("text_targets", meta->text_targets)); + } + else { + throw "invalid broke_for!"; + } + } // flush + + msg = msg_stream.str(); + } + + void vp_json_kafka_broker_node::broke_msg(const std::string& msg) { + // broke msg to kafka by kafka client api + if (kafka_producer != nullptr) { + kafka_producer->pushMessage(msg); + } + } +} +#endif \ No newline at end of file diff --git a/nodes/broker/vp_json_kafka_broker_node.h b/nodes/broker/vp_json_kafka_broker_node.h new file mode 100644 index 0000000..77b1ced --- /dev/null +++ b/nodes/broker/vp_json_kafka_broker_node.h @@ -0,0 +1,32 @@ +#pragma once + +#ifdef VP_WITH_KAFKA +#include + +#include "vp_msg_broker_node.h" +#include "kafka_utils/KafkaProducer.h" +#include "cereal_archive/vp_objects_cereal_archive.h" + +namespace vp_nodes { + // message broker node, broke json data to kafka. + class vp_json_kafka_broker_node: public vp_msg_broker_node + { + private: + /* data */ + std::shared_ptr kafka_producer = nullptr; + protected: + // to json + virtual void format_msg(const std::shared_ptr& meta, std::string& msg) override; + // to console + virtual void broke_msg(const std::string& msg) override; + public: + vp_json_kafka_broker_node(std::string node_name, + std::string kafka_servers = "127.0.0.1:9092", + std::string topic_name = "videopipe_topic", + vp_broke_for broke_for = vp_broke_for::NORMAL, + int broking_cache_warn_threshold = 50, + int broking_cache_ignore_threshold = 200); + ~vp_json_kafka_broker_node(); + }; +} +#endif \ No newline at end of file diff --git a/nodes/broker/vp_msg_broker_node.cpp b/nodes/broker/vp_msg_broker_node.cpp new file mode 100644 index 0000000..4786ff4 --- /dev/null +++ b/nodes/broker/vp_msg_broker_node.cpp @@ -0,0 +1,85 @@ + +#include "vp_msg_broker_node.h" + + +namespace vp_nodes { + + vp_msg_broker_node::vp_msg_broker_node(std::string node_name, + vp_broke_for broke_for, + int broking_cache_warn_threshold, + int broking_cache_ignore_threshold): + vp_node(node_name), + broke_for(broke_for), + broking_cache_warn_threshold(broking_cache_warn_threshold), + broking_cache_ignore_threshold(broking_cache_ignore_threshold) { + broking_th = std::thread(&vp_msg_broker_node::broking_run, this); + } + + vp_msg_broker_node::~vp_msg_broker_node() { + + } + + void vp_msg_broker_node::stop_broking() { + broking = false; + frames_to_broke.push(nullptr); // send dead flag to broking_thread + broking_cache_semaphore.signal(); + if (broking_th.joinable()) { + broking_th.join(); + } + } + + std::shared_ptr vp_msg_broker_node::handle_frame_meta(std::shared_ptr meta) { + // cache frame meta only if cache size is not greater than threshold + if (frames_to_broke.size() < broking_cache_ignore_threshold) { + // it is a producer + frames_to_broke.push(meta); + broking_cache_semaphore.signal(); + } + + // warning 1 time in log + auto size = frames_to_broke.size(); + if (size > broking_cache_warn_threshold && !broking_cache_warned) { + broking_cache_warned = true; + VP_WARN(vp_utils::string_format("[%s] [message broker] cache size is exceeding threshold! cache size is [%d], threshold is [%d]", node_name.c_str(), size, broking_cache_warn_threshold)); + } + + if (size <= broking_cache_warn_threshold) { + broking_cache_warned = false; + } + + return meta; + } + + std::shared_ptr vp_msg_broker_node::handle_control_meta(std::shared_ptr meta) { + return meta; + } + + void vp_msg_broker_node::broking_run() { + while (broking) { + // it is a consumer + broking_cache_semaphore.wait(); + + auto frame_meta = frames_to_broke.front(); + frames_to_broke.pop(); + + // dead flag + if (frame_meta == nullptr) { + continue; + } + + // message to be broked + std::string message; + + // step 1, format message + format_msg(frame_meta, message); // MUST be implemented in child class + + // ignore if message is empty, because no broking occurs is allowed for some frames if some conditions not satisfied + if (message.empty()) { + continue; + } + + // step 2, broke message + broke_msg(message); // MUST be implemented in child class + } + } +} \ No newline at end of file diff --git a/nodes/broker/vp_msg_broker_node.h b/nodes/broker/vp_msg_broker_node.h new file mode 100644 index 0000000..26821d2 --- /dev/null +++ b/nodes/broker/vp_msg_broker_node.h @@ -0,0 +1,65 @@ + +#pragma once + +#include "../vp_node.h" + +namespace vp_nodes { + // broke for what type of data (vp_frame_target, vp_frame_face_target or others) + enum class vp_broke_for { + NORMAL, // vp_frame_target + FACE, // vp_frame_face_target + TEXT, // vp_frame_text_target + POSE // vp_frame_pose_target + // others to extend + }; + + // base node for message brokers, + // used to serialize objects (inside vp_frame_meta) to structured data and then push them to external modules like kafka, file or sockets. + // note: + // 1. this node works asynchronously which would not block pipeline. + // 2. this class can not be initialized directly. + class vp_msg_broker_node: public vp_node + { + private: + // warning if cache size greater than threshold + int broking_cache_warn_threshold = 50; + bool broking_cache_warned = false; + + // ignore if cache size greater than threshold (skip directly) + int broking_cache_ignore_threshold = 200; + + // cache frames to be broked + std::queue> frames_to_broke; + vp_utils::vp_semaphore broking_cache_semaphore; + + // broking thread + std::thread broking_th; + // broking function + void broking_run(); + bool broking = true; + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override final; + virtual std::shared_ptr handle_control_meta(std::shared_ptr meta) override final; + + // serialize objects to message which SHOULD be implemented in child class. + virtual void format_msg(const std::shared_ptr& meta, std::string& msg) = 0; + // broke message to external modules which SHOULD be implemented in child class. + virtual void broke_msg(const std::string& msg) = 0; + // wait thread exits in vp_msg_broker_node + void stop_broking(); + // node applied for what type of target + vp_broke_for broke_for = vp_broke_for::NORMAL; + + // string for broke_for + std::map broke_fors = {{vp_broke_for::NORMAL, "normal"}, + {vp_broke_for::FACE, "face"}, + {vp_broke_for::TEXT, "text"}, + {vp_broke_for::POSE, "pose"}}; + public: + vp_msg_broker_node(std::string node_name, + vp_broke_for broke_for = vp_broke_for::NORMAL, + int broking_cache_warn_threshold = 50, + int broking_cache_ignore_threshold = 200); + ~vp_msg_broker_node(); + }; +} \ No newline at end of file diff --git a/nodes/broker/vp_plate_socket_broker_node.cpp b/nodes/broker/vp_plate_socket_broker_node.cpp new file mode 100644 index 0000000..a4ad2d1 --- /dev/null +++ b/nodes/broker/vp_plate_socket_broker_node.cpp @@ -0,0 +1,160 @@ + + +#include "vp_plate_socket_broker_node.h" + + +namespace vp_nodes { + + vp_plate_socket_broker_node::vp_plate_socket_broker_node(std::string node_name, + std::string des_ip, + int des_port, + std::string plates_dir, + int min_crop_width, + int min_crop_height, + vp_broke_for broke_for, + bool only_for_tracked, + int broking_cache_warn_threshold, + int broking_cache_ignore_threshold): + vp_msg_broker_node(node_name, broke_for, broking_cache_warn_threshold, broking_cache_ignore_threshold), + des_ip(des_ip), + des_port(des_port), + plates_dir(plates_dir), + min_crop_width(min_crop_width), + min_crop_height(min_crop_height), + only_for_tracked(only_for_tracked) { + // only for vp_frame_target + assert(broke_for == vp_broke_for::NORMAL); + udp_writer = kissnet::udp_socket(kissnet::endpoint(des_ip, des_port)); + VP_INFO(vp_utils::string_format("[%s] [message broker] set des_ip as `%s` and des_port as [%d]", node_name.c_str(), des_ip.c_str(), des_port)); + this->initialized(); + } + + vp_plate_socket_broker_node::~vp_plate_socket_broker_node() { + deinitialized(); + stop_broking(); + } + + void vp_plate_socket_broker_node::format_msg(const std::shared_ptr& meta, std::string& msg) { + /* format: + line 0, <-- + line 1, 1st time + line 2, 1st channel index, frame index + line 3, 1st the cropped image's path + line 4, 1st the whole image's path + line 5, 1st plate color + line 6, 1st plate text + line 7, --> + line 8, <-- + line 9, 2nd time + line 10, 2nd channel index, frame index + line 11, 2nd the cropped image's path + line 12, 2nd the whole image's path + line 13, 2nd plate color + line 14, 2nd plate text + line 15, --> + line 16, ... + */ + auto& broked_ids = all_broked_ids[meta->channel_index]; + auto& broked_texts = all_broked_texts[meta->channel_index]; + // remove 50 elements every 100 ids + if (broked_ids.size() > 100) { + broked_ids.erase(broked_ids.begin(), broked_ids.begin() + 50); + } + if (broked_texts.size() > 100) { + broked_texts.erase(broked_texts.begin(), broked_texts.begin() + 50); + } + + std::stringstream msg_stream; + auto format_basic_info = [&](int channel_index, int frame_index) { + msg_stream << vp_utils::time_format(NOW) << std::endl; // line1 + msg_stream << channel_index << "," << frame_index << std::endl; // line2 + }; + auto save_cropped_image = [&](cv::Mat& frame, cv::Rect rect, std::string name) { + auto cropped = frame(rect); + cv::imwrite(name, cropped); + msg_stream << name << std::endl; // line3 + }; + auto save_whole_image = [&](cv::Mat& frame, std::string name) { + cv::imwrite(name, frame); + msg_stream << name << std::endl; // line4 + }; + auto format_plate = [&](const std::string& plate_color, const std::string& plate_text) { + msg_stream << plate_color << std::endl; // line5 + msg_stream << plate_text << std::endl; // line6 + }; + if (broke_for == vp_broke_for::NORMAL) { + for (int i = 0; i < meta->targets.size(); i++) { + auto& t = meta->targets[i]; + // only broke for tracked targets and have enough frames + if ((only_for_tracked && t->track_id < 0) || (only_for_tracked && t->tracks.size() < min_tracked_frames)) { + continue; + } + + // only broke 1 time for specific track id if it has been tracked, or broke many times + if (t->track_id >= 0 && + std::find(broked_ids.begin(), broked_ids.end(), t->track_id) != broked_ids.end()) { + broked_ids.push_back(t->track_id); + continue; + } + + // size filter + if (t->width < min_crop_width || t->height < min_crop_height) { + continue; + } + + auto color_and_text = vp_utils::string_split(t->primary_label, '_'); + // make sure plate text and color detected + if(color_and_text.size() != 2) { + continue; + } + auto& color = color_and_text[0]; + auto& text = color_and_text[1]; + // check for length, 7 or 3 + 6 or 3 + 7 + if (text.length() != 7 && text.length() != 9 && text.length() != 10) { + continue; + } + + // only broke 1 time for specific plate text in small period + if (std::find(broked_texts.begin(), broked_texts.end(), text) != broked_texts.end()) { + broked_texts.push_back(text); + continue; + } + + // cache for texts + broked_texts.push_back(text); + // cache for ids + if (t->track_id >= 0) { + broked_ids.push_back(t->track_id); + } + + auto cropped_name = plates_dir + "/" + std::to_string(t->channel_index) + "_" + std::to_string(t->frame_index) + "_" + color + "_" + text + "_cropped.jpg"; + auto whole_name = plates_dir + "/" + std::to_string(t->channel_index) + "_" + std::to_string(t->frame_index) + "_" + color + "_" + text + "_whole.jpg"; + // start flag + msg_stream << "<--" << std::endl; + // basic info + format_basic_info(meta->channel_index, meta->frame_index); + // save small cropped image + save_cropped_image(meta->frame, cv::Rect(t->x, t->y, t->width, t->height), cropped_name); + // save whole image + save_whole_image(meta->frame, whole_name); + // format color and text + format_plate(color, text); + // end flag + msg_stream << "-->"; + + if (i != meta->targets.size() - 1) { + msg_stream << std::endl; // not the last one + } + } + } + + msg = msg_stream.str(); + } + + void vp_plate_socket_broker_node::broke_msg(const std::string& msg) { + // broke msg to socket by udp + auto bytes_2_send = reinterpret_cast(msg.c_str()); + auto bytes_2_send_len = msg.size(); + udp_writer.send(bytes_2_send, bytes_2_send_len); + } +} \ No newline at end of file diff --git a/nodes/broker/vp_plate_socket_broker_node.h b/nodes/broker/vp_plate_socket_broker_node.h new file mode 100644 index 0000000..d04faf2 --- /dev/null +++ b/nodes/broker/vp_plate_socket_broker_node.h @@ -0,0 +1,56 @@ +#pragma once + +#include "vp_msg_broker_node.h" +#include "cereal_archive/vp_objects_cereal_archive.h" + +// light weight socket support +#include "../../third_party/kissnet/kissnet.hpp" + +namespace vp_nodes { + // message broker node, broke text & color for license plate (hold by vp_frame_target) to socket via udp. + // which could be used for lpr camera. + class vp_plate_socket_broker_node: public vp_msg_broker_node + { + private: + // save dir for cropped images and whole images, which would be used for displaying in lpr camera + std::string plates_dir = "plate_images"; + // min width to crop (license plate will be ignored if target's width is smaller than this value) + int min_crop_width = 50; + // min height to crop (license plate will be ignored if target's height is smaller than this value) + int min_crop_height = 50; + // only broke for tracked targets (track_id is not -1) + bool only_for_tracked = false; + + // min tracked frames if only_for_tracked is true + int min_tracked_frames = 25; + + // host the data sent to via udp + std::string des_ip = ""; + // port the data sent to via udp + int des_port = 0; + + // udp socket writer + kissnet::udp_socket udp_writer; + + // support multi-channel, used for avoid duplicate data + std::map> all_broked_ids; // channel -> target ids which have been broked + std::map> all_broked_texts; // channel -> plate texts which have been broked (filter using similiarity comparison) + protected: + // to custom format + virtual void format_msg(const std::shared_ptr& meta, std::string& msg) override; + // to socket via udp + virtual void broke_msg(const std::string& msg) override; + public: + vp_plate_socket_broker_node(std::string node_name, + std::string des_ip = "", + int des_port = 0, + std::string plates_dir = "plate_images", + int min_crop_width = 100, + int min_crop_height = 0, + vp_broke_for broke_for = vp_broke_for::NORMAL, + bool only_for_tracked = true, + int broking_cache_warn_threshold = 50, + int broking_cache_ignore_threshold = 200); + ~vp_plate_socket_broker_node(); + }; +} \ No newline at end of file diff --git a/nodes/broker/vp_xml_file_broker_node.cpp b/nodes/broker/vp_xml_file_broker_node.cpp new file mode 100644 index 0000000..7f1321c --- /dev/null +++ b/nodes/broker/vp_xml_file_broker_node.cpp @@ -0,0 +1,70 @@ +#include "vp_xml_file_broker_node.h" + +namespace vp_nodes { + + vp_xml_file_broker_node::vp_xml_file_broker_node(std::string node_name, + vp_broke_for broke_for, + std::string file_path_and_name, + int broking_cache_warn_threshold, + int broking_cache_ignore_threshold): + vp_msg_broker_node(node_name, broke_for, broking_cache_warn_threshold, broking_cache_ignore_threshold) { + // open file + if (file_path_and_name.empty()) { + file_path_and_name = "vp_broker_" + vp_utils::time_format(NOW, "---") + ".xml"; + } + VP_INFO(vp_utils::string_format("[%s] [message broker] set broking file path as `%s`", node_name.c_str(), file_path_and_name.c_str())); + xml_writer.open(file_path_and_name); + + this->initialized(); + } + + vp_xml_file_broker_node::~vp_xml_file_broker_node() { + deinitialized(); + stop_broking(); + } + + void vp_xml_file_broker_node::format_msg(const std::shared_ptr& meta, std::string& msg) { + // serialize objects to xml by cereal + std::stringstream msg_stream; + { + cereal::XMLOutputArchive xml_archive(msg_stream); + + // global values + xml_archive(cereal::make_nvp("channel_index", meta->channel_index), + cereal::make_nvp("frame_index", meta->frame_index), + cereal::make_nvp("width", meta->frame.cols), + cereal::make_nvp("height", meta->frame.rows), + cereal::make_nvp("fps", meta->fps), + cereal::make_nvp("broke_for", broke_fors.at(broke_for))); + + // serialize values according to broke_for + if (broke_for == vp_broke_for::NORMAL) { + xml_archive(cereal::make_nvp("target_size", meta->targets.size()), + cereal::make_nvp("targets", meta->targets)); + } + else if (broke_for == vp_broke_for::FACE) { + xml_archive(cereal::make_nvp("face_target_size", meta->face_targets.size()), + cereal::make_nvp("face_targets", meta->face_targets)); + } + else if (broke_for == vp_broke_for::TEXT) { + xml_archive(cereal::make_nvp("text_target_size", meta->text_targets.size()), + cereal::make_nvp("text_targets", meta->text_targets)); + } + else { + throw "invalid broke_for!"; + } + } // flush + + msg = msg_stream.str(); + } + + void vp_xml_file_broker_node::broke_msg(const std::string& msg) { + // broke msg to file by ofstream + if (xml_writer.is_open()) { + xml_writer << msg << std::endl; + } + else { + // TO-DO + } + } +} \ No newline at end of file diff --git a/nodes/broker/vp_xml_file_broker_node.h b/nodes/broker/vp_xml_file_broker_node.h new file mode 100644 index 0000000..e25713c --- /dev/null +++ b/nodes/broker/vp_xml_file_broker_node.h @@ -0,0 +1,28 @@ +#pragma once + +#include + +#include "vp_msg_broker_node.h" +#include "cereal_archive/vp_objects_cereal_archive.h" + +namespace vp_nodes { + // message broker node (for demo/debug purpose), broke xml data to file. + class vp_xml_file_broker_node: public vp_msg_broker_node + { + private: + // xml file writer + ofstream xml_writer; + protected: + // to xml + virtual void format_msg(const std::shared_ptr& meta, std::string& msg) override; + // to file + virtual void broke_msg(const std::string& msg) override; + public: + vp_xml_file_broker_node(std::string node_name, + vp_broke_for broke_for = vp_broke_for::NORMAL, + std::string file_path_and_name = "", + int broking_cache_warn_threshold = 50, + int broking_cache_ignore_threshold = 200); + ~vp_xml_file_broker_node(); + }; +} \ No newline at end of file diff --git a/nodes/broker/vp_xml_socket_broker_node.cpp b/nodes/broker/vp_xml_socket_broker_node.cpp new file mode 100644 index 0000000..935b7af --- /dev/null +++ b/nodes/broker/vp_xml_socket_broker_node.cpp @@ -0,0 +1,68 @@ + + +#include "vp_xml_socket_broker_node.h" + + +namespace vp_nodes { + + vp_xml_socket_broker_node::vp_xml_socket_broker_node(std::string node_name, + std::string des_ip, + int des_port, + vp_broke_for broke_for, + int broking_cache_warn_threshold, + int broking_cache_ignore_threshold): + vp_msg_broker_node(node_name, broke_for, broking_cache_warn_threshold, broking_cache_ignore_threshold), + des_ip(des_ip), + des_port(des_port) { + udp_writer = kissnet::udp_socket(kissnet::endpoint(des_ip, des_port)); + VP_INFO(vp_utils::string_format("[%s] [message broker] set des_ip as `%s` and des_port as [%d]", node_name.c_str(), des_ip.c_str(), des_port)); + this->initialized(); + } + + vp_xml_socket_broker_node::~vp_xml_socket_broker_node() { + deinitialized(); + stop_broking(); + } + + void vp_xml_socket_broker_node::format_msg(const std::shared_ptr& meta, std::string& msg) { + // serialize objects to xml by cereal + std::stringstream msg_stream; + { + cereal::XMLOutputArchive xml_archive(msg_stream); + + // global values + xml_archive(cereal::make_nvp("channel_index", meta->channel_index), + cereal::make_nvp("frame_index", meta->frame_index), + cereal::make_nvp("width", meta->frame.cols), + cereal::make_nvp("height", meta->frame.rows), + cereal::make_nvp("fps", meta->fps), + cereal::make_nvp("broke_for", broke_fors.at(broke_for))); + + // serialize values according to broke_for + if (broke_for == vp_broke_for::NORMAL) { + xml_archive(cereal::make_nvp("target_size", meta->targets.size()), + cereal::make_nvp("targets", meta->targets)); + } + else if (broke_for == vp_broke_for::FACE) { + xml_archive(cereal::make_nvp("face_target_size", meta->face_targets.size()), + cereal::make_nvp("face_targets", meta->face_targets)); + } + else if (broke_for == vp_broke_for::TEXT) { + xml_archive(cereal::make_nvp("text_target_size", meta->text_targets.size()), + cereal::make_nvp("text_targets", meta->text_targets)); + } + else { + throw "invalid broke_for!"; + } + } // flush + + msg = msg_stream.str(); + } + + void vp_xml_socket_broker_node::broke_msg(const std::string& msg) { + // broke msg to socket by udp + auto bytes_2_send = reinterpret_cast(msg.c_str()); + auto bytes_2_send_len = msg.size(); + udp_writer.send(bytes_2_send, bytes_2_send_len); + } +} \ No newline at end of file diff --git a/nodes/broker/vp_xml_socket_broker_node.h b/nodes/broker/vp_xml_socket_broker_node.h new file mode 100644 index 0000000..2ea265c --- /dev/null +++ b/nodes/broker/vp_xml_socket_broker_node.h @@ -0,0 +1,35 @@ +#pragma once + +#include "vp_msg_broker_node.h" +#include "cereal_archive/vp_objects_cereal_archive.h" + +// light weight socket support +#include "../../third_party/kissnet/kissnet.hpp" + +namespace vp_nodes { + // message broker node, broke xml data to socket via udp. + class vp_xml_socket_broker_node: public vp_msg_broker_node + { + private: + // host the data sent to via udp + std::string des_ip = ""; + // port the data sent to via udp + int des_port = 0; + + // udp socket writer + kissnet::udp_socket udp_writer; + protected: + // to xml + virtual void format_msg(const std::shared_ptr& meta, std::string& msg) override; + // to socket via udp + virtual void broke_msg(const std::string& msg) override; + public: + vp_xml_socket_broker_node(std::string node_name, + std::string des_ip = "", + int des_port = 0, + vp_broke_for broke_for = vp_broke_for::NORMAL, + int broking_cache_warn_threshold = 50, + int broking_cache_ignore_threshold = 200); + ~vp_xml_socket_broker_node(); + }; +} \ No newline at end of file diff --git a/nodes/ffio/ff_common.cpp b/nodes/ffio/ff_common.cpp new file mode 100644 index 0000000..2c61b0d --- /dev/null +++ b/nodes/ffio/ff_common.cpp @@ -0,0 +1,55 @@ + +#ifdef VP_WITH_FFMPEG +#include "ff_common.h" + +namespace vp_nodes { + ff_scaler::ff_scaler(int src_width, + int src_height, + AVPixelFormat src_fmt, + int dst_width, + int dst_height, + AVPixelFormat dst_fmt): + m_src_width(src_width), + m_src_height(src_height), + m_src_fmt(src_fmt), + m_dst_width(dst_width), + m_dst_height(dst_height), + m_dst_fmt(dst_fmt) { + sws_ctx = sws_getContext(src_width, src_height, src_fmt, dst_width,dst_height, dst_fmt, SWS_FAST_BILINEAR, NULL, NULL, NULL); + } + + ff_scaler::~ff_scaler() { + if (!sws_ctx) { + sws_freeContext(sws_ctx); + } + } + + bool ff_scaler::scale(ff_av_frame_ptr src, ff_av_frame_ptr& dst) { + if (!sws_ctx || !src || !dst) { + return false; + } + + if (src->format != m_src_fmt || src->width != m_src_width || src->height != m_src_height || + dst->format != m_dst_fmt || dst->width != m_dst_width || dst->height != m_dst_height) { + return false; + } + + if (dst->format == AV_PIX_FMT_BGR24) { + dst->linesize[0] = m_dst_width * 3; + } + + /* allocate buffer if need for dst */ + if (dst->data[0] == NULL) { + auto ret = av_frame_get_buffer(dst.get(), 0); + if (ret < 0) { + return false; + } + } + + auto ret = sws_scale(sws_ctx, src->data, src->linesize, 0, src->height, dst->data, dst->linesize); + dst->pts = src->pts; + dst->pkt_dts = src->pkt_dts; + return ret >= 0; + } +} +#endif \ No newline at end of file diff --git a/nodes/ffio/ff_common.h b/nodes/ffio/ff_common.h new file mode 100644 index 0000000..c69b90c --- /dev/null +++ b/nodes/ffio/ff_common.h @@ -0,0 +1,64 @@ +#pragma once +#ifdef VP_WITH_FFMPEG +#include +#include +#include +#include +#include +#include +#include +extern "C" { +#include +#include +#include +#include +#include +#include +#include +} + +#define ff_av_packet_ptr std::shared_ptr +#define ff_av_frame_ptr std::shared_ptr +#define ff_src_ptr std::shared_ptr +#define ff_des_ptr std::shared_ptr +#define ff_scaler_ptr std::shared_ptr + +#define alloc_ff_src(channel_index) std::make_shared(channel_index) +#define alloc_ff_des(channel_index) std::make_shared(channel_index) +#define alloc_ff_scaler(src_width, src_height, src_fmt, dst_width, dst_height, dst_fmt) \ + std::make_shared(src_width, src_height, src_fmt, dst_width, dst_height, dst_fmt) + +#define alloc_ff_av_frame() \ + std::shared_ptr(av_frame_alloc(), [](AVFrame *frame) { av_frame_free(&frame); }) +#define alloc_ff_av_packet() \ + std::shared_ptr(av_packet_alloc(), [](AVPacket *pkt) { av_packet_free(&pkt); }) + +namespace vp_nodes { + /** + * tools for color conversion & image resize using FFmpeg on CPUs. + * + */ + class ff_scaler { + private: + SwsContext* sws_ctx = NULL; + int m_src_width; + int m_src_height; + AVPixelFormat m_src_fmt; + int m_dst_width; + int m_dst_height; + AVPixelFormat m_dst_fmt; + public: + /* disable copy and assignment operations */ + ff_scaler(const ff_scaler&) = delete; + ff_scaler& operator=(const ff_scaler&) = delete; + ff_scaler(int src_width, + int src_height, + AVPixelFormat src_fmt, + int dst_width, + int dst_height, + AVPixelFormat dst_fmt); + ~ff_scaler(); + bool scale(ff_av_frame_ptr src, ff_av_frame_ptr& dst); + }; +} +#endif \ No newline at end of file diff --git a/nodes/ffio/ff_des.cpp b/nodes/ffio/ff_des.cpp new file mode 100644 index 0000000..00c99fe --- /dev/null +++ b/nodes/ffio/ff_des.cpp @@ -0,0 +1,482 @@ +#ifdef VP_WITH_FFMPEG +#include +#include +#include "ff_des.h" + +namespace vp_nodes { + ff_des::ff_des(int channel_index): m_channel_index(channel_index) { + } + + ff_des::~ff_des() { + inner_close(); + } + + void ff_des::enmux_run() { + auto ret = 0; + while (m_enmux_running) { + ff_av_packet_ptr ff_packet = nullptr; + m_enmux_semaphore.wait(); + { + std::lock_guard g(m_enmux_packets_m); + ff_packet = m_enmux_packets_q.front(); + m_enmux_packets_q.pop(); + } + + if (!ff_packet) { + VP_INFO(vp_utils::string_format("[ffio/ff_des][%d][enmux_run] get exit flag.", + get_channel_index())); + break; + } else { + /* set parameters for packets */ + av_packet_rescale_ts(ff_packet.get(), m_enc_ctx->time_base, m_ofmt_ctx->streams[m_inner_stream_index]->time_base); + ff_packet->stream_index = m_inner_stream_index; + ret = av_interleaved_write_frame(m_ofmt_ctx, ff_packet.get()); + } + + if (ret < 0) { + VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][enmux_run] av_interleaved_write_frame failed. ret: %d", + get_channel_index(), ret)); + break; + } + } + /* write the trailer for output stream */ + ret = av_write_trailer(m_ofmt_ctx); + if (ret != 0) { + VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][enmux_run] av_write_trailer failed. ret: %d", + get_channel_index(), ret)); + } + + /* set enmux running flag to false in case of abnormal exit */ + m_enmux_running = false; + VP_INFO(vp_utils::string_format("[ffio/ff_des][%d][enmux_run] enmux thread exits.", + get_channel_index())); + } + + void ff_des::encode_run() { + while (m_encode_running) { + ff_av_frame_ptr ff_frame; + m_encode_semaphore.wait(); + { + std::lock_guard g(m_encode_frames_m); + ff_frame = m_encode_frames_q.front(); + m_encode_frames_q.pop(); + } + + auto ret = 0; + if (!ff_frame) { + VP_INFO(vp_utils::string_format("[ffio/ff_des][%d][encode_run] get exit flag.", + get_channel_index())); + break; + } + ff_frame->pts = m_enc_ctx->frame_number; + ret = avcodec_send_frame(m_enc_ctx, ff_frame.get()); + if (ret < 0) { + VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][encode_run] avcodec_send_frame failed. ret: %d", + get_channel_index(), ret)); + break; + } + + while (ret >= 0) { + auto ff_packet = alloc_ff_av_packet(); + ret = avcodec_receive_packet(m_enc_ctx, ff_packet.get()); + if (ret == AVERROR(EAGAIN)) { + break; + } else if (ret < 0) { + VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][encode_run] avcodec_receive_packet failed. ret: %d", + get_channel_index(), ret)); + ff_frame = nullptr; + break; + } + + bool notify = true; + { + std::lock_guard g(m_enmux_packets_m); + m_enmux_packets_q.push(ff_packet); + if (m_enmux_packets_q.size() > m_enmux_packets_q_max_size) { + m_enmux_packets_q.pop(); + notify = false; + VP_WARN(vp_utils::string_format("[ffio/ff_des][%d][encode_run] exceed m_enmux_packets_q_max_size(%d), discard the front in queue.", + get_channel_index(), m_enmux_packets_q_max_size)); + } + } + if (notify) { + m_enmux_semaphore.signal(); + } + } + + if (!ff_frame) { + break; + } + } + /* set encode running flag to false in case of abnormal exit */ + m_encode_running = false; + { + // send exit flag to notify enmux thread + std::lock_guard g(m_enmux_packets_m); + m_enmux_packets_q.push(nullptr); + VP_INFO(vp_utils::string_format("[ffio/ff_des][%d][encode_run] send exit flag to enmux thread.", + get_channel_index())); + } + m_enmux_semaphore.signal(); + VP_INFO(vp_utils::string_format("[ffio/ff_des][%d][encode_run] encode thread exits.", + get_channel_index())); + } + + bool ff_des:: + open(const std::string& uri, + int width, int height, int fps, int bitrate, int max_b_frames, + const std::string& encoder_name, + AVPixelFormat sw_pix_fmt, + AVHWDeviceType hw_type, + AVPixelFormat hw_pix_fmt) { + inner_close(); + + auto uri_valid = false; + if (!uri_valid) { + auto uri_parts = vp_utils::string_split(uri, '.'); // file + if (std::find(m_supported_files.begin(), m_supported_files.end(), uri_parts[uri_parts.size() - 1]) != m_supported_files.end()) { + uri_valid = true; + m_live_stream = false; + } + } + if (!uri_valid) { + auto uri_parts = vp_utils::string_split(uri, ':'); // live stream + if (std::find(m_supported_protocols.begin(), m_supported_protocols.end(), uri_parts[0]) != m_supported_protocols.end()) { + uri_valid = true; + m_live_stream = true; + } + } + if (!uri_valid) { + VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][open] uri invalid! uri MUST start with `rtsp/rtmp/..`(network streams) or end with `mp4/mkv/..`(file streams).", + get_channel_index())); + return false; + } + + return inner_open(uri, width, height, fps, bitrate, max_b_frames, encoder_name, sw_pix_fmt, hw_type, hw_pix_fmt); + } + + void ff_des::close() { + inner_close(); + } + + bool ff_des::is_opened() const { + return m_encode_running && m_enmux_running; + } + + void ff_des::inner_close() { + /* set running flags to false */ + m_encode_running = false; + m_enmux_running = false; + + inner_exit_signal(); + /* waiting for threads exist */ + if (m_encode_th != nullptr && m_encode_th->joinable()) { + m_encode_th->join(); + } + if (m_enmux_th != nullptr && m_enmux_th->joinable()) { + m_enmux_th->join(); + } + + /* free format & codec context */ + if (m_ofmt_ctx) { + if (m_ofmt_ctx->pb) { + avio_closep(&m_ofmt_ctx->pb); + } + avformat_free_context(m_ofmt_ctx); + } + if (m_enc_ctx) { + avcodec_free_context(&m_enc_ctx); + } + + /* clear queue */ + { + // clear enmux_packets_q + std::lock_guard g(m_enmux_packets_m); + m_enmux_packets_q = {}; + m_enmux_semaphore.reset(); + } + { + // clear encode_frames_q + std::lock_guard g(m_encode_frames_m); + m_encode_frames_q = {}; + m_encode_semaphore.reset(); + } + VP_INFO(vp_utils::string_format("[ffio/ff_des][%d][inner_close] close successfully.", + get_channel_index())); + } + + std::string ff_des::print_summary() { + return ""; + } + + bool ff_des:: + inner_open(const std::string& uri, + int width, int height, int fps, int bitrate, int max_b_frames, + const std::string& encoder_name, + AVPixelFormat sw_pix_fmt, + AVHWDeviceType hw_type, + AVPixelFormat hw_pix_fmt) { + auto ret = 0; + /* allocate the output media context */ + ret = avformat_alloc_output_context2(&m_ofmt_ctx, NULL, NULL, uri.c_str()); + if (ret < 0) { + VP_WARN(vp_utils::string_format("[ffio/ff_des][%d][inner_open] could not deduce output format from uri, try to use FLV.", + get_channel_index())); + ret = avformat_alloc_output_context2(&m_ofmt_ctx, NULL, "flv", uri.c_str()); + } + if (ret < 0) { + VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][inner_open] could not deduce output format from uri(used FLV).", + get_channel_index())); + return false; + } + auto ofmt = m_ofmt_ctx->oformat; + + /* find encoder for the output stream */ + const AVCodec* enc = nullptr; + if (encoder_name.empty()) { + enc = avcodec_find_encoder(ofmt->video_codec); // get default encoder by AVCodecID + } else { + enc = avcodec_find_encoder_by_name(encoder_name.c_str()); // get by encoder name + } + if (!enc) { + VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][inner_open] could not find proper encoder for output stream.", + get_channel_index())); + avformat_free_context(m_ofmt_ctx); + m_ofmt_ctx = NULL; + return false; + } + /* allocate a codec context for the encoder */ + m_enc_ctx = avcodec_alloc_context3(enc); + if (!m_enc_ctx) { + VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][inner_open] could not allocate context for encoder(%s).", + get_channel_index(), enc->name)); + avformat_free_context(m_ofmt_ctx); + m_ofmt_ctx = NULL; + return false; + } + /* set parameters for encoder */ + m_enc_ctx->framerate = (AVRational) {fps, 1}; + m_enc_ctx->time_base = av_inv_q(m_enc_ctx->framerate); + m_enc_ctx->pix_fmt = sw_pix_fmt; + m_enc_ctx->width = width; + m_enc_ctx->height = height; + m_enc_ctx->bit_rate = bitrate * 1024; + m_enc_ctx->max_b_frames = max_b_frames; + /* check if need init hwaccels context for encoder */ + if (hw_type != AVHWDeviceType::AV_HWDEVICE_TYPE_NONE) { + m_enc_ctx->pix_fmt = hw_pix_fmt; + if ((ret = hw_encoder_init(m_enc_ctx, hw_type, sw_pix_fmt)) < 0) { + VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][inner_open] hw_encoder_init failed. ret: %d", + get_channel_index(), ret)); + avcodec_free_context(&m_enc_ctx); + avformat_free_context(m_ofmt_ctx); + m_ofmt_ctx = NULL; + return false; + } + } + /* open the encoder */ + if ((ret = avcodec_open2(m_enc_ctx, enc, NULL)) < 0) { + VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][inner_open] could not open encoder. ret: %d", + get_channel_index(), ret)); + avcodec_free_context(&m_enc_ctx); + avformat_free_context(m_ofmt_ctx); + m_ofmt_ctx = NULL; + return false; + } + /* create new stream for output */ + auto out_stream = avformat_new_stream(m_ofmt_ctx, NULL); + if (!out_stream) { + VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][inner_open] could not create new stream for output uri.", + get_channel_index())); + avcodec_free_context(&m_enc_ctx); + avformat_free_context(m_ofmt_ctx); + m_ofmt_ctx = NULL; + return false; + } + m_inner_stream_index = out_stream->index; + out_stream->time_base = m_enc_ctx->time_base; + ret = avcodec_parameters_from_context(out_stream->codecpar, m_enc_ctx); + if (ret < 0) { + VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][inner_open] failed to copy codec parameters from encode context. ret: %d", + get_channel_index(), ret)); + avcodec_free_context(&m_enc_ctx); + avformat_free_context(m_ofmt_ctx); + m_ofmt_ctx = NULL; + return false; + } + if (!(m_ofmt_ctx->flags & AVFMT_NOFILE)) { + ret = avio_open(&m_ofmt_ctx->pb, uri.c_str(), AVIO_FLAG_WRITE); + m_ofmt_ctx->flush_packets = 1; + m_ofmt_ctx->flags |= AVFMT_FLAG_FLUSH_PACKETS; + if (ret < 0) { + VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][inner_open] avio_open failed on output uri(%s). ret: %d", + get_channel_index(), uri.c_str(), ret)); + avcodec_free_context(&m_enc_ctx); + avformat_free_context(m_ofmt_ctx); + m_ofmt_ctx = NULL; + return false; + } + } + + /* write the stream header */ + if ((ret = avformat_write_header(m_ofmt_ctx, NULL)) < 0) { + VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][inner_open] avformat_write_header failed. ret: %d", + get_channel_index(), ret)); + avcodec_free_context(&m_enc_ctx); + avformat_free_context(m_ofmt_ctx); + m_ofmt_ctx = NULL; + return false; + } + + /* initialize video & ff_des properties */ + m_hw_type_name = hw_type != AV_HWDEVICE_TYPE_NONE ? std::string(av_hwdevice_get_type_name(hw_type)) : "none"; + m_encoder_name = encoder_name.empty() ? std::string(m_enc_ctx->codec->name) : encoder_name; + m_uri = uri; + m_fps = fps; + m_width = width; + m_height = height; + m_bitrate = bitrate; + m_codec_name = std::string(avcodec_get_name(out_stream->codecpar->codec_id));; + m_pixel_format = std::string(av_get_pix_fmt_name(static_cast(out_stream->codecpar->format))); + m_max_b_frames = max_b_frames; + + /* collect summary notify caller */ + auto summary = print_summary(); + + /* go! */ + m_enmux_running = true; + m_encode_running = true; + m_enmux_th = std::make_shared(&ff_des::enmux_run, this); + m_encode_th = std::make_shared(&ff_des::encode_run, this); + VP_INFO(vp_utils::string_format("[ffio/ff_des][%d][inner_open] open successfully.", + get_channel_index())); + return true; + } + + bool ff_des::write(const ff_av_frame_ptr& frame) { + if (!is_opened()) { + return false; + } + + bool notify = true; + { + std::lock_guard g(m_encode_frames_m); + m_encode_frames_q.push(frame); + if (m_encode_frames_q.size() > m_encode_frames_q_max_size) { + m_encode_frames_q.pop(); + notify = false; + VP_WARN(vp_utils::string_format("[ffio/ff_des][%d][write] exceed m_encode_frames_q_max_size(%d), discard the front in queue.", + get_channel_index(), m_encode_frames_q_max_size)); + } + } + if (notify) { + m_encode_semaphore.signal(); + } + return true; + } + + ff_des& ff_des::operator<<(const ff_av_frame_ptr& frame) { + write(frame); + return *this; + } + + void ff_des::inner_exit_signal() { + { + // send exit flag to notify encode thread + std::lock_guard g(m_encode_frames_m); + m_encode_frames_q.push(nullptr); + } + m_encode_semaphore.signal(); + } + + int ff_des::get_video_fps() const { + return m_fps; + } + + int ff_des::get_video_width() const { + return m_width; + } + + int ff_des::get_video_height() const { + return m_height; + } + + long ff_des::get_video_bitrate() const { + return m_bitrate; + } + + std::string ff_des::get_video_codec_name() const { + return m_codec_name; + } + + std::string ff_des::get_video_pixel_format_name() const { + return m_pixel_format; + } + + bool ff_des::is_live_stream() const { + return m_live_stream; + } + + std::string ff_des::get_hw_type_name() const { + return m_hw_type_name; + } + + std::string ff_des::get_encoder_name() const { + return m_encoder_name; + } + + std::string ff_des::get_uri() const { + return m_uri; + } + + int ff_des::get_channel_index() const { + return m_channel_index; + } + + const AVCodecContext* ff_des::get_encode_ctx() const { + return m_enc_ctx; + } + + int ff_des:: + hw_encoder_init(AVCodecContext* enc_ctx, + const enum AVHWDeviceType hw_type, + const enum AVPixelFormat sw_pix_fmt) { + AVBufferRef* hw_device_ctx = nullptr; + AVBufferRef* hw_frames_ref = nullptr; + AVHWFramesContext* frames_ctx = nullptr; + int err = 0; + if ((err = av_hwdevice_ctx_create(&hw_device_ctx, hw_type, + NULL, NULL, 0)) < 0) { + VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][hw_encoder_init] failed to create specified(%s) HW device context for encoder.", + get_channel_index(), av_hwdevice_get_type_name(hw_type))); + return err; + } + + if (!(hw_frames_ref = av_hwframe_ctx_alloc(hw_device_ctx))) { + VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][hw_encoder_init] failed to create specified(%s) HW frame context for encoder.", + get_channel_index(), av_hwdevice_get_type_name(hw_type))); + return err; + } + frames_ctx = (AVHWFramesContext* )(hw_frames_ref->data); + frames_ctx->format = enc_ctx->pix_fmt; + frames_ctx->sw_format = sw_pix_fmt; + frames_ctx->width = enc_ctx->width; + frames_ctx->height = enc_ctx->height; + frames_ctx->initial_pool_size = 20; + if ((err = av_hwframe_ctx_init(hw_frames_ref)) < 0) { + VP_ERROR(vp_utils::string_format("[ffio/ff_des][%d][hw_encoder_init] failed to initialize specified(%s) HW frame context for encoder.", + get_channel_index(), av_hwdevice_get_type_name(hw_type))); + av_buffer_unref(&hw_frames_ref); + av_buffer_unref(&hw_device_ctx); + return err; + } + enc_ctx->hw_frames_ctx = av_buffer_ref(hw_frames_ref); + if (!enc_ctx->hw_frames_ctx) + err = AVERROR(ENOMEM); + + av_buffer_unref(&hw_frames_ref); + av_buffer_unref(&hw_device_ctx); + return err; + } +} +#endif \ No newline at end of file diff --git a/nodes/ffio/ff_des.h b/nodes/ffio/ff_des.h new file mode 100644 index 0000000..9cf069d --- /dev/null +++ b/nodes/ffio/ff_des.h @@ -0,0 +1,241 @@ +#pragma once +#ifdef VP_WITH_FFMPEG +#include +#include +#include +#include +#include +#include +#include "ff_common.h" +#include "../../utils/vp_semaphore.h" +#include "../../utils/vp_utils.h" +#include "../../utils/logger/vp_logger.h" + +namespace vp_nodes { + /** + * encode and enmux using FFmpeg. + * used to encode & enmux network streams or file streams. + */ + class ff_des: public std::enable_shared_from_this { + private: + /* core members */ + const std::vector m_supported_files = {"mp4", "mkv", "flv", "h265", "h264"}; + const std::vector m_supported_protocols = {"rtsp", "rtmp", "udp", "rtp"}; + AVFormatContext* m_ofmt_ctx = nullptr; + AVCodecContext* m_enc_ctx = nullptr; + std::queue m_enmux_packets_q; + std::queue m_encode_frames_q; + std::mutex m_enmux_packets_m; + std::mutex m_encode_frames_m; + int m_enmux_packets_q_max_size = 25; + int m_encode_frames_q_max_size = 25; + std::shared_ptr m_enmux_th = nullptr; + std::shared_ptr m_encode_th = nullptr; + vp_utils::vp_semaphore m_enmux_semaphore; + vp_utils::vp_semaphore m_encode_semaphore; + + /** + * live stream or not for output. + */ + bool m_live_stream = false; + + /** + * fps of output stream. + */ + int m_fps = 0; + + /** + * width of output stream in pixels. + */ + int m_width = 0; + + /** + * height of output stream in pixels. + */ + int m_height = 0; + + /** + * bitrate of output stream (kbit/s). + */ + long m_bitrate = 0; + + /** + * codec type name of output stream (strings like `h264/hevc/vp8/...`). + */ + std::string m_codec_name = ""; + + /** + * pixel format of output stream (strings like `YUV420P/NV12/...`). + */ + std::string m_pixel_format = ""; + + /** + * max B frames for encode. + */ + int m_max_b_frames = 0; + + /** + * channel index of output. + */ + int m_channel_index = -1; + + /** + * type name of hardware used for encode (`none` if no hardware used). + */ + std::string m_hw_type_name = ""; + + /** + * encoder name used for encoding. + */ + std::string m_encoder_name = ""; + + /** + * uri for output (rtmp/file/...). + */ + std::string m_uri = ""; + + /* inner flags */ + int m_inner_stream_index = -1; + bool m_enmux_running = false; + bool m_encode_running = false; + + /* inner methods */ + void enmux_run(); + void encode_run(); + void inner_close(); + void inner_exit_signal(); + std::string print_summary(); + bool inner_open(const std::string& uri, + int width, int height, int fps, int bitrate, int max_b_frames, + const std::string& encoder_name = "", + AVPixelFormat sw_pix_fmt = AVPixelFormat::AV_PIX_FMT_YUV420P, + AVHWDeviceType hw_type = AVHWDeviceType::AV_HWDEVICE_TYPE_NONE, + AVPixelFormat hw_pix_fmt = AVPixelFormat::AV_PIX_FMT_NONE); + int hw_encoder_init(AVCodecContext* enc_ctx, const enum AVHWDeviceType hw_type, const enum AVPixelFormat sw_pix_fmt); + + public: + /** + * create ff_des instance using initial parameters. + * + * @param channel_index specify the channel index for output stream. + */ + ff_des(int channel_index); + ~ff_des(); + + /* disable copy and assignment operations */ + ff_des(const ff_des&) = delete; + ff_des& operator=(const ff_des&) = delete; + + /** + * try to open ff_des, not thread-safe. + * + * @param + */ + bool open(const std::string& uri, + int width, int height, int fps, int bitrate, int max_b_frames = 0, + const std::string& encoder_name = "", + AVPixelFormat sw_pix_fmt = AVPixelFormat::AV_PIX_FMT_YUV420P, + AVHWDeviceType hw_type = AVHWDeviceType::AV_HWDEVICE_TYPE_NONE, + AVPixelFormat hw_pix_fmt = AVPixelFormat::AV_PIX_FMT_NONE); + + /** + * close ff_des. + */ + void close(); + + /** + * if working or not. + * + * @return + * true if it is working. + */ + bool is_opened() const; + + /** + * write the next frame to ff_des, keep as same as cv::VideoWriter. + * + * @param frame frame to be written. + * + * @return + * true if write successfully. + * + * @note + * return false means: + * 1. write too quickly no space prepared, please try to write again. + * 2. ff_des not opened yet or not working. + */ + bool write(const ff_av_frame_ptr& frame); + + /** + * write the next frame to ff_des, support for `<<` operator. + * + * @param frame frame to be written. + * + * @return + * reference for ff_des. + */ + ff_des& operator<<(const ff_av_frame_ptr& frame); + + /** + * get fps of output stream in pixels. + */ + int get_video_fps() const; + + /** + * get width of output stream in pixels. + */ + int get_video_width() const; + + /** + * get height of output stream in pixels. + */ + int get_video_height() const; + + /** + * get bitrate of output stream (kbit/s). + */ + long get_video_bitrate() const; + + /** + * get codec type name of output stream (strings like `h264/hevc/vp8/...`). + */ + std::string get_video_codec_name() const; + + /** + * get pixel format of output stream (strings like `YUV420P/NV12/...`). + */ + std::string get_video_pixel_format_name() const; + + /** + * check is live stream or not. + */ + bool is_live_stream() const; + + /** + * get type name of hardware used for encoding (strings like 'cuda/vaapi', 'none' if no hardware used). + */ + std::string get_hw_type_name() const; + + /** + * get encoder name used for encoding. + */ + std::string get_encoder_name() const; + + /** + * get uri of output. + */ + std::string get_uri() const; + + /** + * get channel index of output. + */ + int get_channel_index() const; + + /** + * get pointer of encode context from FFmpeg. + * used to allocate hardware AVFrame outside of ff_des. + */ + const AVCodecContext* get_encode_ctx() const; + }; +} +#endif \ No newline at end of file diff --git a/nodes/ffio/ff_src.cpp b/nodes/ffio/ff_src.cpp new file mode 100644 index 0000000..2471469 --- /dev/null +++ b/nodes/ffio/ff_src.cpp @@ -0,0 +1,475 @@ +#ifdef VP_WITH_FFMPEG +#include +#include +#include "ff_src.h" + +namespace vp_nodes { + ff_src::ff_src(int channel_index): m_channel_index(channel_index) { + } + + ff_src::~ff_src() { + inner_close(); + } + + void ff_src::demux_run() { + auto start_time = std::chrono::system_clock::now(); + auto total_bytes = 0; + while (m_demux_running) { + auto demux_start_t = std::chrono::system_clock::now(); + auto ff_packet = alloc_ff_av_packet(); + auto ret = 0; + + if ((ret = av_read_frame(m_ifmt_ctx, ff_packet.get())) < 0) { + VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][demux_run] av_read_frame failed. ret: %d", + get_channel_index(), + ret)); + break; + } + + // only demux the right video stream + if (ff_packet->stream_index != m_inner_stream_index) { + continue; + } +/* + // analyse + vp_media::stream_analyser analyser(ff_packet->data, ff_packet->size, true); + std::vector nal_units; + analyser.analyse(nal_units); + std::ostringstream oss; + for (auto& nal: nal_units) { + oss << std::setw(8) << std::setfill(' ') << nal.index + << std::setw(16) << std::setfill(' ') << nal.offset + << std::setw(8) << std::setfill(' ') << nal.nal_length + << std::setw(16) << std::setfill(' ') << vp_media::stream_analyser::to_hex(nal.start_bytes) << vp_media::stream_analyser::to_hex(nal.head_bytes, false) + << std::setw(4) << std::setfill(' ') << nal.nal_type + << std::setw(24) << std::setfill(' ') << nal.nal_type_name << std::endl; + } + std::cout << oss.str(); +*/ + // need calculate bitrate manually + if (m_bitrate <= 0) { + total_bytes += ff_packet->size; + auto current_time = std::chrono::system_clock::now(); + auto elapsed_seconds = std::chrono::duration_cast(current_time - start_time); + if (elapsed_seconds.count() > 5.0) { + m_bitrate = (total_bytes * 8) / elapsed_seconds.count() / 1024; // convert to kbit/s + total_bytes = 0; + start_time = std::chrono::system_clock::now(); + } + } + + bool notify = true; + { + std::lock_guard g(m_demux_packets_m); + m_demux_packets_q.push(ff_packet); + if (m_demux_packets_q.size() > m_demux_packets_q_max_size) { + m_demux_packets_q.pop(); + notify = false; + VP_WARN(vp_utils::string_format("[ffio/ff_src][%d][demux_run] exceed m_demux_packets_q_max_size(%d), discard the front in queue.", + get_channel_index(), + m_demux_packets_q_max_size)); + } + } + if (notify) { + m_demux_semaphore.signal(); + } + + /* 1. wait for a while for video file + * 2. demux as soon as possible for live stream + */ + if (!m_live_stream) { + auto cost_t = std::chrono::duration_cast(std::chrono::system_clock::now() - demux_start_t); + auto duration_t = std::chrono::milliseconds(int(1000.0 / m_fps)); + if (cost_t < duration_t) { + std::this_thread::sleep_for(duration_t - cost_t); + } + } + } + + /* set demux running flag to false in case of abnormal exit */ + m_demux_running = false; + { + // send exit flag to notify decode thread + std::lock_guard g(m_demux_packets_m); + m_demux_packets_q.push(nullptr); + VP_INFO(vp_utils::string_format("[ffio/ff_src][%d][demux_run] send exit flag to decode thread.", + get_channel_index())); + } + m_demux_semaphore.signal(); + VP_INFO(vp_utils::string_format("[ffio/ff_src][%d][demux_run] demux thread exits.", + get_channel_index())); + } + + void ff_src::decode_run() { + while (m_decode_running) { + m_demux_semaphore.wait(); + ff_av_packet_ptr ff_packet = nullptr; + { + std::lock_guard g(m_demux_packets_m); + ff_packet = m_demux_packets_q.front(); + m_demux_packets_q.pop(); + } + + auto ret = 0; + /* get exit flag */ + if (!ff_packet) { + VP_INFO(vp_utils::string_format("[ffio/ff_src][%d][decode_run] get exit flag, go to flush decoder.", + get_channel_index())); + ret = avcodec_send_packet(m_dec_ctx, NULL); // flush decoder + } else { + ret = avcodec_send_packet(m_dec_ctx, ff_packet.get()); + } + + if (ret < 0) { + VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][decode_run] avcodec_send_packet failed. ret: %d", + get_channel_index(), ret)); + break; + } + + while (ret >= 0) { + auto ff_frame = alloc_ff_av_frame(); + ret = avcodec_receive_frame(m_dec_ctx, ff_frame.get()); + if (ret == AVERROR(EAGAIN)) { + break; + } else if (ret < 0) { + VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][decode_run] avcodec_receive_frame failed. ret: %d", + get_channel_index(), + ret)); + ff_packet = nullptr; + break; + } + { + std::lock_guard g(m_decode_frames_m); + m_decode_frames_q.push(ff_frame); + if (m_decode_frames_q.size() > m_decode_frames_q_max_size) { + m_decode_frames_q.pop(); + VP_WARN(vp_utils::string_format("[ffio/ff_src][%d][decode_run] exceed m_decode_frames_q_max_size(%d), discard the front in queue.", + get_channel_index(), + m_decode_frames_q_max_size)); + } + } + } + + if (!ff_packet) { + break; + } + } + + /* set decode running flag to false in case of abnormal exit */ + m_decode_running = false; + VP_INFO(vp_utils::string_format("[ffio/ff_src][%d][decode_run] decode thread exits.", + get_channel_index())); + } + + bool ff_src::open(const std::string& uri, const std::string& decoder_name, AVHWDeviceType hw_type) { + inner_close(); + auto uri_valid = false; + if (!uri_valid) { + auto uri_parts = vp_utils::string_split(uri, '.'); // file + if (std::find(m_supported_files.begin(), m_supported_files.end(), uri_parts[uri_parts.size() - 1]) != m_supported_files.end()) { + uri_valid = true; + m_live_stream = false; + } + } + if (!uri_valid) { + auto uri_parts = vp_utils::string_split(uri, ':'); // live stream + if (std::find(m_supported_protocols.begin(), m_supported_protocols.end(), uri_parts[0]) != m_supported_protocols.end()) { + uri_valid = true; + m_live_stream = true; + } + } + if (!uri_valid) { + VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][open] uri invalid! uri MUST start with `rtsp/rtmp/..`(network streams) or end with `mp4/mkv/..`(file streams).", + get_channel_index())); + return false; + } + + return inner_open(uri, decoder_name, hw_type); + } + + bool ff_src::inner_open(const std::string& uri, const std::string& decoder_name, AVHWDeviceType hw_type) { + auto ret = 0; + AVDictionary *options = NULL; + // Set options + av_dict_set(&options, "max_delay", "100000", 0); // 设置最大延迟为100ms + av_dict_set(&options, "buffer_size", "2000000", 0); // 设置缓冲区大小为2MB + av_dict_set(&options, "timeout", "3000000", 0); // 设置超时时间为3秒 + av_dict_set(&options, "rtsp_transport", "tcp", 0); // 使用TCP传输 + av_dict_set(&options, "max_interleave_delta", "1000000", 0); // 设置最大间隔时间为1秒 + + /* open input uri(file/stream), and allocate format context */ + if (avformat_open_input(&m_ifmt_ctx, uri.c_str(), NULL, NULL) < 0) { + VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][inner_open] could not open input uri.", + get_channel_index())); + return false; + } + /* retrieve stream information */ + if (avformat_find_stream_info(m_ifmt_ctx, NULL) < 0) { + VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][inner_open] could not find stream information.", + get_channel_index())); + avformat_close_input(&m_ifmt_ctx); + return false; + } + /* find video stream */ + if ((ret = av_find_best_stream(m_ifmt_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, NULL, 0)) < 0) { + VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][inner_open] could not find proper VIDEO stream.", + get_channel_index())); + avformat_close_input(&m_ifmt_ctx); + return false; + } + m_inner_stream_index = ret; + auto in_stream = m_ifmt_ctx->streams[m_inner_stream_index]; + + /* initialize video properties */ + m_fps = int(av_q2d(in_stream->r_frame_rate)); + m_width = in_stream->codecpar->width; + m_height = in_stream->codecpar->height; + m_bitrate = in_stream->codecpar->bit_rate / 1024; // kb/s + m_codec_name = std::string(avcodec_get_name(in_stream->codecpar->codec_id)); + m_duration = av_q2d(in_stream->time_base) * in_stream->duration; + m_duration = m_duration < 0 ? 0 : m_duration; + m_pixel_format = std::string(av_get_pix_fmt_name(static_cast(in_stream->codecpar->format))); + + /* find decoder for the input video stream */ + const AVCodec* dec = nullptr; + if (decoder_name.empty()) { + dec = avcodec_find_decoder(in_stream->codecpar->codec_id); // get default decoder by AVCodecID + } else { + dec = avcodec_find_decoder_by_name(decoder_name.c_str()); // get by decoder name + } + if (!dec) { + VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][inner_open] could not find the proper decoder for input stream.", + get_channel_index())); + avformat_close_input(&m_ifmt_ctx); + return false; + } + /* allocate codec context for the decoder */ + m_dec_ctx = avcodec_alloc_context3(dec); + if (!m_dec_ctx) { + VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][inner_open] could not allocate context for decoder(%s).", + get_channel_index(), dec->name)); + avformat_close_input(&m_ifmt_ctx); + return false; + } + /* copy codec parameters from input stream to codec context */ + if ((ret = avcodec_parameters_to_context(m_dec_ctx, in_stream->codecpar)) < 0) { + VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][inner_open] could not copy codec parameters to decode context. ret: %d", + get_channel_index(), ret)); + avcodec_free_context(&m_dec_ctx); + avformat_close_input(&m_ifmt_ctx); + return false; + } + /* check if need init hwaccels context for decoder */ + if (hw_type != AVHWDeviceType::AV_HWDEVICE_TYPE_NONE + && hw_decoder_init(m_dec_ctx, hw_type) < 0) { + VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][inner_open] failed to create specified(%s) HW device context for decoder.", + get_channel_index(), av_hwdevice_get_type_name(hw_type))); + avcodec_free_context(&m_dec_ctx); + avformat_close_input(&m_ifmt_ctx); + return false; + } + /* open the decoder */ + if ((ret = avcodec_open2(m_dec_ctx, dec, NULL)) < 0) { + VP_ERROR(vp_utils::string_format("[ffio/ff_src][%d][inner_open] could not open decoder. ret: %d", + get_channel_index(), ret)); + avcodec_free_context(&m_dec_ctx); + avformat_close_input(&m_ifmt_ctx); + return false; + } + /* initialize node properties */ + m_decoder_name = decoder_name.empty() ? std::string(m_dec_ctx->codec->name) : decoder_name; + m_uri = uri; + m_hw_type_name = hw_type != AV_HWDEVICE_TYPE_NONE ? std::string(av_hwdevice_get_type_name(hw_type)) : "none"; + + /* collect summary notify caller */ + auto summary = print_summary(); + if (m_src_opened_hooker) { + m_src_opened_hooker(shared_from_this(), summary); + } + + /* go! */ + m_demux_running = true; + m_decode_running = true; + m_demux_th = std::make_shared(&ff_src::demux_run, this); + m_decode_th = std::make_shared(&ff_src::decode_run, this); + VP_INFO(vp_utils::string_format("[ffio/ff_src][%d][inner_open] open successfully.", + get_channel_index())); + return true; + } + + void ff_src::close() { + inner_close(); + } + + void ff_src::inner_close() { + /* set running flags to false */ + m_demux_running = false; + m_decode_running = false; + + /* waiting for threads exist */ + if (m_demux_th != nullptr && m_demux_th->joinable()) { + m_demux_th->join(); + } + if (m_decode_th != nullptr && m_decode_th->joinable()) { + m_decode_th->join(); + } + + /* free format & codec context */ + if (m_ifmt_ctx) { + avformat_close_input(&m_ifmt_ctx); + } + if (m_dec_ctx) { + avcodec_free_context(&m_dec_ctx); + } + + /* clear queue */ + { + // clear m_demux_packets_q + std::lock_guard g(m_demux_packets_m); + m_demux_packets_q = {}; + m_demux_semaphore.reset(); + } + { + // clear m_decode_frames_q + std::lock_guard g(m_decode_frames_m); + m_decode_frames_q = {}; + } + VP_INFO(vp_utils::string_format("[ffio/ff_src][%d][inner_close] close successfully.", + get_channel_index())); + } + + bool ff_src::is_opened() const { + return m_demux_running && m_decode_running; + } + + int ff_src::get_video_fps() const { + // no check is_opened or not + return m_fps; + } + + int ff_src::get_video_width() const { + // no check is_opened or not + return m_width; + } + + int ff_src::get_video_height() const { + // no check is opened or not + return m_height; + } + + long ff_src::get_video_bitrate() const { + // no check is opened or not + return m_bitrate; + } + + std::string ff_src::get_video_codec_name() const { + // no check is_opened or not + return m_codec_name; + } + + double ff_src::get_video_duration() const { + // no check is opened or not + return m_duration; + } + + bool ff_src::is_live_stream() const { + // no check is opened or not + return m_live_stream; + } + + std::string ff_src::get_hw_type_name() const { + // no check is opened or not + return m_hw_type_name; + } + + std::string ff_src::get_decoder_name() const { + // no check is opened or not + return m_decoder_name; + } + + std::string ff_src::get_uri() const { + // no check is opened or not + return m_uri; + } + + int ff_src::get_channel_index() const { + // no check is opened or not + return m_channel_index; + } + + std::string ff_src::get_video_pixel_format_name() const { + // no check is opened or not + return m_pixel_format; + } + + std::string ff_src::print_summary() { + std::ostringstream s_stream; + s_stream << std::endl; + s_stream << "################# summary for [ff_src] #################" << std::endl; + s_stream << "|channel_index |: " << get_channel_index() << std::endl; + s_stream << "|uri |: " << get_uri() << std::endl; + s_stream << "|is_live_stream |: " << (is_live_stream() ? std::string("YES") : std::string("NO")) << std::endl; + s_stream << "|decoder_name |: " << get_decoder_name() << std::endl; + s_stream << "|hw_type_name |: " << get_hw_type_name() << std::endl; + s_stream << "|------------------| " << std::endl; + s_stream << "|video_codec |: " << get_video_codec_name() << std::endl; + s_stream << "|video_pic_format |: " << get_video_pixel_format_name() << std::endl; + s_stream << "|video_fps |: " << get_video_fps() << std::endl; + s_stream << "|video_width |: " << get_video_width() << std::endl; + s_stream << "|video_height |: " << get_video_height() << std::endl; + s_stream << "|video_bitrate |: " << get_video_bitrate() << " [kbit/s]" << std::endl; + s_stream << "|video_duration |: " << get_video_duration() << " [seconds]" << std::endl; + s_stream << "|------------------| " << std::endl; + s_stream << "################# summary for [ff_src] #################" << std::endl; + + auto summary = s_stream.str(); + VP_INFO(summary); + return summary; + } + + bool ff_src::read(ff_av_frame_ptr& frame) { + auto ret = false; + { + std::lock_guard g(m_decode_frames_m); + if (!m_decode_frames_q.empty()) { + frame = m_decode_frames_q.front(); + m_decode_frames_q.pop(); + ret = true; + } else { + frame = nullptr; + } + } + if (!ret) { + // switch control of CPUs if no frame returned, avoid of occupying CPUs for a long time by caller outside + //std::this_thread::sleep_for(std::chrono::milliseconds(1)); + //VP_DEBUG(vp_utils::string_format("[ffio/ff_src][%d][read] read frame failed, sleep for 1 millisecond.", + // get_channel_index())); + } + + /* false means no frame returned (not opened or read too quickly so data not prepared) */ + return ret; + } + + ff_src& ff_src::operator>>(ff_av_frame_ptr& frame) { + read(frame); + return *this; + } + + int ff_src::hw_decoder_init(AVCodecContext* dec_ctx, const enum AVHWDeviceType type) { + int err = 0; + AVBufferRef* hw_device_ctx = nullptr; + if ((err = av_hwdevice_ctx_create(&hw_device_ctx, type, + NULL, NULL, 0)) < 0) { + return err; + } + // update using AVHWDeviceContext*(hw_device_ctx->data) + dec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx); + av_buffer_unref(&hw_device_ctx); + return err; + } + + void ff_src::set_src_opened_hooker(ff_src_opened_hooker src_opened_hooker) { + m_src_opened_hooker = src_opened_hooker; + } +} +#endif \ No newline at end of file diff --git a/nodes/ffio/ff_src.h b/nodes/ffio/ff_src.h new file mode 100644 index 0000000..ea134c3 --- /dev/null +++ b/nodes/ffio/ff_src.h @@ -0,0 +1,246 @@ +#pragma once +#ifdef VP_WITH_FFMPEG +#include +#include +#include +#include +#include +#include +#include "ff_common.h" +#include "../../utils/vp_semaphore.h" +#include "../../utils/vp_utils.h" +#include "../../utils/logger/vp_logger.h" + +namespace vp_nodes { + class ff_src; + typedef std::function ff_src_opened_hooker; + + /** + * demux and decode using FFmpeg. + * used to demux & decode network streams or file streams. + */ + class ff_src: public std::enable_shared_from_this { + private: + /* core members */ + const std::vector m_supported_files = {"mp4", "mkv", "flv", "avi", "h264"}; + const std::vector m_supported_protocols = {"rtsp", "rtmp", "http", "rtp"}; + AVFormatContext* m_ifmt_ctx = nullptr; + AVCodecContext* m_dec_ctx = nullptr; + std::queue m_demux_packets_q; + std::queue m_decode_frames_q; + std::mutex m_demux_packets_m; + std::mutex m_decode_frames_m; + int m_demux_packets_q_max_size = 25; + int m_decode_frames_q_max_size = 25; + std::shared_ptr m_demux_th = nullptr; + std::shared_ptr m_decode_th = nullptr; + vp_utils::vp_semaphore m_demux_semaphore; + + /** + * live stream or not for input. + */ + bool m_live_stream = false; + + /** + * fps of input stream. + */ + int m_fps = 0; + + /** + * width of input stream in pixels. + */ + int m_width = 0; + + /** + * height of input stream in pixels. + */ + int m_height = 0; + + /** + * bitrate of input stream (kbit/s). + */ + long m_bitrate = 0; + + /** + * codec type name of input stream (strings like `h264/hevc/vp8/...`). + */ + std::string m_codec_name = ""; + + /** + * pixel format of input stream (strings like `YUV420P/NV12/...`). + */ + std::string m_pixel_format = ""; + + /** + * duration of input stream (seconds), only for file stream. + */ + double m_duration = 0.0; + + /** + * channel index of input. + */ + int m_channel_index = -1; + + /** + * type name of hardware used for decoding (`none` if no hardware used). + */ + std::string m_hw_type_name = ""; + + /** + * decoder name used for decoding. + */ + std::string m_decoder_name = ""; + + /** + * uri of input (rtsp/file/...). + */ + std::string m_uri = ""; + + /* inner flags */ + int m_inner_stream_index = -1; + bool m_demux_running = false; + bool m_decode_running = false; + + /* inner methods */ + void demux_run(); + void decode_run(); + void inner_close(); + std::string print_summary(); + bool inner_open(const std::string& uri, const std::string& decoder_name, AVHWDeviceType hw_type); + int hw_decoder_init(AVCodecContext* dec_ctx, const enum AVHWDeviceType type); + + /* callbacks */ + ff_src_opened_hooker m_src_opened_hooker; + public: + /** + * create ff_src instance using initial parameters. + * + * @param channel_index specify the channel index for input stream. + */ + ff_src(int channel_index); + ~ff_src(); + + /* disable copy and assignment operations */ + ff_src(const ff_src&) = delete; + ff_src& operator=(const ff_src&) = delete; + + /** + * try to open ff_src, not thread-safe. + * + * @param uri uri to open, url for network streams or file path for file streams. + * @param decoder_name specify the decoder name used for decoding, MUST supported by FFmpeg. + * @param hw_type type of hardware for decoding, MUST supported by FFmpeg. + * + * @note + * if `decoder_name` not specified, ff_src will choose the default decoder accordding to the codec type of input stream. + */ + bool open(const std::string& uri, + const std::string& decoder_name = "", + AVHWDeviceType hw_type = AVHWDeviceType::AV_HWDEVICE_TYPE_NONE); + /** + * close ff_src. + */ + void close(); + + /** + * if working or not. + * + * @return + * true if it is working. + */ + bool is_opened() const; + + /** + * read the next frame from ff_src, keep as same as cv::VideoCapture. + * + * @param frame frame to be returned (return nullptr if read failed). + * + * @return + * true if read successfully. + * + * @note + * return false means: + * 1. read too quickly no data prepared, please try to read again. + * 2. ff_src not opened yet or not working. + */ + bool read(ff_av_frame_ptr& frame); + + /** + * read the next frame from ff_src, support for `>>` operator. + * + * @param frame frame to be returned (return nullptr if read failed). + * + * @return + * reference for ff_src. + */ + ff_src& operator>>(ff_av_frame_ptr& frame); + + /** + * get fps of input stream. + */ + int get_video_fps() const; + + /** + * get width of input stream in pixels. + */ + int get_video_width() const; + + /** + * get height of input stream in pixels. + */ + int get_video_height() const; + + /** + * get bitrate of input stream (kbit/s). + */ + long get_video_bitrate() const; + + /** + * get codec type name of input stream (strings like `h264/hevc/vp8/...`). + */ + std::string get_video_codec_name() const; + + /** + * get pixel format of input stream (strings like `YUV420P/NV12/...`). + */ + std::string get_video_pixel_format_name() const; + + /** + * get duration of input stream (seconds), only for file stream. + */ + double get_video_duration() const; + + /** + * check is live stream or not. + */ + bool is_live_stream() const; + + /** + * get type name of hardware used for decoding (strings like 'cuda/vaapi', 'none' if no hardware used). + */ + std::string get_hw_type_name() const; + + /** + * get decoder name used for decoding. + */ + std::string get_decoder_name() const; + + /** + * get uri of input. + */ + std::string get_uri() const; + + /** + * get channel index of input. + */ + int get_channel_index() const; + + /** + * set callback for opened event. would be activated every time ff_src opened. + * + * @param src_opened_hooker callback activated when ff_src opened. + */ + void set_src_opened_hooker(ff_src_opened_hooker src_opened_hooker); + }; +} +#endif \ No newline at end of file diff --git a/nodes/ffio/vp_ff_des_node.cpp b/nodes/ffio/vp_ff_des_node.cpp new file mode 100644 index 0000000..b55d322 --- /dev/null +++ b/nodes/ffio/vp_ff_des_node.cpp @@ -0,0 +1,97 @@ +#ifdef VP_WITH_FFMPEG +#include "vp_ff_des_node.h" + +namespace vp_nodes { + vp_ff_des_node:: + vp_ff_des_node(const std::string& node_name, + int channel_index, + const std::string& out_uri, + vp_objects::vp_size resolution_w_h, + int bitrate, + bool osd, + std::string encoder_name): + vp_des_node(node_name, channel_index), + m_out_uri(out_uri), + m_resolution_w_h(resolution_w_h), + m_out_bitrate(bitrate), + m_use_osd(osd), + m_encoder_name(encoder_name) { + m_ff_des = alloc_ff_des(channel_index); + VP_INFO(vp_utils::string_format("[%s] [%s]", node_name.c_str(), out_uri.c_str())); + this->initialized(); + } + + vp_ff_des_node::~vp_ff_des_node() { + deinitialized(); + if (sws_ctx) { + sws_freeContext(sws_ctx); + } + } + + std::shared_ptr vp_ff_des_node::handle_frame_meta(std::shared_ptr meta) { + VP_DEBUG(vp_utils::string_format("[%s] received frame meta, channel_index=>%d, frame_index=>%d", node_name.c_str(), meta->channel_index, meta->frame_index)); + + cv::Mat resize_frame = (m_use_osd && !meta->osd_frame.empty()) ? meta->osd_frame : meta->frame; + auto out_width = resize_frame.cols; + auto out_height = resize_frame.rows; + if (m_resolution_w_h.width != 0 && m_resolution_w_h.height != 0) { + out_width = m_resolution_w_h.width; + out_height = m_resolution_w_h.height; + } + + /* try to open ff_dec. */ + if (!m_ff_des->is_opened()) { + if(!m_ff_des->open(m_out_uri, + out_width, + out_height, + meta->fps, + m_out_bitrate, + 0, + m_encoder_name, + AV_PIX_FMT_YUV420P)) { + VP_WARN(vp_utils::string_format("[%s] could not open ff_des.", node_name.c_str())); + /* general works in vp_des_node. */ + return vp_des_node::handle_frame_meta(meta); + } + } + + /* initialize sws_ctx. */ + if (!sws_ctx) { + sws_ctx = sws_getContext(resize_frame.cols, + resize_frame.rows, + AV_PIX_FMT_BGR24, + out_width, + out_height, + AV_PIX_FMT_YUV420P, + 0, NULL, NULL, NULL); + } + if (!sws_ctx) { + VP_WARN(vp_utils::string_format("[%s] could not initialize sws_ctx.", node_name.c_str())); + /* general works in vp_des_node. */ + return vp_des_node::handle_frame_meta(meta); + } + + /* cv::Mat -> AVFrame. */ + /* resize and convert to AV_PIX_FMT_YUV420P. */ + auto dst_frame = alloc_ff_av_frame(); + dst_frame->width = out_width; + dst_frame->height = out_height; + dst_frame->format= AV_PIX_FMT_YUV420P; + av_frame_get_buffer(dst_frame.get(), 0); + + auto p = resize_frame.data; + int linesize[1] = {resize_frame.cols * 3}; + sws_scale(sws_ctx, &p, linesize, 0, resize_frame.rows, dst_frame->data, dst_frame->linesize); + + /* write to ff_des. */ + m_ff_des->write(dst_frame); + + /* general works in vp_des_node. */ + return vp_des_node::handle_frame_meta(meta); + } + + std::string vp_ff_des_node::to_string() { + return m_out_uri; + } +} +#endif \ No newline at end of file diff --git a/nodes/ffio/vp_ff_des_node.h b/nodes/ffio/vp_ff_des_node.h new file mode 100644 index 0000000..788563e --- /dev/null +++ b/nodes/ffio/vp_ff_des_node.h @@ -0,0 +1,70 @@ +#pragma once +#ifdef VP_WITH_FFMPEG +#include "ff_des.h" +#include "../vp_des_node.h" + +namespace vp_nodes { + /** + * universal DES node using FFmpeg. + * + * support output uri: + * 1. path of file streams like `./vp_data/out_vp_test.mp4`. + * 2. url of network streams like `rtmp://192.168.77.68/live/stream`. + */ + class vp_ff_des_node final: public vp_des_node { + private: + /* inner members. */ + std::string m_out_uri = ""; + bool m_use_osd = true; + int m_out_bitrate = 1024; + std::string m_encoder_name = ""; + vp_objects::vp_size m_resolution_w_h; + + /** + * encode & enmux. + */ + ff_des_ptr m_ff_des = nullptr; + /** + * SwsContext used fot scale by FFmpeg. + */ + SwsContext* sws_ctx = NULL; + protected: + // re-implementation, return nullptr. + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + public: + /** + * create vp_ff_des_node instance using initial parameters. + * + * @param node_name specify the name of DES node. + * @param channel_index specify the channel index of DES node. + * @param out_uri specify the uri to be written. + * @param use_osd specify use osd frame as output or not. + * @param out_width specify the final width of output, 0 means use the width of frame flowing in pipeline. + * @param out_height specify the final height of output, 0 means use the height of frame flowing in pipeline. + * @param out_fps specify the fps of output, 0 means use the fps of original stream in pipeline. + * @param out_bitrate specify the bitrate of output (kbit/s). + * @param out_max_b_frames specify the max B frames in a GOP for encoding. + * @param encoder_name specify the encoder name (`libx264`/`libx265`/`h264_nvenc`/`hevc_nvenc`) used for encoding in FFmpeg. + * @param out_sw_pix_fmt specify the pixel format of output. + * + * @note + * the encoder specified by `encoder_name` MUST be supported already in FFmpeg, + * we can run `ffmpeg -encoders` to show list of encoders supported in FFmpeg. + * if the encoder not found, please reconfigure & rebuild your FFmpeg. + */ + vp_ff_des_node(const std::string& node_name, + int channel_index, + const std::string& out_uri, + vp_objects::vp_size resolution_w_h = {}, + int bitrate = 1024, + bool osd = true, + std::string encoder_name = "libx264"); + ~vp_ff_des_node(); + + /** + * return out uri of DES node. + */ + virtual std::string to_string() override; + }; +} +#endif \ No newline at end of file diff --git a/nodes/ffio/vp_ff_src_node.cpp b/nodes/ffio/vp_ff_src_node.cpp new file mode 100644 index 0000000..538594a --- /dev/null +++ b/nodes/ffio/vp_ff_src_node.cpp @@ -0,0 +1,149 @@ +#ifdef VP_WITH_FFMPEG +#include "vp_ff_src_node.h" + +namespace vp_nodes { + + vp_ff_src_node:: + vp_ff_src_node(const std::string& node_name, + int channel_index, + const std::string& uri, + const std::string& decoder_name, + float resize_ratio, + int skip_interval): + vp_src_node(node_name, channel_index, resize_ratio), + m_uri(uri), + m_decoder_name(decoder_name), + m_skip_interval(skip_interval) { + assert(skip_interval >= 0 && skip_interval <= 9); + m_ff_src = alloc_ff_src(channel_index); + VP_INFO(vp_utils::string_format("[%s] [%s]", node_name.c_str(), uri.c_str())); + this->initialized(); + } + + vp_ff_src_node::~vp_ff_src_node() { + deinitialized(); + } + + void vp_ff_src_node::handle_run() { + SwsContext* sws_ctx = NULL; + auto free_sws_ctx = [&]() { + /* free swsContext. */ + if (sws_ctx) { + sws_freeContext(sws_ctx); + sws_ctx = NULL; + } + }; + auto reopen_wait = 10; + auto reopen_times = 0; + ff_av_frame_ptr src_frame; + int video_width = 0; + int video_height = 0; + int fps = 0; + int skip = 0; + while (alive) { + /* wait for data coming. */ + gate.knock(); + if (!m_ff_src->is_opened()) { + if(!m_ff_src->open(m_uri, m_decoder_name)) { + reopen_times++; + if (reopen_times < 5) { + VP_WARN(vp_utils::string_format("[%s] open uri failed, try again right now...", node_name.c_str())); + } + else { + VP_WARN(vp_utils::string_format("[%s] open uri failed too many times, wait for %d seconds then try again...", node_name.c_str(), reopen_wait)); + std::this_thread::sleep_for(std::chrono::milliseconds(1000 * reopen_wait)); + reopen_times = 0; + } + continue; + } + free_sws_ctx(); + } + + // video properties + if (video_width == 0 || video_height == 0 || fps == 0) { + video_width = m_ff_src->get_video_width(); + video_height = m_ff_src->get_video_height(); + fps = m_ff_src->get_video_fps(); + + original_fps = fps; + original_width = video_width; + original_height = video_height; + + // set true fps because skip some frames + fps = fps / (m_skip_interval + 1); + } + // stream_info_hooker activated if need + vp_stream_info stream_info {channel_index, original_fps, original_width, original_height, to_string()}; + invoke_stream_info_hooker(node_name, stream_info); + + /* try to read next frame from ff_src. */ + if (!m_ff_src->read(src_frame)) { + //VP_WARN(vp_utils::string_format("[%s] reading frame failed, total frame==>%d", node_name.c_str(), frame_index)); + continue; + } + + // need skip + if (skip < m_skip_interval) { + skip++; + continue; + } + skip = 0; + + /* AVFrame -> cv::Mat. */ + /* resize and convert to BGR24. */ + auto n_width = int(src_frame->width * resize_ratio); + auto n_height = int(src_frame->height * resize_ratio); + if (!sws_ctx) { + sws_ctx = sws_getContext(src_frame->width, + src_frame->height, + AVPixelFormat(src_frame->format), + n_width, n_height, + AV_PIX_FMT_BGR24, + 0, NULL, NULL, NULL); + } + if (!sws_ctx) { + VP_WARN(vp_utils::string_format("[%s] could not initialize sws_ctx.", node_name.c_str())); + continue; + } + auto buffer_size = n_width * n_height * 3; + uchar* bgr24 = new uchar[buffer_size]; + int linesize[1] = {n_width * 3}; + sws_scale(sws_ctx, src_frame->data, src_frame->linesize, 0, src_frame->height, &bgr24, linesize); + + cv::Mat frame(n_height, n_width, CV_8UC3, bgr24); + auto c_frame = frame.clone(); + delete[] bgr24; + // set true size because resize + video_width = c_frame.cols; + video_height = c_frame.rows; + + this->frame_index++; + // create frame meta + auto out_meta = + std::make_shared(c_frame, this->frame_index, this->channel_index, video_width, video_height, fps); + + if (out_meta != nullptr) { + this->out_queue.push(out_meta); + + // handled hooker activated if need + if (this->meta_handled_hooker) { + meta_handled_hooker(node_name, out_queue.size(), out_meta); + } + + // important! notify consumer of out_queue in case it is waiting. + this->out_queue_semaphore.signal(); + VP_DEBUG(vp_utils::string_format("[%s] after handling meta, out_queue.size()==>%d", node_name.c_str(), out_queue.size())); + } + } + + free_sws_ctx(); + // send dead flag for dispatch_thread + this->out_queue.push(nullptr); + this->out_queue_semaphore.signal(); + } + + std::string vp_ff_src_node::to_string() { + return m_uri; + } +} +#endif \ No newline at end of file diff --git a/nodes/ffio/vp_ff_src_node.h b/nodes/ffio/vp_ff_src_node.h new file mode 100644 index 0000000..f17fca4 --- /dev/null +++ b/nodes/ffio/vp_ff_src_node.h @@ -0,0 +1,61 @@ + +#pragma once +#ifdef VP_WITH_FFMPEG +#include "ff_src.h" +#include "../vp_src_node.h" + +namespace vp_nodes { + /** + * universal SRC node using FFmpeg. + * + * support uri: + * 1. path of file streams like `./vp_data/vp_test.mp4`. + * 2. url of network streams like `rtsp://192.168.77.68/main_stream`. + */ + class vp_ff_src_node final: public vp_src_node { + private: + /* inner members. */ + std::string m_decoder_name = ""; + std::string m_uri = ""; + // 0 means no skip + int m_skip_interval = 0; + + /** + * demux & decode. + */ + ff_src_ptr m_ff_src = nullptr; + protected: + /** + * get frames using FFmpeg. + */ + virtual void handle_run() override; + public: + /** + * create vp_ff_src_node instance using initial parameters. + * + * @param node_name specify the name of SRC node. + * @param channel_index specify the channel index of SRC node. + * @param uri specify the uri to be opened by SRC node. + * @param decoder_name specify the decoder name (`h264`/`hevc`/`h264_cuvid`/`hevc_cuvid`) used for decoding in FFmpeg. + * @param resize_ratio specify the resize ratio applied to frames. + * + * @note + * the decoder specified by `decoder_name` MUST be supported already in FFmpeg, + * we can run `ffmpeg -decoders` to show list of decoders supported in FFmpeg. + * if the decoder not found, please reconfigure & rebuild your FFmpeg. + */ + vp_ff_src_node(const std::string& node_name, + int channel_index, + const std::string& uri, + const std::string& decoder_name = "h264", + float resize_ratio = 1.0, + int skip_interval = 0); + ~vp_ff_src_node(); + + /** + * return uri of SRC node. + */ + virtual std::string to_string() override; + }; +} +#endif \ No newline at end of file diff --git a/nodes/infers/README.md b/nodes/infers/README.md new file mode 100644 index 0000000..a13c0b8 --- /dev/null +++ b/nodes/infers/README.md @@ -0,0 +1,5 @@ +# Summary + +1. If the inference node works on the whole frame, let it derived from `vp_primary_infer_node` class. +2. If the inference node works on the small cropped images, let it derived from `vp_secondary_infer_node` class. +3. We can define multi derived classes to handle different types of dl models (`detector/pose estimation/classification`). Also, if they work on the same type of target AND with the same logic (for example, hava the same `preprocess/postprocess`), we can use a unique class to load different deep learning models (like resnet18 and resnet50). diff --git a/nodes/infers/vp_classifier_node.cpp b/nodes/infers/vp_classifier_node.cpp new file mode 100755 index 0000000..f9b06bb --- /dev/null +++ b/nodes/infers/vp_classifier_node.cpp @@ -0,0 +1,99 @@ + + +#include "vp_classifier_node.h" + +namespace vp_nodes { + + vp_classifier_node::vp_classifier_node(std::string node_name, + std::string model_path, + std::string model_config_path, + std::string labels_path, + int input_width, + int input_height, + int batch_size, + std::vector p_class_ids_applied_to, + int min_width_applied_to, + int min_height_applied_to, + int crop_padding, + bool need_softmax, + float scale, + cv::Scalar mean, + cv::Scalar std, + bool swap_rb, + bool swap_chn): + vp_secondary_infer_node(node_name, + model_path, + model_config_path, + labels_path, + input_width, + input_height, + batch_size, + p_class_ids_applied_to, + min_width_applied_to, + min_height_applied_to, + crop_padding, + scale, + mean, + std, + swap_rb, + swap_chn), need_softmax(need_softmax){ + this->initialized(); + } + + vp_classifier_node::~vp_classifier_node() { + deinitialized(); + } + + void vp_classifier_node::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + // make sure heads of output are not zero + assert(raw_outputs.size() > 0); + assert(frame_meta_with_batch.size() == 1); + + // just one head of output + auto& output = raw_outputs[0]; + assert(output.dims == 2); + + auto count = output.rows; + auto index = 0; + + auto& frame_meta = frame_meta_with_batch[0]; + cv::Point class_id_point; + int class_id; + double score; + + for (int i = 0; i < count; i++) { + for (int j = index; j < frame_meta->targets.size(); j++) + { + if (!need_apply(frame_meta->targets[j]->primary_class_id, frame_meta->targets[j]->width, frame_meta->targets[j]->height)) { + // continue as its primary_class_id is not in p_class_ids_applied_to + continue; + } + + auto prob = output.row(i); + cv::minMaxLoc(prob, 0, &score, 0, &class_id_point); + + if (need_softmax) { + float maxProb = 0.0; + float sum = 0.0; + cv::Mat softmaxProb; + maxProb = *std::max_element(prob.begin(), prob.end()); + cv::exp(prob - maxProb, softmaxProb); + sum = (float)cv::sum(softmaxProb)[0]; + softmaxProb /= sum; + minMaxLoc(softmaxProb.reshape(1, 1), 0, &score, 0, &class_id_point); + } + class_id = class_id_point.x; + auto label = (labels.size() < class_id + 1) ? "" : labels[class_id]; + + // update back to frame meta + frame_meta->targets[j]->secondary_class_ids.push_back(class_id); + frame_meta->targets[j]->secondary_scores.push_back(score); + frame_meta->targets[j]->secondary_labels.push_back(label); + + // break as we found the right target! + index = j + 1; + break; + } + } + } +} \ No newline at end of file diff --git a/nodes/infers/vp_classifier_node.h b/nodes/infers/vp_classifier_node.h new file mode 100755 index 0000000..be30403 --- /dev/null +++ b/nodes/infers/vp_classifier_node.h @@ -0,0 +1,38 @@ + + +#pragma once + +#include "../vp_secondary_infer_node.h" + +namespace vp_nodes { + // common classifier for image classification task. + // used for image classification, update secondary_class_ids/secondary_labels/secondary_scores of vp_frame_target. + class vp_classifier_node: public vp_secondary_infer_node + { + private: + // softmax logic applied on output or not + bool need_softmax; + protected: + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_classifier_node(std::string node_name, + std::string model_path, + std::string model_config_path = "", + std::string labels_path = "", + int input_width = 128, + int input_height = 128, + int batch_size = 1, + std::vector p_class_ids_applied_to = std::vector(), + int min_width_applied_to = 0, + int min_height_applied_to = 0, + int crop_padding = 10, + bool need_softmax = true, // imagenet dataset + float scale = 1 / 255.0, + cv::Scalar mean = cv::Scalar(123.675, 116.28, 103.53), // imagenet dataset + cv::Scalar std = cv::Scalar(0.229, 0.224, 0.225), + bool swap_rb = true, + bool swap_chn = false); + ~vp_classifier_node(); + }; + +} \ No newline at end of file diff --git a/nodes/infers/vp_enet_seg_node.cpp b/nodes/infers/vp_enet_seg_node.cpp new file mode 100644 index 0000000..b11901d --- /dev/null +++ b/nodes/infers/vp_enet_seg_node.cpp @@ -0,0 +1,33 @@ + +#include "vp_enet_seg_node.h" + +namespace vp_nodes { + + vp_enet_seg_node::vp_enet_seg_node(std::string node_name, + std::string model_path, + std::string model_config_path, + std::string labels_path, + int input_width, + int input_height, + int batch_size, + int class_id_offset, + float scale, + cv::Scalar mean, + cv::Scalar std, + bool swap_rb): + vp_primary_infer_node(node_name, model_path, model_config_path, labels_path, input_width, input_height, batch_size, class_id_offset, scale, mean, std, swap_rb) { + this->initialized(); + } + + vp_enet_seg_node::~vp_enet_seg_node() { + deinitialized(); + } + + void vp_enet_seg_node::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + auto& frame_meta = frame_meta_with_batch[0]; + auto& mask = raw_outputs[0]; + + // save output as mask directly， this is the mask for the whole frame. + frame_meta->mask = mask.clone(); + } +} \ No newline at end of file diff --git a/nodes/infers/vp_enet_seg_node.h b/nodes/infers/vp_enet_seg_node.h new file mode 100644 index 0000000..93aaa27 --- /dev/null +++ b/nodes/infers/vp_enet_seg_node.h @@ -0,0 +1,30 @@ +#pragma once + +#include "../vp_primary_infer_node.h" + + +namespace vp_nodes { + // semantic segmentation based on ENet + // + class vp_enet_seg_node: public vp_primary_infer_node + { + private: + /* data */ + protected: + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_enet_seg_node(std::string node_name, + std::string model_path, + std::string model_config_path = "", + std::string labels_path = "", + int input_width = 1024, + int input_height = 512, + int batch_size = 1, + int class_id_offset = 0, + float scale = 1 / 255.0, + cv::Scalar mean = cv::Scalar(0), + cv::Scalar std = cv::Scalar(1), + bool swap_rb = true); + ~vp_enet_seg_node(); + }; +} \ No newline at end of file diff --git a/nodes/infers/vp_face_swap_node.cpp b/nodes/infers/vp_face_swap_node.cpp new file mode 100644 index 0000000..d4a1935 --- /dev/null +++ b/nodes/infers/vp_face_swap_node.cpp @@ -0,0 +1,528 @@ + +#include "vp_face_swap_node.h" + +namespace vp_nodes { + + vp_face_swap_node::vp_face_swap_node(std::string node_name, + std::string yunet_face_detect_model, + std::string buffalo_l_face_encoding_model, + std::string emap_file_for_embeddings, + std::string insightface_swap_model, + std::string swap_source_image, + int swap_source_face_index, + int min_face_w_h, + bool swap_on_osd, + bool act_as_primary_detector): + vp_primary_infer_node(node_name, ""), + act_as_primary_detector(act_as_primary_detector), + swap_on_osd(swap_on_osd), + min_face_w_h(min_face_w_h) { + /* init net*/ + face_extract_net = cv::dnn::readNetFromONNX(yunet_face_detect_model); + face_encoding_net = cv::dnn::readNetFromONNX(buffalo_l_face_encoding_model); + face_swap_net = cv::dnn::readNetFromONNX(insightface_swap_model); + #ifdef VP_WITH_CUDA + face_extract_net.setPreferableBackend(cv::dnn::DNN_BACKEND_CUDA); + face_extract_net.setPreferableTarget(cv::dnn::DNN_TARGET_CUDA); + face_encoding_net.setPreferableBackend(cv::dnn::DNN_BACKEND_CUDA); + face_encoding_net.setPreferableTarget(cv::dnn::DNN_TARGET_CUDA); + face_swap_net.setPreferableBackend(cv::dnn::DNN_BACKEND_CUDA); + face_swap_net.setPreferableTarget(cv::dnn::DNN_TARGET_CUDA); + #endif + init_source_face_embeddings(swap_source_image, swap_source_face_index, emap_file_for_embeddings); + this->initialized(); + } + + vp_face_swap_node::~vp_face_swap_node() { + deinitialized(); + } + + // please refer to vp_infer_node::run_infer_combinations + void vp_face_swap_node::run_infer_combinations(const std::vector>& frame_meta_with_batch) { + assert(frame_meta_with_batch.size() == 1); + auto& frame_meta = frame_meta_with_batch[0]; + + if (frame_meta->face_targets.size() == 0) { + if (!act_as_primary_detector) { + return; + } + // extract faces and update back to frame meta + // to-do + } + + auto start_time = std::chrono::system_clock::now(); + // iterate each face target + for (int i = 0; i < frame_meta->face_targets.size(); i++) { + if (frame_meta->face_targets[i]->width < min_face_w_h || frame_meta->face_targets[i]->height < min_face_w_h) { + continue; + } + + cv::Mat aligned_face, swapped_face; + + //auto t = std::chrono::system_clock::now(); + // align and crop first + auto warp_mat = align_crop(frame_meta->frame, frame_meta->face_targets[i]->key_points, aligned_face); + //auto T = std::chrono::duration_cast(std::chrono::system_clock::now() - t).count(); + + //std::cout << "1st T:" << T << std::endl; + //t = std::chrono::system_clock::now(); + // swap + swap(aligned_face, swapped_face); + //T = std::chrono::duration_cast(std::chrono::system_clock::now() - t).count(); + //std::cout << "2nd T:" << T << std::endl; + + //t = std::chrono::system_clock::now(); + // past back to frame or osd frame + if (swap_on_osd) { + if (frame_meta->osd_frame.empty()) { + frame_meta->osd_frame = frame_meta->frame.clone(); + } + } + auto& bg = swap_on_osd ? frame_meta->osd_frame : frame_meta->frame; + paste_back(bg, swapped_face, warp_mat); + //T = std::chrono::duration_cast(std::chrono::system_clock::now() - t).count(); + //std::cout << "3rd T:" << T << std::endl; + } + + //auto total_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time).count(); + //std::cout << "total T:" << total_time << std::endl; + } + + cv::Mat vp_face_swap_node::read_emap_mat_from_txt(const std::string& emap_file_for_embeddings, int rows, int cols) { + std::ifstream file(emap_file_for_embeddings); + + cv::Mat matrix(rows, cols, CV_32F); + std::string line; + for (int i = 0; i < rows; ++i) { + std::getline(file, line); + std::istringstream iss(line); + for (int j = 0; j < cols; ++j) { + float value; + assert(iss >> value); + matrix.at(i, j) = value; + } + } + file.close(); + return matrix; + } + + cv::Mat vp_face_swap_node::process_embeddings_using_emap(const cv::Mat& source_face_normed_embedding, const cv::Mat& emap) { + cv::Mat latent = source_face_normed_embedding.clone().reshape(1, 1); + cv::Mat result = latent * emap; + cv::normalize(result, result); + return result; + } + + cv::Mat vp_face_swap_node::get_similarity_transform_matrix(float src[5][2]) { + using namespace cv; + //float dst[5][2] = { {38.2946f, 51.6963f}, {73.5318f, 51.5014f}, {56.0252f, 71.7366f}, {41.5493f, 92.3655f}, {70.7299f, 92.2041f} }; // for 112*112 + float dst[5][2] = { {43.0f, 58.0f}, {85.0f, 58.0f}, {64.0f, 81.0f}, {47.0f, 105.0f}, {80.0f, 105.0f} }; // for 128*128 + //float dst[5][2] = { {38.0f, 54.0f}, {90.0f, 54.0f}, {64.0f, 85.0f}, {47.0f, 109.0f}, {80.0f, 109.0f} }; // for 128*128, zoom out + float avg0 = (src[0][0] + src[1][0] + src[2][0] + src[3][0] + src[4][0]) / 5; + float avg1 = (src[0][1] + src[1][1] + src[2][1] + src[3][1] + src[4][1]) / 5; + //Compute mean of src and dst. + float src_mean[2] = { avg0, avg1 }; + float dst_mean[2] = { 56.0262f, 71.9008f }; + //Subtract mean from src and dst. + float src_demean[5][2]; + for (int i = 0; i < 2; i++) + { + for (int j = 0; j < 5; j++) + { + src_demean[j][i] = src[j][i] - src_mean[i]; + } + } + float dst_demean[5][2]; + for (int i = 0; i < 2; i++) + { + for (int j = 0; j < 5; j++) + { + dst_demean[j][i] = dst[j][i] - dst_mean[i]; + } + } + double A00 = 0.0, A01 = 0.0, A10 = 0.0, A11 = 0.0; + for (int i = 0; i < 5; i++) + A00 += dst_demean[i][0] * src_demean[i][0]; + A00 = A00 / 5; + for (int i = 0; i < 5; i++) + A01 += dst_demean[i][0] * src_demean[i][1]; + A01 = A01 / 5; + for (int i = 0; i < 5; i++) + A10 += dst_demean[i][1] * src_demean[i][0]; + A10 = A10 / 5; + for (int i = 0; i < 5; i++) + A11 += dst_demean[i][1] * src_demean[i][1]; + A11 = A11 / 5; + Mat A = (Mat_(2, 2) << A00, A01, A10, A11); + double d[2] = { 1.0, 1.0 }; + double detA = A00 * A11 - A01 * A10; + if (detA < 0) + d[1] = -1; + double T[3][3] = { {1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0} }; + Mat s, u, vt, v; + SVD::compute(A, s, u, vt); + double smax = s.ptr(0)[0]>s.ptr(1)[0] ? s.ptr(0)[0] : s.ptr(1)[0]; + double tol = smax * 2 * FLT_MIN; + int rank = 0; + if (s.ptr(0)[0]>tol) + rank += 1; + if (s.ptr(1)[0]>tol) + rank += 1; + double arr_u[2][2] = { {u.ptr(0)[0], u.ptr(0)[1]}, {u.ptr(1)[0], u.ptr(1)[1]} }; + double arr_vt[2][2] = { {vt.ptr(0)[0], vt.ptr(0)[1]}, {vt.ptr(1)[0], vt.ptr(1)[1]} }; + double det_u = arr_u[0][0] * arr_u[1][1] - arr_u[0][1] * arr_u[1][0]; + double det_vt = arr_vt[0][0] * arr_vt[1][1] - arr_vt[0][1] * arr_vt[1][0]; + if (rank == 1) + { + if ((det_u*det_vt) > 0) + { + Mat uvt = u*vt; + T[0][0] = uvt.ptr(0)[0]; + T[0][1] = uvt.ptr(0)[1]; + T[1][0] = uvt.ptr(1)[0]; + T[1][1] = uvt.ptr(1)[1]; + } + else + { + double temp = d[1]; + d[1] = -1; + Mat D = (Mat_(2, 2) << d[0], 0.0, 0.0, d[1]); + Mat Dvt = D*vt; + Mat uDvt = u*Dvt; + T[0][0] = uDvt.ptr(0)[0]; + T[0][1] = uDvt.ptr(0)[1]; + T[1][0] = uDvt.ptr(1)[0]; + T[1][1] = uDvt.ptr(1)[1]; + d[1] = temp; + } + } + else + { + Mat D = (Mat_(2, 2) << d[0], 0.0, 0.0, d[1]); + Mat Dvt = D*vt; + Mat uDvt = u*Dvt; + T[0][0] = uDvt.ptr(0)[0]; + T[0][1] = uDvt.ptr(0)[1]; + T[1][0] = uDvt.ptr(1)[0]; + T[1][1] = uDvt.ptr(1)[1]; + } + double var1 = 0.0; + for (int i = 0; i < 5; i++) + var1 += src_demean[i][0] * src_demean[i][0]; + var1 = var1 / 5; + double var2 = 0.0; + for (int i = 0; i < 5; i++) + var2 += src_demean[i][1] * src_demean[i][1]; + var2 = var2 / 5; + double scale = 1.0 / (var1 + var2)* (s.ptr(0)[0] * d[0] + s.ptr(1)[0] * d[1]); + double TS[2]; + TS[0] = T[0][0] * src_mean[0] + T[0][1] * src_mean[1]; + TS[1] = T[1][0] * src_mean[0] + T[1][1] * src_mean[1]; + T[0][2] = dst_mean[0] - scale*TS[0]; + T[1][2] = dst_mean[1] - scale*TS[1]; + T[0][0] *= scale; + T[0][1] *= scale; + T[1][0] *= scale; + T[1][1] *= scale; + Mat transform_mat = (Mat_(2, 3) << T[0][0], T[0][1], T[0][2], T[1][0], T[1][1], T[1][2]); + return transform_mat; + } + + cv::Mat vp_face_swap_node::align_crop(cv::Mat& src_img, std::vector>& kps, cv::Mat& aligned_image) { + float face_keypoints[5][2] = + {{kps[0].first, kps[0].second}, + {kps[1].first, kps[1].second}, + {kps[2].first, kps[2].second}, + {kps[3].first, kps[3].second}, + {kps[4].first, kps[4].second}}; + + cv::Mat warp_mat = get_similarity_transform_matrix(face_keypoints); + cv::warpAffine(src_img, aligned_image, warp_mat, cv::Size(128, 128), cv::INTER_LINEAR); + return warp_mat; + } + + void vp_face_swap_node::extract_faces(const cv::Mat& frame, std::vector& face_boxes) { + auto blob_to_infer = cv::dnn::blobFromImage(frame); + face_extract_net.setInput(blob_to_infer); + const std::vector out_names = {"loc", "conf", "iou"}; + std::vector raw_outputs; + face_extract_net.forward(raw_outputs, out_names); + + using namespace cv; + float scoreThreshold = 0.7; + float nmsThreshold = 0.5; + int topK = 50; + int inputW = frame.cols; + int inputH = frame.rows; + + // 3 heads of output + assert(raw_outputs.size() == 3); + + // Extract from output_blobs + Mat loc = raw_outputs[0]; + Mat conf = raw_outputs[1]; + Mat iou = raw_outputs[2]; + + // we need generate priors if input size changed or priors is not initialized + if (loc.rows != priors.size()) { + generatePriors(inputW, inputH); + } + + assert(loc.rows == priors.size()); + assert(loc.rows == conf.rows); + assert(loc.rows == iou.rows); + + // Decode from deltas and priors + const std::vector variance = {0.1f, 0.2f}; + float* loc_v = (float*)(loc.data); + float* conf_v = (float*)(conf.data); + float* iou_v = (float*)(iou.data); + Mat faces; + // (tl_x, tl_y, w, h, re_x, re_y, le_x, le_y, nt_x, nt_y, rcm_x, rcm_y, lcm_x, lcm_y, score) + // 'tl': top left point of the bounding box + // 're': right eye, 'le': left eye + // 'nt': nose tip + // 'rcm': right corner of mouth, 'lcm': left corner of mouth + Mat face(1, 15, CV_32FC1); + for (size_t i = 0; i < priors.size(); ++i) { + // Get score + float clsScore = conf_v[i*2+1]; + float iouScore = iou_v[i]; + // Clamp + if (iouScore < 0.f) { + iouScore = 0.f; + } + else if (iouScore > 1.f) { + iouScore = 1.f; + } + float score = std::sqrt(clsScore * iouScore); + face.at(0, 14) = score; + + // Get bounding box + float cx = (priors[i].x + loc_v[i*14+0] * variance[0] * priors[i].width) * inputW; + float cy = (priors[i].y + loc_v[i*14+1] * variance[0] * priors[i].height) * inputH; + float w = priors[i].width * exp(loc_v[i*14+2] * variance[0]) * inputW; + float h = priors[i].height * exp(loc_v[i*14+3] * variance[1]) * inputH; + float x1 = cx - w / 2; + float y1 = cy - h / 2; + face.at(0, 0) = x1; + face.at(0, 1) = y1; + face.at(0, 2) = w; + face.at(0, 3) = h; + + // Get landmarks + face.at(0, 4) = (priors[i].x + loc_v[i*14+ 4] * variance[0] * priors[i].width) * inputW; // right eye, x + face.at(0, 5) = (priors[i].y + loc_v[i*14+ 5] * variance[0] * priors[i].height) * inputH; // right eye, y + face.at(0, 6) = (priors[i].x + loc_v[i*14+ 6] * variance[0] * priors[i].width) * inputW; // left eye, x + face.at(0, 7) = (priors[i].y + loc_v[i*14+ 7] * variance[0] * priors[i].height) * inputH; // left eye, y + face.at(0, 8) = (priors[i].x + loc_v[i*14+ 8] * variance[0] * priors[i].width) * inputW; // nose tip, x + face.at(0, 9) = (priors[i].y + loc_v[i*14+ 9] * variance[0] * priors[i].height) * inputH; // nose tip, y + face.at(0, 10) = (priors[i].x + loc_v[i*14+10] * variance[0] * priors[i].width) * inputW; // right corner of mouth, x + face.at(0, 11) = (priors[i].y + loc_v[i*14+11] * variance[0] * priors[i].height) * inputH; // right corner of mouth, y + face.at(0, 12) = (priors[i].x + loc_v[i*14+12] * variance[0] * priors[i].width) * inputW; // left corner of mouth, x + face.at(0, 13) = (priors[i].y + loc_v[i*14+13] * variance[0] * priors[i].height) * inputH; // left corner of mouth, y + + faces.push_back(face); + } + + if (faces.rows > 1) { + // Retrieve boxes and scores + std::vector faceBoxes; + std::vector faceScores; + for (int rIdx = 0; rIdx < faces.rows; rIdx++) + { + faceBoxes.push_back(Rect2i(int(faces.at(rIdx, 0)), + int(faces.at(rIdx, 1)), + int(faces.at(rIdx, 2)), + int(faces.at(rIdx, 3)))); + faceScores.push_back(faces.at(rIdx, 14)); + } + + std::vector keepIdx; + dnn::NMSBoxes(faceBoxes, faceScores, scoreThreshold, nmsThreshold, keepIdx, 1.f, topK); + + // Get NMS results + Mat nms_faces; + for (int idx: keepIdx) + { + nms_faces.push_back(faces.row(idx)); + } + + // insert face target back to frame meta + for (int i = 0; i < nms_faces.rows; i++) { + auto x = int(nms_faces.at(i, 0)); + auto y = int(nms_faces.at(i, 1)); + auto w = int(nms_faces.at(i, 2)); + auto h = int(nms_faces.at(i, 3)); + + // check value range + x = std::max(x, 0); + y = std::max(y, 0); + w = std::min(w, frame.cols - x); + h = std::min(h, frame.rows - y); + + auto kp1 = std::pair(int(nms_faces.at(i, 4)), int(nms_faces.at(i, 5))); + auto kp2 = std::pair(int(nms_faces.at(i, 6)), int(nms_faces.at(i, 7))); + auto kp3 = std::pair(int(nms_faces.at(i, 8)), int(nms_faces.at(i, 9))); + auto kp4 = std::pair(int(nms_faces.at(i, 10)), int(nms_faces.at(i, 11))); + auto kp5 = std::pair(int(nms_faces.at(i, 12)), int(nms_faces.at(i, 13))); + auto score = nms_faces.at(i, 14); + + face_box face; + face.x = x; + face.y = y; + face.width = w; + face.height = h; + face.score = score; + face.kps = std::vector>{kp1, kp2, kp3, kp4, kp5}; + face_boxes.push_back(face); + } + } + } + + void vp_face_swap_node::init_source_face_embeddings(std::string& swap_source_image, int swap_source_face_index, std::string& emap_file_for_embeddings) { + std::vector source_faces; + auto source_mat = cv::imread(swap_source_image); + + // extract faces + extract_faces(source_mat, source_faces); + + assert(source_faces.size() > 0); + // sort from left to right + std::sort(source_faces.begin(), source_faces.end(), [](face_box a, face_box b){ return a.x < b.x;}); + + auto the_selected_face = (swap_source_face_index < 0 || swap_source_face_index >= source_faces.size()) ? source_faces[0] : source_faces[swap_source_face_index]; + cv::Mat aligned_face; + auto warp_mat = align_crop(source_mat, the_selected_face.kps, aligned_face); + // for debug + cv::imwrite("selected_source_face.jpg", aligned_face); + + // read emap from file + auto emap = read_emap_mat_from_txt(emap_file_for_embeddings); + + // encoding for the selected face (infer for only 1 time) + cv::Mat source_blob = cv::dnn::blobFromImage(aligned_face, 1 / 127.5, cv::Size(112, 112), (127.5, 127.5, 127.5), true); + std::vector source_outputs; + face_encoding_net.setInput(source_blob); + face_encoding_net.forward(source_outputs, face_encoding_net.getUnconnectedOutLayersNames()); + + auto& source_output = source_outputs[0]; + // process using emap + source_face_embeddings = process_embeddings_using_emap(source_output, emap); + } + + void vp_face_swap_node::swap(cv::Mat& aligned_face, cv::Mat& swapped_face) { + cv::Mat target_blob = cv::dnn::blobFromImage(aligned_face, 1 / 255.0, cv::Size(128, 128), (0, 0, 0), true); + std::vector target_outputs; + face_swap_net.setInput(target_blob, "target"); + face_swap_net.setInput(source_face_embeddings, "source"); + face_swap_net.forward(target_outputs, face_swap_net.getUnconnectedOutLayersNames()); + + auto& output = target_outputs[0]; + cv::Mat output_channel_last; + cv::transposeND(output, {0,2,3,1}, output_channel_last); + + cv::Mat img_fake(output_channel_last.size[1], output_channel_last.size[2], CV_32FC3, output_channel_last.data); + cv::cvtColor(img_fake, swapped_face, cv::COLOR_RGB2BGR); + } + + void vp_face_swap_node::paste_back(cv::Mat& bg, cv::Mat& swapped_face, const cv::Mat& transform_matrix) { + bg.convertTo(bg, CV_32FC3, 1.0 / 255); + + cv::Mat IM; + cv::invertAffineTransform(transform_matrix, IM); + cv::Mat img_mask(swapped_face.rows, swapped_face.cols, CV_32FC1, 1); + cv::warpAffine(swapped_face, swapped_face, IM, bg.size()); + cv::warpAffine(img_mask, img_mask, IM, bg.size()); + + // create mask + cv::threshold(img_mask, img_mask, 0, 1, cv::THRESH_BINARY); + cv::Point min_loc, max_loc; + double min_val, max_val; + cv::minMaxLoc(img_mask, &min_val, &max_val, &min_loc, &max_loc); + + int mask_h = max_loc.y - min_loc.y; + int mask_w = max_loc.x - min_loc.x; + + int mask_size = std::sqrt(mask_h * mask_w); + int k = std::max(mask_size / 10, 10); + + cv::Mat kernel = cv::getStructuringElement(cv::MORPH_RECT, cv::Size(k, k)); + cv::erode(img_mask, img_mask, kernel); + + k = std::max(mask_size / 20, 5); + cv::GaussianBlur(img_mask, img_mask, cv::Size(2 * k + 1, 2 * k + 1), 0); + cv::cvtColor(img_mask, img_mask, cv::COLOR_GRAY2BGR); + + // merge swapped face and original background + cv::Mat ones(bg.rows, bg.cols, CV_32FC3, cv::Scalar(1, 1, 1)); + bg = img_mask.mul(swapped_face) + (ones - img_mask).mul(bg); + bg.convertTo(bg, CV_8U, 255); + } + + void vp_face_swap_node::generatePriors(int inputW, int inputH) { + using namespace cv; + // Calculate shapes of different scales according to the shape of input image + Size feature_map_2nd = { + int(int((inputW+1)/2)/2), int(int((inputH+1)/2)/2) + }; + Size feature_map_3rd = { + int(feature_map_2nd.width/2), int(feature_map_2nd.height/2) + }; + Size feature_map_4th = { + int(feature_map_3rd.width/2), int(feature_map_3rd.height/2) + }; + Size feature_map_5th = { + int(feature_map_4th.width/2), int(feature_map_4th.height/2) + }; + Size feature_map_6th = { + int(feature_map_5th.width/2), int(feature_map_5th.height/2) + }; + + std::vector feature_map_sizes; + feature_map_sizes.push_back(feature_map_3rd); + feature_map_sizes.push_back(feature_map_4th); + feature_map_sizes.push_back(feature_map_5th); + feature_map_sizes.push_back(feature_map_6th); + + // Fixed params for generating priors + const std::vector> min_sizes = { + {10.0f, 16.0f, 24.0f}, + {32.0f, 48.0f}, + {64.0f, 96.0f}, + {128.0f, 192.0f, 256.0f} + }; + CV_Assert(min_sizes.size() == feature_map_sizes.size()); // just to keep vectors in sync + const std::vector steps = { 8, 16, 32, 64 }; + + // Generate priors + priors.clear(); + for (size_t i = 0; i < feature_map_sizes.size(); ++i) + { + Size feature_map_size = feature_map_sizes[i]; + std::vector min_size = min_sizes[i]; + + for (int _h = 0; _h < feature_map_size.height; ++_h) + { + for (int _w = 0; _w < feature_map_size.width; ++_w) + { + for (size_t j = 0; j < min_size.size(); ++j) + { + float s_kx = min_size[j] / inputW; + float s_ky = min_size[j] / inputH; + + float cx = (_w + 0.5f) * steps[i] / inputW; + float cy = (_h + 0.5f) * steps[i] / inputH; + + Rect2f prior = { cx, cy, s_kx, s_ky }; + priors.push_back(prior); + } + } + } + } + } + + void vp_face_swap_node::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + + } +} \ No newline at end of file diff --git a/nodes/infers/vp_face_swap_node.h b/nodes/infers/vp_face_swap_node.h new file mode 100644 index 0000000..9009b66 --- /dev/null +++ b/nodes/infers/vp_face_swap_node.h @@ -0,0 +1,65 @@ +#pragma once + +#include "../vp_primary_infer_node.h" + +namespace vp_nodes { + // face swap node + // used to swap faces in videos/images using a specific face + class vp_face_swap_node: public vp_primary_infer_node + { + private: + // inner temporary use + struct face_box { + int x; + int y; + int width; + int height; + float score; + std::vector> kps; + }; + + /* onnx network using opencv::dnn as backend */ + cv::dnn::Net face_extract_net; + cv::dnn::Net face_encoding_net; + cv::dnn::Net face_swap_net; + + cv::Mat read_emap_mat_from_txt(const std::string& emap_file_for_embeddings, int rows = 512, int cols = 512); + cv::Mat process_embeddings_using_emap(const cv::Mat& source_face_normed_embedding, const cv::Mat& emap); + + cv::Mat get_similarity_transform_matrix(float src[5][2]); + cv::Mat align_crop(cv::Mat& _src_img, std::vector>& kps, cv::Mat& _aligned_img); + + void extract_faces(const cv::Mat& frame, std::vector& faces); + void init_source_face_embeddings(std::string& swap_source_image, int swap_source_face_index, std::string& emap_file_for_embeddings); + void swap(cv::Mat& aligned_face, cv::Mat& swapped_face); + void paste_back(cv::Mat& bg, cv::Mat& swapped_face, const cv::Mat& transform_matrix); + + cv::Mat source_face_embeddings; + bool act_as_primary_detector = false; + bool swap_on_osd = true; + + int min_face_w_h = 50; + + /* used for extract faces */ + std::vector priors; + void generatePriors(int inputW, int inputH); + protected: + // we need a totally new logic for the whole infer combinations + // no separate step pre-defined needed in base class + virtual void run_infer_combinations(const std::vector>& frame_meta_with_batch) override; + // override pure virtual method, for compile pass + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_face_swap_node(std::string node_name, + std::string yunet_face_detect_model, + std::string buffalo_l_face_encoding_model, + std::string emap_file_for_embeddings, + std::string insightface_swap_model, + std::string swap_source_image, + int swap_source_face_index = 0, + int min_face_w_h = 50, + bool swap_on_osd = true, + bool act_as_primary_detector = false); + ~vp_face_swap_node(); + }; +} \ No newline at end of file diff --git a/nodes/infers/vp_feature_encoder_node.cpp b/nodes/infers/vp_feature_encoder_node.cpp new file mode 100755 index 0000000..6836bd5 --- /dev/null +++ b/nodes/infers/vp_feature_encoder_node.cpp @@ -0,0 +1,15 @@ + +#include "vp_feature_encoder_node.h" + + +namespace vp_nodes { + + vp_feature_encoder_node::vp_feature_encoder_node(std::string node_name, std::string model_path): + vp_secondary_infer_node(node_name, model_path) { + this->initialized(); + } + + vp_feature_encoder_node::~vp_feature_encoder_node() { + deinitialized(); + } +} \ No newline at end of file diff --git a/nodes/infers/vp_feature_encoder_node.h b/nodes/infers/vp_feature_encoder_node.h new file mode 100755 index 0000000..eefcce5 --- /dev/null +++ b/nodes/infers/vp_feature_encoder_node.h @@ -0,0 +1,18 @@ + +#pragma once + +#include "../vp_secondary_infer_node.h" + +namespace vp_nodes { + // common feature encoder for image feature extraction. + // used for feature extraction, update embeddings of vp_frame_target. + class vp_feature_encoder_node: public vp_secondary_infer_node + { + private: + /* data */ + public: + vp_feature_encoder_node(std::string node_name, std::string model_path); + ~vp_feature_encoder_node(); + }; + +} \ No newline at end of file diff --git a/nodes/infers/vp_lane_detector_node.cpp b/nodes/infers/vp_lane_detector_node.cpp new file mode 100644 index 0000000..8ba056c --- /dev/null +++ b/nodes/infers/vp_lane_detector_node.cpp @@ -0,0 +1,34 @@ + +#include "vp_lane_detector_node.h" + +namespace vp_nodes { + + vp_lane_detector_node::vp_lane_detector_node(std::string node_name, + std::string model_path, + std::string model_config_path, + std::string labels_path, + int input_width, + int input_height, + int batch_size, + int class_id_offset, + float scale, + cv::Scalar mean, + cv::Scalar std, + bool swap_rb, + bool swap_chn): + vp_primary_infer_node(node_name, model_path, model_config_path, labels_path, input_width, input_height, batch_size, class_id_offset, scale, mean, std, swap_rb, swap_chn) { + this->initialized(); + } + + vp_lane_detector_node::~vp_lane_detector_node() { + deinitialized(); + } + + void vp_lane_detector_node::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + auto& frame_meta = frame_meta_with_batch[0]; + auto& mask = raw_outputs[0]; + + // save output as mask directly， this is the mask for the whole frame. + frame_meta->mask = mask.clone(); + } +} \ No newline at end of file diff --git a/nodes/infers/vp_lane_detector_node.h b/nodes/infers/vp_lane_detector_node.h new file mode 100644 index 0000000..a24847f --- /dev/null +++ b/nodes/infers/vp_lane_detector_node.h @@ -0,0 +1,31 @@ +#pragma once + +#include "../vp_primary_infer_node.h" + + +namespace vp_nodes { + // lane detect based on CenterNet + // + class vp_lane_detector_node: public vp_primary_infer_node + { + private: + /* data */ + protected: + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_lane_detector_node(std::string node_name, + std::string model_path, + std::string model_config_path = "", + std::string labels_path = "", + int input_width = 736, + int input_height = 416, + int batch_size = 1, + int class_id_offset = 0, + float scale = 1, + cv::Scalar mean = cv::Scalar(0), + cv::Scalar std = cv::Scalar(1), + bool swap_rb = true, + bool swap_chn = true); + ~vp_lane_detector_node(); + }; +} \ No newline at end of file diff --git a/nodes/infers/vp_mask_rcnn_detector_node.cpp b/nodes/infers/vp_mask_rcnn_detector_node.cpp new file mode 100644 index 0000000..a74f620 --- /dev/null +++ b/nodes/infers/vp_mask_rcnn_detector_node.cpp @@ -0,0 +1,89 @@ + +#include "vp_mask_rcnn_detector_node.h" + + +namespace vp_nodes { + vp_mask_rcnn_detector_node::vp_mask_rcnn_detector_node(std::string node_name, + std::string model_path, + std::string model_config_path, + std::string labels_path, + int input_width, + int input_height, + int batch_size, + int class_id_offset, + float score_threshold, + float scale, + cv::Scalar mean, + cv::Scalar std, + bool swap_rb): + vp_primary_infer_node(node_name, model_path, model_config_path, labels_path, input_width, input_height, batch_size, class_id_offset, scale, mean, std, swap_rb), + score_threshold(score_threshold) { + this->initialized(); + } + + vp_mask_rcnn_detector_node::~vp_mask_rcnn_detector_node() { + deinitialized(); + } + + + void vp_mask_rcnn_detector_node::infer(const cv::Mat& blob_to_infer, std::vector& raw_outputs) { + // blob_to_infer is a 4D matrix + // the first dim is number of batch, MUST be 1 + assert(blob_to_infer.dims == 4); + assert(blob_to_infer.size[0] == 1); + assert(!net.empty()); + + net.setInput(blob_to_infer); + net.forward(raw_outputs, out_names); + } + + + void vp_mask_rcnn_detector_node::preprocess(const std::vector& mats_to_infer, cv::Mat& blob_to_infer) { + // ignore preprocess logic in base class + cv::dnn::blobFromImages(mats_to_infer, blob_to_infer, 1.0, cv::Size(), cv::Scalar(), true, false); + } + + void vp_mask_rcnn_detector_node::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + auto& frame_meta = frame_meta_with_batch[0]; + + auto outDetections = raw_outputs[0]; + auto outMasks = raw_outputs[1]; + + // Output size of masks is NxCxHxW where + // N - number of detected boxes + // C - number of classes (excluding background) + // HxW - segmentation shape + const int numDetections = outDetections.size[2]; + const int numClasses = outMasks.size[1]; + + auto outDetections_ = outDetections.reshape(1, outDetections.total() / 7); + auto& frame = frame_meta->frame; + for (int i = 0; i < numDetections; ++i) { + float score = outDetections_.at(i, 2); + if (score > score_threshold) { + // Extract the bounding box + int classId = static_cast(outDetections_.at(i, 1)); + int left = static_cast(frame.cols * outDetections_.at(i, 3)); + int top = static_cast(frame.rows * outDetections_.at(i, 4)); + int right = static_cast(frame.cols * outDetections_.at(i, 5)); + int bottom = static_cast(frame.rows * outDetections_.at(i, 6)); + + left = max(0, min(left, frame.cols - 1)); + top = max(0, min(top, frame.rows - 1)); + right = max(0, min(right, frame.cols - 1)); + bottom = max(0, min(bottom, frame.rows - 1)); + + auto label = (labels.size() < classId + 1) ? "" : labels[classId]; + + // Extract the mask for the object + cv::Mat objectMask(outMasks.size[2], outMasks.size[3], CV_32F, outMasks.ptr(i, classId)); + + classId += class_id_offset; + auto target = std::make_shared(left, top, right - left, bottom - top, classId, score, frame_meta->frame_index, frame_meta->channel_index, label); + target->mask = objectMask; + + frame_meta->targets.push_back(target); + } + } + } +} \ No newline at end of file diff --git a/nodes/infers/vp_mask_rcnn_detector_node.h b/nodes/infers/vp_mask_rcnn_detector_node.h new file mode 100644 index 0000000..7aa3e56 --- /dev/null +++ b/nodes/infers/vp_mask_rcnn_detector_node.h @@ -0,0 +1,41 @@ + +#pragma once + +#include "../vp_primary_infer_node.h" +#include "../../objects/vp_frame_target.h" + + + +namespace vp_nodes { + // image segmentation based on Mask RCNN + // https://github.com/matterport/Mask_RCNN + class vp_mask_rcnn_detector_node: public vp_primary_infer_node + { + private: + /* data */ + // names of output layers in mask_rcnn + const std::vector out_names = {"detection_out_final", "detection_masks"}; + float score_threshold = 0.5; + protected: + // override infer and preprocess as mask_rcnn has a different logic + virtual void infer(const cv::Mat& blob_to_infer, std::vector& raw_outputs) override; + virtual void preprocess(const std::vector& mats_to_infer, cv::Mat& blob_to_infer) override; + + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_mask_rcnn_detector_node(std::string node_name, + std::string model_path, + std::string model_config_path = "", + std::string labels_path = "", + int input_width = 416, + int input_height = 416, + int batch_size = 1, + int class_id_offset = 0, + float score_threshold = 0.5, + float scale = 1 / 255.0, + cv::Scalar mean = cv::Scalar(0), + cv::Scalar std = cv::Scalar(1), + bool swap_rb = true); + ~vp_mask_rcnn_detector_node(); + }; +} \ No newline at end of file diff --git a/nodes/infers/vp_mllm_analyser_node.cpp b/nodes/infers/vp_mllm_analyser_node.cpp new file mode 100644 index 0000000..e413a1b --- /dev/null +++ b/nodes/infers/vp_mllm_analyser_node.cpp @@ -0,0 +1,34 @@ +#include "vp_mllm_analyser_node.h" + +#ifdef VP_WITH_LLM +namespace vp_nodes { + vp_mllm_analyser_node::vp_mllm_analyser_node(std::string node_name, + std::string model_name, + std::string prompt, + std::string api_base_url, + std::string api_key, + llmlib::LLMBackendType backend_type): + vp_primary_infer_node(node_name, ""), + llm_model_name(model_name), + llm_prompt(prompt) { + cli = llmlib::LLMClient(api_base_url, api_key, backend_type); + this->initialized(); + } + + vp_mllm_analyser_node::~vp_mllm_analyser_node() { + deinitialized(); + } + + void vp_mllm_analyser_node::run_infer_combinations(const std::vector>& frame_meta_with_batch) { + assert(frame_meta_with_batch.size() == 1); + auto& frame_meta = frame_meta_with_batch[0]; + + auto output = cli.simple_chat(llm_model_name, llm_prompt, {frame_meta->frame}, {}); + frame_meta->description = output; + } + + void vp_mllm_analyser_node::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + + } +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_mllm_analyser_node.h b/nodes/infers/vp_mllm_analyser_node.h new file mode 100644 index 0000000..6447ed4 --- /dev/null +++ b/nodes/infers/vp_mllm_analyser_node.h @@ -0,0 +1,32 @@ +#pragma once + +#ifdef VP_WITH_LLM +#include "../vp_primary_infer_node.h" +#include "../../third_party/cpp_llmlib/llmlib.hpp" + +namespace vp_nodes { + // image(frame) analyser based on Multimodal Large Language Model + class vp_mllm_analyser_node: public vp_primary_infer_node + { + private: + /* data */ + llmlib::LLMClient cli; + std::string llm_prompt; + std::string llm_model_name; + protected: + // we need a totally new logic for the whole infer combinations + // no separate step pre-defined needed in base class + virtual void run_infer_combinations(const std::vector>& frame_meta_with_batch) override; + // override pure virtual method, for compile pass + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_mllm_analyser_node(std::string node_name, + std::string model_name, + std::string prompt, + std::string api_base_url, + std::string api_key = "", + llmlib::LLMBackendType backend_type = llmlib::LLMBackendType::Ollama); + ~vp_mllm_analyser_node(); + }; +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_openpose_detector_node.cpp b/nodes/infers/vp_openpose_detector_node.cpp new file mode 100755 index 0000000..33b249a --- /dev/null +++ b/nodes/infers/vp_openpose_detector_node.cpp @@ -0,0 +1,107 @@ +#include + +#include "vp_openpose_detector_node.h" + +namespace vp_nodes { + + vp_openpose_detector_node::vp_openpose_detector_node(std::string node_name, + std::string model_path, + std::string model_config_path, + std::string labels_path, + int input_width, + int input_height, + int batch_size, + int class_id_offset, + float score_threshold, + vp_objects::vp_pose_type type, + float scale, + cv::Scalar mean, + cv::Scalar std, + bool swap_rb): + vp_primary_infer_node(node_name, + model_path, + model_config_path, + labels_path, + input_width, + input_height, + batch_size, + class_id_offset, + scale, mean, + std, + swap_rb), + score_threshold(score_threshold), type(type) { + this->initialized(); + } + + vp_openpose_detector_node::~vp_openpose_detector_node() { + deinitialized(); + } + + void vp_openpose_detector_node::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + // make sure heads of output are not zero + assert(raw_outputs.size() > 0); + // as it just one head of output + auto& netOutputBlobs = raw_outputs[0]; + // batch, number of heatmaps, h, w + assert(netOutputBlobs.dims == 4); + + auto start1 = std::chrono::system_clock::now(); + + int netOutputBlob_dims[3] = {netOutputBlobs.size[1], netOutputBlobs.size[2], netOutputBlobs.size[3]}; + // scan batch + for (int b = 0; b < netOutputBlobs.size[0]; b++) { + auto& frame_meta = frame_meta_with_batch[b]; + //auto h_scale = frame_meta->frame.rows * 1.0 / netOutputBlobs.size[2]; + //auto w_scale = frame_meta->frame.cols * 1.0 / netOutputBlobs.size[3]; + + cv::Mat netOutputBlob = cv::Mat(3, netOutputBlob_dims, CV_32F, const_cast(netOutputBlobs.ptr(b))); + + int keyPointId = 0; + std::vector> detectedKeypoints; + std::vector keyPointsList; + + auto start2 = std::chrono::system_clock::now(); + std::vector netOutputParts; + splitNetOutputBlobToParts(netOutputBlob, cv::Size(frame_meta->frame.cols, frame_meta->frame.rows), netOutputParts); + + auto start3 = std::chrono::system_clock::now(); + for(int i = 0; i < points_map.at(type); ++i) { + std::vector keyPoints; + getKeyPoints(netOutputParts[i], score_threshold, keyPoints); + + for(int i = 0; i< keyPoints.size(); ++i, ++keyPointId) { + keyPoints[i].id = keyPointId; + } + + detectedKeypoints.push_back(keyPoints); + keyPointsList.insert(keyPointsList.end(), keyPoints.begin(), keyPoints.end()); + } + + std::vector> validPairs; + std::set invalidPairs; + getValidPairs(netOutputParts, detectedKeypoints, validPairs, invalidPairs); + + std::vector> personwiseKeypoints; + getPersonwiseKeypoints(validPairs, invalidPairs, personwiseKeypoints); + + // insert pose targets back into frame meta + for(int n = 0; n < personwiseKeypoints.size(); ++n) { + std::vector kps; + for (int i = 0; i < personwiseKeypoints[n].size(); i++) { + auto index = personwiseKeypoints[n][i]; + if (index != -1) { + auto& p = keyPointsList[index]; + kps.push_back(vp_objects::vp_pose_keypoint {i, p.point.x, p.point.y, p.probability}); + } + else { + // point not detected + kps.push_back(vp_objects::vp_pose_keypoint {i, -1, -1, 0}); + } + } + + auto pose_target = std::make_shared(type, kps); + frame_meta->pose_targets.push_back(pose_target); + } + } + } +} \ No newline at end of file diff --git a/nodes/infers/vp_openpose_detector_node.h b/nodes/infers/vp_openpose_detector_node.h new file mode 100755 index 0000000..1ed0442 --- /dev/null +++ b/nodes/infers/vp_openpose_detector_node.h @@ -0,0 +1,293 @@ + +#pragma once + +#include +#include +#include +#include + +#include "../vp_primary_infer_node.h" +#include "../../objects/vp_frame_pose_target.h" + + +namespace vp_nodes { + // body keypoints detector using openpose + // https://github.com/CMU-Perceptual-Computing-Lab/openpose + class vp_openpose_detector_node: public vp_primary_infer_node + { + private: + float score_threshold; + // pose type (model type) + vp_objects::vp_pose_type type; + + // map indexs for PAFs + const std::map>> mapIdxes_map = { + {vp_objects::vp_pose_type::body_25, {{0,1}, {14,15}, {22,23}, {16,17}, {18,19}, {24,25}, {26,27}, {6,7}, {2,3}, {4,5}, {8,9}, {10,11}, {12,13}, {30,31}, {32,33}, {36,37}, {34,35}, + {38,39}, {40,41} ,{42,43}, {44,45}, {46,47}, {48,49}, {50,51}}}, + {vp_objects::vp_pose_type::coco, {{12,13}, {20,21}, {14,15}, {16,17}, {22,23}, {24,25}, {0,1}, + {2,3}, {4,5}, {6,7}, {8,9}, {10,11}, {28,29}, {30,31}, {34,35}, {32,33}, {36,37}, {18,19}, {26,27}}}, + {vp_objects::vp_pose_type::mpi_15, {{0,1}, {2,3}, {4,5}, {6,7}, {8,9}, {10,11}, {12,13}, {14,15}, {16,17}, {18,19}, {20,21}, {22,23}, {24,25}, {26,27}}}, + {vp_objects::vp_pose_type::hand, std::vector>()}, + {vp_objects::vp_pose_type::face, std::vector>()} + }; + + // pose pairs for PAFs + const std::map>> posePairs_map = { + {vp_objects::vp_pose_type::body_25, {{1,8}, {1,2}, {1,5}, {2,3}, {3,4}, {5,6}, {6,7}, {8,9}, + {9,10}, {10,11}, {8,12}, {12,13}, {13,14}, {1,0}, {0,15}, {15,17}, {0,16}, {16,18}, + {14,19}, {19,20}, {14,21}, {11,22}, {22,23}, {11,24}}}, + {vp_objects::vp_pose_type::coco, {{1,2}, {1,5}, {2,3}, {3,4}, {5,6}, {6,7}, + {1,8}, {8,9}, {9,10}, {1,11}, {11,12}, {12,13}, + {1,0}, {0,14}, {14,16}, {0,15}, {15,17}, {2,16}, + {5,17}}}, + {vp_objects::vp_pose_type::mpi_15, {{0,1}, {1,2}, {2,3}, {3,4}, {1,5}, {5,6}, {6,7}, {1,14}, {14,8}, {8,9}, {9,10}, {14,11}, {11,12}, {12,13}, {0, 2}, {0, 5}}}, + {vp_objects::vp_pose_type::hand, std::vector>()}, + {vp_objects::vp_pose_type::face, std::vector>()} + }; + + // points count for each type of pose model + const std::map points_map = { + {vp_objects::vp_pose_type::body_25, 25}, + {vp_objects::vp_pose_type::coco, 18}, + {vp_objects::vp_pose_type::mpi_15, 15}, + {vp_objects::vp_pose_type::hand, 0}, + {vp_objects::vp_pose_type::face, 0} + }; + + // for postprocess purpose + struct ValidPair{ + ValidPair(int aId,int bId,float score) { + this->aId = aId; + this->bId = bId; + this->score = score; + } + + int aId; + int bId; + float score; + }; + struct KeyPoint{ + KeyPoint(cv::Point point, float probability) { + this->id = -1; + this->point = point; + this->probability = probability; + } + + int id; + cv::Point point; + float probability; + }; + // for postprocess purpose end + + void getKeyPoints(cv::Mat& probMap, double threshold, std::vector& keyPoints) { + cv::Mat smoothProbMap; + cv::GaussianBlur(probMap, smoothProbMap, cv::Size( 3, 3 ), 0, 0); + + cv::Mat maskedProbMap; + cv::threshold(smoothProbMap, maskedProbMap, threshold, 255, cv::THRESH_BINARY); + + maskedProbMap.convertTo(maskedProbMap, CV_8U, 1); + + std::vector > contours; + cv::findContours(maskedProbMap, contours, cv::RETR_TREE, cv::CHAIN_APPROX_SIMPLE); + + for(int i = 0; i < contours.size();++i) { + cv::Mat blobMask = cv::Mat::zeros(smoothProbMap.rows, smoothProbMap.cols, smoothProbMap.type()); + cv::fillConvexPoly(blobMask, contours[i], cv::Scalar(1)); + + double maxVal; + cv::Point maxLoc; + + cv::minMaxLoc(smoothProbMap.mul(blobMask), 0, &maxVal, 0, &maxLoc); + keyPoints.push_back(KeyPoint(maxLoc, probMap.at(maxLoc.y, maxLoc.x))); + } + } + + void splitNetOutputBlobToParts(cv::Mat& netOutputBlob, const cv::Size& targetSize, std::vector& netOutputParts) { + int nParts = netOutputBlob.size[0]; + int h = netOutputBlob.size[1]; + int w = netOutputBlob.size[2]; + + for(int i = 0; i< nParts; ++i) { + cv::Mat part(h, w, CV_32F, netOutputBlob.ptr(i)); + + cv::Mat resizedPart; + + cv::resize(part, resizedPart, targetSize); + + netOutputParts.push_back(resizedPart); + } + } + + void populateInterpPoints(const cv::Point& a,const cv::Point& b,int numPoints,std::vector& interpCoords){ + float xStep = ((float)(b.x - a.x)) / (float)(numPoints - 1); + float yStep = ((float)(b.y - a.y)) / (float)(numPoints - 1); + interpCoords.push_back(a); + + for(int i = 1; i< numPoints - 1; ++i) { + interpCoords.push_back(cv::Point(a.x + xStep * i, a.y + yStep * i)); + } + interpCoords.push_back(b); + } + + void getValidPairs(const std::vector& netOutputParts, + const std::vector>& detectedKeypoints, + std::vector>& validPairs, + std::set& invalidPairs) { + + int nInterpSamples = 10; + float confTh = 0.7; + + for(int k = 0; k < mapIdxes_map.at(type).size(); ++k) { + //A->B constitute a limb + cv::Mat pafA = netOutputParts[mapIdxes_map.at(type)[k].first + points_map.at(type) + 1]; + cv::Mat pafB = netOutputParts[mapIdxes_map.at(type)[k].second + points_map.at(type) + 1]; + + //Find the keypoints for the first and second limb + const std::vector& candA = detectedKeypoints[posePairs_map.at(type)[k].first]; + const std::vector& candB = detectedKeypoints[posePairs_map.at(type)[k].second]; + + int nA = candA.size(); + int nB = candB.size(); + + /* + # If keypoints for the joint-pair is detected + # check every joint in candA with every joint in candB + # Calculate the distance vector between the two joints + # Find the PAF values at a set of interpolated points between the joints + # Use the above formula to compute a score to mark the connection valid + */ + + if(nA != 0 && nB != 0){ + std::vector localValidPairs; + + for(int i = 0; i< nA; ++i) { + int maxJ = -1; + float maxScore = -1; + bool found = false; + + for(int j = 0; j < nB; ++j) { + std::pair distance(candB[j].point.x - candA[i].point.x, candB[j].point.y - candA[i].point.y); + + float norm = std::sqrt(distance.first * distance.first + distance.second * distance.second); + + if(!norm) { + continue; + } + + distance.first /= norm; + distance.second /= norm; + + //Find p(u) + std::vector interpCoords; + populateInterpPoints(candA[i].point, candB[j].point, nInterpSamples, interpCoords); + //Find L(p(u)) + std::vector> pafInterp; + for(int l = 0; l < interpCoords.size();++l) { + pafInterp.push_back( + std::pair( + pafA.at(interpCoords[l].y, interpCoords[l].x), + pafB.at(interpCoords[l].y, interpCoords[l].x) + )); + } + + std::vector pafScores; + float sumOfPafScores = 0; + int numOverTh = 0; + for(int l = 0; l< pafInterp.size(); ++l){ + float score = pafInterp[l].first * distance.first + pafInterp[l].second * distance.second; + sumOfPafScores += score; + if(score > score_threshold){ + ++numOverTh; + } + + pafScores.push_back(score); + } + + float avgPafScore = sumOfPafScores / ((float)pafInterp.size()); + + if(((float)numOverTh)/((float)nInterpSamples) > confTh){ + if(avgPafScore > maxScore) { + maxJ = j; + maxScore = avgPafScore; + found = true; + } + } + }/* j */ + + if(found) { + localValidPairs.push_back(ValidPair(candA[i].id, candB[maxJ].id, maxScore)); + } + + }/* i */ + + validPairs.push_back(localValidPairs); + + } else { + invalidPairs.insert(k); + validPairs.push_back(std::vector()); + } + }/* k */ + } + + void getPersonwiseKeypoints(const std::vector>& validPairs, + const std::set& invalidPairs, + std::vector>& personwiseKeypoints) { + for(int k = 0; k < mapIdxes_map.at(type).size(); ++k) { + if(invalidPairs.find(k) != invalidPairs.end()) { + continue; + } + + const std::vector& localValidPairs(validPairs[k]); + + int indexA(posePairs_map.at(type)[k].first); + int indexB(posePairs_map.at(type)[k].second); + + for(int i = 0; i< localValidPairs.size(); ++i) { + bool found = false; + int personIdx = -1; + + for(int j = 0; !found && j < personwiseKeypoints.size();++j) { + if(indexA < personwiseKeypoints[j].size() && + personwiseKeypoints[j][indexA] == localValidPairs[i].aId) { + personIdx = j; + found = true; + } + }/* j */ + + if(found) { + personwiseKeypoints[personIdx].at(indexB) = localValidPairs[i].bId; + } + else if(k < points_map.at(type) - 1) { + std::vector lpkp(std::vector(points_map.at(type), -1)); + + lpkp.at(indexA) = localValidPairs[i].aId; + lpkp.at(indexB) = localValidPairs[i].bId; + + personwiseKeypoints.push_back(lpkp); + } + + }/* i */ + }/* k */ + } + + protected: + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_openpose_detector_node(std::string node_name, + std::string model_path, + std::string model_config_path = "", + std::string labels_path = "", + int input_width = 368, + int input_height = 368, + int batch_size = 1, + int class_id_offset = 0, + float score_threshold = 0.1, + vp_objects::vp_pose_type type = vp_objects::vp_pose_type::coco, + float scale = 1 / 255.0, + cv::Scalar mean = cv::Scalar(0), + cv::Scalar std = cv::Scalar(1), + bool swap_rb = false); + ~vp_openpose_detector_node(); + }; +} \ No newline at end of file diff --git a/nodes/infers/vp_ppocr_text_detector_node.cpp b/nodes/infers/vp_ppocr_text_detector_node.cpp new file mode 100644 index 0000000..9f79aa6 --- /dev/null +++ b/nodes/infers/vp_ppocr_text_detector_node.cpp @@ -0,0 +1,66 @@ + +#ifdef VP_WITH_PADDLE +#include "vp_ppocr_text_detector_node.h" +#include "../../objects/vp_frame_text_target.h" + +namespace vp_nodes { + + vp_ppocr_text_detector_node::vp_ppocr_text_detector_node(std::string node_name, + std::string det_model_dir, + std::string cls_model_dir, + std::string rec_model_dir, + std::string rec_char_dict_path): + vp_primary_infer_node(node_name, "") { + // to make the code simpler, paddle_ocr has no more config other than model path + // we need modify source code at ../../third_party/paddle_ocr/ if we need tune the parameters + ocr = std::make_shared(det_model_dir, cls_model_dir, rec_model_dir, rec_char_dict_path); + this->initialized(); + } + + vp_ppocr_text_detector_node::~vp_ppocr_text_detector_node() { + deinitialized(); + } + + void vp_ppocr_text_detector_node::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + + } + + // please refer to vp_infer_node::run_infer_combinations + void vp_ppocr_text_detector_node::run_infer_combinations(const std::vector>& frame_meta_with_batch) { + assert(frame_meta_with_batch.size() == 1); + std::vector mats_to_infer; + + // start + auto start_time = std::chrono::system_clock::now(); + + // prepare data, as same as base class + vp_primary_infer_node::prepare(frame_meta_with_batch, mats_to_infer); + auto prepare_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + start_time = std::chrono::system_clock::now(); + // call paddle ocr + auto ocr_results = ocr->ocr(mats_to_infer); + assert(ocr_results.size() == 1); + + auto& ocr_result = ocr_results[0]; + auto& frame_meta = frame_meta_with_batch[0]; + + // scan text detected in frame + for (int i = 0; i < ocr_result.size(); i++) { + /* code */ + auto& text = ocr_result[i]; + std::vector> region_vertexes = {{text.box[0][0], text.box[0][1]}, + {text.box[1][0], text.box[1][1]}, + {text.box[2][0], text.box[2][1]}, + {text.box[3][0], text.box[3][1]}}; + // create text target and update back into frame meta + auto text_target = std::make_shared(region_vertexes, text.text, text.score); + frame_meta->text_targets.push_back(text_target); + } + auto infer_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + // can not calculate preprocess time and postprocess time, set 0 by default. + vp_infer_node::infer_combinations_time_cost(mats_to_infer.size(), prepare_time.count(), 0, infer_time.count(), 0); + } +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_ppocr_text_detector_node.h b/nodes/infers/vp_ppocr_text_detector_node.h new file mode 100644 index 0000000..0a13711 --- /dev/null +++ b/nodes/infers/vp_ppocr_text_detector_node.h @@ -0,0 +1,33 @@ +#pragma once + +#ifdef VP_WITH_PADDLE +#include "../vp_primary_infer_node.h" +#include "../../third_party/paddle_ocr/include/paddleocr.h" + +namespace vp_nodes { + // ocr based on paddle ocr + // paddle ocr project(official): https://github.com/PaddlePaddle/PaddleOCR + // source code(modified based on official): ../../third_party/paddle_ocr + // note: + // this class is not based on opencv::dnn module but paddle, a few data members declared in base class are not usable any more(just ignore), such as vp_infer_node::net. + class vp_ppocr_text_detector_node: public vp_primary_infer_node + { + private: + // paddle ocr instance + std::shared_ptr ocr; + protected: + // we need a totally new logic for the whole infer combinations + // no separate step pre-defined needed in base class + virtual void run_infer_combinations(const std::vector>& frame_meta_with_batch) override; + // override pure virtual method, for compile pass + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_ppocr_text_detector_node(std::string node_name, + std::string det_model_dir = "", + std::string cls_model_dir = "", + std::string rec_model_dir = "", + std::string rec_char_dict_path = ""); + ~vp_ppocr_text_detector_node(); + }; +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_restoration_node.cpp b/nodes/infers/vp_restoration_node.cpp new file mode 100644 index 0000000..b3122a6 --- /dev/null +++ b/nodes/infers/vp_restoration_node.cpp @@ -0,0 +1,52 @@ + +#include "vp_restoration_node.h" + +namespace vp_nodes { + + vp_restoration_node::vp_restoration_node(std::string node_name, + std::string realesrgan_bg_restoration_model, + std::string face_restoration_model, + bool restoration_to_osd): + vp_primary_infer_node(node_name, ""), + restoration_to_osd(restoration_to_osd) { + /* init net*/ + restoration_net = cv::dnn::readNetFromONNX(realesrgan_bg_restoration_model); + /* to-do, load face restoration model*/ + #ifdef VP_WITH_CUDA + //restoration_net.setPreferableBackend(cv::dnn::DNN_BACKEND_CUDA); + //restoration_net.setPreferableTarget(cv::dnn::DNN_TARGET_CUDA); + #endif + this->initialized(); + } + + vp_restoration_node::~vp_restoration_node() { + deinitialized(); + } + + // please refer to vp_infer_node::run_infer_combinations + void vp_restoration_node::run_infer_combinations(const std::vector>& frame_meta_with_batch) { + assert(frame_meta_with_batch.size() == 1); + auto& frame_meta = frame_meta_with_batch[0]; + + // 4x larger for RealESRGAN_x4plus model + cv::Mat target_blob = cv::dnn::blobFromImage(frame_meta->frame, 1 / 255.0, cv::Size(frame_meta->frame.cols, frame_meta->frame.rows), (0, 0, 0), true); + std::vector target_outputs; + restoration_net.setInput(target_blob); + restoration_net.forward(target_outputs, restoration_net.getUnconnectedOutLayersNames()); + + // parse to image + auto& output = target_outputs[0]; + cv::Mat output_channel_last; + cv::transposeND(output, {0,2,3,1}, output_channel_last); + cv::Mat img_result(output_channel_last.size[1], output_channel_last.size[2], CV_32FC3, output_channel_last.data); + img_result.convertTo(img_result, CV_8U, 255); + + // update back to frame meta + auto& bg = restoration_to_osd ? frame_meta->osd_frame : frame_meta->frame; + cv::cvtColor(img_result, bg, cv::COLOR_RGB2BGR); + } + + void vp_restoration_node::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + + } +} \ No newline at end of file diff --git a/nodes/infers/vp_restoration_node.h b/nodes/infers/vp_restoration_node.h new file mode 100644 index 0000000..da02d26 --- /dev/null +++ b/nodes/infers/vp_restoration_node.h @@ -0,0 +1,27 @@ +#pragma once + +#include "../vp_primary_infer_node.h" + +namespace vp_nodes { + // general image restoration node using Real-ESRGAN + // used to enhance quality of frames in video， see more: https://github.com/xinntao/Real-ESRGAN + class vp_restoration_node: public vp_primary_infer_node + { + private: + /* onnx network using opencv::dnn as backend */ + cv::dnn::Net restoration_net; + bool restoration_to_osd = true; + protected: + // we need a totally new logic for the whole infer combinations + // no separate step pre-defined needed in base class + virtual void run_infer_combinations(const std::vector>& frame_meta_with_batch) override; + // override pure virtual method, for compile pass + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_restoration_node(std::string node_name, + std::string realesrgan_bg_restoration_model, + std::string face_restoration_model = "", + bool restoration_to_osd = true); + ~vp_restoration_node(); + }; +} \ No newline at end of file diff --git a/nodes/infers/vp_sface_feature_encoder_node.cpp b/nodes/infers/vp_sface_feature_encoder_node.cpp new file mode 100644 index 0000000..93f24ca --- /dev/null +++ b/nodes/infers/vp_sface_feature_encoder_node.cpp @@ -0,0 +1,183 @@ + +#include +#include +#include "vp_sface_feature_encoder_node.h" + +namespace vp_nodes { + + vp_sface_feature_encoder_node::vp_sface_feature_encoder_node(std::string node_name, std::string model_path): + vp_secondary_infer_node(node_name, model_path, + "", "", + 112, 112, + 1, std::vector(), 0, 0, + 0, 1, cv::Scalar()) { + this->initialized(); + } + + vp_sface_feature_encoder_node::~vp_sface_feature_encoder_node() { + deinitialized(); + } + + void vp_sface_feature_encoder_node::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + // make sure heads of output are not zero + assert(raw_outputs.size() > 0); + assert(frame_meta_with_batch.size() == 1); + + // just one head of output + auto& output = raw_outputs[0]; + assert(output.dims == 2); + + auto count = output.rows; + auto& frame_meta = frame_meta_with_batch[0]; + + // update feature data back into frame meta + for (int i = 0; i < count; i++) { + cv::Mat feature = output.row(i); + + for (int j = 0; j < feature.cols; j++) { + frame_meta->face_targets[i]->embeddings.push_back(feature.at(0, j)); + } + } + } + + // refer to vp_secondary_infer_node::prepare + void vp_sface_feature_encoder_node::prepare(const std::vector>& frame_meta_with_batch, std::vector& mats_to_infer) { + // only one by one for secondary infer node + assert(frame_meta_with_batch.size() == 1); + + // align face and crop + auto& frame_meta = frame_meta_with_batch[0]; + + // batch by batch inside single frame + for (auto& i : frame_meta->face_targets) { + // align and crop + float face_keypoints[5][2] = + {{i->key_points[0].first, i->key_points[0].second}, + {i->key_points[1].first, i->key_points[1].second}, + {i->key_points[2].first, i->key_points[2].second}, + {i->key_points[3].first, i->key_points[3].second}, + {i->key_points[4].first, i->key_points[4].second}}; + cv::Mat aligned_face; + alignCrop(frame_meta->frame, face_keypoints, aligned_face); + mats_to_infer.push_back(aligned_face); + } + } + + cv::Mat vp_sface_feature_encoder_node::getSimilarityTransformMatrix(float src[5][2]) { + using namespace cv; + float dst[5][2] = { {38.2946f, 51.6963f}, {73.5318f, 51.5014f}, {56.0252f, 71.7366f}, {41.5493f, 92.3655f}, {70.7299f, 92.2041f} }; + float avg0 = (src[0][0] + src[1][0] + src[2][0] + src[3][0] + src[4][0]) / 5; + float avg1 = (src[0][1] + src[1][1] + src[2][1] + src[3][1] + src[4][1]) / 5; + //Compute mean of src and dst. + float src_mean[2] = { avg0, avg1 }; + float dst_mean[2] = { 56.0262f, 71.9008f }; + //Subtract mean from src and dst. + float src_demean[5][2]; + for (int i = 0; i < 2; i++) + { + for (int j = 0; j < 5; j++) + { + src_demean[j][i] = src[j][i] - src_mean[i]; + } + } + float dst_demean[5][2]; + for (int i = 0; i < 2; i++) + { + for (int j = 0; j < 5; j++) + { + dst_demean[j][i] = dst[j][i] - dst_mean[i]; + } + } + double A00 = 0.0, A01 = 0.0, A10 = 0.0, A11 = 0.0; + for (int i = 0; i < 5; i++) + A00 += dst_demean[i][0] * src_demean[i][0]; + A00 = A00 / 5; + for (int i = 0; i < 5; i++) + A01 += dst_demean[i][0] * src_demean[i][1]; + A01 = A01 / 5; + for (int i = 0; i < 5; i++) + A10 += dst_demean[i][1] * src_demean[i][0]; + A10 = A10 / 5; + for (int i = 0; i < 5; i++) + A11 += dst_demean[i][1] * src_demean[i][1]; + A11 = A11 / 5; + Mat A = (Mat_(2, 2) << A00, A01, A10, A11); + double d[2] = { 1.0, 1.0 }; + double detA = A00 * A11 - A01 * A10; + if (detA < 0) + d[1] = -1; + double T[3][3] = { {1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0} }; + Mat s, u, vt, v; + SVD::compute(A, s, u, vt); + double smax = s.ptr(0)[0]>s.ptr(1)[0] ? s.ptr(0)[0] : s.ptr(1)[0]; + double tol = smax * 2 * FLT_MIN; + int rank = 0; + if (s.ptr(0)[0]>tol) + rank += 1; + if (s.ptr(1)[0]>tol) + rank += 1; + double arr_u[2][2] = { {u.ptr(0)[0], u.ptr(0)[1]}, {u.ptr(1)[0], u.ptr(1)[1]} }; + double arr_vt[2][2] = { {vt.ptr(0)[0], vt.ptr(0)[1]}, {vt.ptr(1)[0], vt.ptr(1)[1]} }; + double det_u = arr_u[0][0] * arr_u[1][1] - arr_u[0][1] * arr_u[1][0]; + double det_vt = arr_vt[0][0] * arr_vt[1][1] - arr_vt[0][1] * arr_vt[1][0]; + if (rank == 1) + { + if ((det_u*det_vt) > 0) + { + Mat uvt = u*vt; + T[0][0] = uvt.ptr(0)[0]; + T[0][1] = uvt.ptr(0)[1]; + T[1][0] = uvt.ptr(1)[0]; + T[1][1] = uvt.ptr(1)[1]; + } + else + { + double temp = d[1]; + d[1] = -1; + Mat D = (Mat_(2, 2) << d[0], 0.0, 0.0, d[1]); + Mat Dvt = D*vt; + Mat uDvt = u*Dvt; + T[0][0] = uDvt.ptr(0)[0]; + T[0][1] = uDvt.ptr(0)[1]; + T[1][0] = uDvt.ptr(1)[0]; + T[1][1] = uDvt.ptr(1)[1]; + d[1] = temp; + } + } + else + { + Mat D = (Mat_(2, 2) << d[0], 0.0, 0.0, d[1]); + Mat Dvt = D*vt; + Mat uDvt = u*Dvt; + T[0][0] = uDvt.ptr(0)[0]; + T[0][1] = uDvt.ptr(0)[1]; + T[1][0] = uDvt.ptr(1)[0]; + T[1][1] = uDvt.ptr(1)[1]; + } + double var1 = 0.0; + for (int i = 0; i < 5; i++) + var1 += src_demean[i][0] * src_demean[i][0]; + var1 = var1 / 5; + double var2 = 0.0; + for (int i = 0; i < 5; i++) + var2 += src_demean[i][1] * src_demean[i][1]; + var2 = var2 / 5; + double scale = 1.0 / (var1 + var2)* (s.ptr(0)[0] * d[0] + s.ptr(1)[0] * d[1]); + double TS[2]; + TS[0] = T[0][0] * src_mean[0] + T[0][1] * src_mean[1]; + TS[1] = T[1][0] * src_mean[0] + T[1][1] * src_mean[1]; + T[0][2] = dst_mean[0] - scale*TS[0]; + T[1][2] = dst_mean[1] - scale*TS[1]; + T[0][0] *= scale; + T[0][1] *= scale; + T[1][0] *= scale; + T[1][1] *= scale; + Mat transform_mat = (Mat_(2, 3) << T[0][0], T[0][1], T[0][2], T[1][0], T[1][1], T[1][2]); + return transform_mat; + } + + void vp_sface_feature_encoder_node::alignCrop(cv::Mat& _src_img, float _src_point[5][2], cv::Mat& _aligned_img) { + cv::Mat warp_mat = getSimilarityTransformMatrix(_src_point); + cv::warpAffine(_src_img, _aligned_img, warp_mat, cv::Size(input_width, input_height), cv::INTER_LINEAR); + } +} \ No newline at end of file diff --git a/nodes/infers/vp_sface_feature_encoder_node.h b/nodes/infers/vp_sface_feature_encoder_node.h new file mode 100644 index 0000000..30940e7 --- /dev/null +++ b/nodes/infers/vp_sface_feature_encoder_node.h @@ -0,0 +1,27 @@ + +#pragma once + +#include "../vp_secondary_infer_node.h" + +namespace vp_nodes { + // face feature encoder based on SFace, update embeddings of vp_frame_face_target + // https://github.com/opencv/opencv/blob/4.x/modules/objdetect/src/face_recognize.cpp + // https://github.com/zhongyy/SFace + class vp_sface_feature_encoder_node: public vp_secondary_infer_node + { + private: + // get transform matrix for aglin face + cv::Mat getSimilarityTransformMatrix(float src[5][2]); + // align and crop + void alignCrop(cv::Mat& _src_img, float _src_point[5][2], cv::Mat& _aligned_img); + protected: + // override prepare as sface has an additional logic for face align before preprocess + virtual void prepare(const std::vector>& frame_meta_with_batch, std::vector& mats_to_infer) override; + + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_sface_feature_encoder_node(std::string node_name, std::string model_path); + ~vp_sface_feature_encoder_node(); + }; + + } \ No newline at end of file diff --git a/nodes/infers/vp_trt_vehicle_color_classifier.cpp b/nodes/infers/vp_trt_vehicle_color_classifier.cpp new file mode 100644 index 0000000..85188ec --- /dev/null +++ b/nodes/infers/vp_trt_vehicle_color_classifier.cpp @@ -0,0 +1,64 @@ +#ifdef VP_WITH_TRT +#include "vp_trt_vehicle_color_classifier.h" + +namespace vp_nodes { + vp_trt_vehicle_color_classifier::vp_trt_vehicle_color_classifier(std::string node_name, + std::string vehicle_color_cls_model_path, + std::vector p_class_ids_applied_to, + int min_width_applied_to, int min_height_applied_to): + vp_secondary_infer_node(node_name, "", "", "", 1, 1, 1, p_class_ids_applied_to, min_width_applied_to, min_height_applied_to) { + vehicle_color_classifier = std::make_shared(vehicle_color_cls_model_path); + this->initialized(); + } + + vp_trt_vehicle_color_classifier::~vp_trt_vehicle_color_classifier() { + deinitialized(); + } + + void vp_trt_vehicle_color_classifier::run_infer_combinations(const std::vector>& frame_meta_with_batch) { + assert(frame_meta_with_batch.size() == 1); + std::vector mats_to_infer; + + // start + auto start_time = std::chrono::system_clock::now(); + + // prepare data, as same as base class + vp_secondary_infer_node::prepare(frame_meta_with_batch, mats_to_infer); + auto prepare_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + // infer using trt_vehicle library + start_time = std::chrono::system_clock::now(); + std::vector vehicles_color; + vehicle_color_classifier->classify(mats_to_infer, vehicles_color); + + auto& frame_meta = frame_meta_with_batch[0]; + auto index = 0; + for (int i = 0; i < vehicles_color.size(); i++) { + for (int j = index; j < frame_meta->targets.size(); j++) { + // need apply or not? + if (!need_apply(frame_meta->targets[j]->primary_class_id, frame_meta->targets[j]->width, frame_meta->targets[j]->height)) { + // continue as its primary_class_id is not in p_class_ids_applied_to + continue; + } + + // update back to frame meta + frame_meta->targets[j]->secondary_class_ids.push_back(vehicles_color[i].class_); + frame_meta->targets[j]->secondary_scores.push_back(vehicles_color[i].score); + frame_meta->targets[j]->secondary_labels.push_back(vehicles_color[i].label); + + // break as we found the right target! + index = j + 1; + break; + } + } + auto infer_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + // can not calculate preprocess time and postprocess time, set 0 by default. + vp_infer_node::infer_combinations_time_cost(mats_to_infer.size(), prepare_time.count(), 0, infer_time.count(), 0); + } + + void vp_trt_vehicle_color_classifier::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + + } +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_trt_vehicle_color_classifier.h b/nodes/infers/vp_trt_vehicle_color_classifier.h new file mode 100644 index 0000000..1d0dba3 --- /dev/null +++ b/nodes/infers/vp_trt_vehicle_color_classifier.h @@ -0,0 +1,26 @@ +#pragma once + +#ifdef VP_WITH_TRT +#include "../vp_secondary_infer_node.h" +#include "../../third_party/trt_vehicle/models/vehicle_color_classifier.h" + +namespace vp_nodes { + // vehicle color classifier based on tensorrt using trt_vehicle library + // update secondary_class_ids/secondary_labels/secondary_scores of vp_frame_target. + class vp_trt_vehicle_color_classifier: public vp_secondary_infer_node + { + private: + /* data */ + std::shared_ptr vehicle_color_classifier = nullptr; + protected: + // we need a totally new logic for the whole infer combinations + // no separate step pre-defined needed in base class + virtual void run_infer_combinations(const std::vector>& frame_meta_with_batch) override; + // override pure virtual method, for compile pass + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_trt_vehicle_color_classifier(std::string node_name, std::string vehicle_color_cls_model_path = "", std::vector p_class_ids_applied_to = std::vector(), int min_width_applied_to = 0, int min_height_applied_to = 0); + ~vp_trt_vehicle_color_classifier(); + }; +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_trt_vehicle_detector.cpp b/nodes/infers/vp_trt_vehicle_detector.cpp new file mode 100644 index 0000000..fda2c05 --- /dev/null +++ b/nodes/infers/vp_trt_vehicle_detector.cpp @@ -0,0 +1,64 @@ +#ifdef VP_WITH_TRT +#include "vp_trt_vehicle_detector.h" + +namespace vp_nodes { + + vp_trt_vehicle_detector::vp_trt_vehicle_detector(std::string node_name, + std::string vehicle_det_model_path): + vp_primary_infer_node(node_name, "") { + vehicle_detector = std::make_shared(vehicle_det_model_path); + this->initialized(); + } + + vp_trt_vehicle_detector::~vp_trt_vehicle_detector() { + deinitialized(); + } + + // please refer to vp_infer_node::run_infer_combinations + void vp_trt_vehicle_detector::run_infer_combinations(const std::vector>& frame_meta_with_batch) { + assert(frame_meta_with_batch.size() == 1); + std::vector mats_to_infer; + + // start + auto start_time = std::chrono::system_clock::now(); + + // prepare data, as same as base class + vp_primary_infer_node::prepare(frame_meta_with_batch, mats_to_infer); + auto prepare_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + start_time = std::chrono::system_clock::now(); + std::vector> vehicles; + vehicle_detector->detect(mats_to_infer, vehicles); + + assert(vehicles.size() == 1); + auto& vehicle_list = vehicles[0]; + auto& frame_meta = frame_meta_with_batch[0]; + + for (int i = 0; i < vehicle_list.size(); i++) { + auto& objbox = vehicle_list[i]; + + // check value range + objbox.x = std::max(objbox.x, 0); + objbox.y = std::max(objbox.y, 0); + objbox.width = std::min(objbox.width, frame_meta->frame.cols - objbox.x); + objbox.height = std::min(objbox.height, frame_meta->frame.rows - objbox.y); + if (objbox.width <= 0 || objbox.height <= 0) { + continue; + } + + auto target = std::make_shared(objbox.x, objbox.y, objbox.width, objbox.height, + objbox.class_, objbox.score, frame_meta->frame_index, frame_meta->channel_index, objbox.label); + // create target and update back into frame meta + frame_meta->targets.push_back(target); + } + auto infer_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + // can not calculate preprocess time and postprocess time, set 0 by default. + vp_infer_node::infer_combinations_time_cost(mats_to_infer.size(), prepare_time.count(), 0, infer_time.count(), 0); + } + + void vp_trt_vehicle_detector::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + + } +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_trt_vehicle_detector.h b/nodes/infers/vp_trt_vehicle_detector.h new file mode 100644 index 0000000..2fdddf0 --- /dev/null +++ b/nodes/infers/vp_trt_vehicle_detector.h @@ -0,0 +1,24 @@ +#pragma once + +#ifdef VP_WITH_TRT +#include "../vp_primary_infer_node.h" +#include "../../third_party/trt_vehicle/models/vehicle_detector.h" + +namespace vp_nodes { + // vehicle detector based on tensorrt using trt_vehicle library + class vp_trt_vehicle_detector: public vp_primary_infer_node + { + private: + std::shared_ptr vehicle_detector = nullptr; + protected: + // we need a totally new logic for the whole infer combinations + // no separate step pre-defined needed in base class + virtual void run_infer_combinations(const std::vector>& frame_meta_with_batch) override; + // override pure virtual method, for compile pass + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_trt_vehicle_detector(std::string node_name, std::string vehicle_det_model_path = ""); + ~vp_trt_vehicle_detector(); + }; +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_trt_vehicle_feature_encoder.cpp b/nodes/infers/vp_trt_vehicle_feature_encoder.cpp new file mode 100644 index 0000000..24a05a3 --- /dev/null +++ b/nodes/infers/vp_trt_vehicle_feature_encoder.cpp @@ -0,0 +1,65 @@ +#ifdef VP_WITH_TRT +#include "vp_trt_vehicle_feature_encoder.h" + +namespace vp_nodes { + vp_trt_vehicle_feature_encoder::vp_trt_vehicle_feature_encoder(std::string node_name, + std::string vehicle_feature_model_path, + std::vector p_class_ids_applied_to, + int min_width_applied_to, int min_height_applied_to): + vp_secondary_infer_node(node_name, "", "", "", 1, 1, 1, p_class_ids_applied_to, min_width_applied_to, min_height_applied_to) { + vehicle_feature_encoder = std::make_shared(vehicle_feature_model_path); + this->initialized(); + } + + vp_trt_vehicle_feature_encoder::~vp_trt_vehicle_feature_encoder() { + deinitialized(); + } + + void vp_trt_vehicle_feature_encoder::run_infer_combinations(const std::vector>& frame_meta_with_batch) { + assert(frame_meta_with_batch.size() == 1); + std::vector mats_to_infer; + + // start + auto start_time = std::chrono::system_clock::now(); + + // prepare data, as same as base class + vp_secondary_infer_node::prepare(frame_meta_with_batch, mats_to_infer); + auto prepare_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + // infer using trt_vehicle library + start_time = std::chrono::system_clock::now(); + std::vector> vehicles_feature; + vehicle_feature_encoder->encode(mats_to_infer, vehicles_feature); + + auto& frame_meta = frame_meta_with_batch[0]; + auto index = 0; + for (int i = 0; i < vehicles_feature.size(); i++) { + for (int j = index; j < frame_meta->targets.size(); j++) { + // need apply or not? + if (!need_apply(frame_meta->targets[j]->primary_class_id, frame_meta->targets[j]->width, frame_meta->targets[j]->height)) { + // continue as its primary_class_id is not in p_class_ids_applied_to + continue; + } + + // update back to frame meta + auto& features = vehicles_feature[i]; + for(int k = 0; k < features.size(); k++) { + frame_meta->targets[j]->embeddings.push_back(features[k]); + } + + // break as we found the right target! + index = j + 1; + break; + } + } + auto infer_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + // can not calculate preprocess time and postprocess time, set 0 by default. + vp_infer_node::infer_combinations_time_cost(mats_to_infer.size(), prepare_time.count(), 0, infer_time.count(), 0); + } + + void vp_trt_vehicle_feature_encoder::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + + } +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_trt_vehicle_feature_encoder.h b/nodes/infers/vp_trt_vehicle_feature_encoder.h new file mode 100644 index 0000000..99c7a2d --- /dev/null +++ b/nodes/infers/vp_trt_vehicle_feature_encoder.h @@ -0,0 +1,26 @@ +#pragma once + +#ifdef VP_WITH_TRT +#include "../vp_secondary_infer_node.h" +#include "../../third_party/trt_vehicle/models/vehicle_feature_encoder.h" + +namespace vp_nodes { + // vehicle feature encoder based on tensorrt using trt_vehicle library + // update embeddings of vp_frame_target + class vp_trt_vehicle_feature_encoder: public vp_secondary_infer_node + { + private: + /* data */ + std::shared_ptr vehicle_feature_encoder = nullptr; + protected: + // we need a totally new logic for the whole infer combinations + // no separate step pre-defined needed in base class + virtual void run_infer_combinations(const std::vector>& frame_meta_with_batch) override; + // override pure virtual method, for compile pass + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_trt_vehicle_feature_encoder(std::string node_name, std::string vehicle_feature_model_path = "", std::vector p_class_ids_applied_to = std::vector(), int min_width_applied_to = 0, int min_height_applied_to = 0); + ~vp_trt_vehicle_feature_encoder(); + }; +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_trt_vehicle_plate_detector.cpp b/nodes/infers/vp_trt_vehicle_plate_detector.cpp new file mode 100644 index 0000000..15e49e3 --- /dev/null +++ b/nodes/infers/vp_trt_vehicle_plate_detector.cpp @@ -0,0 +1,71 @@ +#ifdef VP_WITH_TRT +#include "vp_trt_vehicle_plate_detector.h" + +namespace vp_nodes { + + vp_trt_vehicle_plate_detector::vp_trt_vehicle_plate_detector(std::string node_name, + std::string plate_det_model_path, + std::string char_rec_model_path): + vp_secondary_infer_node(node_name, "") { + plate_detector = std::make_shared(plate_det_model_path, char_rec_model_path); + this->initialized(); + } + + vp_trt_vehicle_plate_detector::~vp_trt_vehicle_plate_detector() { + deinitialized(); + } + + void vp_trt_vehicle_plate_detector::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + + } + + // please refer to vp_infer_node::run_infer_combinations + void vp_trt_vehicle_plate_detector::run_infer_combinations(const std::vector>& frame_meta_with_batch) { + assert(frame_meta_with_batch.size() == 1); + std::vector mats_to_infer; + + // start + auto start_time = std::chrono::system_clock::now(); + + // prepare data, as same as base class + vp_secondary_infer_node::prepare(frame_meta_with_batch, mats_to_infer); + auto prepare_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + start_time = std::chrono::system_clock::now(); + // here we detect vehicle plate for all target(like car/bus) in frame + std::vector> plates; + plate_detector->detect(mats_to_infer, plates, true); // detect at most 1 plate for each image + + auto& frame_meta = frame_meta_with_batch[0]; + for (int i = 0; i < plates.size(); i++) { + for (int j = 0; j < plates[i].size(); j++) { + auto& plate = plates[i][j]; + + // only plate detection but no recognition result + if (plate.text.empty()) { + continue; + } + + // check value range + auto x = std::max(0, plate.x + frame_meta->targets[i]->x - 10); // offset + auto y = std::max(0, plate.y + frame_meta->targets[i]->y - 8); // offset + auto w = std::min(plate.width, frame_meta->frame.cols - x); + auto h = std::min(plate.height, frame_meta->frame.rows - y); + if (w <= 0 || h <=0) { + continue; + } + + // create sub target and update back into frame meta + // we treat vehicle plate as sub target of those in vp_frame_meta.targets + auto sub_target = std::make_shared(x, y, w, h, + -1, 0, plate.color + "_" + plate.text, frame_meta->frame_index, frame_meta->channel_index); + frame_meta->targets[i]->sub_targets.push_back(sub_target); + } + } + auto infer_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + // can not calculate preprocess time and postprocess time, set 0 by default. + vp_infer_node::infer_combinations_time_cost(mats_to_infer.size(), prepare_time.count(), 0, infer_time.count(), 0); + } +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_trt_vehicle_plate_detector.h b/nodes/infers/vp_trt_vehicle_plate_detector.h new file mode 100644 index 0000000..ad56b87 --- /dev/null +++ b/nodes/infers/vp_trt_vehicle_plate_detector.h @@ -0,0 +1,29 @@ +#pragma once + +#ifdef VP_WITH_TRT +#include "../vp_secondary_infer_node.h" +#include "../../objects/vp_sub_target.h" +#include "../../third_party/trt_vehicle/models/vehicle_plate_detector.h" + +namespace vp_nodes { + // vehicle plate detector based on tensorrt using trt_vehicle library + // source code: ../../third_party/trt_vehicle + // note: derived from vp_secondary_infer_node since it detects plates on small cropped images, which is different from vp_trt_vehicle_plate_detector_v2 class + // this class is not based on opencv::dnn module but tensorrt, a few data members declared in base class are not usable any more(just ignore), such as vp_infer_node::net. + class vp_trt_vehicle_plate_detector: public vp_secondary_infer_node + { + private: + /* data */ + std::shared_ptr plate_detector = nullptr; + protected: + // we need a totally new logic for the whole infer combinations + // no separate step pre-defined needed in base class + virtual void run_infer_combinations(const std::vector>& frame_meta_with_batch) override; + // override pure virtual method, for compile pass + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_trt_vehicle_plate_detector(std::string node_name, std::string plate_det_model_path = "", std::string char_rec_model_path = ""); + ~vp_trt_vehicle_plate_detector(); + }; +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_trt_vehicle_plate_detector_v2.cpp b/nodes/infers/vp_trt_vehicle_plate_detector_v2.cpp new file mode 100644 index 0000000..a7ee273 --- /dev/null +++ b/nodes/infers/vp_trt_vehicle_plate_detector_v2.cpp @@ -0,0 +1,75 @@ +#ifdef VP_WITH_TRT +#include "vp_trt_vehicle_plate_detector_v2.h" + +namespace vp_nodes { + + vp_trt_vehicle_plate_detector_v2::vp_trt_vehicle_plate_detector_v2(std::string node_name, + std::string plate_det_model_path, + std::string char_rec_model_path): + vp_primary_infer_node(node_name, "") { + plate_detector = std::make_shared(plate_det_model_path, char_rec_model_path); + this->initialized(); + } + + vp_trt_vehicle_plate_detector_v2::~vp_trt_vehicle_plate_detector_v2() { + deinitialized(); + } + + void vp_trt_vehicle_plate_detector_v2::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + + } + + // please refer to vp_infer_node::run_infer_combinations + void vp_trt_vehicle_plate_detector_v2::run_infer_combinations(const std::vector>& frame_meta_with_batch) { + std::vector mats_to_infer; + + // start + auto start_time = std::chrono::system_clock::now(); + + // prepare data, as same as base class + vp_primary_infer_node::prepare(frame_meta_with_batch, mats_to_infer); + auto prepare_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + start_time = std::chrono::system_clock::now(); + // here we detect vehicle plate on the whole big frame + std::vector> plates; + plate_detector->detect(mats_to_infer, plates); + + for (int i = 0; i < plates.size(); i++) { + auto& frame_meta = frame_meta_with_batch[i]; + for (int j = 0; j < plates[i].size(); j++) { + auto& plate = plates[i][j]; + + // only plate detection but no recognition result + if (plate.text.empty()) { + continue; + } + + // check value range + auto x = std::max(0, plate.x); + auto y = std::max(0, plate.y); + auto w = std::min(plate.width, frame_meta->frame.cols - x); + auto h = std::min(plate.height, frame_meta->frame.rows - y); + if (w <= 0 || h <=0) { + continue; + } + + // create target and update back into frame meta + /* + we treat vehicle plate as target in vp_frame_meta.targets, + 1. x,y,width,height in vp_frame_target stand for location of plate in frame + 2. primary_label in vp_frame_target stands for plate color and plate text, which combined with '_' + */ + auto target = std::make_shared(x, y, w, h, + -1, 0, frame_meta->frame_index, frame_meta->channel_index, plate.color + "_" + plate.text); + frame_meta->targets.push_back(target); + VP_INFO("plate-color:" + plate.color + " plate-text:" + plate.text); + } + } + auto infer_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + // can not calculate preprocess time and postprocess time, set 0 by default. + vp_infer_node::infer_combinations_time_cost(mats_to_infer.size(), prepare_time.count(), 0, infer_time.count(), 0); + } +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_trt_vehicle_plate_detector_v2.h b/nodes/infers/vp_trt_vehicle_plate_detector_v2.h new file mode 100644 index 0000000..ac8e171 --- /dev/null +++ b/nodes/infers/vp_trt_vehicle_plate_detector_v2.h @@ -0,0 +1,28 @@ +#pragma once + +#ifdef VP_WITH_TRT +#include "../vp_primary_infer_node.h" +#include "../../third_party/trt_vehicle/models/vehicle_plate_detector.h" + +namespace vp_nodes { + // vehicle plate detector based on tensorrt using trt_vehicle library + // source code: ../../third_party/trt_vehicle + // note: derived from vp_primary_infer_node since it detects plates on the whole big frame, which is different from vp_trt_vehicle_plate_detector class + // this class is not based on opencv::dnn module but tensorrt, a few data members declared in base class are not usable any more(just ignore), such as vp_infer_node::net. + class vp_trt_vehicle_plate_detector_v2: public vp_primary_infer_node + { + private: + /* data */ + std::shared_ptr plate_detector = nullptr; + protected: + // we need a totally new logic for the whole infer combinations + // no separate step pre-defined needed in base class + virtual void run_infer_combinations(const std::vector>& frame_meta_with_batch) override; + // override pure virtual method, for compile pass + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_trt_vehicle_plate_detector_v2(std::string node_name, std::string plate_det_model_path = "", std::string char_rec_model_path = ""); + ~vp_trt_vehicle_plate_detector_v2(); + }; +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_trt_vehicle_scanner.cpp b/nodes/infers/vp_trt_vehicle_scanner.cpp new file mode 100644 index 0000000..201dc3c --- /dev/null +++ b/nodes/infers/vp_trt_vehicle_scanner.cpp @@ -0,0 +1,64 @@ +#ifdef VP_WITH_TRT +#include "vp_trt_vehicle_scanner.h" + +namespace vp_nodes { + + vp_trt_vehicle_scanner::vp_trt_vehicle_scanner(std::string node_name, + std::string vehicle_scan_model_path): + vp_primary_infer_node(node_name, "") { + vehicle_scanner = std::make_shared(vehicle_scan_model_path); + this->initialized(); + } + + vp_trt_vehicle_scanner::~vp_trt_vehicle_scanner() { + deinitialized(); + } + + // please refer to vp_infer_node::run_infer_combinations + void vp_trt_vehicle_scanner::run_infer_combinations(const std::vector>& frame_meta_with_batch) { + assert(frame_meta_with_batch.size() == 1); + std::vector mats_to_infer; + + // start + auto start_time = std::chrono::system_clock::now(); + + // prepare data, as same as base class + vp_primary_infer_node::prepare(frame_meta_with_batch, mats_to_infer); + auto prepare_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + start_time = std::chrono::system_clock::now(); + std::vector> vehicles; + vehicle_scanner->detect(mats_to_infer, vehicles); + + assert(vehicles.size() == 1); + auto& vehicle_list = vehicles[0]; + auto& frame_meta = frame_meta_with_batch[0]; + + for (int i = 0; i < vehicle_list.size(); i++) { + auto& objbox = vehicle_list[i]; + + // check value range + objbox.x = std::max(objbox.x, 0); + objbox.y = std::max(objbox.y, 0); + objbox.width = std::min(objbox.width, frame_meta->frame.cols - objbox.x); + objbox.height = std::min(objbox.height, frame_meta->frame.rows - objbox.y); + if (objbox.width <= 0 || objbox.height <= 0) { + continue; + } + + auto target = std::make_shared(objbox.x, objbox.y, objbox.width, objbox.height, + objbox.class_, objbox.score, frame_meta->frame_index, frame_meta->channel_index, objbox.label); + // create target and update back into frame meta + frame_meta->targets.push_back(target); + } + auto infer_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + // can not calculate preprocess time and postprocess time, set 0 by default. + vp_infer_node::infer_combinations_time_cost(mats_to_infer.size(), prepare_time.count(), 0, infer_time.count(), 0); + } + + void vp_trt_vehicle_scanner::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + + } +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_trt_vehicle_scanner.h b/nodes/infers/vp_trt_vehicle_scanner.h new file mode 100644 index 0000000..980b2db --- /dev/null +++ b/nodes/infers/vp_trt_vehicle_scanner.h @@ -0,0 +1,24 @@ +#pragma once + +#ifdef VP_WITH_TRT +#include "../vp_primary_infer_node.h" +#include "../../third_party/trt_vehicle/models/vehicle_scanner.h" + +namespace vp_nodes { + // vehicle scanner based on tensorrt using trt_vehicle library + class vp_trt_vehicle_scanner: public vp_primary_infer_node + { + private: + std::shared_ptr vehicle_scanner = nullptr; + protected: + // we need a totally new logic for the whole infer combinations + // no separate step pre-defined needed in base class + virtual void run_infer_combinations(const std::vector>& frame_meta_with_batch) override; + // override pure virtual method, for compile pass + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_trt_vehicle_scanner(std::string node_name, std::string vehicle_scan_model_path = ""); + ~vp_trt_vehicle_scanner(); + }; +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_trt_vehicle_type_classifier.cpp b/nodes/infers/vp_trt_vehicle_type_classifier.cpp new file mode 100644 index 0000000..53e3811 --- /dev/null +++ b/nodes/infers/vp_trt_vehicle_type_classifier.cpp @@ -0,0 +1,64 @@ +#ifdef VP_WITH_TRT +#include "vp_trt_vehicle_type_classifier.h" + +namespace vp_nodes { + vp_trt_vehicle_type_classifier::vp_trt_vehicle_type_classifier(std::string node_name, + std::string vehicle_type_cls_model_path, + std::vector p_class_ids_applied_to, + int min_width_applied_to, int min_height_applied_to): + vp_secondary_infer_node(node_name, "", "", "", 1, 1, 1, p_class_ids_applied_to, min_width_applied_to, min_height_applied_to) { + vehicle_type_classifier = std::make_shared(vehicle_type_cls_model_path); + this->initialized(); + } + + vp_trt_vehicle_type_classifier::~vp_trt_vehicle_type_classifier() { + deinitialized(); + } + + void vp_trt_vehicle_type_classifier::run_infer_combinations(const std::vector>& frame_meta_with_batch) { + assert(frame_meta_with_batch.size() == 1); + std::vector mats_to_infer; + + // start + auto start_time = std::chrono::system_clock::now(); + + // prepare data, as same as base class + vp_secondary_infer_node::prepare(frame_meta_with_batch, mats_to_infer); + auto prepare_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + // infer using trt_vehicle library + start_time = std::chrono::system_clock::now(); + std::vector vehicles_type; + vehicle_type_classifier->classify(mats_to_infer, vehicles_type); + + auto& frame_meta = frame_meta_with_batch[0]; + auto index = 0; + for (int i = 0; i < vehicles_type.size(); i++) { + for (int j = index; j < frame_meta->targets.size(); j++) { + // need apply or not? + if (!need_apply(frame_meta->targets[j]->primary_class_id, frame_meta->targets[j]->width, frame_meta->targets[j]->height)) { + // continue as its primary_class_id is not in p_class_ids_applied_to + continue; + } + + // update back to frame meta + frame_meta->targets[j]->secondary_class_ids.push_back(vehicles_type[i].class_); + frame_meta->targets[j]->secondary_scores.push_back(vehicles_type[i].score); + frame_meta->targets[j]->secondary_labels.push_back(vehicles_type[i].label); + + // break as we found the right target! + index = j + 1; + break; + } + } + auto infer_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + // can not calculate preprocess time and postprocess time, set 0 by default. + vp_infer_node::infer_combinations_time_cost(mats_to_infer.size(), prepare_time.count(), 0, infer_time.count(), 0); + } + + void vp_trt_vehicle_type_classifier::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + + } +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_trt_vehicle_type_classifier.h b/nodes/infers/vp_trt_vehicle_type_classifier.h new file mode 100644 index 0000000..2a5e015 --- /dev/null +++ b/nodes/infers/vp_trt_vehicle_type_classifier.h @@ -0,0 +1,26 @@ +#pragma once + +#ifdef VP_WITH_TRT +#include "../vp_secondary_infer_node.h" +#include "../../third_party/trt_vehicle/models/vehicle_type_classifier.h" + +namespace vp_nodes { + // vehicle type classifier based on tensorrt using trt_vehicle library + // update secondary_class_ids/secondary_labels/secondary_scores of vp_frame_target. + class vp_trt_vehicle_type_classifier: public vp_secondary_infer_node + { + private: + /* data */ + std::shared_ptr vehicle_type_classifier = nullptr; + protected: + // we need a totally new logic for the whole infer combinations + // no separate step pre-defined needed in base class + virtual void run_infer_combinations(const std::vector>& frame_meta_with_batch) override; + // override pure virtual method, for compile pass + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_trt_vehicle_type_classifier(std::string node_name, std::string vehicle_type_cls_model_path = "", std::vector p_class_ids_applied_to = std::vector(), int min_width_applied_to = 0, int min_height_applied_to = 0); + ~vp_trt_vehicle_type_classifier(); + }; +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_trt_yolov8_classifier.cpp b/nodes/infers/vp_trt_yolov8_classifier.cpp new file mode 100644 index 0000000..10c7765 --- /dev/null +++ b/nodes/infers/vp_trt_yolov8_classifier.cpp @@ -0,0 +1,67 @@ + +#ifdef VP_WITH_TRT +#include "vp_trt_yolov8_classifier.h" + +namespace vp_nodes { + vp_trt_yolov8_classifier::vp_trt_yolov8_classifier(std::string node_name, + std::string model_path, + std::string labels_path, + std::vector p_class_ids_applied_to, + int min_width_applied_to, int min_height_applied_to): + vp_secondary_infer_node(node_name, "", "", labels_path, 1, 1, 1, p_class_ids_applied_to, min_width_applied_to, min_height_applied_to) { + yolov8_classifier = std::make_shared(model_path); + this->initialized(); + } + + vp_trt_yolov8_classifier::~vp_trt_yolov8_classifier() { + deinitialized(); + } + + void vp_trt_yolov8_classifier::run_infer_combinations(const std::vector>& frame_meta_with_batch) { + assert(frame_meta_with_batch.size() == 1); + std::vector mats_to_infer; + + // start + auto start_time = std::chrono::system_clock::now(); + + // prepare data, as same as base class + vp_secondary_infer_node::prepare(frame_meta_with_batch, mats_to_infer); + auto prepare_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + // infer using trt_vehicle library + start_time = std::chrono::system_clock::now(); + std::vector> classidications; + yolov8_classifier->classify(mats_to_infer, classidications); + + auto& frame_meta = frame_meta_with_batch[0]; + auto index = 0; + for (int i = 0; i < classidications.size(); i++) { + for (int j = index; j < frame_meta->targets.size(); j++) { + // need apply or not? + if (!need_apply(frame_meta->targets[j]->primary_class_id, frame_meta->targets[j]->width, frame_meta->targets[j]->height)) { + // continue as its primary_class_id is not in p_class_ids_applied_to + continue; + } + + // update back to frame meta + frame_meta->targets[j]->secondary_class_ids.push_back(classidications[i][0].class_id); + frame_meta->targets[j]->secondary_scores.push_back(classidications[i][0].conf); + auto label = labels.size() == 0 ? "" : labels[classidications[i][0].class_id]; + frame_meta->targets[j]->secondary_labels.push_back(label); + + // break as we found the right target! + index = j + 1; + break; + } + } + auto infer_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + // can not calculate preprocess time and postprocess time, set 0 by default. + vp_infer_node::infer_combinations_time_cost(mats_to_infer.size(), prepare_time.count(), 0, infer_time.count(), 0); + } + + void vp_trt_yolov8_classifier::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + + } +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_trt_yolov8_classifier.h b/nodes/infers/vp_trt_yolov8_classifier.h new file mode 100644 index 0000000..cb34460 --- /dev/null +++ b/nodes/infers/vp_trt_yolov8_classifier.h @@ -0,0 +1,30 @@ +#pragma once + +#ifdef VP_WITH_TRT +#include "../vp_secondary_infer_node.h" +#include "../../third_party/trt_yolov8/trt_yolov8_classifier.h" + +namespace vp_nodes { + // universal yolov8 classifier based on tensorrt using third_party/trt_yolov8 library + class vp_trt_yolov8_classifier: public vp_secondary_infer_node + { + private: + /* data */ + std::shared_ptr yolov8_classifier = nullptr; + protected: + // we need a totally new logic for the whole infer combinations + // no separate step pre-defined needed in base class + virtual void run_infer_combinations(const std::vector>& frame_meta_with_batch) override; + // override pure virtual method, for compile pass + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_trt_yolov8_classifier(std::string node_name, + std::string model_path, + std::string labels_path = "", + std::vector p_class_ids_applied_to = std::vector(), + int min_width_applied_to = 0, + int min_height_applied_to = 0); + ~vp_trt_yolov8_classifier(); + }; +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_trt_yolov8_detector.cpp b/nodes/infers/vp_trt_yolov8_detector.cpp new file mode 100644 index 0000000..4273ea0 --- /dev/null +++ b/nodes/infers/vp_trt_yolov8_detector.cpp @@ -0,0 +1,69 @@ + +#ifdef VP_WITH_TRT +#include "vp_trt_yolov8_detector.h" + +namespace vp_nodes { + + vp_trt_yolov8_detector::vp_trt_yolov8_detector(std::string node_name, + std::string model_path, + std::string labels_path): + vp_primary_infer_node(node_name, "", "", labels_path) { + yolov8_detector = std::make_shared(model_path); + this->initialized(); + } + + vp_trt_yolov8_detector::~vp_trt_yolov8_detector() { + deinitialized(); + } + + // please refer to vp_infer_node::run_infer_combinations + void vp_trt_yolov8_detector::run_infer_combinations(const std::vector>& frame_meta_with_batch) { + assert(frame_meta_with_batch.size() == 1); + std::vector mats_to_infer; + + // start + auto start_time = std::chrono::system_clock::now(); + + // prepare data, as same as base class + vp_primary_infer_node::prepare(frame_meta_with_batch, mats_to_infer); + auto prepare_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + start_time = std::chrono::system_clock::now(); + std::vector> detections; + yolov8_detector->detect(mats_to_infer, detections); + + assert(detections.size() == 1); + auto& detection_list = detections[0]; + auto& frame_meta = frame_meta_with_batch[0]; + + for (int i = 0; i < detection_list.size(); i++) { + auto& objbox = detection_list[i]; + + // objbox.bbox: center_x center_y width height + auto rect = get_rect(frame_meta->frame, objbox.bbox); // convert to: x, y, width,height + + // check value range + int x = std::max(rect.x, 0); + int y = std::max(rect.y, 0); + int width = std::min(rect.width, frame_meta->frame.cols - x); + int height = std::min(rect.height, frame_meta->frame.rows - y); + if (width <= 0 || height <= 0) { + continue; + } + auto label = labels.size() == 0 ? "" : labels[objbox.class_id]; + auto target = std::make_shared(x, y, width, height, + objbox.class_id, objbox.conf, frame_meta->frame_index, frame_meta->channel_index, label); + // create target and update back into frame meta + frame_meta->targets.push_back(target); + } + auto infer_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + // can not calculate preprocess time and postprocess time, set 0 by default. + vp_infer_node::infer_combinations_time_cost(mats_to_infer.size(), prepare_time.count(), 0, infer_time.count(), 0); + } + + void vp_trt_yolov8_detector::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + + } +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_trt_yolov8_detector.h b/nodes/infers/vp_trt_yolov8_detector.h new file mode 100644 index 0000000..d0d3154 --- /dev/null +++ b/nodes/infers/vp_trt_yolov8_detector.h @@ -0,0 +1,24 @@ +#pragma once + +#ifdef VP_WITH_TRT +#include "../vp_primary_infer_node.h" +#include "../../third_party/trt_yolov8/trt_yolov8_detector.h" + +namespace vp_nodes { + // universal yolov8 detector based on tensorrt using third_party/trt_yolov8 library + class vp_trt_yolov8_detector: public vp_primary_infer_node + { + private: + std::shared_ptr yolov8_detector = nullptr; + protected: + // we need a totally new logic for the whole infer combinations + // no separate step pre-defined needed in base class + virtual void run_infer_combinations(const std::vector>& frame_meta_with_batch) override; + // override pure virtual method, for compile pass + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_trt_yolov8_detector(std::string node_name, std::string model_path, std::string labels_path = ""); + ~vp_trt_yolov8_detector(); + }; +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_trt_yolov8_pose_detector.cpp b/nodes/infers/vp_trt_yolov8_pose_detector.cpp new file mode 100644 index 0000000..9b4bca1 --- /dev/null +++ b/nodes/infers/vp_trt_yolov8_pose_detector.cpp @@ -0,0 +1,60 @@ + +#ifdef VP_WITH_TRT +#include "vp_trt_yolov8_pose_detector.h" + +namespace vp_nodes { + + vp_trt_yolov8_pose_detector::vp_trt_yolov8_pose_detector(std::string node_name, + std::string model_path): + vp_primary_infer_node(node_name, "") { + yolov8_pose_detector = std::make_shared(model_path); + this->initialized(); + } + + vp_trt_yolov8_pose_detector::~vp_trt_yolov8_pose_detector() { + deinitialized(); + } + + // please refer to vp_infer_node::run_infer_combinations + void vp_trt_yolov8_pose_detector::run_infer_combinations(const std::vector>& frame_meta_with_batch) { + assert(frame_meta_with_batch.size() == 1); + std::vector mats_to_infer; + + // start + auto start_time = std::chrono::system_clock::now(); + + // prepare data, as same as base class + vp_primary_infer_node::prepare(frame_meta_with_batch, mats_to_infer); + auto prepare_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + start_time = std::chrono::system_clock::now(); + std::vector> detections; + yolov8_pose_detector->detect(mats_to_infer, detections); + + assert(detections.size() == 1); + auto& detection_list = detections[0]; + auto& frame_meta = frame_meta_with_batch[0]; + + for (int i = 0; i < detection_list.size(); i++) { + auto& objbox = detection_list[i]; + auto rect = get_rect_adapt_landmark(frame_meta->frame, objbox.bbox, objbox.keypoints); + + std::vector kps; + for (int j = 0; j < 51; j += 3) { + kps.push_back(vp_objects::vp_pose_keypoint {j, int(objbox.keypoints[j]), int(objbox.keypoints[j + 1]), objbox.keypoints[j + 2]}); + } + + auto pose_target = std::make_shared(vp_objects::vp_pose_type::yolov8_pose_17, kps); + frame_meta->pose_targets.push_back(pose_target); + } + auto infer_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + // can not calculate preprocess time and postprocess time, set 0 by default. + vp_infer_node::infer_combinations_time_cost(mats_to_infer.size(), prepare_time.count(), 0, infer_time.count(), 0); + } + + void vp_trt_yolov8_pose_detector::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + + } +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_trt_yolov8_pose_detector.h b/nodes/infers/vp_trt_yolov8_pose_detector.h new file mode 100644 index 0000000..184607c --- /dev/null +++ b/nodes/infers/vp_trt_yolov8_pose_detector.h @@ -0,0 +1,24 @@ +#pragma once + +#ifdef VP_WITH_TRT +#include "../vp_primary_infer_node.h" +#include "../../third_party/trt_yolov8/trt_yolov8_pose_detector.h" + +namespace vp_nodes { + // universal yolov8 pose detector based on tensorrt using third_party/trt_yolov8 library + class vp_trt_yolov8_pose_detector: public vp_primary_infer_node + { + private: + std::shared_ptr yolov8_pose_detector = nullptr; + protected: + // we need a totally new logic for the whole infer combinations + // no separate step pre-defined needed in base class + virtual void run_infer_combinations(const std::vector>& frame_meta_with_batch) override; + // override pure virtual method, for compile pass + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_trt_yolov8_pose_detector(std::string node_name, std::string model_path); + ~vp_trt_yolov8_pose_detector(); + }; +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_trt_yolov8_seg_detector.cpp b/nodes/infers/vp_trt_yolov8_seg_detector.cpp new file mode 100644 index 0000000..3defbce --- /dev/null +++ b/nodes/infers/vp_trt_yolov8_seg_detector.cpp @@ -0,0 +1,78 @@ + +#ifdef VP_WITH_TRT +#include "vp_trt_yolov8_seg_detector.h" + +namespace vp_nodes { + + vp_trt_yolov8_seg_detector::vp_trt_yolov8_seg_detector(std::string node_name, + std::string model_path, + std::string labels_path): + vp_primary_infer_node(node_name, "", "", labels_path) { + yolov8_seg_detector = std::make_shared(model_path); + this->initialized(); + } + + vp_trt_yolov8_seg_detector::~vp_trt_yolov8_seg_detector() { + deinitialized(); + } + + // please refer to vp_infer_node::run_infer_combinations + void vp_trt_yolov8_seg_detector::run_infer_combinations(const std::vector>& frame_meta_with_batch) { + assert(frame_meta_with_batch.size() == 1); + std::vector mats_to_infer; + + // start + auto start_time = std::chrono::system_clock::now(); + + // prepare data, as same as base class + vp_primary_infer_node::prepare(frame_meta_with_batch, mats_to_infer); + auto prepare_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + start_time = std::chrono::system_clock::now(); + std::vector> detections; + std::vector> masks; + yolov8_seg_detector->detect(mats_to_infer, detections, masks); + + assert(detections.size() == 1); + assert(masks.size() == 1); + auto& detection_list = detections[0]; + auto& mask_list = masks[0]; + assert(detection_list.size() == mask_list.size()); + auto& frame_meta = frame_meta_with_batch[0]; + + for (int i = 0; i < detection_list.size(); i++) { + auto& objbox = detection_list[i]; + auto& mask = mask_list[i]; + auto scaled_mask = scale_mask(mask, frame_meta->frame); + + // objbox.bbox: center_x center_y width height + auto rect = get_rect(frame_meta->frame, objbox.bbox); // convert to: x, y, width,height + // check value range + rect.x = std::max(rect.x, 0); + rect.y = std::max(rect.y, 0); + rect.width = std::min(rect.width, frame_meta->frame.cols - rect.x); + rect.height = std::min(rect.height, frame_meta->frame.rows - rect.y); + if (rect.width <= 0 || rect.height <= 0) { + continue; + } + + auto label = labels.size() == 0 ? "" : labels[objbox.class_id]; + auto target = std::make_shared(rect.x, rect.y, rect.width, rect.height, + objbox.class_id, objbox.conf, frame_meta->frame_index, frame_meta->channel_index, label); + auto rect_mask = scaled_mask(rect); + target->mask = rect_mask; + + // create target and update back into frame meta + frame_meta->targets.push_back(target); + } + auto infer_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + // can not calculate preprocess time and postprocess time, set 0 by default. + vp_infer_node::infer_combinations_time_cost(mats_to_infer.size(), prepare_time.count(), 0, infer_time.count(), 0); + } + + void vp_trt_yolov8_seg_detector::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + + } +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_trt_yolov8_seg_detector.h b/nodes/infers/vp_trt_yolov8_seg_detector.h new file mode 100644 index 0000000..fbe1875 --- /dev/null +++ b/nodes/infers/vp_trt_yolov8_seg_detector.h @@ -0,0 +1,24 @@ +#pragma once + +#ifdef VP_WITH_TRT +#include "../vp_primary_infer_node.h" +#include "../../third_party/trt_yolov8/trt_yolov8_seg_detector.h" + +namespace vp_nodes { + // universal yolov8 segmentation detector based on tensorrt using third_party/trt_yolov8 library + class vp_trt_yolov8_seg_detector: public vp_primary_infer_node + { + private: + std::shared_ptr yolov8_seg_detector = nullptr; + protected: + // we need a totally new logic for the whole infer combinations + // no separate step pre-defined needed in base class + virtual void run_infer_combinations(const std::vector>& frame_meta_with_batch) override; + // override pure virtual method, for compile pass + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_trt_yolov8_seg_detector(std::string node_name, std::string model_path, std::string labels_path = ""); + ~vp_trt_yolov8_seg_detector(); + }; +} +#endif \ No newline at end of file diff --git a/nodes/infers/vp_yolo5_seg_node.cpp b/nodes/infers/vp_yolo5_seg_node.cpp new file mode 100644 index 0000000..c8c55a0 --- /dev/null +++ b/nodes/infers/vp_yolo5_seg_node.cpp @@ -0,0 +1,307 @@ + +#include +#include +#include +#include "vp_yolo5_seg_node.h" + + +namespace vp_nodes { + vp_yolo5_seg_node::vp_yolo5_seg_node(std::string node_name, + std::string model_path, + std::string labels_path, + int input_width, + int input_height, + int batch_size, + int class_id_offset, + float conf_threshold, + float iou_threshold): + vp_primary_infer_node(node_name, model_path, "", labels_path, input_width, input_height, batch_size, class_id_offset), + conf_threshold(conf_threshold), + iou_threshold(iou_threshold) { + this->initialized(); + } + + vp_yolo5_seg_node::~vp_yolo5_seg_node() { + deinitialized(); + } + + void vp_yolo5_seg_node::prepare(const std::vector>& frame_meta_with_batch, std::vector& mats_to_infer) { + for (auto& i: frame_meta_with_batch) { + auto letterboxed = letterbox(i->frame, cv::Size(input_width, input_height), scale_ratio, padding); + mats_to_infer.push_back(letterboxed); + } + } + + void vp_yolo5_seg_node::infer(const cv::Mat& blob_to_infer, std::vector& raw_outputs) { + // blob_to_infer is a 4D matrix + // the first dim is number of batch, MUST be 1 + assert(blob_to_infer.dims == 4); + assert(blob_to_infer.size[0] == 1); + assert(!net.empty()); + + net.setInput(blob_to_infer); + net.forward(raw_outputs, net.getUnconnectedOutLayersNames()); + } + + void vp_yolo5_seg_node::preprocess(const std::vector& mats_to_infer, cv::Mat& blob_to_infer) { + // ignore preprocess logic in base class + cv::dnn::blobFromImages(mats_to_infer, blob_to_infer, 1.0 / 255.0, cv::Size(), cv::Scalar(), true, false); + } + + void vp_yolo5_seg_node::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + auto& frame_meta = frame_meta_with_batch[0]; + auto& frame = frame_meta->frame; + auto pred = raw_outputs[0]; // [1, N, 4+1+NUM_CLASSES+32] + auto proto = raw_outputs[1]; // [1, 32, 96, 160] + + std::vector boxes; + std::vector confidences; + std::vector class_ids; + std::vector mask_coeffs; + + int num_boxes = pred.size[1]; + float* data = (float*)pred.data; + + // decode + for (int i = 0; i < num_boxes; ++i) { + float obj_conf = data[4]; + if (obj_conf < conf_threshold) { + data += (5 + labels.size() + MASK_CHANNELS); + continue; + } + + int class_id = -1; + float max_cls_conf = 0; + for (int c = 0; c < labels.size(); ++c) { + float cls_conf = data[5 + c]; + if (cls_conf > max_cls_conf) { + max_cls_conf = cls_conf; + class_id = c; + } + } + + float total_conf = obj_conf * max_cls_conf; + if (total_conf < conf_threshold || class_id == -1) { + data += (5 + labels.size() + MASK_CHANNELS); + continue; + } + + float cx = data[0]; + float cy = data[1]; + float w = data[2]; + float h = data[3]; + + float x1 = cx - w * 0.5f; + float y1 = cy - h * 0.5f; + float x2 = cx + w * 0.5f; + float y2 = cy + h * 0.5f; + + if (x2 <= x1 || y2 <= y1 || w <= 0 || h <= 0) { + data += (5 + labels.size() + MASK_CHANNELS); + continue; + } + + float x_scale = scale_ratio.at(0); + float y_scale = scale_ratio.at(1); + int orig_x1 = cvRound((x1 - padding.x) * x_scale); + int orig_y1 = cvRound((y1 - padding.y) * y_scale); + int orig_x2 = cvRound((x2 - padding.x) * x_scale); + int orig_y2 = cvRound((y2 - padding.y) * y_scale); + + orig_x1 = max(0, orig_x1); + orig_y1 = max(0, orig_y1); + orig_x2 = min(frame.cols, orig_x2); + orig_y2 = min(frame.rows, orig_y2); + + if (orig_x2 <= orig_x1 || orig_y2 <= orig_y1) { + data += (5 + labels.size() + MASK_CHANNELS); + continue; + } + + cv::Rect box(orig_x1, orig_y1, orig_x2 - orig_x1, orig_y2 - orig_y1); + boxes.push_back(box); + confidences.push_back(total_conf); + class_ids.push_back(class_id); + + cv::Mat coeff(1, MASK_CHANNELS, CV_32F); + for (int m = 0; m < MASK_CHANNELS; ++m) { + coeff.at(m) = data[5 + labels.size() + m]; + } + mask_coeffs.push_back(coeff); + + data += (5 + labels.size() + MASK_CHANNELS); + } + + std::vector final_indices; + std::unordered_map> class_boxes; + for (int i = 0; i < boxes.size(); ++i) { + class_boxes[class_ids[i]].push_back(i); + } + + // NMS + for (auto& kv : class_boxes) { + const std::vector& indices_in_class = kv.second; + std::vector boxes_cls; + std::vector scores_cls; + for (int idx : indices_in_class) { + boxes_cls.push_back(boxes[idx]); + scores_cls.push_back(confidences[idx]); + } + + std::vector nms_indices = nms(boxes_cls, scores_cls, iou_threshold); + for (int local_idx : nms_indices) { + final_indices.push_back(indices_in_class[local_idx]); + } + } + + // create target push back to frame meta + for (int idx : final_indices) { + cv::Rect box = boxes[idx]; + float conf = confidences[idx]; + int cls = class_ids[idx]; + + auto label = (labels.size() < cls + 1) ? "" : labels[cls]; + cls += class_id_offset; + auto target = std::make_shared(box.x, box.y, box.width, box.height, cls, conf, frame_meta->frame_index, frame_meta->channel_index, label); + + // try to extract mask + auto local_mask = process_mask(proto, mask_coeffs[idx], box, + cv::Size(input_width, input_height), + padding, scale_ratio.at(0), scale_ratio.at(1)); + target->mask = local_mask; + + frame_meta->targets.push_back(target); + } + + std::cout << "Detected: " << final_indices.size() << " instances" << std::endl; + } + + cv::Mat vp_yolo5_seg_node::letterbox(const cv::Mat& src, cv::Size target_size, cv::Mat& scale_ratio, cv::Point& padding) { + float scale = min(static_cast(target_size.height) / src.rows, + static_cast(target_size.width) / src.cols); + int new_width = cvRound(src.cols * scale); + int new_height = cvRound(src.rows * scale); + + cv::Mat resized; + resize(src, resized, cv::Size(new_width, new_height), 0, 0, cv::INTER_LINEAR); + + cv::Mat letterboxed(target_size, CV_8UC3, cv::Scalar(114, 114, 114)); + int top = (target_size.height - new_height) / 2; + int left = (target_size.width - new_width) / 2; + cv::Rect roi(left, top, new_width, new_height); + resized.copyTo(letterboxed(roi)); + + scale_ratio = cv::Mat(1, 2, CV_32F); + scale_ratio.at(0) = 1.0f / scale; // x_scale + scale_ratio.at(1) = 1.0f / scale; // y_scale + + padding = cv::Point(left, top); + return letterboxed; + } + + // Sigmoid + cv::Mat vp_yolo5_seg_node::sigmoid(const cv::Mat& x) { + cv::Mat result; + cv::exp(-x, result); + result = 1.0f / (1.0f + result); + return result; + } + + cv::Mat vp_yolo5_seg_node::process_mask(const cv::Mat& proto, // [1, 32, H/4, W/4] + const cv::Mat& coeffs, // [1, 32] + const cv::Rect& bbox, + const cv::Size& input_size, // e.g., 640x384 + const cv::Point& padding, + const float x_scale, + const float y_scale) { + + // 1. 获取 proto 尺寸 [1,32,96,160] → h=96, w=160 + int h_proto = proto.size[2]; + int w_proto = proto.size[3]; + + // 2. 重塑为 [32, h, w] + cv::Mat proto_3d = proto.reshape(1, {MASK_CHANNELS, h_proto, w_proto}); + + // 3. 展平为 [32, h*w] + cv::Mat proto_flat = proto_3d.reshape(1, MASK_CHANNELS); + + // 4. 矩阵乘: [1,32] × [32, h*w] → [1, h*w] + cv::Mat mask_raw = coeffs * proto_flat; + + // 5. 重塑为 [h, w] + cv::Mat mask_2d = mask_raw.reshape(0, h_proto); + + // 6. Sigmoid + cv::Mat mask_sigmoid = sigmoid(mask_2d); + + // 7. 原图 bbox → 输入图坐标（含 padding） + float x1_in = bbox.x / x_scale + padding.x; + float y1_in = bbox.y / y_scale + padding.y; + float x2_in = (bbox.x + bbox.width) / x_scale + padding.x; + float y2_in = (bbox.y + bbox.height) / y_scale + padding.y; + + // 8. 输入图 → proto 坐标（/4） + float x1_p = x1_in / 4.0f; + float y1_p = y1_in / 4.0f; + float x2_p = x2_in / 4.0f; + float y2_p = y2_in / 4.0f; + + int px1 = max(0, (int)floor(x1_p)); + int py1 = max(0, (int)floor(y1_p)); + int px2 = min(w_proto, (int)ceil(x2_p)); + int py2 = min(h_proto, (int)ceil(y2_p)); + + if (px2 <= px1 || py2 <= py1) { + return cv::Mat::zeros(bbox.size(), CV_8UC1); + } + + // 9. 裁剪 + cv::Mat cropped = mask_sigmoid(cv::Rect(px1, py1, px2 - px1, py2 - py1)).clone(); + + // 10. 上采样到 bbox 尺寸 + cv::Mat resized; + resize(cropped, resized, bbox.size(), 0, 0, cv::INTER_LINEAR); + + // 11. 二值化 + cv::Mat mask_bin; + threshold(resized, mask_bin, 0.5, 255, cv::THRESH_BINARY); + mask_bin.convertTo(mask_bin, CV_8UC1); + + return mask_bin; + } + + // NMS + std::vector vp_yolo5_seg_node::nms(const std::vector& boxes, const std::vector& scores, float iou_threshold) { + std::vector indices; + std::vector areas(boxes.size()); + for (size_t i = 0; i < boxes.size(); ++i) { + areas[i] = boxes[i].area(); + } + + std::vector order(scores.size()); + iota(order.begin(), order.end(), 0); + sort(order.begin(), order.end(), [&](int a, int b) { + return scores[a] > scores[b]; + }); + + std::vector keep(scores.size(), true); + for (int i = 0; i < order.size(); ++i) { + int idx = order[i]; + if (!keep[idx]) continue; + for (int j = i + 1; j < order.size(); ++j) { + int idx2 = order[j]; + if (!keep[idx2]) continue; + cv::Rect intersect = boxes[idx] & boxes[idx2]; + float iou = intersect.area() / (areas[idx] + areas[idx2] - intersect.area() + 1e-6f); + if (iou > iou_threshold) { + keep[idx2] = false; + } + } + } + + for (int i = 0; i < keep.size(); ++i) { + if (keep[i]) indices.push_back(i); + } + return indices; + } +} \ No newline at end of file diff --git a/nodes/infers/vp_yolo5_seg_node.h b/nodes/infers/vp_yolo5_seg_node.h new file mode 100644 index 0000000..271e3b9 --- /dev/null +++ b/nodes/infers/vp_yolo5_seg_node.h @@ -0,0 +1,48 @@ + +#pragma once + +#include "../vp_primary_infer_node.h" +#include "../../objects/vp_frame_target.h" + + + +namespace vp_nodes { + // driving area segmentation based on yolov5s-seg-v7.0 + // https://github.com/ultralytics/yolov5/releases/tag/v7.0 + class vp_yolo5_seg_node: public vp_primary_infer_node + { + private: + /* data */ + float conf_threshold = 0.5; + float iou_threshold = 0.7; + int MASK_CHANNELS = 32; + cv::Mat scale_ratio; + cv::Point padding; + cv::Mat letterbox(const cv::Mat& src, cv::Size target_size, cv::Mat& scale_ratio, cv::Point& padding); + cv::Mat sigmoid(const cv::Mat& x); + cv::Mat process_mask(const cv::Mat& proto, // [1, 32, H/4, W/4] + const cv::Mat& coeffs, // [1, 32] + const cv::Rect& bbox, + const cv::Size& input_size, // e.g., 640x384 + const cv::Point& padding, + const float x_scale, + const float y_scale); + std::vector nms(const std::vector& boxes, const std::vector& scores, float iou_threshold); + protected: + virtual void infer(const cv::Mat& blob_to_infer, std::vector& raw_outputs) override; + virtual void preprocess(const std::vector& mats_to_infer, cv::Mat& blob_to_infer) override; + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + virtual void prepare(const std::vector>& frame_meta_with_batch, std::vector& mats_to_infer) override; + public: + vp_yolo5_seg_node(std::string node_name, + std::string model_path, + std::string labels_path = "", + int input_width = 640, + int input_height = 384, + int batch_size = 1, + int class_id_offset = 0, + float conf_threshold = 0.5, + float iou_threshold = 0.7); + ~vp_yolo5_seg_node(); + }; +} \ No newline at end of file diff --git a/nodes/infers/vp_yolo_detector_node.cpp b/nodes/infers/vp_yolo_detector_node.cpp new file mode 100755 index 0000000..64e24e9 --- /dev/null +++ b/nodes/infers/vp_yolo_detector_node.cpp @@ -0,0 +1,118 @@ + +#include "vp_yolo_detector_node.h" + +namespace vp_nodes { + + vp_yolo_detector_node::vp_yolo_detector_node(std::string node_name, + std::string model_path, + std::string model_config_path, + std::string labels_path, + int input_width, + int input_height, + int batch_size, + int class_id_offset, + float score_threshold, + float confidence_threshold, + float nms_threshold, + float scale, + cv::Scalar mean, + cv::Scalar std, + bool swap_rb): + vp_primary_infer_node(node_name, model_path, model_config_path,labels_path,input_width, input_height, batch_size, class_id_offset, scale, mean, std, swap_rb), + score_threshold(score_threshold), + confidence_threshold(confidence_threshold), + nms_threshold(nms_threshold) { + this->initialized(); + } + + vp_yolo_detector_node::~vp_yolo_detector_node() { + deinitialized(); + } + + void vp_yolo_detector_node::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + // make sure heads of output are not zero + assert(raw_outputs.size() > 0); + + // check dims of each output + auto dim_offset = raw_outputs[0].dims == 2 ? 0 : 1; + auto batch = dim_offset == 0 ? 1 : raw_outputs[0].size[0]; + + std::vector class_ids; + std::vector confidences; + std::vector boxes; + + for (int k = 0; k < batch; k++) { // scan batch + auto& frame_meta = frame_meta_with_batch[k]; + for (int i = 0; i < raw_outputs.size(); ++i) { // scan heads of output + cv::Mat output = cv::Mat(raw_outputs[i].size[0 + dim_offset], raw_outputs[i].size[1 + dim_offset], CV_32F, const_cast(raw_outputs[i].ptr(k))); + + auto data = (float*)output.data; + for (int j = 0; j < output.rows; ++j, data += output.cols) { + + float confidence = data[4]; + // check confidence threshold + if (confidence < confidence_threshold) { + continue; + } + + cv::Mat scores = output.row(j).colRange(5, output.cols); + cv::Point class_id; + double max_score; + // Get the value and location of the maximum score + cv::minMaxLoc(scores, 0, &max_score, 0, &class_id); + + // check score threshold + if (max_score >= score_threshold) { + // smaller than 1.0 means relative value + // greater than 1.0 means absolute value which need to be converted to relative value + int centerX = (int)((data[0] <= 1 ? data[0] : data[0] / input_width) * frame_meta->frame.cols); + int centerY = (int)((data[1] <= 1 ? data[1] : data[1] / input_height) * frame_meta->frame.rows); + int width = (int)((data[2] <= 1 ? data[2] : data[2] / input_width) * frame_meta->frame.cols); + int height = (int)((data[3] <= 1 ? data[3] : data[3] / input_height) * frame_meta->frame.rows); + int left = centerX - width / 2; + int top = centerY - height / 2; + + class_ids.push_back(class_id.x); + confidences.push_back((float)confidence); + boxes.push_back(cv::Rect(left, top, width, height)); + } + } + } + + // nms + std::vector indices; + cv::dnn::NMSBoxes(boxes, confidences, confidence_threshold, nms_threshold, indices); + + // create target + for (int i = 0; i < indices.size(); ++i) { + int idx = indices[i]; + auto box = boxes[idx]; + auto confidence = confidences[idx]; + auto class_id = class_ids[idx]; + auto label = (labels.size() < class_id + 1) ? "" : labels[class_id]; + + // check value range + box.x = std::max(box.x, 0); + box.y = std::max(box.y, 0); + box.width = std::min(box.width, frame_meta->frame.cols - box.x); + box.height = std::min(box.height, frame_meta->frame.rows - box.y); + if (box.width <= 0 || box.height <= 0) { + continue; + } + + // apply offset to class id since multi detectors can exist at front of this one. + // later we MUST use the new class id (applied offset) instead of orignal one. + class_id += class_id_offset; + + auto target = std::make_shared(box.x, box.y, box.width, box.height, class_id, confidence, frame_meta->frame_index, frame_meta->channel_index, label); + + // insert target back to frame meta + frame_meta->targets.push_back(target); + } + + class_ids.clear(); + confidences.clear(); + boxes.clear(); + } + } +} \ No newline at end of file diff --git a/nodes/infers/vp_yolo_detector_node.h b/nodes/infers/vp_yolo_detector_node.h new file mode 100755 index 0000000..6e6ef85 --- /dev/null +++ b/nodes/infers/vp_yolo_detector_node.h @@ -0,0 +1,35 @@ + +#pragma once + +#include "../vp_primary_infer_node.h" + +namespace vp_nodes { + // yolo detector, support yolov3/4/5 + // https://github.com/pjreddie/darknet + class vp_yolo_detector_node: public vp_primary_infer_node + { + private: + float score_threshold; + float confidence_threshold; + float nms_threshold; + protected: + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_yolo_detector_node(std::string node_name, + std::string model_path, + std::string model_config_path = "", + std::string labels_path = "", + int input_width = 416, + int input_height = 416, + int batch_size = 1, + int class_id_offset = 0, + float score_threshold = 0.5, + float confidence_threshold = 0.5, + float nms_threshold = 0.5, + float scale = 1 / 255.0, + cv::Scalar mean = cv::Scalar(0), + cv::Scalar std = cv::Scalar(1), + bool swap_rb = true); + ~vp_yolo_detector_node(); + }; +} \ No newline at end of file diff --git a/nodes/infers/vp_yunet_face_detector_node.cpp b/nodes/infers/vp_yunet_face_detector_node.cpp new file mode 100644 index 0000000..f97b4be --- /dev/null +++ b/nodes/infers/vp_yunet_face_detector_node.cpp @@ -0,0 +1,229 @@ + +#include "vp_yunet_face_detector_node.h" + + +namespace vp_nodes { + + vp_yunet_face_detector_node::vp_yunet_face_detector_node(std::string node_name, + std::string model_path, + float score_threshold, + float nms_threshold, + int top_k): + vp_primary_infer_node(node_name, model_path), + scoreThreshold(score_threshold), + nmsThreshold(nms_threshold), + topK(top_k) { + this->initialized(); + } + + vp_yunet_face_detector_node::~vp_yunet_face_detector_node() { + deinitialized(); + } + + void vp_yunet_face_detector_node::postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) { + using namespace cv; + // 3 heads of output + assert(raw_outputs.size() == 3); + assert(frame_meta_with_batch.size() == 1); + auto& frame_meta = frame_meta_with_batch[0]; + + // Extract from output_blobs + Mat loc = raw_outputs[0]; + Mat conf = raw_outputs[1]; + Mat iou = raw_outputs[2]; + + // we need generate priors if input size changed or priors is not initialized + if (loc.rows != priors.size()) { + inputW = frame_meta->frame.cols; + inputH = frame_meta->frame.rows; + generatePriors(); + } + + assert(loc.rows == priors.size()); + assert(loc.rows == conf.rows); + assert(loc.rows == iou.rows); + + // Decode from deltas and priors + const std::vector variance = {0.1f, 0.2f}; + float* loc_v = (float*)(loc.data); + float* conf_v = (float*)(conf.data); + float* iou_v = (float*)(iou.data); + Mat faces; + // (tl_x, tl_y, w, h, re_x, re_y, le_x, le_y, nt_x, nt_y, rcm_x, rcm_y, lcm_x, lcm_y, score) + // 'tl': top left point of the bounding box + // 're': right eye, 'le': left eye + // 'nt': nose tip + // 'rcm': right corner of mouth, 'lcm': left corner of mouth + Mat face(1, 15, CV_32FC1); + for (size_t i = 0; i < priors.size(); ++i) { + // Get score + float clsScore = conf_v[i*2+1]; + float iouScore = iou_v[i]; + // Clamp + if (iouScore < 0.f) { + iouScore = 0.f; + } + else if (iouScore > 1.f) { + iouScore = 1.f; + } + float score = std::sqrt(clsScore * iouScore); + face.at(0, 14) = score; + + // Get bounding box + float cx = (priors[i].x + loc_v[i*14+0] * variance[0] * priors[i].width) * inputW; + float cy = (priors[i].y + loc_v[i*14+1] * variance[0] * priors[i].height) * inputH; + float w = priors[i].width * exp(loc_v[i*14+2] * variance[0]) * inputW; + float h = priors[i].height * exp(loc_v[i*14+3] * variance[1]) * inputH; + float x1 = cx - w / 2; + float y1 = cy - h / 2; + face.at(0, 0) = x1; + face.at(0, 1) = y1; + face.at(0, 2) = w; + face.at(0, 3) = h; + + // Get landmarks + face.at(0, 4) = (priors[i].x + loc_v[i*14+ 4] * variance[0] * priors[i].width) * inputW; // right eye, x + face.at(0, 5) = (priors[i].y + loc_v[i*14+ 5] * variance[0] * priors[i].height) * inputH; // right eye, y + face.at(0, 6) = (priors[i].x + loc_v[i*14+ 6] * variance[0] * priors[i].width) * inputW; // left eye, x + face.at(0, 7) = (priors[i].y + loc_v[i*14+ 7] * variance[0] * priors[i].height) * inputH; // left eye, y + face.at(0, 8) = (priors[i].x + loc_v[i*14+ 8] * variance[0] * priors[i].width) * inputW; // nose tip, x + face.at(0, 9) = (priors[i].y + loc_v[i*14+ 9] * variance[0] * priors[i].height) * inputH; // nose tip, y + face.at(0, 10) = (priors[i].x + loc_v[i*14+10] * variance[0] * priors[i].width) * inputW; // right corner of mouth, x + face.at(0, 11) = (priors[i].y + loc_v[i*14+11] * variance[0] * priors[i].height) * inputH; // right corner of mouth, y + face.at(0, 12) = (priors[i].x + loc_v[i*14+12] * variance[0] * priors[i].width) * inputW; // left corner of mouth, x + face.at(0, 13) = (priors[i].y + loc_v[i*14+13] * variance[0] * priors[i].height) * inputH; // left corner of mouth, y + + faces.push_back(face); + } + + if (faces.rows > 1) + { + // Retrieve boxes and scores + std::vector faceBoxes; + std::vector faceScores; + for (int rIdx = 0; rIdx < faces.rows; rIdx++) + { + faceBoxes.push_back(Rect2i(int(faces.at(rIdx, 0)), + int(faces.at(rIdx, 1)), + int(faces.at(rIdx, 2)), + int(faces.at(rIdx, 3)))); + faceScores.push_back(faces.at(rIdx, 14)); + } + + std::vector keepIdx; + dnn::NMSBoxes(faceBoxes, faceScores, scoreThreshold, nmsThreshold, keepIdx, 1.f, topK); + + // Get NMS results + Mat nms_faces; + for (int idx: keepIdx) + { + nms_faces.push_back(faces.row(idx)); + } + + // insert face target back to frame meta + for (int i = 0; i < nms_faces.rows; i++) { + auto x = int(nms_faces.at(i, 0)); + auto y = int(nms_faces.at(i, 1)); + auto w = int(nms_faces.at(i, 2)); + auto h = int(nms_faces.at(i, 3)); + + // check value range + x = std::max(x, 0); + y = std::max(y, 0); + w = std::min(w, frame_meta->frame.cols - x); + h = std::min(h, frame_meta->frame.rows - y); + + auto kp1 = std::pair(int(nms_faces.at(i, 4)), int(nms_faces.at(i, 5))); + auto kp2 = std::pair(int(nms_faces.at(i, 6)), int(nms_faces.at(i, 7))); + auto kp3 = std::pair(int(nms_faces.at(i, 8)), int(nms_faces.at(i, 9))); + auto kp4 = std::pair(int(nms_faces.at(i, 10)), int(nms_faces.at(i, 11))); + auto kp5 = std::pair(int(nms_faces.at(i, 12)), int(nms_faces.at(i, 13))); + auto score = nms_faces.at(i, 14); + + auto face_target = std::make_shared(x, y, w, h, score, std::vector>{kp1, kp2, kp3, kp4, kp5}); + + frame_meta->face_targets.push_back(face_target); + } + + } + } + + // refer to vp_infer_node::preprocess + void vp_yunet_face_detector_node::preprocess(const std::vector& mats_to_infer, cv::Mat& blob_to_infer) { + cv::dnn::blobFromImages(mats_to_infer, blob_to_infer); + } + + // refer to vp_infer_node::infer + void vp_yunet_face_detector_node::infer(const cv::Mat& blob_to_infer, std::vector& raw_outputs) { + // blob_to_infer is a 4D matrix + // the first dim is number of batch, MUST be 1 + assert(blob_to_infer.dims == 4); + assert(blob_to_infer.size[0] == 1); + assert(!net.empty()); + + net.setInput(blob_to_infer); + net.forward(raw_outputs, out_names); + } + + void vp_yunet_face_detector_node::generatePriors() { + using namespace cv; + // Calculate shapes of different scales according to the shape of input image + Size feature_map_2nd = { + int(int((inputW+1)/2)/2), int(int((inputH+1)/2)/2) + }; + Size feature_map_3rd = { + int(feature_map_2nd.width/2), int(feature_map_2nd.height/2) + }; + Size feature_map_4th = { + int(feature_map_3rd.width/2), int(feature_map_3rd.height/2) + }; + Size feature_map_5th = { + int(feature_map_4th.width/2), int(feature_map_4th.height/2) + }; + Size feature_map_6th = { + int(feature_map_5th.width/2), int(feature_map_5th.height/2) + }; + + std::vector feature_map_sizes; + feature_map_sizes.push_back(feature_map_3rd); + feature_map_sizes.push_back(feature_map_4th); + feature_map_sizes.push_back(feature_map_5th); + feature_map_sizes.push_back(feature_map_6th); + + // Fixed params for generating priors + const std::vector> min_sizes = { + {10.0f, 16.0f, 24.0f}, + {32.0f, 48.0f}, + {64.0f, 96.0f}, + {128.0f, 192.0f, 256.0f} + }; + CV_Assert(min_sizes.size() == feature_map_sizes.size()); // just to keep vectors in sync + const std::vector steps = { 8, 16, 32, 64 }; + + // Generate priors + priors.clear(); + for (size_t i = 0; i < feature_map_sizes.size(); ++i) + { + Size feature_map_size = feature_map_sizes[i]; + std::vector min_size = min_sizes[i]; + + for (int _h = 0; _h < feature_map_size.height; ++_h) + { + for (int _w = 0; _w < feature_map_size.width; ++_w) + { + for (size_t j = 0; j < min_size.size(); ++j) + { + float s_kx = min_size[j] / inputW; + float s_ky = min_size[j] / inputH; + + float cx = (_w + 0.5f) * steps[i] / inputW; + float cy = (_h + 0.5f) * steps[i] / inputH; + + Rect2f prior = { cx, cy, s_kx, s_ky }; + priors.push_back(prior); + } + } + } + } + } +} \ No newline at end of file diff --git a/nodes/infers/vp_yunet_face_detector_node.h b/nodes/infers/vp_yunet_face_detector_node.h new file mode 100644 index 0000000..477c09e --- /dev/null +++ b/nodes/infers/vp_yunet_face_detector_node.h @@ -0,0 +1,34 @@ + +#pragma once + +#include "../vp_primary_infer_node.h" +#include "../../objects/vp_frame_face_target.h" + +namespace vp_nodes { + // face detector based on YunNet + // https://github.com/opencv/opencv/blob/4.x/modules/objdetect/src/face_detect.cpp + // https://github.com/ShiqiYu/libfacedetection + class vp_yunet_face_detector_node: public vp_primary_infer_node + { + private: + // names of output layers in yunet + const std::vector out_names = {"loc", "conf", "iou"}; + float scoreThreshold = 0.7; + float nmsThreshold = 0.5; + int topK = 50; + int inputW; + int inputH; + std::vector priors; + void generatePriors(); + protected: + // override infer and preprocess as yunet has a different logic + virtual void infer(const cv::Mat& blob_to_infer, std::vector& raw_outputs) override; + virtual void preprocess(const std::vector& mats_to_infer, cv::Mat& blob_to_infer) override; + + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) override; + public: + vp_yunet_face_detector_node(std::string node_name, std::string model_path, float score_threshold = 0.7, float nms_threshold = 0.5, int top_k = 50); + ~vp_yunet_face_detector_node(); + }; + +} \ No newline at end of file diff --git a/nodes/osd/README.md b/nodes/osd/README.md new file mode 100644 index 0000000..1889fe5 --- /dev/null +++ b/nodes/osd/README.md @@ -0,0 +1,5 @@ +# Summary + +Two situations to create a new osd node: +1. Need a new osd style for the same type of target +2. OSD for different types of targets, such as `vp_frame_target` (by default), `vp_frame_pose_target`, `vp_frame_face_target` ... diff --git a/nodes/osd/vp_ba_crossline_osd_node.cpp b/nodes/osd/vp_ba_crossline_osd_node.cpp new file mode 100644 index 0000000..459164a --- /dev/null +++ b/nodes/osd/vp_ba_crossline_osd_node.cpp @@ -0,0 +1,91 @@ + +#include "vp_ba_crossline_osd_node.h" + +namespace vp_nodes { + + vp_ba_crossline_osd_node::vp_ba_crossline_osd_node(std::string node_name, std::string font): vp_node(node_name) { + if (!font.empty()) { + ft2 = cv::freetype::createFreeType2(); + ft2->loadFontData(font, 0); + } + this->initialized(); + } + + vp_ba_crossline_osd_node::~vp_ba_crossline_osd_node() { + deinitialized(); + } + + std::shared_ptr vp_ba_crossline_osd_node::handle_frame_meta(std::shared_ptr meta) { + // operations on osd_frame + if (meta->osd_frame.empty()) { + meta->osd_frame = meta->frame.clone(); + } + + auto& canvas = meta->osd_frame; + // scan targets + for (auto& i : meta->targets) { + // track_id + auto id = std::to_string(i->track_id); + auto labels_to_display = i->primary_label; + + // tracked + if (i->track_id != -1) { + labels_to_display = "#" + id + " " + labels_to_display; + } + + for (auto& label : i->secondary_labels) { + labels_to_display += "|" + label; + } + + // draw tracks if size>=2 + if (i->tracks.size() >= 2) { + for (int n = 0; n < (i->tracks.size() - 1); n++) { + auto p1 = i->tracks[n].track_point(); + auto p2 = i->tracks[n + 1].track_point(); + cv::line(canvas, cv::Point(p1.x, p1.y), cv::Point(p2.x, p2.y), cv::Scalar(0, 255, 255), 1, cv::LINE_AA); + } + } + + cv::rectangle(canvas, cv::Rect(i->x, i->y, i->width, i->height), cv::Scalar(255, 255, 0), 2); + if (ft2 != nullptr) { + ft2->putText(canvas, labels_to_display, cv::Point(i->x, i->y), 20, cv::Scalar(255, 0, 255), cv::FILLED, cv::LINE_AA, true); + } + else { + //cv::putText(canvas, labels_to_display, cv::Point(i->x, i->y), 1, 1, cv::Scalar(255, 0, 255)); + int baseline = 0; + auto size = cv::getTextSize(labels_to_display, 1, 1, 1, &baseline); + vp_utils::put_text_at_center_of_rect(canvas, labels_to_display, cv::Rect(i->x, i->y - size.height, size.width, size.height), true, 1, 1, cv::Scalar(), cv::Scalar(179, 52, 255), cv::Scalar(179, 52, 255)); + } + + // scan sub targets + for (auto& sub_target: i->sub_targets) { + cv::rectangle(canvas, cv::Rect(sub_target->x, sub_target->y, sub_target->width, sub_target->height), cv::Scalar(255)); + if (ft2 != nullptr) { + ft2->putText(canvas, sub_target->label, cv::Point(sub_target->x, sub_target->y), 20, cv::Scalar(0, 0, 255), cv::FILLED, cv::LINE_AA, true); + } + else { + cv::putText(canvas, sub_target->label, cv::Point(sub_target->x, sub_target->y), 1, 1, cv::Scalar(0, 0, 255)); + } + } + } + + /* crossline draw for current channel */ + auto& total_crossline = all_total_crossline[meta->channel_index]; + auto& line = all_lines[meta->channel_index]; + + // scan ba results and ONLY deal with crossline + for (auto& i : meta->ba_results) { + if (i->type == vp_objects::vp_ba_type::CROSSLINE) { + // line has 2 points + assert(i->involve_region_in_frame.size() == 2); + line = vp_objects::vp_line(i->involve_region_in_frame[0], i->involve_region_in_frame[1]); + total_crossline += 1; + } + } + + // draw crossline data + cv::line(canvas, cv::Point(line.start.x, line.start.y), cv::Point(line.end.x, line.end.y), cv::Scalar(0, 255, 0), 2, cv::LINE_AA); + cv::putText(canvas, vp_utils::string_format("total crossline targets: [%d]", total_crossline), cv::Point(20, 20), 1, 2, cv::Scalar(0, 0, 255), 2); + return meta; + } +} \ No newline at end of file diff --git a/nodes/osd/vp_ba_crossline_osd_node.h b/nodes/osd/vp_ba_crossline_osd_node.h new file mode 100644 index 0000000..50ec569 --- /dev/null +++ b/nodes/osd/vp_ba_crossline_osd_node.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include +#include "../vp_node.h" +#include "../../objects/shapes/vp_point.h" +#include "../../objects/shapes/vp_line.h" + +namespace vp_nodes { + // osd node for behaviour analysis of crossline + class vp_ba_crossline_osd_node: public vp_node + { + private: + // support chinese font + cv::Ptr ft2; + + // support multi channels + std::map all_total_crossline; + std::map all_lines; + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + public: + vp_ba_crossline_osd_node(std::string node_name, std::string font = ""); + ~vp_ba_crossline_osd_node(); + }; +} \ No newline at end of file diff --git a/nodes/osd/vp_ba_jam_osd_node.cpp b/nodes/osd/vp_ba_jam_osd_node.cpp new file mode 100644 index 0000000..34445e6 --- /dev/null +++ b/nodes/osd/vp_ba_jam_osd_node.cpp @@ -0,0 +1,109 @@ + +#include "vp_ba_jam_osd_node.h" + +namespace vp_nodes { + + vp_ba_jam_osd_node::vp_ba_jam_osd_node(std::string node_name, std::string font): vp_node(node_name) { + if (!font.empty()) { + ft2 = cv::freetype::createFreeType2(); + ft2->loadFontData(font, 0); + } + this->initialized(); + } + + vp_ba_jam_osd_node::~vp_ba_jam_osd_node() { + deinitialized(); + } + + std::shared_ptr vp_ba_jam_osd_node::handle_frame_meta(std::shared_ptr meta) { + // operations on osd_frame + if (meta->osd_frame.empty()) { + meta->osd_frame = meta->frame.clone(); + } + + auto& canvas = meta->osd_frame; + // scan targets + for (auto& i : meta->targets) { + // track_id + auto id = std::to_string(i->track_id); + auto labels_to_display = i->primary_label; + + // tracked + if (i->track_id != -1) { + labels_to_display = "#" + id + " " + labels_to_display; + } + + for (auto& label : i->secondary_labels) { + labels_to_display += "|" + label; + } + + // draw tracks if size>=2 + if (i->tracks.size() >= 2) { + for (int n = 0; n < (i->tracks.size() - 1); n++) { + auto p1 = i->tracks[n].track_point(); + auto p2 = i->tracks[n + 1].track_point(); + cv::line(canvas, cv::Point(p1.x, p1.y), cv::Point(p2.x, p2.y), cv::Scalar(0, 255, 255), 1, cv::LINE_AA); + } + } + + cv::rectangle(canvas, cv::Rect(i->x, i->y, i->width, i->height), cv::Scalar(255, 255, 0), 2); + if (ft2 != nullptr) { + ft2->putText(canvas, labels_to_display, cv::Point(i->x, i->y), 20, cv::Scalar(255, 0, 255), cv::FILLED, cv::LINE_AA, true); + } + else { + //cv::putText(canvas, labels_to_display, cv::Point(i->x, i->y), 1, 1, cv::Scalar(255, 0, 255)); + int baseline = 0; + auto size = cv::getTextSize(labels_to_display, 1, 1, 1, &baseline); + vp_utils::put_text_at_center_of_rect(canvas, labels_to_display, cv::Rect(i->x, i->y - size.height, size.width, size.height), true, 1, 1, cv::Scalar(), cv::Scalar(179, 52, 255), cv::Scalar(179, 52, 255)); + } + + // scan sub targets + for (auto& sub_target: i->sub_targets) { + cv::rectangle(canvas, cv::Rect(sub_target->x, sub_target->y, sub_target->width, sub_target->height), cv::Scalar(255)); + if (ft2 != nullptr) { + ft2->putText(canvas, sub_target->label, cv::Point(sub_target->x, sub_target->y), 20, cv::Scalar(0, 0, 255), cv::FILLED, cv::LINE_AA, true); + } + else { + cv::putText(canvas, sub_target->label, cv::Point(sub_target->x, sub_target->y), 1, 1, cv::Scalar(0, 0, 255)); + } + } + } + + /* jam draw for current channel */ + auto& region = all_jam_regions[meta->channel_index]; + auto& jam_result = all_jam_results[meta->channel_index]; + auto& involve_ids = all_involve_ids[meta->channel_index]; + + // scan ba results and ONLY deal with stop and unstop + for (auto& i : meta->ba_results) { + if (i->type == vp_objects::vp_ba_type::JAM) { + region = i->involve_region_in_frame; + involve_ids = i->involve_target_ids_in_frame; + jam_result = true; + } + if (i->type == vp_objects::vp_ba_type::UNJAM) { + region = i->involve_region_in_frame; + involve_ids = i->involve_target_ids_in_frame; // not used later + jam_result = false; + } + } + + // draw jam data + auto poly_vertexs = [&]() { + std::vector vertexs; + for(auto& p: region) { + vertexs.push_back(cv::Point(p.x, p.y)); + } + return vertexs; + }; + if (jam_result) { + cv::polylines(canvas, poly_vertexs(), true, cv::Scalar(0, 0, 255), 2, cv::LINE_AA); // region + // targets in jan region, highlight + auto targets = meta->get_targets_by_ids(involve_ids); + for (auto& t: targets) { + cv::rectangle(canvas, cv::Rect(t->x, t->y, t->width, t->height), cv::Scalar(0, 0, 255), 2, cv::LINE_AA); + } + } + return meta; + } +} \ No newline at end of file diff --git a/nodes/osd/vp_ba_jam_osd_node.h b/nodes/osd/vp_ba_jam_osd_node.h new file mode 100644 index 0000000..922bb69 --- /dev/null +++ b/nodes/osd/vp_ba_jam_osd_node.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include "../vp_node.h" +#include "../../objects/shapes/vp_point.h" +#include "../../objects/shapes/vp_line.h" + +namespace vp_nodes { + // osd node for behaviour analysis of stop + class vp_ba_jam_osd_node: public vp_node + { + private: + // support chinese font + cv::Ptr ft2; + + // support multi channels + std::map> all_jam_regions; // channel -> jam region + std::map all_jam_results; // channel -> jam status + std::map> all_involve_ids; // channel -> target ids when enter jam status + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + public: + vp_ba_jam_osd_node(std::string node_name, std::string font = ""); + ~vp_ba_jam_osd_node(); + }; +} \ No newline at end of file diff --git a/nodes/osd/vp_ba_stop_osd_node.cpp b/nodes/osd/vp_ba_stop_osd_node.cpp new file mode 100644 index 0000000..16e87ae --- /dev/null +++ b/nodes/osd/vp_ba_stop_osd_node.cpp @@ -0,0 +1,114 @@ + +#include "vp_ba_stop_osd_node.h" + +namespace vp_nodes { + + vp_ba_stop_osd_node::vp_ba_stop_osd_node(std::string node_name, std::string font): vp_node(node_name) { + if (!font.empty()) { + ft2 = cv::freetype::createFreeType2(); + ft2->loadFontData(font, 0); + } + this->initialized(); + } + + vp_ba_stop_osd_node::~vp_ba_stop_osd_node() { + deinitialized(); + } + + std::shared_ptr vp_ba_stop_osd_node::handle_frame_meta(std::shared_ptr meta) { + // operations on osd_frame + if (meta->osd_frame.empty()) { + meta->osd_frame = meta->frame.clone(); + } + + auto& canvas = meta->osd_frame; + // scan targets + for (auto& i : meta->targets) { + // track_id + auto id = std::to_string(i->track_id); + auto labels_to_display = i->primary_label; + + // tracked + if (i->track_id != -1) { + labels_to_display = "#" + id + " " + labels_to_display; + } + + for (auto& label : i->secondary_labels) { + labels_to_display += "|" + label; + } + + // draw tracks if size>=2 + if (i->tracks.size() >= 2) { + for (int n = 0; n < (i->tracks.size() - 1); n++) { + auto p1 = i->tracks[n].track_point(); + auto p2 = i->tracks[n + 1].track_point(); + cv::line(canvas, cv::Point(p1.x, p1.y), cv::Point(p2.x, p2.y), cv::Scalar(0, 255, 255), 1, cv::LINE_AA); + } + } + + cv::rectangle(canvas, cv::Rect(i->x, i->y, i->width, i->height), cv::Scalar(255, 255, 0), 2); + if (ft2 != nullptr) { + ft2->putText(canvas, labels_to_display, cv::Point(i->x, i->y), 20, cv::Scalar(255, 0, 255), cv::FILLED, cv::LINE_AA, true); + } + else { + //cv::putText(canvas, labels_to_display, cv::Point(i->x, i->y), 1, 1, cv::Scalar(255, 0, 255)); + int baseline = 0; + auto size = cv::getTextSize(labels_to_display, 1, 1, 1, &baseline); + vp_utils::put_text_at_center_of_rect(canvas, labels_to_display, cv::Rect(i->x, i->y - size.height, size.width, size.height), true, 1, 1, cv::Scalar(), cv::Scalar(179, 52, 255), cv::Scalar(179, 52, 255)); + } + + // scan sub targets + for (auto& sub_target: i->sub_targets) { + cv::rectangle(canvas, cv::Rect(sub_target->x, sub_target->y, sub_target->width, sub_target->height), cv::Scalar(255)); + if (ft2 != nullptr) { + ft2->putText(canvas, sub_target->label, cv::Point(sub_target->x, sub_target->y), 20, cv::Scalar(0, 0, 255), cv::FILLED, cv::LINE_AA, true); + } + else { + cv::putText(canvas, sub_target->label, cv::Point(sub_target->x, sub_target->y), 1, 1, cv::Scalar(0, 0, 255)); + } + } + } + + /* stop draw for current channel */ + auto& region = all_stop_regions[meta->channel_index]; + auto& total_stop = all_total_stop[meta->channel_index]; + auto& stop_count = all_stop_count[meta->channel_index]; + + // scan ba results and ONLY deal with stop and unstop + for (auto& i : meta->ba_results) { + if (i->type == vp_objects::vp_ba_type::STOP) { + region = i->involve_region_in_frame; + stop_count++; + total_stop.push_back(i->involve_target_ids_in_frame[0]); // only 1 target + } + if (i->type == vp_objects::vp_ba_type::UNSTOP) { + region = i->involve_region_in_frame; + for (auto j = total_stop.begin(); j < total_stop.end();) { + if (*j == i->involve_target_ids_in_frame[0]) { + j = total_stop.erase(j); + break; + } + j++; + } + } + } + + // draw stop data + auto poly_vertexs = [&]() { + std::vector vertexs; + for(auto& p: region) { + vertexs.push_back(cv::Point(p.x, p.y)); + } + return vertexs; + }; + cv::polylines(canvas, poly_vertexs(), true, cv::Scalar(0, 255, 0), 2, cv::LINE_AA); + // cv::putText(canvas, vp_utils::string_format("total stop targets: [%d]", stop_count), cv::Point(20, 20), 1, 2, cv::Scalar(0, 0, 255), 2); + + // stop targets, highlight + auto stop_targets = meta->get_targets_by_ids(total_stop); + for (auto& t: stop_targets) { + cv::rectangle(canvas, cv::Rect(t->x, t->y, t->width, t->height), cv::Scalar(0, 0, 255), 2, cv::LINE_AA); + } + return meta; + } +} \ No newline at end of file diff --git a/nodes/osd/vp_ba_stop_osd_node.h b/nodes/osd/vp_ba_stop_osd_node.h new file mode 100644 index 0000000..9ed57e8 --- /dev/null +++ b/nodes/osd/vp_ba_stop_osd_node.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include "../vp_node.h" +#include "../../objects/shapes/vp_point.h" +#include "../../objects/shapes/vp_line.h" + +namespace vp_nodes { + // osd node for behaviour analysis of stop + class vp_ba_stop_osd_node: public vp_node + { + private: + // support chinese font + cv::Ptr ft2; + + // support multi channels + std::map> all_total_stop; // channel -> stop target ids + std::map> all_stop_regions; // channel -> stop region + std::map all_stop_count; // channel -> stop count + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + public: + vp_ba_stop_osd_node(std::string node_name, std::string font = ""); + ~vp_ba_stop_osd_node(); + }; +} \ No newline at end of file diff --git a/nodes/osd/vp_cluster_node.cpp b/nodes/osd/vp_cluster_node.cpp new file mode 100644 index 0000000..a763752 --- /dev/null +++ b/nodes/osd/vp_cluster_node.cpp @@ -0,0 +1,214 @@ +#include "vp_cluster_node.h" +#include "../../third_party/bhtsne/tsne.h" + +namespace vp_nodes { + vp_cluster_node::vp_cluster_node(std::string node_name, + bool use_tSNE, + std::vector s_labels_to_display, + int sampling_frequency, + int min_sampling_width, + int min_sampling_height, + int max_sample_num_for_tsne, + int max_sample_num_per_category): + vp_node(node_name), + use_tSNE(use_tSNE), + s_labels_to_display(s_labels_to_display), + sampling_frequency(sampling_frequency), + min_sampling_width(min_sampling_width), + min_sampling_height(min_sampling_height), + max_sample_num_for_tsne(max_sample_num_for_tsne), + max_sample_num_per_category(max_sample_num_per_category) { + this->initialized(); + } + + vp_cluster_node::~vp_cluster_node() { + deinitialized(); + } + + // please refer to ../../third_party/trt_vehicle/main/vehicle_cluster.cpp + void vp_cluster_node::reduce_dims_using_tsne(std::vector>& low_features, + int no_dims, int max_iter, double perplexity, + double theta, int rand_seed, bool skip_random_init, + int stop_lying_iter, int mom_switch_iter) { + assert(cache_high_features.size() != 0); + auto N = cache_high_features.size(); + auto D = cache_high_features[0].second.size(); // all the same as the first feature's dims + + // prepare input + double data[N * D]; + double Y[N * no_dims]; + for (int i = 0; i < N; i++) { + for (int j = 0; j < D; j++) { + data[i * D + j] = cache_high_features[i].second[j]; + } + } + + // call t-SNE + TSNE::run(data, N, D, Y, no_dims, perplexity, theta, rand_seed, skip_random_init, max_iter, stop_lying_iter, mom_switch_iter); + + // prepare output + for (int i = 0; i < N; i++) { + std::vector low_dims_feature; + for (int j = 0; j < no_dims; j++) { + low_dims_feature.push_back(float(Y[i * no_dims + j])); + } + low_features.push_back(low_dims_feature); + } + } + + + std::shared_ptr vp_cluster_node::handle_frame_meta(std::shared_ptr meta) { + // sampling frequency + if (std::chrono::duration_cast(NOW - last_sampling_time) < std::chrono::milliseconds(sampling_frequency)) { + return meta; + } + last_sampling_time = NOW; + + /* t-SNE */ + if (use_tSNE) { + for (int i = 0; i < meta->targets.size(); i++) { + if (meta->targets[i]->width < min_sampling_width || meta->targets[i]->height < min_sampling_height) { + continue; + } + + if (cache_high_features.size() >= max_sample_num_for_tsne) { + // too many cache, we need control number of samples + cache_high_features.erase(cache_high_features.begin()); + } + // save mat->feature pair to cache + cv::Mat m; + cv::Mat roi(meta->frame, cv::Rect(meta->targets[i]->x, meta->targets[i]->y, meta->targets[i]->width, meta->targets[i]->height)); + roi.copyTo(m); + std::pair> p {m, meta->targets[i]->embeddings}; + cache_high_features.push_back(p); + } + + // at least 10 samples to t-SNE + if (cache_high_features.size() >= 10) { + // reduce dims first + std::vector> low_features; + reduce_dims_using_tsne(low_features); + + // now display on screen + // normalize low dims feature to coordinate of [0:1] and display them on 2D screen + auto max_x = 0.0f, max_y = 0.0f, min_x = 0.0f, min_y = 0.0f; + for(int i = 0; i < low_features.size(); i++) { + auto& f = low_features[i]; + // 2 values in f + max_x = std::max(max_x, f[0]); + max_y = std::max(max_y, f[1]); + min_x = std::min(min_x, f[0]); + min_y = std::min(min_y, f[1]); + } + auto x_range = max_x - min_x; + auto y_range = max_y - min_y; + + // draw on (tsne_canvas_w_h + tsne_thumbnail_w_h) * (tsne_canvas_w_h + tsne_thumbnail_w_h) + cv::Mat canvas(tsne_canvas_w_h + tsne_thumbnail_w_h, tsne_canvas_w_h + tsne_thumbnail_w_h, CV_8UC3, cv::Scalar(127, 127, 127)); + for(int i = 0; i < low_features.size(); i++) { + auto& f = low_features[i]; + // convert to [0:1] + f[0] = (f[0] - min_x) / x_range; + f[1] = (f[1] - min_y) / y_range; + + auto& img = cache_high_features[i].first; + cv::Mat img_tmp; + cv::resize(img, img_tmp, cv::Size(tsne_thumbnail_w_h, tsne_thumbnail_w_h)); + cv::rectangle(img_tmp, cv::Rect(0, 0, img_tmp.cols, img_tmp.rows), cv::Scalar(255, 0, 0)); + cv::Mat roi(canvas, cv::Rect(int(f[0] * tsne_canvas_w_h), int(f[1] * tsne_canvas_w_h), tsne_thumbnail_w_h, tsne_thumbnail_w_h)); + img_tmp.copyTo(roi); + } + + cv::imshow("cluster using features powered by t-SNE", canvas); + } + } + + /* categories */ + for (int i = 0; i < meta->targets.size(); i++) { + auto& t = meta->targets[i]; + if (t->width < min_sampling_width || t->height < min_sampling_height) { + continue; + } + + cv::Mat m; + cv::Mat roi(meta->frame, cv::Rect(t->x, t->y, t->width, t->height)); + roi.copyTo(m); + + for (int j = 0; j < t->secondary_labels.size(); j++) { + auto& category = t->secondary_labels[j]; + bool filter_pass = false; + // all + if (s_labels_to_display.size() == 0) { + cache_categories[category].push_back(m); + filter_pass = true; + } + else { + // has a filter + if (std::find(s_labels_to_display.begin(), s_labels_to_display.end(), category) != s_labels_to_display.end()) { + cache_categories[category].push_back(m); + filter_pass = true; + } + } + + if (filter_pass) { + // too many cache, we need control number of samples + if (cache_categories[category].size() >= max_sample_num_per_category) { + cache_categories[category].erase(cache_categories[category].begin()); + } + } + } + } + // display on screen + // calculate total number of rows + auto rows_num = 0; + for (auto& p: cache_categories) { + auto num = p.second.size() / category_num_per_row; + if (p.second.size() % category_num_per_row != 0 || num == 0) { + num++; + } + rows_num += num; + } + + // draw on (category_canvas_w) * (category_canvas_h) + auto category_canvas_w = (category_thumbnail_w_h + category_gap) * (category_num_per_row + 1); + auto category_canvas_h = (category_thumbnail_w_h + category_gap) * (rows_num + 1); + cv::Mat canvas(category_canvas_h, category_canvas_w, CV_8UC3, cv::Scalar(127, 127, 127)); + + auto row_index = 0; + for (auto& p: cache_categories) { + auto col_index = 0; + auto& category_items = p.second; + auto& category_name = p.first; + cv::putText(canvas, category_name, cv::Point(10, (category_gap + category_thumbnail_w_h) * row_index + category_gap), 1, 1, cv::Scalar(0, 0, 255)); + + bool new_row = false; + for (int i = 0; i < category_items.size(); i++) { + cv::Mat img_tmp; + cv::resize(category_items[i], img_tmp, cv::Size(category_thumbnail_w_h, category_thumbnail_w_h)); + cv::Mat roi(canvas, cv::Rect((category_gap + category_thumbnail_w_h) * col_index + category_gap, (category_gap + category_thumbnail_w_h) * row_index + category_gap, category_thumbnail_w_h, category_thumbnail_w_h)); + img_tmp.copyTo(roi); + + col_index++; + if ((col_index + 1) % category_num_per_row == 0) { + col_index = 0; + row_index ++; + new_row = true; + } + else { + new_row = false; + } + } + if (!new_row) { + row_index++; + } + } + + cv::imshow("cluster using labels", canvas); + + return meta; + } + + std::shared_ptr vp_cluster_node::handle_control_meta(std::shared_ptr meta) { + return meta; + } +} \ No newline at end of file diff --git a/nodes/osd/vp_cluster_node.h b/nodes/osd/vp_cluster_node.h new file mode 100644 index 0000000..776de98 --- /dev/null +++ b/nodes/osd/vp_cluster_node.h @@ -0,0 +1,57 @@ +#pragma once + +#include "../vp_node.h" + +namespace vp_nodes { + // cluster node for vp_frame_targets which has ability to display targets on screen according to its embeddings contained in vp_frame_target::embbedings variable or labels contained in vp_frame_target::secondary_labels vector。 + // note!!! + // it is not an osd node which would operates on vp_frame_meta::osd_frame data member. + // it is not a DES node either which can be the last node in pipeline. + // it is just a normal MID node. + class vp_cluster_node: public vp_node + { + private: + // call tSNE algorithm to reduce high dims of feature and display target on 2D screen + bool use_tSNE; + // display target based on categories, empty means for all categories. if you want to disable it, just let vector including just a wrong category name take '123abcd' for example. + std::vector s_labels_to_display; + // how often to sampling (miliseconds). since targets differences in adjacent frames are small, we need not sampling continuously. + int sampling_frequency = 1000; + std::chrono::system_clock::time_point last_sampling_time = NOW; + + // filter for small targets + int min_sampling_width = 40; + int min_sampling_height = 40; + + /* draw parameters for tsne */ + int tsne_canvas_w_h = 600; + int tsne_thumbnail_w_h = 60; + int max_sample_num_for_tsne = 100; + std::vector>> cache_high_features; // mat -> feature + + /* draw parameters for category */ + int category_num_per_row = 10; + int category_thumbnail_w_h = 60; + int category_gap = 10; + int max_sample_num_per_category = 10; + std::map> cache_categories; // label -> mat list + + // reduce dims of features (to xy) so as to display it on 2D screen + void reduce_dims_using_tsne(std::vector>& low_features, + /* default parameters for t-SNE algorithm */ + int no_dims = 2, int max_iter = 500, double perplexity = 2, double theta = 0.5, int rand_seed = -1, bool skip_random_init = false, int stop_lying_iter = 250, int mom_switch_iter = 250); + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + virtual std::shared_ptr handle_control_meta(std::shared_ptr meta) override; + public: + vp_cluster_node(std::string node_name, + bool use_tSNE = true, + std::vector s_labels_to_display = std::vector{}, + int sampling_frequency = 1000, + int min_sampling_width = 40, + int min_sampling_height = 40, + int max_sample_num_for_tsne = 100, + int max_sample_num_per_category = 10); + ~vp_cluster_node(); + }; +} \ No newline at end of file diff --git a/nodes/osd/vp_expr_osd_node.cpp b/nodes/osd/vp_expr_osd_node.cpp new file mode 100644 index 0000000..83e980f --- /dev/null +++ b/nodes/osd/vp_expr_osd_node.cpp @@ -0,0 +1,56 @@ + +#include +#include "vp_expr_osd_node.h" +#include "../../utils/vp_utils.h" + + +namespace vp_nodes { + + vp_expr_osd_node::vp_expr_osd_node(std::string node_name, std::string font):vp_node(node_name) { + assert(font != ""); + ft2 = cv::freetype::createFreeType2(); + ft2->loadFontData(font, 0); + this->initialized(); + } + + vp_expr_osd_node::~vp_expr_osd_node() { + deinitialized(); + } + + std::shared_ptr vp_expr_osd_node::handle_frame_meta(std::shared_ptr meta) { + // operations on osd_frame + if (meta->osd_frame.empty()) { + meta->osd_frame = meta->frame.clone(); + } + auto& canvas = meta->osd_frame; + + for (auto& i: meta->text_targets) { + cv::Point rook_points[4]; + for (int m = 0; m < i->region_vertexes.size(); m++) { + rook_points[m] = + cv::Point(i->region_vertexes[m].first, i->region_vertexes[m].second); + } + + const cv::Point *ppt[1] = {rook_points}; + int npt[] = {4}; + + if (i->flags.find("yes") != std::string::npos) { + cv::polylines(canvas, ppt, npt, 1, 1, CV_RGB(0, 255, 0), 2, cv::LINE_AA, 0); // green + ft2->putText(canvas, "√", rook_points[1], 30, CV_RGB(0, 255, 0), cv::FILLED, cv::LINE_AA, true); + } + else if (i->flags.find("no") != std::string::npos) { + auto right_value = vp_utils::string_split(i->flags, '_')[1]; + cv::polylines(canvas, ppt, npt, 1, 1, CV_RGB(255, 0, 0), 2, cv::LINE_AA, 0); // red + ft2->putText(canvas, "×(" + right_value + ")", rook_points[1], 30, CV_RGB(255, 0, 0), cv::FILLED, cv::LINE_AA, true); + } + else if (i->flags == "invalid") { + cv::polylines(canvas, ppt, npt, 1, 1, CV_RGB(255, 165, 0), 2, cv::LINE_AA, 0); // orange + ft2->putText(canvas, "invalid", rook_points[1], 30, CV_RGB(255, 165, 0), cv::FILLED, cv::LINE_AA, true); + } + else { + // to-do + } + } + return meta; + } +} \ No newline at end of file diff --git a/nodes/osd/vp_expr_osd_node.h b/nodes/osd/vp_expr_osd_node.h new file mode 100644 index 0000000..648d9c2 --- /dev/null +++ b/nodes/osd/vp_expr_osd_node.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#include "../vp_node.h" + +namespace vp_nodes { + // on screen display(short as osd) node. + // mainly used to display math expression which based on vp_frame_text_target on frame. + class vp_expr_osd_node: public vp_node + { + private: + // support chinese font + cv::Ptr ft2; + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + public: + vp_expr_osd_node(std::string node_name, std::string font); + ~vp_expr_osd_node(); + }; + +} \ No newline at end of file diff --git a/nodes/osd/vp_face_osd_node.cpp b/nodes/osd/vp_face_osd_node.cpp new file mode 100644 index 0000000..b6d1bd2 --- /dev/null +++ b/nodes/osd/vp_face_osd_node.cpp @@ -0,0 +1,48 @@ + +#include +#include "vp_face_osd_node.h" + +namespace vp_nodes { + + vp_face_osd_node::vp_face_osd_node(std::string node_name): vp_node(node_name) { + this->initialized(); + } + + vp_face_osd_node::~vp_face_osd_node() { + deinitialized(); + } + + std::shared_ptr vp_face_osd_node::handle_frame_meta(std::shared_ptr meta) { + // operations on osd_frame + if (meta->osd_frame.empty()) { + meta->osd_frame = meta->frame.clone(); + } + auto& canvas = meta->osd_frame; + + // scan face targets + for(auto& i : meta->face_targets) { + cv::rectangle(canvas, cv::Rect(i->x, i->y, i->width, i->height), cv::Scalar(0, 255, 0), 2); + + // track_id + if (i->track_id != -1) { + auto id = std::to_string(i->track_id); + cv::putText(canvas, id, cv::Point(i->x, i->y), 1, 1.5, cv::Scalar(0, 0, 255)); + } + + // just handle 5 keypoints + if (i->key_points.size() >= 5) { + cv::circle(canvas, cv::Point(i->key_points[0].first, i->key_points[0].second), 2, cv::Scalar(255, 0, 0), 2); + cv::circle(canvas, cv::Point(i->key_points[1].first, i->key_points[1].second), 2, cv::Scalar(0, 0, 255), 2); + cv::circle(canvas, cv::Point(i->key_points[2].first, i->key_points[2].second), 2, cv::Scalar(0, 255, 0), 2); + cv::circle(canvas, cv::Point(i->key_points[3].first, i->key_points[3].second), 2, cv::Scalar(255, 0, 255), 2); + cv::circle(canvas, cv::Point(i->key_points[4].first, i->key_points[4].second), 2, cv::Scalar(0, 255, 255), 2); + } + } + + return meta; + } + + std::shared_ptr vp_face_osd_node::handle_control_meta(std::shared_ptr meta) { + return meta; + } +} \ No newline at end of file diff --git a/nodes/osd/vp_face_osd_node.h b/nodes/osd/vp_face_osd_node.h new file mode 100644 index 0000000..e4a51a1 --- /dev/null +++ b/nodes/osd/vp_face_osd_node.h @@ -0,0 +1,20 @@ +#pragma once + +#include "../vp_node.h" + +namespace vp_nodes { + // on screen display(short as osd) node. + // mainly used to display vp_frame_face_target on frame. + class vp_face_osd_node: public vp_node + { + private: + /* data */ + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + virtual std::shared_ptr handle_control_meta(std::shared_ptr meta) override; + public: + vp_face_osd_node(std::string node_name); + ~vp_face_osd_node(); + }; + +} \ No newline at end of file diff --git a/nodes/osd/vp_face_osd_node_v2.cpp b/nodes/osd/vp_face_osd_node_v2.cpp new file mode 100644 index 0000000..bb3c01c --- /dev/null +++ b/nodes/osd/vp_face_osd_node_v2.cpp @@ -0,0 +1,148 @@ + +#include "vp_face_osd_node_v2.h" +#include "../../utils/vp_utils.h" + +namespace vp_nodes { + + vp_face_osd_node_v2::vp_face_osd_node_v2(std::string node_name): vp_node(node_name) { + this->initialized(); + } + + vp_face_osd_node_v2::~vp_face_osd_node_v2() { + deinitialized(); + } + + std::shared_ptr vp_face_osd_node_v2::handle_frame_meta(std::shared_ptr meta) { + // operations on osd_frame + if (meta->osd_frame.empty()) { + // add a gap at the bottom of osd frame + meta->osd_frame = cv::Mat(meta->frame.rows + gap_height + padding * 2, meta->frame.cols, meta->frame.type(), cv::Scalar(128, 128, 128)); + + // initialize by copying frame to osd frame + auto roi = meta->osd_frame(cv::Rect(0, 0, meta->frame.cols, meta->frame.rows)); + meta->frame.copyTo(roi); + } + auto& canvas = meta->osd_frame; + // scan face targets in current frame + for(auto& i : meta->face_targets) { + // draw face rect first + cv::rectangle(canvas, cv::Rect(i->x, i->y, i->width, i->height), cv::Scalar(0, 255, 0), 2); + + // track_id + if (i->track_id != -1) { + auto id = std::to_string(i->track_id); + cv::putText(canvas, id, cv::Point(i->x, i->y), 1, 1.5, cv::Scalar(0, 0, 255)); + } + + // just handle 5 keypoints + if (i->key_points.size() >= 5) { + cv::circle(canvas, cv::Point(i->key_points[0].first, i->key_points[0].second), 2, cv::Scalar(255, 0, 0), 2); + cv::circle(canvas, cv::Point(i->key_points[1].first, i->key_points[1].second), 2, cv::Scalar(0, 0, 255), 2); + cv::circle(canvas, cv::Point(i->key_points[2].first, i->key_points[2].second), 2, cv::Scalar(0, 255, 0), 2); + cv::circle(canvas, cv::Point(i->key_points[3].first, i->key_points[3].second), 2, cv::Scalar(255, 0, 255), 2); + cv::circle(canvas, cv::Point(i->key_points[4].first, i->key_points[4].second), 2, cv::Scalar(0, 255, 255), 2); + } + + // cache the first face + if (the_baseline_face.empty()) { + auto face = meta->frame(cv::Rect(i->x, i->y, i->width, i->height)); + cv::resize(face, the_baseline_face, cv::Size(gap_height, gap_height)); + the_baseline_face_feature = i->embeddings; + } + else { + // check if the face has existed in list by calculating distance between 2 faces + bool exist = false; + for(auto& f : face_features) { + if (match(i->embeddings, f, 0) >= cosine_similar_thresh || + match(i->embeddings, f, 1) <= l2norm_similar_thresh) { + exist = true; + break; + } + } + + if (!exist) { + auto cosine_dis = match(i->embeddings, the_baseline_face_feature, 0); + auto l2_dis = match(i->embeddings, the_baseline_face_feature, 1); + if (cosine_dis >= cosine_similar_thresh || l2_dis <= l2norm_similar_thresh) { + // as same as the_baseline_face + } + else { + // new face, add it to list for dispaly + auto face = meta->frame(cv::Rect(i->x, i->y, i->width, i->height)); + cv::Mat resized_face; + cv::resize(face, resized_face, cv::Size(gap_height, gap_height)); + + faces_list.push_back(resized_face); + face_features.push_back(i->embeddings); + cosine_distances.push_back(cosine_dis); + l2_distances.push_back(l2_dis); + } + } + else { + // has existed in list + } + } + } + + // too many faces, delete the first ones in list + auto width_need = faces_list.size() * (gap_height + padding) + padding; + while (width_need >= canvas.cols) { + faces_list.erase(faces_list.begin()); + face_features.erase(face_features.begin()); + cosine_distances.erase(cosine_distances.begin()); + l2_distances.erase(l2_distances.begin()); + + // check again + width_need = faces_list.size() * (gap_height + padding) + padding; + } + + // make sure the size for each vector + assert(faces_list.size() == face_features.size()); + assert(faces_list.size() == cosine_distances.size()); + assert(faces_list.size() == l2_distances.size()); + + assert(canvas.rows > (gap_height + padding) * 2); + + // display the baseline face + if (!the_baseline_face.empty()) { + auto roi = canvas(cv::Rect(padding, canvas.rows - gap_height * 2 - padding * 2, gap_height, gap_height)); + the_baseline_face.copyTo(roi); + } + + // display faces in list + for (int i = 0; i < faces_list.size(); i++) { + auto roi = canvas(cv::Rect((padding + gap_height) * i + padding, canvas.rows - gap_height - padding, gap_height, gap_height)); + faces_list[i].copyTo(roi); + + // distance between face and the baseline + cv::line(canvas, cv::Point(padding + gap_height / 2, canvas.rows - gap_height - padding * 2 - gap_height / 2), + cv::Point((padding + gap_height) * i + padding + gap_height / 2, canvas.rows - gap_height - padding), cv::Scalar(255, 0, 0), 1, cv::LINE_AA); + cv::putText(canvas, + "cos_dis:" + vp_utils::round_any(cosine_distances[i], 3), + cv::Point((padding + gap_height) * i + padding, canvas.rows - gap_height - padding + 20), 1, 0.9, cv::Scalar(255, 255, 0)); + cv::putText(canvas, + "l2_dis:" + vp_utils::round_any(l2_distances[i], 3), + cv::Point((padding + gap_height) * i + padding, canvas.rows - gap_height - padding + 40), 1, 0.9, cv::Scalar(255, 255, 0)); + } + + return meta; + } + + double vp_face_osd_node_v2::match(std::vector& feature1, std::vector& feature2, int dis_type) { + auto _face_feature1 = cv::Mat(1, feature1.size(), CV_32F, feature1.data()); + auto _face_feature2 = cv::Mat(1, feature2.size(), CV_32F, feature2.data()); + cv::normalize(_face_feature1, _face_feature1); + cv::normalize(_face_feature2, _face_feature2); + + if(dis_type == 0) { + return cv::sum(_face_feature1.mul(_face_feature2))[0]; + } + else if(dis_type == 1) { + return cv::norm(_face_feature1, _face_feature2); + } + else { + throw std::invalid_argument("invalid parameter " + std::to_string(dis_type)); + } + + } +} \ No newline at end of file diff --git a/nodes/osd/vp_face_osd_node_v2.h b/nodes/osd/vp_face_osd_node_v2.h new file mode 100644 index 0000000..e62fce4 --- /dev/null +++ b/nodes/osd/vp_face_osd_node_v2.h @@ -0,0 +1,40 @@ + +#pragma once + +#include +#include +#include "../vp_node.h" + +namespace vp_nodes { + // on screen display(short as osd) node. + // another version for vp_frame_face_target display, including displaying similarity between faces. + class vp_face_osd_node_v2: public vp_node + { + private: + // leave a gap at the bottom of osd frame + int gap_height = 112; + int padding = 5; + + // cache for display + cv::Mat the_baseline_face; // the first face detected in frames + std::vector the_baseline_face_feature; // the feature of first face in frames + std::vector faces_list; // faces detected except the first one + std::vector> face_features; // face features except the first one + std::vector cosine_distances; // cosine distance between face and the first one + std::vector l2_distances; // l2 distance between face and the first one + + double cosine_similar_thresh = 0.363; // higher value means higher similarity, max 1.0 + double l2norm_similar_thresh = 1.128; // lower value means higher similarity, min 0.0 + + // calculate distance + // 0 cosine distance + // 1 l2 distance + double match(std::vector& feature1, std::vector& feature2, int dis_type); + + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + public: + vp_face_osd_node_v2(std::string node_name); + ~vp_face_osd_node_v2(); + }; +} \ No newline at end of file diff --git a/nodes/osd/vp_lane_osd_node.cpp b/nodes/osd/vp_lane_osd_node.cpp new file mode 100644 index 0000000..eed44bc --- /dev/null +++ b/nodes/osd/vp_lane_osd_node.cpp @@ -0,0 +1,51 @@ +#include +#include "vp_lane_osd_node.h" +#include "../../utils/vp_utils.h" + +namespace vp_nodes { + + vp_lane_osd_node::vp_lane_osd_node(std::string node_name): vp_node(node_name) { + this->initialized(); + } + + vp_lane_osd_node::~vp_lane_osd_node() { + deinitialized(); + } + + std::shared_ptr vp_lane_osd_node::handle_frame_meta(std::shared_ptr meta) { + // operations on osd_frame + if (meta->osd_frame.empty()) { + meta->osd_frame = meta->frame.clone(); + } + auto& canvas = meta->osd_frame; + + if (meta->mask.empty()) { + return meta; + } + + // resize mask to the same size of canvas + cv::Mat mask(meta->mask.size[2], meta->mask.size[3], CV_32FC1, meta->mask.data); + cv::Mat mask_big; + cv::resize(mask, mask_big, canvas.size()); + cv::threshold(mask_big, mask_big, 0.5, 1, cv::THRESH_BINARY); + + auto kernel = cv::getStructuringElement(cv::MORPH_RECT, cv::Size(11, 11)); + cv::erode(mask_big, mask_big, kernel); + mask_big.convertTo(mask_big, CV_8U, 255); + + // merge mask and canvas + for (int y = 0; y < canvas.rows; ++y) { + for (int x = 0; x < canvas.cols; ++x) { + canvas.at(y, x)[2] = mask_big.at(y, x); + + canvas.at(y, x)[1] = cv::saturate_cast( + canvas.at(y, x)[1] * 0.8 + mask_big.at(y, x) * 0.2); + + canvas.at(y, x)[0] = cv::saturate_cast( + canvas.at(y, x)[0] * 0.8 + mask_big.at(y, x) * 0.2); + } + } + + return meta; + } +} \ No newline at end of file diff --git a/nodes/osd/vp_lane_osd_node.h b/nodes/osd/vp_lane_osd_node.h new file mode 100644 index 0000000..f26f3f7 --- /dev/null +++ b/nodes/osd/vp_lane_osd_node.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include "../vp_node.h" + +namespace vp_nodes { + class vp_lane_osd_node: public vp_node + { + private: + /* data */ + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + public: + vp_lane_osd_node(std::string node_name); + ~vp_lane_osd_node(); + }; +} \ No newline at end of file diff --git a/nodes/osd/vp_mllm_osd_node.cpp b/nodes/osd/vp_mllm_osd_node.cpp new file mode 100644 index 0000000..3030065 --- /dev/null +++ b/nodes/osd/vp_mllm_osd_node.cpp @@ -0,0 +1,91 @@ + +#include "vp_mllm_osd_node.h" + +#ifdef VP_WITH_LLM +namespace vp_nodes { + + vp_mllm_osd_node::vp_mllm_osd_node(std::string node_name, std::string font): vp_node(node_name) { + ft2 = cv::freetype::createFreeType2(); + ft2->loadFontData(font, 0); + this->initialized(); + } + + vp_mllm_osd_node::~vp_mllm_osd_node() { + deinitialized(); + } + + std::shared_ptr vp_mllm_osd_node::handle_frame_meta(std::shared_ptr meta) { + if (meta->description.empty()) { + return meta; + } + + // operations on osd_frame + if (meta->osd_frame.empty()) { + // add a gap at the bottom of osd frame + meta->osd_frame = cv::Mat(meta->frame.rows + gap_height + padding * 2, meta->frame.cols, meta->frame.type(), cv::Scalar(128, 128, 128)); + + // initialize by copying frame to osd frame + auto roi = meta->osd_frame(cv::Rect(0, 0, meta->frame.cols, meta->frame.rows)); + meta->frame.copyTo(roi); + } + + auto& canvas = meta->osd_frame; + //ft2->putText(canvas, meta->description, cv::Point(10, meta->frame.rows + gap_height + padding * 2), 20, cv::Scalar(255, 0, 0), cv::FILLED, cv::LINE_AA, true); + draw_text_in_rect(canvas, meta->description, cv::Rect(5, meta->frame.rows + 5, meta->frame.cols - 10, gap_height + padding * 2 - 10), 20, cv::Scalar(255, 0, 0)); + + // log using INFO level + VP_INFO(vp_utils::string_format("[%s] [%s]", node_name.c_str(), meta->description.c_str())); + return meta; + } + + std::vector vp_mllm_osd_node::utf8_split(const std::string& text) { + std::vector chars; + for (size_t i = 0; i < text.size();) { + unsigned char c = text[i]; + size_t len = 1; + if ((c & 0x80) == 0x00) len = 1; // ASCII + else if ((c & 0xE0) == 0xC0) len = 2; // 2bytes + else if ((c & 0xF0) == 0xE0) len = 3; // 3bytes + else if ((c & 0xF8) == 0xF0) len = 4; // 4bytes + chars.push_back(text.substr(i, len)); + i += len; + } + return chars; + } + + void vp_mllm_osd_node::draw_text_in_rect(cv::Mat& img, + const std::string& text, + const cv::Rect& rect, + int fontHeight, + cv::Scalar color) { + std::vector chars = utf8_split(text); + std::string currentLine; + int baseline = 0; + + int y = rect.y; + + for (size_t i = 0; i < chars.size(); i++) { + std::string tempLine = currentLine + chars[i]; + cv::Size textSize = ft2->getTextSize(tempLine, fontHeight, -1, &baseline); + + if (textSize.width > rect.width && !currentLine.empty()) { + int drawY = y + textSize.height; + if (drawY > rect.y + rect.height) break; + ft2->putText(img, currentLine, cv::Point(rect.x, drawY), + fontHeight, color, -1, 8, true); + y += textSize.height + 5; + currentLine.clear(); + } + currentLine += chars[i]; + } + + if (!currentLine.empty()) { + int drawY = y + ft2->getTextSize(currentLine, fontHeight, -1, &baseline).height; + if (drawY <= rect.y + rect.height) { + ft2->putText(img, currentLine, cv::Point(rect.x, drawY), + fontHeight, color, -1, 8, true); + } + } + } +} +#endif \ No newline at end of file diff --git a/nodes/osd/vp_mllm_osd_node.h b/nodes/osd/vp_mllm_osd_node.h new file mode 100644 index 0000000..93cf10e --- /dev/null +++ b/nodes/osd/vp_mllm_osd_node.h @@ -0,0 +1,35 @@ + +#pragma once + +#ifdef VP_WITH_LLM +#include +#include + +#include "../vp_node.h" + +namespace vp_nodes { + // on screen display(short as osd) node. + // mainly used to display description(output from LLM) on frame. + class vp_mllm_osd_node: public vp_node + { + private: + // leave a gap at the bottom of osd frame + int gap_height = 112; + int padding = 5; + + // support chinese font + cv::Ptr ft2; + std::vector utf8_split(const std::string& text); + void draw_text_in_rect(cv::Mat& img, + const std::string& text, + const cv::Rect& rect, + int fontHeight, + cv::Scalar color); + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + public: + vp_mllm_osd_node(std::string node_name, std::string font); + ~vp_mllm_osd_node(); + }; +} +#endif \ No newline at end of file diff --git a/nodes/osd/vp_osd_node.cpp b/nodes/osd/vp_osd_node.cpp new file mode 100755 index 0000000..41c6a0b --- /dev/null +++ b/nodes/osd/vp_osd_node.cpp @@ -0,0 +1,82 @@ + + +#include +#include "vp_osd_node.h" + +namespace vp_nodes { + + vp_osd_node::vp_osd_node(std::string node_name, std::string font): + vp_node(node_name) { + if (!font.empty()) { + ft2 = cv::freetype::createFreeType2(); + ft2->loadFontData(font, 0); + } + this->initialized(); + } + + vp_osd_node::~vp_osd_node() { + deinitialized(); + } + + std::shared_ptr vp_osd_node::handle_control_meta(std::shared_ptr meta) { + return meta; + } + + // display logic + std::shared_ptr vp_osd_node::handle_frame_meta(std::shared_ptr meta) { + // operations on osd_frame + if (meta->osd_frame.empty()) { + meta->osd_frame = meta->frame.clone(); + } + + auto& canvas = meta->osd_frame; + // scan targets + for (auto& i : meta->targets) { + // track_id + auto id = std::to_string(i->track_id); + auto labels_to_display = i->primary_label; + + // tracked + if (i->track_id != -1) { + labels_to_display = "#" + id + " " + labels_to_display; + } + + for (auto& label : i->secondary_labels) { + labels_to_display += "|" + label; + } + + // draw tracks if size>=2 + if (i->tracks.size() >= 2) { + for (int n = 0; n < (i->tracks.size() - 1); n++) { + auto p1 = i->tracks[n].track_point(); + auto p2 = i->tracks[n + 1].track_point(); + cv::line(canvas, cv::Point(p1.x, p1.y), cv::Point(p2.x, p2.y), cv::Scalar(0, 255, 255), 1, cv::LINE_AA); + } + } + + cv::rectangle(canvas, cv::Rect(i->x, i->y, i->width, i->height), cv::Scalar(255, 255, 0), 2); + if (ft2 != nullptr) { + ft2->putText(canvas, labels_to_display, cv::Point(i->x, i->y), 20, cv::Scalar(255, 0, 255), cv::FILLED, cv::LINE_AA, true); + } + else { + //cv::putText(canvas, labels_to_display, cv::Point(i->x, i->y), 1, 1, cv::Scalar(255, 0, 255)); + int baseline = 0; + auto size = cv::getTextSize(labels_to_display, 1, 1.5, 1, &baseline); + vp_utils::put_text_at_center_of_rect(canvas, labels_to_display, cv::Rect(i->x, i->y - size.height, size.width, size.height), true, 1, 1, cv::Scalar(), cv::Scalar(179, 52, 255), cv::Scalar(179, 52, 255)); + } + + // scan sub targets + for (auto& sub_target: i->sub_targets) { + cv::rectangle(canvas, cv::Rect(sub_target->x, sub_target->y, sub_target->width, sub_target->height), cv::Scalar(255)); + if (ft2 != nullptr) { + ft2->putText(canvas, sub_target->label, cv::Point(sub_target->x, sub_target->y), 20, cv::Scalar(0, 0, 255), cv::FILLED, cv::LINE_AA, true); + } + else { + cv::putText(canvas, sub_target->label, cv::Point(sub_target->x, sub_target->y), 1, 1, cv::Scalar(0, 0, 255)); + } + } + + } + return meta; + } +} \ No newline at end of file diff --git a/nodes/osd/vp_osd_node.h b/nodes/osd/vp_osd_node.h new file mode 100755 index 0000000..e4ac291 --- /dev/null +++ b/nodes/osd/vp_osd_node.h @@ -0,0 +1,38 @@ + +#pragma once + +#include +#include "../vp_node.h" +/* +* ################################ +* why need osd in our pipeline? +* ################################ +* there are several reasons why we need osd, +* 1. we need debug, for the outputs of infer, tracker and ba, displaying is more straightforward than printing. +* 2. in some situations like cast screen, it is a functional requirement that we need display what we have done on screen to show what we can do to others who do not know about our product. +* +* drawing the targets on current frame is the most common operation for osd. +*/ +namespace vp_nodes { + // config for vp_osd_node, define how to draw + typedef struct vp_osd_node_option + { + int aaa = 0; + } vp_osd_option; + + + // on screen display(short as osd) node. + // mainly used to display vp_frame_target on frame. + class vp_osd_node: public vp_node { + private: + // support chinese font + cv::Ptr ft2; + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + virtual std::shared_ptr handle_control_meta(std::shared_ptr meta) override; + public: + vp_osd_node(std::string node_name, std::string font = ""); + ~vp_osd_node(); + }; + +} \ No newline at end of file diff --git a/nodes/osd/vp_osd_node_v2.cpp b/nodes/osd/vp_osd_node_v2.cpp new file mode 100644 index 0000000..6831183 --- /dev/null +++ b/nodes/osd/vp_osd_node_v2.cpp @@ -0,0 +1,96 @@ + +#include +#include "vp_osd_node_v2.h" +#include "../../utils/vp_utils.h" + + +namespace vp_nodes { + + vp_osd_node_v2::vp_osd_node_v2(std::string node_name, std::string font):vp_node(node_name) { + if (!font.empty()) { + ft2 = cv::freetype::createFreeType2(); + ft2->loadFontData(font, 0); + } + this->initialized(); + } + + vp_osd_node_v2::~vp_osd_node_v2() { + deinitialized(); + } + + std::shared_ptr vp_osd_node_v2::handle_frame_meta(std::shared_ptr meta) { + // operations on osd_frame + if (meta->osd_frame.empty()) { + // add a gap at the bottom of osd frame + meta->osd_frame = cv::Mat(meta->frame.rows + gap_height + padding * 2, meta->frame.cols, meta->frame.type(), cv::Scalar(128, 128, 128)); + + // initialize by copying frame to osd frame + auto roi = meta->osd_frame(cv::Rect(0, 0, meta->frame.cols, meta->frame.rows)); + meta->frame.copyTo(roi); + } + auto& canvas = meta->osd_frame; + + auto base_left = padding; + // scan targets + for (int i = 0; i < meta->targets.size(); i++) { + auto& target = meta->targets[i]; + + // scan sub targets + for (int j = 0; j < target->sub_targets.size(); j++) { + auto& sub_target = target->sub_targets[j]; + cv::rectangle(canvas, cv::Rect(sub_target->x, sub_target->y, sub_target->width, sub_target->height), cv::Scalar(255, 255, 255), 2); + + auto roi = canvas(cv::Rect(base_left, meta->osd_frame.rows - padding - gap_height, gap_height, gap_height)); + // white background + roi = cv::Scalar(255, 255, 255); + + auto ori = canvas(cv::Rect(sub_target->x, sub_target->y, sub_target->width, sub_target->height)); + cv::Mat tmp = ori.clone(); + int offset_w = 0, offset_h = 0; + if (tmp.rows > tmp.cols) { + cv::resize(tmp, tmp, cv::Size(int(float(gap_height) / tmp.rows * tmp.cols) , gap_height)); + offset_w = (gap_height - tmp.cols) / 2; + offset_h = 0; + } + else { + cv::resize(tmp, tmp, cv::Size(gap_height, int(float(gap_height) / tmp.cols * tmp.rows))); + offset_h = (gap_height - tmp.rows) / 2; + offset_w = 0; + } + + // copy sub target to the bottom of screen + roi = canvas(cv::Rect(base_left + offset_w, meta->osd_frame.rows - padding - gap_height + offset_h, tmp.cols, tmp.rows)); + tmp.copyTo(roi); + + // line from target to sub target + cv::line(canvas, cv::Point(target->x + target->width / 2, target->y + target->height), cv::Point(base_left + gap_height / 2, meta->osd_frame.rows - padding - gap_height), cv::Scalar(255, 0, 0), 3, cv::LINE_AA); + + // label of sub target + auto sub_label = vp_utils::string_split(sub_target->label, '_'); + + // !!! + // for plate sub target (color_text) + if (sub_label.size() == 2) { + assert(ft2 != nullptr); + ft2->putText(canvas, sub_label[1], cv::Point(base_left + 10, meta->osd_frame.rows - padding - 5), 36, cv::Scalar(0), cv::FILLED, cv::LINE_AA, true); + } + base_left += gap_height + padding; + } + + // target + auto labels_to_display = target->primary_label; + for (auto& label : target->secondary_labels) { + labels_to_display += "/" + label; + } + + cv::rectangle(canvas, cv::Rect(target->x, target->y, target->width, target->height), cv::Scalar(255, 255, 0), 3); + if (ft2 != nullptr) { + ft2->putText(canvas, labels_to_display, cv::Point(target->x, target->y), 25, cv::Scalar(255, 0, 255), cv::FILLED, cv::LINE_AA, true); + } + else { + cv::putText(canvas, labels_to_display, cv::Point(target->x, target->y), 1, 1, cv::Scalar(255, 0, 255)); + } + } + return meta; + } +} \ No newline at end of file diff --git a/nodes/osd/vp_osd_node_v2.h b/nodes/osd/vp_osd_node_v2.h new file mode 100644 index 0000000..013d781 --- /dev/null +++ b/nodes/osd/vp_osd_node_v2.h @@ -0,0 +1,26 @@ +#pragma once + +#include + +#include "../vp_node.h" + +namespace vp_nodes { + // on screen display(short as osd) node. + // another version for vp_frame_target display, display vp_sub_target at the bottom of screen. + class vp_osd_node_v2: public vp_node + { + private: + // support chinese font + cv::Ptr ft2; + + // leave a gap at the bottom of osd frame + int gap_height = 256; + int padding = 10; + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + public: + vp_osd_node_v2(std::string node_name, std::string font = ""); + ~vp_osd_node_v2(); + }; + +} \ No newline at end of file diff --git a/nodes/osd/vp_osd_node_v3.cpp b/nodes/osd/vp_osd_node_v3.cpp new file mode 100644 index 0000000..f9e0601 --- /dev/null +++ b/nodes/osd/vp_osd_node_v3.cpp @@ -0,0 +1,82 @@ +#include + +#include "vp_osd_node_v3.h" + +namespace vp_nodes { + + vp_osd_node_v3::vp_osd_node_v3(std::string node_name, std::string font, bool show_bbox): + vp_node(node_name), show_bbox(show_bbox) { + if (!font.empty()) { + ft2 = cv::freetype::createFreeType2(); + ft2->loadFontData(font, 0); + } + this->initialized(); + } + + vp_osd_node_v3::~vp_osd_node_v3() { + deinitialized(); + } + + std::shared_ptr vp_osd_node_v3::handle_frame_meta(std::shared_ptr meta) { + // operations on osd_frame + if (meta->osd_frame.empty()) { + meta->osd_frame = meta->frame.clone(); + } + + auto& canvas = meta->osd_frame; + // scan targets + for (auto& i : meta->targets) { + if (show_bbox) { + // track_id + auto id = std::to_string(i->track_id); + auto labels_to_display = i->primary_label; + + // tracked + if (i->track_id != -1) { + labels_to_display = "#" + id + " " + labels_to_display; + } + + for (auto& label : i->secondary_labels) { + labels_to_display += "|" + label; + } + + // draw tracks if size>=2 + if (i->tracks.size() >= 2) { + for (int n = 0; n < (i->tracks.size() - 1); n++) { + auto p1 = i->tracks[n].track_point(); + auto p2 = i->tracks[n + 1].track_point(); + cv::line(canvas, cv::Point(p1.x, p1.y), cv::Point(p2.x, p2.y), cv::Scalar(0, 255, 255), 1, cv::LINE_AA); + } + } + + cv::rectangle(canvas, cv::Rect(i->x, i->y, i->width, i->height), cv::Scalar(255, 255, 0), 2); + if (ft2 != nullptr) { + ft2->putText(canvas, labels_to_display, cv::Point(i->x, i->y), 20, cv::Scalar(255, 0, 255), cv::FILLED, cv::LINE_AA, true); + } + else { + //cv::putText(canvas, labels_to_display, cv::Point(i->x, i->y), 1, 1, cv::Scalar(255, 0, 255)); + int baseline = 0; + auto size = cv::getTextSize(labels_to_display, 1, 1, 1, &baseline); + vp_utils::put_text_at_center_of_rect(canvas, labels_to_display, cv::Rect(i->x, i->y - size.height, size.width, size.height), true); + } + } + /* mask area */ + if (!i->mask.empty()) { + // Resize the mask, threshold, color and apply it on the image + cv::resize(i->mask, i->mask, cv::Size(i->width, i->height)); + cv::Mat mask = (i->mask > mask_threshold); + cv::Mat coloredRoi = (0.3 * cv::Scalar(255, 255, 0) + 0.7 * meta->osd_frame(cv::Rect(i->x, i->y, i->width, i->height))); + coloredRoi.convertTo(coloredRoi, CV_8UC3); + + // Draw the contours on the image + vector contours; + cv::Mat hierarchy; + mask.convertTo(mask, CV_8U); + cv::findContours(mask, contours, hierarchy, cv::RETR_CCOMP, cv::CHAIN_APPROX_SIMPLE); + cv::drawContours(coloredRoi, contours, -1, cv::Scalar(255, 255, 0), 5, cv::LINE_8, hierarchy, 100); + coloredRoi.copyTo(meta->osd_frame(cv::Rect(i->x, i->y, i->width, i->height)), mask); + } + } + return meta; + } +} \ No newline at end of file diff --git a/nodes/osd/vp_osd_node_v3.h b/nodes/osd/vp_osd_node_v3.h new file mode 100644 index 0000000..a719f0b --- /dev/null +++ b/nodes/osd/vp_osd_node_v3.h @@ -0,0 +1,24 @@ +#pragma once + +#include + +#include "../vp_node.h" + + +namespace vp_nodes { + // on screen display(short as osd) node. + // another version for vp_frame_target display, display mask area for image segmentation. + class vp_osd_node_v3: public vp_node + { + private: + // support chinese font + cv::Ptr ft2; + float mask_threshold = 0.3; + bool show_bbox = true; + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + public: + vp_osd_node_v3(std::string node_name, std::string font = "", bool show_bbox = true); + ~vp_osd_node_v3(); + }; +} \ No newline at end of file diff --git a/nodes/osd/vp_plate_osd_node.cpp b/nodes/osd/vp_plate_osd_node.cpp new file mode 100644 index 0000000..73dd1c9 --- /dev/null +++ b/nodes/osd/vp_plate_osd_node.cpp @@ -0,0 +1,83 @@ +#include + +#include "vp_plate_osd_node.h" + +namespace vp_nodes { + + vp_plate_osd_node::vp_plate_osd_node(std::string node_name, std::string font, bool display_his): + vp_node(node_name), display_his(display_his) { + if (!font.empty()) { + ft2 = cv::freetype::createFreeType2(); + ft2->loadFontData(font, 0); + } + this->initialized(); + } + + vp_plate_osd_node::~vp_plate_osd_node() { + deinitialized(); + } + + std::shared_ptr vp_plate_osd_node::handle_frame_meta(std::shared_ptr meta) { + // operations on osd_frame + if (meta->osd_frame.empty()) { + meta->osd_frame = meta->frame.clone(); + } + + auto& canvas = meta->osd_frame; + if (display_his) { + cv::rectangle(canvas, cv::Rect(0, canvas.rows - height_his, canvas.cols, height_his), cv::Scalar(240, 200, 220), cv::FILLED); + } + + // scan targets + for (auto& i : meta->targets) { + // plate color and plate text + auto color_and_text = vp_utils::string_split(i->primary_label, '_'); + if(color_and_text.size() != 2) { + continue; + } + auto& color = color_and_text[0]; + auto& text = color_and_text[1]; + auto text_2_display = text_colors.at(color) + " " + text; + // tracked ? + if (i->track_id != -1) { + text_2_display = "#" + std::to_string(i->track_id) + " " + text_2_display; + } + + cv::rectangle(canvas, cv::Rect(i->x, i->y, i->width, i->height), draw_colors.at(color), 2); + if (ft2 != nullptr) { + ft2->putText(canvas, text_2_display, cv::Point(i->x, i->y), 20, draw_colors.at(color), cv::FILLED, cv::LINE_AA, true); + } + else { + // ignore + } + + if (display_his) { + // put it to cache + auto plate = meta->frame(cv::Rect(i->x, i->y, i->width, i->height)); + cv::Mat resized_plate; + cv::resize(plate, resized_plate, cv::Size(height_his, int((float(height_his) / plate.cols) * plate.rows))); + plates_his.push_back(resized_plate); + } + } + + // display history at the bottom of canvas + if (display_his) { + // remove previous history plates if need, since no space to draw + auto width_need = plates_his.size() * (height_his + gap_his) + gap_his; + while(width_need >= canvas.cols) { + plates_his.erase(plates_his.begin()); + width_need = plates_his.size() * (height_his + gap_his) + gap_his; + } + + // draw history plates at the bottom of screen + for (int i = 0; i < plates_his.size(); i++) { + auto& p = plates_his[i]; + + auto roi = canvas(cv::Rect((gap_his + height_his) * i + gap_his, canvas.rows - height_his / 2 - p.rows / 2 , height_his, p.rows)); + plates_his[i].copyTo(roi); + } + } + + return meta; + } +} \ No newline at end of file diff --git a/nodes/osd/vp_plate_osd_node.h b/nodes/osd/vp_plate_osd_node.h new file mode 100644 index 0000000..21c8d33 --- /dev/null +++ b/nodes/osd/vp_plate_osd_node.h @@ -0,0 +1,42 @@ +#pragma once + +#include + +#include "../vp_node.h" + + +namespace vp_nodes { + // on screen display(short as osd) node. + // used for displaying vehicle plate on frame, draw rectangle according to plate color + class vp_plate_osd_node: public vp_node + { + private: + // support chinese font + cv::Ptr ft2; + float mask_threshold = 0.3; + + // draw color on frame + std::map draw_colors {{"blue", cv::Scalar(255, 0, 0)}, + {"green", cv::Scalar(0, 255, 0)}, + {"yellow", cv::Scalar(0, 255, 255)}, + {"white", cv::Scalar(255, 255, 255)}}; + + // map to Chinese + std::map text_colors {{"blue", "蓝"}, + {"green", "绿"}, + {"yellow", "黄"}, + {"white", "白"}}; + + // history plates at the bottom of screen + std::vector plates_his; + int height_his = 100; + int gap_his = 10; + bool display_his = true; + + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + public: + vp_plate_osd_node(std::string node_name, std::string font = "", bool display_his = true); + ~vp_plate_osd_node(); + }; +} \ No newline at end of file diff --git a/nodes/osd/vp_pose_osd_node.cpp b/nodes/osd/vp_pose_osd_node.cpp new file mode 100644 index 0000000..b108970 --- /dev/null +++ b/nodes/osd/vp_pose_osd_node.cpp @@ -0,0 +1,46 @@ + +#include "vp_pose_osd_node.h" + +namespace vp_nodes { + + vp_pose_osd_node::vp_pose_osd_node(std::string node_name): vp_node(node_name) { + populateColorPalette(colors, 100); // generate colors + this->initialized(); + } + + vp_pose_osd_node::~vp_pose_osd_node() { + deinitialized(); + } + + std::shared_ptr vp_pose_osd_node::handle_frame_meta(std::shared_ptr meta) { + // operations on osd_frame + if (meta->osd_frame.empty()) { + meta->osd_frame = meta->frame.clone(); + } + + // scan pose targets + for (int i = 0; i < meta->pose_targets.size(); i++) { + auto& pose_target = meta->pose_targets[i]; + + auto nPairs = posePairs_map.at(pose_target->type).size(); + + for (int j = 0; j < nPairs; j++) { + auto& a = pose_target->key_points[posePairs_map.at(pose_target->type)[j].first]; + auto& b = pose_target->key_points[posePairs_map.at(pose_target->type)[j].second]; + // some points not detected + if (a.x < 0 || a.y < 0 || b.x < 0 || b.y < 0) { + continue; + } + cv::line(meta->osd_frame, cv::Point(a.x, a.y), cv::Point(b.x, b.y), colors[j], 2, cv::LINE_AA); + cv::circle(meta->osd_frame, cv::Point(a.x, a.y), 3, colors[j], -1, cv::LINE_AA); + cv::circle(meta->osd_frame, cv::Point(b.x, b.y), 3, colors[j], -1, cv::LINE_AA); + } + } + + return meta; + } + + std::shared_ptr vp_pose_osd_node::handle_control_meta(std::shared_ptr meta) { + return meta; + } +} \ No newline at end of file diff --git a/nodes/osd/vp_pose_osd_node.h b/nodes/osd/vp_pose_osd_node.h new file mode 100644 index 0000000..24672be --- /dev/null +++ b/nodes/osd/vp_pose_osd_node.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include +#include + +#include "../vp_node.h" + +namespace vp_nodes { + // on screen display(short as osd) node. + // mainly used to display vp_frame_pose_target on frame. + class vp_pose_osd_node: public vp_node + { + private: + // pose pairs for PAFs + const std::map>> posePairs_map = { + {vp_objects::vp_pose_type::body_25, {{1,8}, {1,2}, {1,5}, {2,3}, {3,4}, {5,6}, {6,7}, {8,9}, + {9,10}, {10,11}, {8,12}, {12,13}, {13,14}, {1,0}, {0,15}, {15,17}, {0,16}, {16,18}, + {14,19}, {19,20}, {14,21}, {11,22}, {22,23}, {11,24}}}, + {vp_objects::vp_pose_type::coco, {{1,2}, {1,5}, {2,3}, {3,4}, {5,6}, {6,7}, + {1,8}, {8,9}, {9,10}, {1,11}, {11,12}, {12,13}, + {1,0}, {0,14}, {14,16}, {0,15}, {15,17}, {2,16}, + {5,17}}}, + {vp_objects::vp_pose_type::mpi_15, {{0,1}, {1,2}, {2,3}, {3,4}, {1,5}, {5,6}, {6,7}, {1,14}, {14,8}, {8,9}, {9,10}, {14,11}, {11,12}, {12,13}, {0, 2}, {0, 5}}}, + {vp_objects::vp_pose_type::hand, std::vector>()}, + {vp_objects::vp_pose_type::face, std::vector>()}, + {vp_objects::vp_pose_type::yolov8_pose_17, {{0, 1}, {0, 2}, {0, 5}, {0, 6}, {1, 2}, {1, 3}, {2, 4}, {5, 6}, {5, 7}, {5, 11}, + {6, 8}, {6, 12}, {7, 9}, {8, 10}, {11, 12}, {11, 13}, {12, 14}, {13, 15}, {14, 16}}} + }; + + void populateColorPalette(std::vector& colors, int nColors) { + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> dis1(64, 200); + std::uniform_int_distribution<> dis2(100, 255); + std::uniform_int_distribution<> dis3(100, 255); + + for(int i = 0; i < nColors; ++i){ + colors.push_back(cv::Scalar(dis1(gen),dis2(gen),dis3(gen))); + } + } + std::vector colors; + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + virtual std::shared_ptr handle_control_meta(std::shared_ptr meta) override; + public: + vp_pose_osd_node(std::string node_name); + ~vp_pose_osd_node(); + }; + +} \ No newline at end of file diff --git a/nodes/osd/vp_seg_osd_node.cpp b/nodes/osd/vp_seg_osd_node.cpp new file mode 100644 index 0000000..7dba7df --- /dev/null +++ b/nodes/osd/vp_seg_osd_node.cpp @@ -0,0 +1,127 @@ +#include +#include "vp_seg_osd_node.h" +#include "../../utils/vp_utils.h" + +namespace vp_nodes { + + vp_seg_osd_node::vp_seg_osd_node(std::string node_name, std::string classes_file, std::string colors_file): vp_node(node_name) { + // load classes names if possible + if (!classes_file.empty()) { + std::ifstream ifs(classes_file.c_str()); + assert(ifs.is_open()); + std::string line; + while (std::getline(ifs, line)) { + classes.push_back(line); + } + ifs.close(); + } + + // load colors if possible + if (!colors_file.empty()) { + std::ifstream ifs(colors_file.c_str()); + assert(ifs.is_open()); + std::string line; + while (std::getline(ifs, line)) { + auto color_s = vp_utils::string_split(line, ','); + cv::Vec3b color(static_cast(std::stoi(color_s[0])), static_cast(std::stoi(color_s[1])), static_cast(std::stoi(color_s[2]))); + colors.push_back(color); + } + ifs.close(); + } + + this->initialized(); + } + + vp_seg_osd_node::~vp_seg_osd_node() { + deinitialized(); + } + + std::shared_ptr vp_seg_osd_node::handle_frame_meta(std::shared_ptr meta) { + // operations on osd_frame + if (meta->osd_frame.empty()) { + // add a gap at the left of osd frame + meta->osd_frame = cv::Mat(meta->frame.rows, meta->frame.cols + gap, meta->frame.type(), cv::Scalar(255, 255, 255)); + + // initialize by copying frame to osd frame + auto roi = meta->osd_frame(cv::Rect(gap, 0, meta->frame.cols, meta->frame.rows)); + meta->frame.copyTo(roi); + } + + // left for display color/class pairs + auto canvas_left = meta->osd_frame(cv::Rect(0, 0, gap, meta->osd_frame.rows)); + // right for display result + auto canvas_right = meta->osd_frame(cv::Rect(gap, 0, meta->frame.cols, meta->osd_frame.rows)); + + if (!meta->mask.empty()) { + cv::Mat segm; + colorizeSegmentation(meta->mask, segm); + + cv::resize(segm, segm, canvas_right.size(), 0, 0, cv::INTER_NEAREST); + cv::addWeighted(canvas_right, 0, segm, 1, 0.0, canvas_right); + } + + if (!classes.empty()) { + showLegend(canvas_left); + } + + return meta; + } + + + void vp_seg_osd_node::colorizeSegmentation(const cv::Mat &score, cv::Mat &segm) { + using namespace cv; + const int rows = score.size[2]; + const int cols = score.size[3]; + const int chns = score.size[1]; + + if (colors.empty()) { + // Generate colors. + colors.push_back(Vec3b()); + for (int i = 1; i < chns; ++i) { + Vec3b color; + for (int j = 0; j < 3; ++j) + color[j] = (colors[i - 1][j] + rand() % 256) / 2; + colors.push_back(color); + } + } + + assert(colors.size() == chns); + + Mat maxCl = Mat::zeros(rows, cols, CV_8UC1); + Mat maxVal(rows, cols, CV_32FC1, score.data); + for (int ch = 1; ch < chns; ch++) { + for (int row = 0; row < rows; row++) { + const float *ptrScore = score.ptr(0, ch, row); + uint8_t *ptrMaxCl = maxCl.ptr(row); + float *ptrMaxVal = maxVal.ptr(row); + for (int col = 0; col < cols; col++) { + if (ptrScore[col] > ptrMaxVal[col]) { + ptrMaxVal[col] = ptrScore[col]; + ptrMaxCl[col] = (uchar)ch; + } + } + } + } + + segm.create(rows, cols, CV_8UC3); + for (int row = 0; row < rows; row++) { + const uchar *ptrMaxCl = maxCl.ptr(row); + Vec3b *ptrSegm = segm.ptr(row); + for (int col = 0; col < cols; col++) { + ptrSegm[col] = colors[ptrMaxCl[col]]; + } + } + } + + void vp_seg_osd_node::showLegend(cv::Mat& board) { + using namespace cv; + auto kBlockHeight = 30; + const int numClasses = (int)classes.size(); + + for (int i = 0; i < numClasses; i++) { + Mat block = board.rowRange(i * kBlockHeight, (i + 1) * kBlockHeight); + block.setTo(colors[i]); + putText(block, classes[i], Point(0, kBlockHeight / 2), FONT_HERSHEY_SIMPLEX, 0.5, Vec3b(255, 255, 255)); + } + } +} \ No newline at end of file diff --git a/nodes/osd/vp_seg_osd_node.h b/nodes/osd/vp_seg_osd_node.h new file mode 100644 index 0000000..9a779f7 --- /dev/null +++ b/nodes/osd/vp_seg_osd_node.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include "../vp_node.h" + +namespace vp_nodes { + class vp_seg_osd_node: public vp_node + { + private: + /* data */ + int gap = 60; + + // classs names of semantic segmentation + std::vector classes; + // colors of semantic segmentation + std::vector colors; + void colorizeSegmentation(const cv::Mat &score, cv::Mat &segm); + void showLegend(cv::Mat& board); + + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + public: + vp_seg_osd_node(std::string node_name, std::string classes_file = "", std::string colors_file = ""); + ~vp_seg_osd_node(); + }; +} \ No newline at end of file diff --git a/nodes/osd/vp_text_osd_node.cpp b/nodes/osd/vp_text_osd_node.cpp new file mode 100644 index 0000000..4d6e31a --- /dev/null +++ b/nodes/osd/vp_text_osd_node.cpp @@ -0,0 +1,53 @@ + +#include "vp_text_osd_node.h" + +namespace vp_nodes { + + vp_text_osd_node::vp_text_osd_node(std::string node_name, std::string font): vp_node(node_name) { + ft2 = cv::freetype::createFreeType2(); + ft2->loadFontData(font, 0); + this->initialized(); + } + + vp_text_osd_node::~vp_text_osd_node() { + deinitialized(); + } + + std::shared_ptr vp_text_osd_node::handle_frame_meta(std::shared_ptr meta) { + // operations on osd_frame + if (meta->osd_frame.empty()) { + // double times higher than frame + meta->osd_frame = cv::Mat(meta->frame.rows * 2, meta->frame.cols, meta->frame.type()); + } + + auto canvas1 = meta->frame.clone(); + auto canvas2 = cv::Mat(meta->frame.rows, meta->frame.cols, meta->frame.type(), cv::Scalar(255, 255, 255)); + + for (int i = 0; i < meta->text_targets.size(); i++) { + auto& text = meta->text_targets[i]; + + cv::Point rook_points[4]; + for (int m = 0; m < text->region_vertexes.size(); m++) { + rook_points[m] = + cv::Point(text->region_vertexes[m].first, text->region_vertexes[m].second); + } + + const cv::Point *ppt[1] = {rook_points}; + int npt[] = {4}; + + cv::polylines(canvas1, ppt, npt, 1, 1, CV_RGB(0, 255, 0), 2, cv::LINE_AA, 0); + cv::polylines(canvas2, ppt, npt, 1, 1, CV_RGB(0, 255, 0), 1, cv::LINE_AA, 0); + + ft2->putText(canvas2, text->text, rook_points[3], 20, cv::Scalar(255, 0, 0), cv::FILLED, cv::LINE_AA, true); + } + + // copy back to osd frame + auto roi1 = meta->osd_frame(cv::Rect(0, 0, meta->frame.cols, meta->frame.rows)); + auto roi2 = meta->osd_frame(cv::Rect(0, meta->frame.rows, meta->frame.cols, meta->frame.rows)); + + canvas1.copyTo(roi1); + canvas2.copyTo(roi2); + + return meta; + } +} \ No newline at end of file diff --git a/nodes/osd/vp_text_osd_node.h b/nodes/osd/vp_text_osd_node.h new file mode 100644 index 0000000..0aa7e4e --- /dev/null +++ b/nodes/osd/vp_text_osd_node.h @@ -0,0 +1,23 @@ + +#pragma once +#include +#include + +#include "../vp_node.h" + +namespace vp_nodes { + // on screen display(short as osd) node. + // mainly used to display vp_frame_text_target on frame. + class vp_text_osd_node: public vp_node + { + private: + // support chinese font + cv::Ptr ft2; + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + public: + vp_text_osd_node(std::string node_name, std::string font); + ~vp_text_osd_node(); + }; + +} \ No newline at end of file diff --git a/nodes/proc/vp_expr_check_node.cpp b/nodes/proc/vp_expr_check_node.cpp new file mode 100644 index 0000000..42eee3b --- /dev/null +++ b/nodes/proc/vp_expr_check_node.cpp @@ -0,0 +1,77 @@ + +#include "vp_expr_check_node.h" +#include "../../third_party/tinyexpr/tinyexpr.h" + +namespace vp_nodes { + + vp_expr_check_node::vp_expr_check_node(std::string node_name):vp_node(node_name) { + this->initialized(); + } + + vp_expr_check_node::~vp_expr_check_node() { + deinitialized(); + } + + std::shared_ptr vp_expr_check_node::handle_frame_meta(std::shared_ptr meta) { + for (auto& i: meta->text_targets) { + auto& text = i->text; + auto left_right = vp_utils::string_split(text, '='); + + // only deal with equation such as `1+1=2`, excluding `1+1` or `1+1=` or `=1+1` + /* + * 1. yes means equation is right + * 2. no means equation is wrong + * 3. invalid means expression is not valid + */ + if (left_right.size() == 2) { + auto left = left_right[0]; + auto right = left_right[1]; + + // replace specific symbols + left = std::regex_replace(left, std::regex("x"), "*"); + left = std::regex_replace(left, std::regex("÷"), "/"); + //... + + if (left.empty()) { + i->flags = "invalid"; + continue; + } + + if (right.empty()) { + i->flags = "invalid"; + continue; + } + + double right_value; + try { + right_value = std::stod(right); + } + catch(const std::exception& e) { + i->flags = "invalid"; + continue; + } + + // parse and calculate the left part + int error; + auto cal_value = te_interp(left.c_str(), &error); + + if (error != 0) { + i->flags = "invalid"; + continue; + } + + if (cal_value == right_value) { + i->flags = "yes_" + vp_utils::round_any(cal_value, 2); + } + else { + i->flags = "no_" + vp_utils::round_any(cal_value, 2); + } + } + else { + i->flags = "invalid"; + } + } + + return meta; + } +} \ No newline at end of file diff --git a/nodes/proc/vp_expr_check_node.h b/nodes/proc/vp_expr_check_node.h new file mode 100644 index 0000000..21d0aff --- /dev/null +++ b/nodes/proc/vp_expr_check_node.h @@ -0,0 +1,17 @@ +#pragma once + +#include "../vp_node.h" + +namespace vp_nodes { + // math expression checker, give right for `1+1=2` and wrong for `sqrt(4)=4`. + // note: this node works based on vp_frame_text_target, it will parse expression at the left of `=` and calculate it then compare with the right side of `=` . + class vp_expr_check_node: public vp_node + { + private: + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + public: + vp_expr_check_node(std::string node_name); + ~vp_expr_check_node(); + }; +} \ No newline at end of file diff --git a/nodes/proc/vp_frame_fusion_node.cpp b/nodes/proc/vp_frame_fusion_node.cpp new file mode 100644 index 0000000..1485891 --- /dev/null +++ b/nodes/proc/vp_frame_fusion_node.cpp @@ -0,0 +1,77 @@ + +#include "vp_frame_fusion_node.h" + +namespace vp_nodes { + + vp_frame_fusion_node::vp_frame_fusion_node(std::string node_name, + std::vector src_points, // 4 calibration points of the source frame + std::vector des_points, // 4 calibration points of the destination frame + int src_channel_index, + int des_channel_index): vp_node(node_name), src_channel_index(src_channel_index), des_channel_index(des_channel_index) { + assert(src_channel_index != des_channel_index); + assert(src_points.size() == 4 && des_points.size() == 4); // cv::getPerspectiveTransform(...) need 4 pairs of point + + cv::Point2f src_ps[4], des_ps[4]; + for (int i = 0; i < 4; i++) { + src_ps[i] = cv::Point2f(src_points[i].x, src_points[i].y); + des_ps[i] = cv::Point2f(des_points[i].x, des_points[i].y); + } + // get transform matrix + trans_mat = cv::getPerspectiveTransform(src_ps, des_ps); + this->initialized(); + } + + vp_frame_fusion_node::~vp_frame_fusion_node() { + deinitialized(); + } + + std::shared_ptr vp_frame_fusion_node::handle_frame_meta(std::shared_ptr meta) { + // ignore for unrelated channels + if (meta->channel_index != src_channel_index && meta->channel_index != des_channel_index) { + return meta; + } + + // for the source channel + if (meta->channel_index == src_channel_index) { + if (tmp_des == nullptr) { + // not ready, return directly + return meta; + } + else { + // ready to fuse, put result to osd_frame + if (tmp_des->osd_frame.empty()) { + tmp_des->osd_frame = tmp_des->frame.clone(); + } + fuse(meta->frame, tmp_des->osd_frame); + + // destination channel first + pendding_meta(tmp_des); + tmp_des = nullptr; + + // source channel last + return meta; + } + } + + // for the destination channel + if (meta->channel_index == des_channel_index) { + if (tmp_des != nullptr) { + // push previous one to downstream first + pendding_meta(tmp_des); + } + // cache current one + tmp_des = meta; + // return nullptr since we need fuse next time + return nullptr; + } + } + + void vp_frame_fusion_node::fuse(cv::Mat& src_canvas, cv::Mat& des_canvas) { + // transform source image + cv::Mat trans_result; + cv::warpPerspective(src_canvas, trans_result, trans_mat, des_canvas.size()); + + // merge pixels by weights + cv::addWeighted(trans_result, 0.7, des_canvas, 0.3, 0, des_canvas); + } +} \ No newline at end of file diff --git a/nodes/proc/vp_frame_fusion_node.h b/nodes/proc/vp_frame_fusion_node.h new file mode 100644 index 0000000..0d30a4a --- /dev/null +++ b/nodes/proc/vp_frame_fusion_node.h @@ -0,0 +1,27 @@ +#pragma once + +#include "../vp_node.h" + +namespace vp_nodes { + // fuse video frames from 2 channels based on the given calibration points. + // only support to fuse 2 channels at the same time so far, fuse the first to second or vice versa, just fuse directly did not check the timestamp of frame. + class vp_frame_fusion_node: public vp_node + { + private: + std::shared_ptr tmp_des = nullptr; + cv::Mat trans_mat; + int src_channel_index = 0; + int des_channel_index = 1; + + void fuse(cv::Mat& src_canvas, cv::Mat& des_canvas); + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + public: + vp_frame_fusion_node(std::string node_name, + std::vector src_points, // 4 calibration points of the source frame + std::vector des_points, // 4 calibration points of the destination frame + int src_channel_index = 0, + int des_channel_index = 1); + ~vp_frame_fusion_node(); + }; +} \ No newline at end of file diff --git a/nodes/record/README.md b/nodes/record/README.md new file mode 100644 index 0000000..6d3d8ee --- /dev/null +++ b/nodes/record/README.md @@ -0,0 +1,20 @@ +# Summary + +`vp_record_node` is used to record video and image, save them to local disk after it finished. It's a middle node but works asynchronously, so recording would not block the pipeline. + +``` +record + ┣ README.md + ┣ vp_image_record_task.cpp + ┣ vp_image_record_task.h // image record task + ┣ vp_record_node.cpp + ┣ vp_record_node.h // record node + ┣ vp_record_task.cpp + ┣ vp_record_task.h // base class for record task, work async + ┣ vp_video_record_task.cpp + ┗ vp_video_record_task.h // video record task +``` + +below is showing how to record video: + +![](../../doc/p5.png) diff --git a/nodes/record/vp_image_record_task.cpp b/nodes/record/vp_image_record_task.cpp new file mode 100644 index 0000000..e05a078 --- /dev/null +++ b/nodes/record/vp_image_record_task.cpp @@ -0,0 +1,62 @@ + +#include "vp_image_record_task.h" + +namespace vp_nodes { + vp_image_record_task::vp_image_record_task(int channel_index, + std::string file_name_without_ext, + std::string save_dir, + bool auto_sub_dir, + bool osd, + vp_objects::vp_size resolution_w_h, + std::string host_node_name, + bool auto_start): + vp_record_task(channel_index, file_name_without_ext, save_dir, auto_sub_dir, resolution_w_h, osd, host_node_name) { + // start automatically when initializing + if (auto_start) { + start(); + } + } + + vp_image_record_task::~vp_image_record_task() { + stop_task(); + } + + void vp_image_record_task::record_task_run() { + /* Below Code Run In A Separate Thread! */ + // get valid path + auto full_record_path = get_full_record_path(); + + while (status == vp_record_task_status::STARTED) { + // it is a consumer + cache_semaphore.wait(); + + auto frame_to_record = frames_to_record.front(); + if (frame_to_record == nullptr) { + //dead flag + continue; + } + + cv::Mat frame_data; + // preprocess, vp_frame_meta -> cv::Mat + preprocess(frame_to_record, frame_data); + frames_to_record.pop_front(); + + // write to disk + cv::imwrite(full_record_path, frame_data); + VP_DEBUG(vp_utils::string_format("[%s] [record] already written frame for `%s`", host_node_name.c_str(), get_full_record_path().c_str())); + /* for image recording, just saving only 1 frame and then complete + * refer to vp_video_record_task + */ + + vp_record_info record_info; + record_info.record_type = vp_record_type::IMAGE; + notify_task_complete(record_info); + // loop end + } + } + + std::string vp_image_record_task::get_file_ext() { + // save as jpg + return ".jpg"; + } +} \ No newline at end of file diff --git a/nodes/record/vp_image_record_task.h b/nodes/record/vp_image_record_task.h new file mode 100644 index 0000000..f10efc2 --- /dev/null +++ b/nodes/record/vp_image_record_task.h @@ -0,0 +1,28 @@ + +#pragma once + +#include "vp_record_task.h" + +namespace vp_nodes { + // image record task, each task instance responsible for recording only 1 image file. + // create multi instances if multi images need to be record at the same time, and maintain these tasks in a list. + // note, the cost of image record is very low but still let it work asynchronously (derived from vp_record_task). + class vp_image_record_task: public vp_record_task { + private: + protected: + // define how to record image + virtual void record_task_run() override; + // retrive .jpg as file extension + virtual std::string get_file_ext() override; + public: + vp_image_record_task(int channel_index, + std::string file_name_without_ext, + std::string save_dir, + bool auto_sub_dir, + bool osd, + vp_objects::vp_size resolution_w_h, + std::string host_node_name = "host_node_not_specified", + bool auto_start = true); + ~vp_image_record_task(); + }; +} \ No newline at end of file diff --git a/nodes/record/vp_record_node.cpp b/nodes/record/vp_record_node.cpp new file mode 100644 index 0000000..765ee8c --- /dev/null +++ b/nodes/record/vp_record_node.cpp @@ -0,0 +1,149 @@ +#include "vp_record_node.h" + + +namespace vp_nodes { + + vp_record_node::vp_record_node(std::string node_name, + std::string video_save_dir, + std::string image_save_dir, + vp_objects::vp_size resolution_w_h, + bool osd, + int pre_record_video_duration, + int record_video_duration, + bool auto_sub_dir, + int bitrate): + vp_node(node_name), + video_save_dir(video_save_dir), + image_save_dir(image_save_dir), + resolution_w_h(resolution_w_h), + osd(osd), + pre_record_video_duration(pre_record_video_duration), + record_video_duration(record_video_duration), + auto_sub_dir(auto_sub_dir), + bitrate(bitrate) { + this->initialized(); + } + + vp_record_node::~vp_record_node() { + deinitialized(); + } + + std::shared_ptr vp_record_node::handle_frame_meta(std::shared_ptr meta) { + // cache fps for current channel + if (all_fps.count(meta->channel_index) == 0) { + all_fps[meta->channel_index] = meta->fps; + } + auto& fps = all_fps[meta->channel_index]; + + // first time for current channel + if (all_pre_records.count(meta->channel_index) == 0) { + all_pre_records[meta->channel_index] = std::deque>(); + } + auto& pre_records = all_pre_records[meta->channel_index]; + + // update pre_records for current channel + pre_records.push_back(meta); + auto frames_need_pre_record = fps * pre_record_video_duration; + // keep max frames + if (pre_records.size() > frames_need_pre_record) { + pre_records.pop_front(); + } + + // first time for current channel + if (all_record_tasks.count(meta->channel_index) == 0) { + all_record_tasks[meta->channel_index] = std::list>(); + } + auto& record_tasks = all_record_tasks[meta->channel_index]; + + // then append data to all tasks of current channel + for (auto i = record_tasks.begin(); i != record_tasks.end();) { + if ((*i)->status == vp_nodes::vp_record_task_status::COMPLETE) { + i = record_tasks.erase(i); // remove task which is complete already + } + else { + (*i)->append_async(meta); // no block here + i++; + } + } + + // done + return meta; + } + + std::shared_ptr vp_record_node::handle_control_meta(std::shared_ptr meta) { + + if (meta->control_type == vp_objects::vp_control_type::IMAGE_RECORD || + meta->control_type == vp_objects::vp_control_type::VIDEO_RECORD) { + /* code */ + // create record task, it will start asynchronously. + auto_new_record_task(meta); + + /* no return since already handle it, do not need pass to next nodes. */ + return nullptr; + } + else { + return vp_node::handle_control_meta(meta); + } + } + + void vp_record_node::auto_new_record_task(std::shared_ptr& meta) { + auto& record_tasks = all_record_tasks[meta->channel_index]; + auto& pre_records = all_pre_records[meta->channel_index]; + auto& fps = all_fps[meta->channel_index]; + + // image record + if (meta->control_type == vp_objects::vp_control_type::IMAGE_RECORD) { + auto image_record_control_meta = std::dynamic_pointer_cast(meta); + + // create image record task + auto file_name_without_ext = image_record_control_meta->image_file_name_without_ext; + auto _osd = image_record_control_meta->osd; + VP_INFO(vp_utils::string_format("[%s] [record] create new image record task, file_name_without_ext is: `%s`", node_name.c_str(), file_name_without_ext.c_str())); + auto image_record_task = std::make_shared(meta->channel_index, + file_name_without_ext, + image_save_dir, + auto_sub_dir, + _osd, + resolution_w_h, node_name); + image_record_task->set_task_complete_hooker([this](int channel_index, vp_record_info record_info) { + // just notify hooker which has attached on the node + if (this->image_record_complete_hooker) { + this->image_record_complete_hooker(channel_index, record_info); + } + VP_INFO(vp_utils::string_format("[%s] [record] image record task completed, file_name_without_ext is: `%s`", node_name.c_str(), record_info.file_name_without_ext.c_str())); + }); + record_tasks.push_back(image_record_task); + } + + // video record + if (meta->control_type == vp_objects::vp_control_type::VIDEO_RECORD) { + auto video_record_control_meta = std::dynamic_pointer_cast(meta); + + // create video record task + auto file_name_without_ext = video_record_control_meta->video_file_name_without_ext; + auto _osd = video_record_control_meta->osd; + auto _record_video_duration = video_record_control_meta->record_video_duration; + if (_record_video_duration == 0) { + // use default value + _record_video_duration = record_video_duration; + } + VP_INFO(vp_utils::string_format("[%s] [record] create new video record task, file_name_without_ext is: `%s`", node_name.c_str(), file_name_without_ext.c_str())); + auto video_record_task = std::make_shared(meta->channel_index, + pre_records, + file_name_without_ext, + video_save_dir, + auto_sub_dir, + _osd, + resolution_w_h, + bitrate, fps, pre_record_video_duration, _record_video_duration, node_name); + video_record_task->set_task_complete_hooker([this](int channel_index, vp_record_info record_info) { + // just notify hooker which has attached on the node + if (this->video_record_complete_hooker) { + this->video_record_complete_hooker(channel_index, record_info); + } + VP_INFO(vp_utils::string_format("[%s] [record] video record task completed, file_name_without_ext is: `%s`", node_name.c_str(), record_info.file_name_without_ext.c_str())); + }); + record_tasks.push_back(video_record_task); + } + } +} \ No newline at end of file diff --git a/nodes/record/vp_record_node.h b/nodes/record/vp_record_node.h new file mode 100644 index 0000000..1e0ae3a --- /dev/null +++ b/nodes/record/vp_record_node.h @@ -0,0 +1,72 @@ +#pragma once + +#include +#include +#include + +#include "../vp_node.h" +#include "../../objects/vp_image_record_control_meta.h" +#include "../../objects/vp_video_record_control_meta.h" + +#include "vp_image_record_task.h" +#include "vp_video_record_task.h" +#include "vp_record_status_hookable.h" + +namespace vp_nodes { + // video/image recording node, save it to local disk. + // it is a middle node but works asynchronously, so recording would not block the pipeline. + // note record node could work on multi channels at the same time. + class vp_record_node: public vp_node, public vp_record_status_hookable + { + private: + /* config data */ + // video save directory + std::string video_save_dir; + // image save directory + std::string image_save_dir; + // pre record time for video (seconds) + int pre_record_video_duration; + // record time for video (seconds), not including pre_record_video_duration + int record_video_duration; + // auto create sub directory by date and channel or not, such as `./video_save_dir/2022-10-8/1/**.mp4` + bool auto_sub_dir; + // width and height + vp_objects::vp_size resolution_w_h = {}; + // bitrate for video record + int bitrate = 1024; + // record osd frame or not + bool osd = false; + + // fps for current video + // int fps = 0; + std::map all_fps; + + /* record task list */ + // std::list> record_tasks; + std::map>> all_record_tasks; + + /* pre-record for video */ + // std::deque> pre_records; + std::map>> all_pre_records; + + // new record task + void auto_new_record_task(std::shared_ptr& meta); + + protected: + // re-implementation + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + // re-implementation + virtual std::shared_ptr handle_control_meta(std::shared_ptr meta) override; + public: + vp_record_node(std::string node_name, + std::string video_save_dir, + std::string image_save_dir, + vp_objects::vp_size resolution_w_h = {}, + bool osd = false, + int pre_record_video_duration = 5, + int record_video_duration = 20, + bool auto_sub_dir = true, + int bitrate = 1024); + ~vp_record_node(); + }; +} diff --git a/nodes/record/vp_record_status_hookable.h b/nodes/record/vp_record_status_hookable.h new file mode 100644 index 0000000..5981ac7 --- /dev/null +++ b/nodes/record/vp_record_status_hookable.h @@ -0,0 +1,24 @@ +#pragma once + +#include "vp_record_task.h" + +namespace vp_nodes { + // callback when record task complete. + class vp_record_status_hookable + { + protected: + vp_record_task_complete_hooker image_record_complete_hooker; + vp_record_task_complete_hooker video_record_complete_hooker; + public: + vp_record_status_hookable(/* args */) {} + ~vp_record_status_hookable() {} + + void set_image_record_complete_hooker(vp_record_task_complete_hooker image_record_complete_hooker) { + this->image_record_complete_hooker = image_record_complete_hooker; + } + + void set_video_record_complete_hooker(vp_record_task_complete_hooker video_record_complete_hooker) { + this->video_record_complete_hooker = video_record_complete_hooker; + } + }; +} \ No newline at end of file diff --git a/nodes/record/vp_record_task.cpp b/nodes/record/vp_record_task.cpp new file mode 100644 index 0000000..8765654 --- /dev/null +++ b/nodes/record/vp_record_task.cpp @@ -0,0 +1,144 @@ + +#include "vp_record_task.h" + +namespace vp_nodes { + vp_record_task::vp_record_task(int channel_index, + std::string file_name_without_ext, + std::string save_dir, + bool auto_sub_dir, + vp_objects::vp_size resolution_w_h, + bool osd, + std::string host_node_name): + channel_index(channel_index), + file_name_without_ext(file_name_without_ext), + save_dir(save_dir), + auto_sub_dir(auto_sub_dir), + resolution_w_h(resolution_w_h), + osd(osd), + host_node_name(host_node_name) { + + } + + vp_record_task::~vp_record_task() { + + } + + void vp_record_task::stop_task() { + status = vp_record_task_status::NOSTRAT; + frames_to_record.push_back(nullptr); + cache_semaphore.signal(); + if (record_task_th.joinable()) { + record_task_th.join(); + } + } + std::string vp_record_task::get_full_record_path() { + // full_record_path already generated + if (!full_record_path.empty()) { + return full_record_path; + } + + // check save dir + if (!std::experimental::filesystem::exists(save_dir)) { + VP_INFO(vp_utils::string_format("[%s] [record] save dir not exists, now creating save dir: `%s`", host_node_name.c_str(), save_dir.c_str())); + std::experimental::filesystem::create_directories(save_dir); + } + + // do not generate sub folder + if (!auto_sub_dir) { + std::experimental::filesystem::path p1(save_dir); + std::experimental::filesystem::path p2(file_name_without_ext + get_file_ext()); + + // ./video/abc.mp4 + auto p = p1 / p2; + if (std::experimental::filesystem::exists(p)) { + // just check once + auto new_file_name = file_name_without_ext + "_" + std::to_string(NOW.time_since_epoch().count()) + get_file_ext(); + VP_WARN(vp_utils::string_format("[%s] [record] `%s` already exists, changing to: `%s`", host_node_name.c_str(), p2.string().c_str(), new_file_name.c_str())); + p = p1 / new_file_name; + } + full_record_path = p.string(); + } + else { + // generate sub folder by date and channel + std::experimental::filesystem::path p1(save_dir); + // just use year-mon-day + std::experimental::filesystem::path p2(vp_utils::time_format(std::chrono::system_clock::now(), "--")); + std::experimental::filesystem::path p3(std::to_string(channel_index)); + std::experimental::filesystem::path p4(file_name_without_ext + get_file_ext()); + + auto p1_3 = p1 / p2 / p3; + if (!std::experimental::filesystem::exists(p1_3)) { + VP_INFO(vp_utils::string_format("[%s] [record] sub dir not exists, now creating sub dir: `%s`", host_node_name.c_str(), p1_3.string().c_str())); + std::experimental::filesystem::create_directories(p1_3); + } + + // ./video/2022-10-10/1/abc.mp4 + auto p = p1_3 / p4; + if (std::experimental::filesystem::exists(p)) { + // just check once + auto new_file_name = file_name_without_ext + "_" + std::to_string(NOW.time_since_epoch().count()) + get_file_ext(); + VP_WARN(vp_utils::string_format("[%s] [record] `%s` already exists, changing to: `%s`", host_node_name.c_str(), p4.string().c_str(), new_file_name.c_str())); + p = p1_3 / new_file_name; + } + + full_record_path = p.string(); + } + VP_INFO(vp_utils::string_format("[%s] [record] get full record path: `%s`", host_node_name.c_str(), full_record_path.c_str())); + return full_record_path; + } + + void vp_record_task::preprocess(std::shared_ptr& frame_to_record, cv::Mat& data) { + cv::Mat resize_frame; + if (this->resolution_w_h.width != 0 && this->resolution_w_h.height != 0) { + cv::resize((osd && !frame_to_record->osd_frame.empty()) ? frame_to_record->osd_frame : frame_to_record->frame, + resize_frame, + cv::Size(resolution_w_h.width, resolution_w_h.height)); + } + else { + resize_frame = (osd && !frame_to_record->osd_frame.empty()) ? frame_to_record->osd_frame : frame_to_record->frame; + } + + resize_frame.copyTo(data); + } + + void vp_record_task::set_task_complete_hooker(vp_record_task_complete_hooker task_complete_hooker) { + // override if already set + this->task_complete_hooker = task_complete_hooker; + } + + void vp_record_task::notify_task_complete(vp_record_info record_info) { + status = vp_record_task_status::COMPLETE; + if (task_complete_hooker) { + // notify to host + // fill fields defined in base class + record_info.channel_index = channel_index; + record_info.file_name_without_ext = file_name_without_ext; + record_info.full_record_path = get_full_record_path(); + record_info.osd = osd; + + task_complete_hooker(channel_index, record_info); + } + } + + void vp_record_task::start() { + // check status + if (status != vp_record_task_status::NOSTRAT) { + return; + } + status = vp_record_task_status::STARTED; + record_task_th = std::thread(&vp_record_task::record_task_run, this); + } + + + void vp_record_task::append_async(std::shared_ptr frame_meta) { + // can append data only if NOSTART or STARTED + if (status == vp_record_task_status::COMPLETE) { + return; + } + + // just push data into queue, it is a producer + // note, since only one thread push, no lock needed here + frames_to_record.push_back(frame_meta); + cache_semaphore.signal(); + } +} \ No newline at end of file diff --git a/nodes/record/vp_record_task.h b/nodes/record/vp_record_task.h new file mode 100644 index 0000000..6e6a50c --- /dev/null +++ b/nodes/record/vp_record_task.h @@ -0,0 +1,111 @@ +#pragma once + +#include +#include +#include +#include +// compile tips: +// remove experimental/ if gcc >= 8.0 +#include +#include +#include +#include +#include + +#include "../../objects/vp_frame_meta.h" +#include "../../utils/vp_utils.h" +#include "../../utils/vp_semaphore.h" +#include "../../utils/logger/vp_logger.h" + +namespace vp_nodes { + // record type + enum vp_record_type { + IMAGE, + VIDEO + }; + + // record information, used to notify others. + struct vp_record_info { + int channel_index; + vp_record_type record_type = vp_record_type::IMAGE; + std::string file_name_without_ext; + std::string full_record_path; + bool osd; + + int pre_record_video_duration = 0; // ignore for image + int record_video_duration = 0; // ignore for image + }; + + // hooker for recording complete + typedef std::function vp_record_task_complete_hooker; + + // status of record task + enum vp_record_task_status { + NOSTRAT, // task initialized but have not called start() + STARTED, // called start() and task is working (writing/saving data to file) + COMPLETE // record task is complete. task instance is un-reusable + }; + + // base class for record task (video & image), works asynchronously and mainly responsible for: + // 1. preprocess frame before recording + // 2. generate valid full record path, including path, name with extension + // 3. run working thread + // 4. notify caller when recording complete + class vp_record_task { + private: + int channel_index; + std::string file_name_without_ext; + std::string save_dir; + bool auto_sub_dir; + vp_objects::vp_size resolution_w_h; + bool osd; + + std::string full_record_path = ""; + vp_record_task_complete_hooker task_complete_hooker; + protected: + // record thread + std::thread record_task_th; + // record thread func, implemented by child class + virtual void record_task_run() = 0; + // wait thread exit in vp_record_task + void stop_task(); + // preprocess, choose frame type (osd or not) and resize + void preprocess(std::shared_ptr& frame_to_record, cv::Mat& data); + // get file extension override by specific class (for example, .mp4 for video and .jpg for image) + virtual std::string get_file_ext() = 0; + + // notify to host when task complete + void notify_task_complete(vp_record_info record_info); + + // cache frames to be recorded (video or image) + // 1. include pre-record frames for video + // 2. just one frame enough for image + std::deque> frames_to_record; + + // synchronize for cache + vp_utils::vp_semaphore cache_semaphore; + std::string host_node_name; // the node name of host, vp_record_task is mainly used inside node. + public: + // status + vp_record_task_status status = vp_record_task_status::NOSTRAT; + // get full record path for file, include path, name with extension + std::string get_full_record_path(); + // register hooker for recording complete + void set_task_complete_hooker(vp_record_task_complete_hooker task_complete_hooker); + // start task async + void start(); + + // append asynchronously, just write frame to cache + void append_async(std::shared_ptr frame_meta); + + vp_record_task(int channel_index, + std::string file_name_without_ext, + std::string save_dir, + bool auto_sub_dir, + vp_objects::vp_size resolution_w_h, + bool osd, + std::string host_node_name); + virtual ~vp_record_task(); // keep virtual since we need destruct child class via base pointer + }; + +} \ No newline at end of file diff --git a/nodes/record/vp_video_record_task.cpp b/nodes/record/vp_video_record_task.cpp new file mode 100644 index 0000000..816afb6 --- /dev/null +++ b/nodes/record/vp_video_record_task.cpp @@ -0,0 +1,104 @@ + +#include "vp_video_record_task.h" +#include "../../utils/vp_utils.h" + +namespace vp_nodes { + vp_video_record_task::vp_video_record_task(int channel_index, + std::deque> pre_record_frames, + std::string file_name_without_ext, + std::string save_dir, + bool auto_sub_dir, + bool osd, + vp_objects::vp_size resolution_w_h, + int bitrate, + int fps, + int pre_record_video_duration, + int record_video_duration, + std::string host_node_name, + bool auto_start): + vp_record_task(channel_index, file_name_without_ext, save_dir, auto_sub_dir, resolution_w_h, osd, host_node_name), + bitrate(bitrate), + fps(fps), + pre_record_video_duration(pre_record_video_duration), + record_video_duration(record_video_duration) { + assert(bitrate > 0); + // transfer to inner cache + for (auto i: pre_record_frames) { + frames_to_record.push_back(i); + cache_semaphore.signal(); + } + + // start automatically when initializing + if (auto_start) { + start(); + } + } + + vp_video_record_task::~vp_video_record_task() { + stop_task(); + } + + void vp_video_record_task::record_task_run() { + /* Below Code Run In A Separate Thread! */ + // get valid path + auto full_record_path = get_full_record_path(); + // format string used by gstreamer + auto gst = vp_utils::string_format(gst_template, bitrate, full_record_path.c_str()); + + // get total frames to record + frames_need_record = (pre_record_video_duration + record_video_duration) * fps; + frames_already_record = 0; + // video duration at least 1 second + assert(frames_need_record > fps * 1); + + // pop data from cache and write into file + while (status == vp_record_task_status::STARTED) { + // it is a consumer + cache_semaphore.wait(); + + auto frame_to_record = frames_to_record.front(); + if (frame_to_record == nullptr) { + // dead flag + continue; + } + + cv::Mat frame_data; + // preprocess, vp_frame_meta -> cv::Mat + preprocess(frame_to_record, frame_data); + frames_to_record.pop_front(); + + // we open video writer in while loop, since we need width and height of frame when open a VideoWriter. + if (!video_writer.isOpened()) { + assert(video_writer.open(gst, cv::CAP_GSTREAMER, 0, fps, {frame_data.cols, frame_data.rows})); + } + + // write cv::Mat to file + video_writer.write(frame_data); + frames_already_record++; + VP_DEBUG(vp_utils::string_format("[%s] [record] already written %d frames for `%s`", host_node_name.c_str(), frames_already_record, get_full_record_path().c_str())); + + /* for video recording, need saving multi frames + * refer to vp_image_record_task + */ + + // check if complete + if (frames_already_record >= frames_need_record) { + // here release writer at once mannually make sure the video file can be used by others. + video_writer.release(); + + vp_record_info record_info; + record_info.record_type = vp_record_type::VIDEO; + record_info.pre_record_video_duration = pre_record_video_duration; + record_info.record_video_duration = record_video_duration; + notify_task_complete(record_info); + // loop end + } + } + } + + + std::string vp_video_record_task::get_file_ext() { + // save as mp4 + return ".mp4"; + } +} \ No newline at end of file diff --git a/nodes/record/vp_video_record_task.h b/nodes/record/vp_video_record_task.h new file mode 100644 index 0000000..9dc7cd6 --- /dev/null +++ b/nodes/record/vp_video_record_task.h @@ -0,0 +1,43 @@ +#pragma once + +#include "vp_record_task.h" + +namespace vp_nodes { + // video record task, each task instance responsible for recording only 1 video file. + // create multi instances if multi videos need to be record at the same time, and maintain these tasks in a list. + class vp_video_record_task: public vp_record_task { + private: + // video writer + cv::VideoWriter video_writer; + // gst template + std::string gst_template = "appsrc ! videoconvert ! x264enc bitrate=%d ! mp4mux ! filesink location=%s"; + + int frames_already_record = -1; + int frames_need_record = 0; + int bitrate; + int fps; + int record_video_duration; + int pre_record_video_duration; + + protected: + // define how to record video + virtual void record_task_run() override; + // retrive .mp4 as file extension + virtual std::string get_file_ext() override; + public: + vp_video_record_task(int channel_index, + std::deque> pre_record_frames, + std::string file_name_without_ext, + std::string save_dir, + bool auto_sub_dir, + bool osd, + vp_objects::vp_size resolution_w_h, + int bitrate, + int fps, + int pre_record_video_duration, + int record_video_duration, + std::string host_node_name = "host_node_not_specified", + bool auto_start = true); + ~vp_video_record_task(); + }; +} \ No newline at end of file diff --git a/nodes/track/deep_sort/README.md b/nodes/track/deep_sort/README.md new file mode 100644 index 0000000..6d212b6 --- /dev/null +++ b/nodes/track/deep_sort/README.md @@ -0,0 +1,6 @@ + + +summary +------------- + +deep sort algorithm for tarcking \ No newline at end of file diff --git a/nodes/track/sort/Hungarian.cpp b/nodes/track/sort/Hungarian.cpp new file mode 100644 index 0000000..86955bd --- /dev/null +++ b/nodes/track/sort/Hungarian.cpp @@ -0,0 +1,392 @@ +/////////////////////////////////////////////////////////////////////////////// +// Hungarian.cpp: Implementation file for Class HungarianAlgorithm. +// +// This is a C++ wrapper with slight modification of a hungarian algorithm implementation by Markus Buehren. +// The original implementation is a few mex-functions for use in MATLAB, found here: +// http://www.mathworks.com/matlabcentral/fileexchange/6543-functions-for-the-rectangular-assignment-problem +// +// Both this code and the orignal code are published under the BSD license. +// by Cong Ma, 2016 +// + +#include "Hungarian.h" +#include +#include +HungarianAlgorithm::HungarianAlgorithm(){} +HungarianAlgorithm::~HungarianAlgorithm(){} + + +//********************************************************// +// A single function wrapper for solving assignment problem. +//********************************************************// +double HungarianAlgorithm::Solve(vector>& DistMatrix, vector& Assignment) +{ + unsigned int nRows = DistMatrix.size(); + unsigned int nCols = DistMatrix[0].size(); + + double *distMatrixIn = new double[nRows * nCols]; + int *assignment = new int[nRows]; + double cost = 0.0; + + // Fill in the distMatrixIn. Mind the index is "i + nRows * j". + // Here the cost matrix of size MxN is defined as a double precision array of N*M elements. + // In the solving functions matrices are seen to be saved MATLAB-internally in row-order. + // (i.e. the matrix [1 2; 3 4] will be stored as a vector [1 3 2 4], NOT [1 2 3 4]). + for (unsigned int i = 0; i < nRows; i++) + for (unsigned int j = 0; j < nCols; j++) + distMatrixIn[i + nRows * j] = DistMatrix[i][j]; + + // call solving function + assignmentoptimal(assignment, &cost, distMatrixIn, nRows, nCols); + + Assignment.clear(); + for (unsigned int r = 0; r < nRows; r++) + Assignment.push_back(assignment[r]); + + delete[] distMatrixIn; + delete[] assignment; + return cost; +} + + +//********************************************************// +// Solve optimal solution for assignment problem using Munkres algorithm, also known as Hungarian Algorithm. +//********************************************************// +void HungarianAlgorithm::assignmentoptimal(int *assignment, double *cost, double *distMatrixIn, int nOfRows, int nOfColumns) +{ + double *distMatrix, *distMatrixTemp, *distMatrixEnd, *columnEnd, value, minValue; + bool *coveredColumns, *coveredRows, *starMatrix, *newStarMatrix, *primeMatrix; + int nOfElements, minDim, row, col; + + /* initialization */ + *cost = 0; + for (row = 0; row nOfColumns) */ + { + minDim = nOfColumns; + + for (col = 0; col= 0) + *cost += distMatrix[row + nOfRows*col]; + } +} + +/********************************************************/ +void HungarianAlgorithm::step2a(int *assignment, double *distMatrix, bool *starMatrix, bool *newStarMatrix, bool *primeMatrix, bool *coveredColumns, bool *coveredRows, int nOfRows, int nOfColumns, int minDim) +{ + bool *starMatrixTemp, *columnEnd; + int col; + + /* cover every column containing a starred zero */ + for (col = 0; col +#include + +using namespace std; + + +class HungarianAlgorithm +{ +public: + HungarianAlgorithm(); + ~HungarianAlgorithm(); + double Solve(vector>& DistMatrix, vector& Assignment); + +private: + void assignmentoptimal(int *assignment, double *cost, double *distMatrix, int nOfRows, int nOfColumns); + void buildassignmentvector(int *assignment, bool *starMatrix, int nOfRows, int nOfColumns); + void computeassignmentcost(int *assignment, double *cost, double *distMatrix, int nOfRows); + void step2a(int *assignment, double *distMatrix, bool *starMatrix, bool *newStarMatrix, bool *primeMatrix, bool *coveredColumns, bool *coveredRows, int nOfRows, int nOfColumns, int minDim); + void step2b(int *assignment, double *distMatrix, bool *starMatrix, bool *newStarMatrix, bool *primeMatrix, bool *coveredColumns, bool *coveredRows, int nOfRows, int nOfColumns, int minDim); + void step3(int *assignment, double *distMatrix, bool *starMatrix, bool *newStarMatrix, bool *primeMatrix, bool *coveredColumns, bool *coveredRows, int nOfRows, int nOfColumns, int minDim); + void step4(int *assignment, double *distMatrix, bool *starMatrix, bool *newStarMatrix, bool *primeMatrix, bool *coveredColumns, bool *coveredRows, int nOfRows, int nOfColumns, int minDim, int row, int col); + void step5(int *assignment, double *distMatrix, bool *starMatrix, bool *newStarMatrix, bool *primeMatrix, bool *coveredColumns, bool *coveredRows, int nOfRows, int nOfColumns, int minDim); +}; diff --git a/nodes/track/sort/KalmanTracker.cpp b/nodes/track/sort/KalmanTracker.cpp new file mode 100644 index 0000000..7b594b2 --- /dev/null +++ b/nodes/track/sort/KalmanTracker.cpp @@ -0,0 +1,176 @@ +/////////////////////////////////////////////////////////////////////////////// +// KalmanTracker.cpp: KalmanTracker Class Implementation Declaration + +#include "KalmanTracker.h" + + +int KalmanTracker::kf_count = 0; + + +// initialize Kalman filter +void KalmanTracker::init_kf(StateType stateMat) +{ + int stateNum = 7; + int measureNum = 4; + kf = KalmanFilter(stateNum, measureNum, 0); + + measurement = cv::Mat::zeros(measureNum, 1, CV_32F); + + kf.transitionMatrix = (cv::Mat_(stateNum, stateNum) << 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1); + + setIdentity(kf.measurementMatrix); + setIdentity(kf.processNoiseCov, Scalar::all(1e-2)); + setIdentity(kf.measurementNoiseCov, Scalar::all(1e-1)); + setIdentity(kf.errorCovPost, Scalar::all(1)); + + // initialize state vector with bounding box in [cx,cy,s,r] style + kf.statePost.at(0, 0) = stateMat.x + stateMat.width / 2; + kf.statePost.at(1, 0) = stateMat.y + stateMat.height / 2; + kf.statePost.at(2, 0) = stateMat.area(); + kf.statePost.at(3, 0) = stateMat.width / stateMat.height; +} + + +// Predict the estimated bounding box. +StateType KalmanTracker::predict() +{ + // predict + Mat p = kf.predict(); + m_age += 1; + + if (m_time_since_update > 0) + m_hit_streak = 0; + m_time_since_update += 1; + + StateType predictBox = get_rect_xysr(p.at(0, 0), p.at(1, 0), p.at(2, 0), p.at(3, 0)); + + m_history.push_back(predictBox); + return m_history.back(); +} + + +// Update the state vector with observed bounding box. +void KalmanTracker::update(StateType stateMat) +{ + m_time_since_update = 0; + m_history.clear(); + m_hits += 1; + m_hit_streak += 1; + + // measurement + measurement.at(0, 0) = stateMat.x + stateMat.width / 2; + measurement.at(1, 0) = stateMat.y + stateMat.height / 2; + measurement.at(2, 0) = stateMat.area(); + measurement.at(3, 0) = stateMat.width / stateMat.height; + + // update + kf.correct(measurement); +} + + +// Return the current state vector +StateType KalmanTracker::get_state() +{ + Mat s = kf.statePost; + return get_rect_xysr(s.at(0, 0), s.at(1, 0), s.at(2, 0), s.at(3, 0)); +} + + +// Convert bounding box from [cx,cy,s,r] to [x,y,w,h] style. +StateType KalmanTracker::get_rect_xysr(float cx, float cy, float s, float r) +{ + float w = sqrt(s * r); + float h = s / w; + float x = (cx - w / 2); + float y = (cy - h / 2); + + if (x < 0 && cx > 0) + x = 0; + if (y < 0 && cy > 0) + y = 0; + + return StateType(x, y, w, h); +} + + + +/* +// -------------------------------------------------------------------- +// Kalman Filter Demonstrating, a 2-d ball demo +// -------------------------------------------------------------------- + +const int winHeight = 600; +const int winWidth = 800; + +Point mousePosition = Point(winWidth >> 1, winHeight >> 1); + +// mouse event callback +void mouseEvent(int event, int x, int y, int flags, void *param) +{ + if (event == CV_EVENT_MOUSEMOVE) { + mousePosition = Point(x, y); + } +} + +void TestKF(); + +void main() +{ + TestKF(); +} + + +void TestKF() +{ + int stateNum = 4; + int measureNum = 2; + KalmanFilter kf = KalmanFilter(stateNum, measureNum, 0); + + // initialization + Mat processNoise(stateNum, 1, CV_32F); + Mat measurement = Mat::zeros(measureNum, 1, CV_32F); + + kf.transitionMatrix = *(Mat_(stateNum, stateNum) << + 1, 0, 1, 0, + 0, 1, 0, 1, + 0, 0, 1, 0, + 0, 0, 0, 1); + + setIdentity(kf.measurementMatrix); + setIdentity(kf.processNoiseCov, Scalar::all(1e-2)); + setIdentity(kf.measurementNoiseCov, Scalar::all(1e-1)); + setIdentity(kf.errorCovPost, Scalar::all(1)); + + randn(kf.statePost, Scalar::all(0), Scalar::all(winHeight)); + + namedWindow("Kalman"); + setMouseCallback("Kalman", mouseEvent); + Mat img(winHeight, winWidth, CV_8UC3); + + while (1) + { + // predict + Mat prediction = kf.predict(); + Point predictPt = Point(prediction.at(0, 0), prediction.at(1, 0)); + + // generate measurement + Point statePt = mousePosition; + measurement.at(0, 0) = statePt.x; + measurement.at(1, 0) = statePt.y; + + // update + kf.correct(measurement); + + // visualization + img.setTo(Scalar(255, 255, 255)); + circle(img, predictPt, 8, CV_RGB(0, 255, 0), -1); // predicted point as green + circle(img, statePt, 8, CV_RGB(255, 0, 0), -1); // current position as red + + imshow("Kalman", img); + char code = (char)waitKey(100); + if (code == 27 || code == 'q' || code == 'Q') + break; + } + destroyWindow("Kalman"); +} +*/ diff --git a/nodes/track/sort/KalmanTracker.h b/nodes/track/sort/KalmanTracker.h new file mode 100644 index 0000000..702e8bd --- /dev/null +++ b/nodes/track/sort/KalmanTracker.h @@ -0,0 +1,72 @@ +/////////////////////////////////////////////////////////////////////////////// +// KalmanTracker.h: KalmanTracker Class Declaration + +#ifndef KALMAN_H +#define KALMAN_H 2 + +#include "opencv2/video/tracking.hpp" +#include "opencv2/highgui/highgui.hpp" + +using namespace std; +using namespace cv; + +#define StateType Rect_ + + +// This class represents the internel state of individual tracked objects observed as bounding box. +class KalmanTracker +{ +public: + KalmanTracker() + { + init_kf(StateType()); + m_time_since_update = 0; + m_hits = 0; + m_hit_streak = 0; + m_age = 0; + m_id = kf_count; + //kf_count++; + } + KalmanTracker(StateType initRect) + { + init_kf(initRect); + m_time_since_update = 0; + m_hits = 0; + m_hit_streak = 0; + m_age = 0; + m_id = kf_count; + kf_count++; + } + + ~KalmanTracker() + { + m_history.clear(); + } + + StateType predict(); + void update(StateType stateMat); + + StateType get_state(); + StateType get_rect_xysr(float cx, float cy, float s, float r); + + static int kf_count; + + int m_time_since_update; + int m_hits; + int m_hit_streak; + int m_age; + int m_id; + +private: + void init_kf(StateType stateMat); + + cv::KalmanFilter kf; + cv::Mat measurement; + + std::vector m_history; +}; + + + + +#endif \ No newline at end of file diff --git a/nodes/track/sort/README.md b/nodes/track/sort/README.md new file mode 100644 index 0000000..d7256f9 --- /dev/null +++ b/nodes/track/sort/README.md @@ -0,0 +1,6 @@ + + +summary +----------- + +sort algorithm for tracking \ No newline at end of file diff --git a/nodes/track/vp_dsort_track_node.cpp b/nodes/track/vp_dsort_track_node.cpp new file mode 100644 index 0000000..4a2c74a --- /dev/null +++ b/nodes/track/vp_dsort_track_node.cpp @@ -0,0 +1,21 @@ +#include "vp_dsort_track_node.h" + +namespace vp_nodes { + + vp_dsort_track_node::vp_dsort_track_node(std::string node_name, + vp_track_for track_for): + vp_track_node(node_name, track_for) { + this->initialized(); + } + + vp_dsort_track_node::~vp_dsort_track_node() { + deinitialized(); + } + + void vp_dsort_track_node::track(int channel_index, const std::vector& target_rects, + const std::vector>& target_embeddings, + std::vector& track_ids) { + // fill track_ids according to target_rects & target_embeddings + // deep sort logic here ... + } +} \ No newline at end of file diff --git a/nodes/track/vp_dsort_track_node.h b/nodes/track/vp_dsort_track_node.h new file mode 100644 index 0000000..ffeb9d5 --- /dev/null +++ b/nodes/track/vp_dsort_track_node.h @@ -0,0 +1,19 @@ +#pragma once +#include "vp_track_node.h" + +namespace vp_nodes { + // track node using deep sort + class vp_dsort_track_node: public vp_track_node + { + private: + /* config data for deep sort algo*/ + protected: + // fill track_ids using deep sort algo + virtual void track(int channel_index, const std::vector& target_rects, + const std::vector>& target_embeddings, + std::vector& track_ids) override; + public: + vp_dsort_track_node(std::string node_name, vp_track_for track_for = vp_track_for::NORMAL); + virtual ~vp_dsort_track_node(); + }; +} diff --git a/nodes/track/vp_sort_track_node.cpp b/nodes/track/vp_sort_track_node.cpp new file mode 100644 index 0000000..f7f92b9 --- /dev/null +++ b/nodes/track/vp_sort_track_node.cpp @@ -0,0 +1,189 @@ +#include "vp_sort_track_node.h" + +namespace vp_nodes { + + vp_sort_track_node::vp_sort_track_node(std::string node_name, + vp_track_for track_for): + vp_track_node(node_name, track_for) { + this->initialized(); + KalmanTracker::kf_count = 0; + } + + vp_sort_track_node::~vp_sort_track_node() { + deinitialized(); + } + + void vp_sort_track_node::track(int channel_index, const std::vector& target_rects, + const std::vector>& target_embeddings, + std::vector& track_ids) { + // fill track_ids according to target_rects (target_embeddings ignored) + track_ids.resize(target_rects.size()); + for (auto& item : track_ids) { + item = -1; + } + + // check if trackers are initialized or not for specific channel + if (all_trackers.count(channel_index) == 0) { + all_trackers[channel_index] = std::vector(); + VP_INFO(vp_utils::string_format("[%s] initialize kalmantracker the first time for channel %d", node_name.c_str(), channel_index)); + } + // track on specific channel + auto& trackers = all_trackers[channel_index]; + + if (trackers.empty()) { + /* first frame*/ + for (unsigned int i = 0; i < target_rects.size(); i++) { + KalmanTracker trk = KalmanTracker(cv::Rect_(target_rects[i].x, target_rects[i].y, target_rects[i].width, target_rects[i].height)); + trackers.push_back(trk); + } + return; + } + //3.1. get predicted locations from existing trackers. + predictedBoxes.clear(); + for (auto it = trackers.begin(); it != trackers.end();) { + Rect_ pBox = (*it).predict(); + if (pBox.x >= 0 && pBox.y >= 0) { + predictedBoxes.push_back(pBox); + it++; + } + else { + it = trackers.erase(it); + } + } + + // 3.2. associate detections to tracked object (both represented as bounding boxes) + // dets : detFrameData[fi] + auto trkNum = predictedBoxes.size(); + auto detNum = target_rects.size(); + + iouMatrix.clear(); + iouMatrix.resize(trkNum, vector(detNum, 0)); + + // compute iou matrix as a distance matrix + for (unsigned int i = 0; i < trkNum; i++) { + for (unsigned int j = 0; j < detNum; j++) { + // use 1-iou because the hungarian algorithm computes a minimum-cost assignment. + iouMatrix[i][j] = 1 - GetIOU(predictedBoxes[i], cv::Rect_(target_rects[j].x, target_rects[j].y, target_rects[j].width, target_rects[j].height)); + } + } + + // solve the assignment problem using hungarian algorithm. + // the resulting assignment is [track(prediction) : detection], with len=preNum + HungarianAlgorithm HungAlgo; + assignment.clear(); + HungAlgo.Solve(iouMatrix, assignment); + + // find matches, unmatched_detections and unmatched_predictions + unmatchedTrajectories.clear(); + unmatchedDetections.clear(); + allItems.clear(); + matchedItems.clear(); + + // there are unmatched detections + if (detNum > trkNum) { + for (unsigned int n = 0; n < detNum; n++) + allItems.insert(n); + + for (unsigned int i = 0; i < trkNum; ++i) + matchedItems.insert(assignment[i]); + + set_difference(allItems.begin(), allItems.end(), + matchedItems.begin(), matchedItems.end(), + insert_iterator>(unmatchedDetections, unmatchedDetections.begin())); + } + // there are unmatched trajectory/predictions + else if (detNum < trkNum) { + for (unsigned int i = 0; i < trkNum; ++i) + if (assignment[i] == -1) // unassigned label will be set as -1 in the assignment algorithm + unmatchedTrajectories.insert(i); + } + else { + + } + + // filter out matched with low IOU + matchedPairs.clear(); + for (unsigned int i = 0; i < trkNum; ++i) { + if (assignment[i] == -1) // pass over invalid values + continue; + if (1 - iouMatrix[i][assignment[i]] < iouThreshold) { + unmatchedTrajectories.insert(i); + unmatchedDetections.insert(assignment[i]); + } + else { + matchedPairs.push_back(cv::Point(i, assignment[i])); + } + } + + + // 3.3. updating trackers + // update matched trackers with assigned detections. + // each prediction is corresponding to a tracker + int detIdx, trkIdx; + for (unsigned int i = 0; i < matchedPairs.size(); i++) { + trkIdx = matchedPairs[i].x; + detIdx = matchedPairs[i].y; + trackers[trkIdx].update(cv::Rect_(target_rects[detIdx].x, + target_rects[detIdx].y, + target_rects[detIdx].width, + target_rects[detIdx].height)); + } + + // create and initialise new trackers for unmatched detections + for (auto& umd : unmatchedDetections) { + KalmanTracker tracker = KalmanTracker(cv::Rect_(target_rects[umd].x, + target_rects[umd].y, + target_rects[umd].width, + target_rects[umd].height)); + trackers.push_back(tracker); + } + + // get trackers' output + frameTrackingResult.clear(); + for (auto it = trackers.begin(); it != trackers.end();) { + if (((*it).m_time_since_update < 1) && + ((*it).m_hit_streak >= min_hits)) { + TrackingBox res; + res.box = (*it).get_state(); + res.id = (*it).m_id + 1; + //res.frame = meta->frame_index; + frameTrackingResult.push_back(res); + it++; + } + else + it++; + + // remove dead tracklet + if (it != trackers.end() && (*it).m_time_since_update > max_age) + it = trackers.erase(it); + } + + for (const auto& tb : frameTrackingResult) { + // id and box need to correspond + for (int i = 0; i < target_rects.size(); ++i) { + /* code */ + if(GetIOU(cv::Rect_(target_rects[i].x, + target_rects[i].y, + target_rects[i].width, + target_rects[i].height), + cv::Rect_(tb.box.x, + tb.box.y, + tb.box.width, + tb.box.height)) > 0.8) { + track_ids[i] = tb.id; + } + } + } + return; + } + + double vp_sort_track_node::GetIOU(cv::Rect_ bb_test, cv::Rect_ bb_gt){ + float in = (bb_test & bb_gt).area(); + float un = bb_test.area() + bb_gt.area() - in; + + if (un < DBL_EPSILON) + return 0; + + return (double)(in / un); + } +} \ No newline at end of file diff --git a/nodes/track/vp_sort_track_node.h b/nodes/track/vp_sort_track_node.h new file mode 100644 index 0000000..c9ae9ae --- /dev/null +++ b/nodes/track/vp_sort_track_node.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include +#include +#include "vp_track_node.h" +#include "sort/Hungarian.h" +#include "sort/KalmanTracker.h" + +namespace vp_nodes { + // track node using sort + class vp_sort_track_node: public vp_track_node + { + private: + /* config data for sort algo */ + /* data */ + typedef struct TrackingBox + { + //int frame; + int id; + Rect_ box; + }TrackingBox; + + int max_age = 1; + int min_hits = 3; + double iouThreshold = 0.5; + // vector trackers; + std::map> all_trackers; + std::vector> predictedBoxes; + std::vector> iouMatrix; + std::vector assignment; + std::set unmatchedDetections; + std::set unmatchedTrajectories; + std::set allItems; + std::set matchedItems; + std::vector matchedPairs; + std::vector frameTrackingResult; + private: + double GetIOU(cv::Rect_ bb_test, cv::Rect_ bb_gt); + protected: + // fill track_ids using sort algo + virtual void track(int channel_index, const std::vector& target_rects, + const std::vector>& target_embeddings, + std::vector& track_ids) override; + public: + vp_sort_track_node(std::string node_name, vp_track_for track_for = vp_track_for::NORMAL); + virtual ~vp_sort_track_node(); + }; + +} + diff --git a/nodes/track/vp_track_node.cpp b/nodes/track/vp_track_node.cpp new file mode 100644 index 0000000..a75eac0 --- /dev/null +++ b/nodes/track/vp_track_node.cpp @@ -0,0 +1,128 @@ + +#include "vp_track_node.h" +//#include "../objects/shapes/vp_rect.h" + +namespace vp_nodes { + + vp_track_node::vp_track_node(std::string node_name, + vp_track_for track_for): + vp_node(node_name), + track_for(track_for) { + } + + vp_track_node::~vp_track_node() { + } + + std::shared_ptr vp_track_node::handle_control_meta(std::shared_ptr meta) { + return meta; + } + + std::shared_ptr vp_track_node::handle_frame_meta(std::shared_ptr meta) { + // channel_index can be different each call + auto channel_index = meta->channel_index; + + // data used for tracking + std::vector rects; // rects of targets + std::vector> embeddings; // embeddings of targets + std::vector track_ids; // track ids of targets + + // step 1, collect data + preprocess(meta, rects, embeddings); + + // step 2, track by channel + track(channel_index, rects, embeddings, track_ids); + + // step 3, postprocess + postprocess(meta, rects, embeddings, track_ids); + + return meta; + } + + void vp_track_node::preprocess(std::shared_ptr frame_meta, + std::vector& target_rects, + std::vector>& target_embeddings) { + if (track_for == vp_track_for::NORMAL) { + for(auto& i: frame_meta->targets) { + target_rects.push_back(i->get_rect()); // rect fo target (via i variable) + target_embeddings.push_back(i->embeddings); // embeddings of target (via i variable) + } + } + + if (track_for == vp_track_for::FACE) { + for(auto& i: frame_meta->face_targets) { + target_rects.push_back(i->get_rect()); // rect of face target (via i variable) + target_embeddings.push_back(i->embeddings); // embeddings of face target (via i variable) + } + } + /* ... extend for more track for... */ + } + + // write track_ids back to frame meta + // we can also cache history rects for each target, and then push them back to tracks field (such as vp_frame_target::tracks) + void vp_track_node::postprocess(std::shared_ptr frame_meta, + const std::vector& target_rects, + const std::vector>& target_embeddings, + const std::vector& track_ids) { + + if (track_ids.empty()) { + return; + } + // assert for length of vectors since they are generated by step1 & step2 separately + // assert(target_rects.size() == target_embeddings.size()); + assert(target_rects.size() == track_ids.size()); + + // support multi channels + auto& tracks_by_id = all_tracks_by_id[frame_meta->channel_index]; + auto& last_tracked_frame_indexes = all_last_tracked_frame_indexes[frame_meta->channel_index]; + + if (track_for == vp_track_for::NORMAL) { + //assert(target_rects.size() == frame_meta->targets.size()); + for (int i = 0; i < frame_meta->targets.size(); i++) { + auto& target = frame_meta->targets[i]; + auto& rect = target_rects[i]; + auto& track_id = track_ids[i]; + + // -1 means no track result returned yet + if (track_id != -1) { + tracks_by_id[track_id].push_back(rect); // cache + last_tracked_frame_indexes[track_id] = frame_meta->frame_index; // update stamp + + target->track_id = track_id; // write track_id back to target + target->tracks = tracks_by_id[track_id]; // write tracks back to target + } + } + } + + if (track_for == vp_track_for::FACE) { + // assert(target_rects.size() == frame_meta->face_targets.size()); + for (int i = 0; i < frame_meta->face_targets.size(); i++) { + auto& face = frame_meta->face_targets[i]; + auto& rect = target_rects[i]; + auto& track_id = track_ids[i]; + + // -1 means no track result returned yet + if (track_id != -1) { + tracks_by_id[track_id].push_back(rect); // cache + last_tracked_frame_indexes[track_id] = frame_meta->frame_index; // update stamp + + face->track_id = track_id; // write track_id back to face target + face->tracks = tracks_by_id[track_id]; // write tracks back to face target + } + } + } + /* ... extend for more track for... */ + + // remove cache tracks if has been long time since last updated (maybe it disappeared already). + for (auto i = last_tracked_frame_indexes.begin(); i != last_tracked_frame_indexes.end();) { + if (frame_meta->frame_index - (i->second) > max_allowed_disappear_frames + || frame_meta->frame_index < i->second) { + VP_DEBUG(vp_utils::string_format("[%s] [tracking] long time no update, so erase cache of tracks for track_id:`%d`, size of tracks is:`%d`", node_name.c_str(), i->first, tracks_by_id[i->first].size())); + tracks_by_id.erase(i->first); // erase tracks first + i = last_tracked_frame_indexes.erase(i); // erase stamp then + } + else { + i++; + } + } + } +} \ No newline at end of file diff --git a/nodes/track/vp_track_node.h b/nodes/track/vp_track_node.h new file mode 100644 index 0000000..7912294 --- /dev/null +++ b/nodes/track/vp_track_node.h @@ -0,0 +1,60 @@ + +#pragma once + +#include +#include +#include "../vp_node.h" + +namespace vp_nodes { + // track node applied to which type of target (vp_frame_target, vp_frame_face_target or others) + enum class vp_track_for { + NORMAL = 1, // vp_frame_target + FACE = 2 // vp_frame_face_target + // others to extend + }; + + // base class for tracking, can not be initialized directly. + // note that a track node can work on different channels at the same time + class vp_track_node: public vp_node { + private: + // track for + vp_track_for track_for = vp_track_for::NORMAL; + + // cache tracks at previous frames + // std::map> tracks_by_id; + std::map>> all_tracks_by_id; + + // stamp + // std::map last_tracked_frame_indexes; + std::map> all_last_tracked_frame_indexes; + + // remove cache tracks if it has been long time since last tracked. + const int max_allowed_disappear_frames = 25; + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override final; + virtual std::shared_ptr handle_control_meta(std::shared_ptr meta) override final; + + // prepare data according to `track_for` + void preprocess(std::shared_ptr frame_meta, + std::vector& target_rects, + std::vector>& target_embeddings); + + // track api + // it is a pure virtual function which should be implemented by derived class. + // In: rects & embeddings whose size() can be zero + // Out: track ids + virtual void track(int channel_index, const std::vector& target_rects, + const std::vector>& target_embeddings, + std::vector& track_ids) = 0; + + // write track_ids back to frame meta + // we can also cache history rects for each target, and then push them back to tracks field (like vp_frame_target::tracks) + void postprocess(std::shared_ptr frame_meta, + const std::vector& target_rects, + const std::vector>& target_embeddings, + const std::vector& track_ids); + public: + vp_track_node(std::string node_name, vp_track_for track_for = vp_track_for::NORMAL); + virtual ~vp_track_node(); + }; +} \ No newline at end of file diff --git a/nodes/vp_app_des_node.cpp b/nodes/vp_app_des_node.cpp new file mode 100644 index 0000000..728ce88 --- /dev/null +++ b/nodes/vp_app_des_node.cpp @@ -0,0 +1,43 @@ + +#include "vp_app_des_node.h" + +namespace vp_nodes { + + vp_app_des_node::vp_app_des_node(std::string node_name, + int channel_index): + vp_des_node(node_name, channel_index) { + this->initialized(); + } + + vp_app_des_node::~vp_app_des_node() { + deinitialized(); + } + + // re-implementation, return nullptr. + std::shared_ptr + vp_app_des_node::handle_frame_meta(std::shared_ptr meta) { + VP_DEBUG(vp_utils::string_format("[%s] received frame meta, channel_index=>%d, frame_index=>%d", node_name.c_str(), meta->channel_index, meta->frame_index)); + + invoke_app_des_result_hooker(meta); + + // for general works defined in base class + return vp_des_node::handle_frame_meta(meta); + } + + // re-implementation, return nullptr. + std::shared_ptr + vp_app_des_node::handle_control_meta(std::shared_ptr meta) { + invoke_app_des_result_hooker(meta); + return vp_des_node::handle_control_meta(meta); + } + + void vp_app_des_node::set_app_des_result_hooker(vp_app_des_result_hooker app_des_result_hooker) { + this->app_des_result_hooker = app_des_result_hooker; + } + + void vp_app_des_node::invoke_app_des_result_hooker(std::shared_ptr meta) { + if (app_des_result_hooker) { + app_des_result_hooker(node_name, meta); + } + } +} \ No newline at end of file diff --git a/nodes/vp_app_des_node.h b/nodes/vp_app_des_node.h new file mode 100644 index 0000000..9ede6f7 --- /dev/null +++ b/nodes/vp_app_des_node.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "vp_des_node.h" +#include "../objects/vp_frame_meta.h" +#include "../objects/vp_control_meta.h" +#include "../utils/vp_utils.h" + + +namespace vp_nodes { + // callback before data disappear inside vp_app_des_node + typedef std::function)> vp_app_des_result_hooker; + + // app des node, send meta data to external host code using callbacks. + class vp_app_des_node: public vp_des_node { + private: + /* data */ + vp_app_des_result_hooker app_des_result_hooker; + + void invoke_app_des_result_hooker(std::shared_ptr meta); + protected: + // re-implementation, return nullptr. + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + // re-implementation, return nullptr. + virtual std::shared_ptr handle_control_meta(std::shared_ptr meta) override; + public: + vp_app_des_node(std::string node_name, + int channel_index); + ~vp_app_des_node(); + + void set_app_des_result_hooker(vp_app_des_result_hooker app_des_result_hooker); + }; +} \ No newline at end of file diff --git a/nodes/vp_app_src_node.cpp b/nodes/vp_app_src_node.cpp new file mode 100644 index 0000000..cef4ea5 --- /dev/null +++ b/nodes/vp_app_src_node.cpp @@ -0,0 +1,81 @@ +#include "vp_app_src_node.h" + +namespace vp_nodes { + vp_app_src_node::vp_app_src_node(std::string node_name, + int channel_index):vp_src_node(node_name, channel_index, 1.0) { + this->initialized(); + } + + vp_app_src_node::~vp_app_src_node() { + deinitialized(); + } + + // host code acts as previous node, call vp_node::meta_flow(...) + bool vp_app_src_node::push_frames(std::vector frames) { + // vp_app_src_node not working + if (!gate.is_open()) { + VP_WARN(vp_utils::string_format("[%s] is not working!", node_name.c_str())); + return false; + } + + if (frames.size() == 0) { + return false; + } + + // MUST have the same size + auto size_warn = [this]() { + VP_WARN(vp_utils::string_format("[%s] frames to be pushed MUST have the same size!", this->node_name.c_str())); + }; + auto w = frames[0].cols; + auto h = frames[0].rows; + for (auto& f: frames) { + if (f.cols != w || f.rows != h) { + size_warn(); + return false; + } + } + + if (original_height != 0 && original_height != h) { + size_warn(); + return false; + } + + if (original_width != 0 && original_width != w) { + size_warn(); + return false; + } + + // initialize video properties + if (original_width == 0 || original_height == 0 || original_fps == 0) { + original_width = w; + original_height = h; + original_fps = 1; // set constant value 1 for vp_app_src_node + } + // stream_info_hooker activated if need + vp_stream_info stream_info {channel_index, original_fps, original_width, original_height, to_string()}; + invoke_stream_info_hooker(node_name, stream_info); + + for (auto& f: frames) { + frame_index++; + auto frame = f.clone(); // cv::Mat::clone() inside pipeline + // create frame meta and meta flow like previous node + auto in_meta = std::make_shared(frame, frame_index, channel_index, original_width, original_height, original_fps); + + vp_node::meta_flow(in_meta); + } + return true; + } + + void vp_app_src_node::handle_run() { + // call vp_node::handle_run() since we assume vp_app_src_node has virtual previous node (from host code) + vp_node::handle_run(); + } + + std::shared_ptr vp_app_src_node::handle_frame_meta(std::shared_ptr meta) { + return vp_node::handle_frame_meta(meta); + } + + std::shared_ptr vp_app_src_node::handle_control_meta(std::shared_ptr meta) { + return vp_node::handle_control_meta(meta); + } +} \ No newline at end of file diff --git a/nodes/vp_app_src_node.h b/nodes/vp_app_src_node.h new file mode 100644 index 0000000..f9308ac --- /dev/null +++ b/nodes/vp_app_src_node.h @@ -0,0 +1,28 @@ + +#pragma once + +#include "vp_src_node.h" + +namespace vp_nodes { + // app src node, receive image data from external host code. + class vp_app_src_node: public vp_src_node + { + private: + protected: + // just call vp_node::handle_run to ignore vp_src_node::handle_run + virtual void handle_run() override; + // just call vp_node::handle_frame_meta to ignore vp_src_node::handle_frame_meta + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + // just call vp_node::handle_control_meta to ignore vp_src_node::handle_control_meta + virtual std::shared_ptr handle_control_meta(std::shared_ptr meta) override; + + public: + vp_app_src_node(std::string node_name, + int channel_index); + ~vp_app_src_node(); + + // push frames into pipeline + // size of frame MUST be the same as the first time pushing to pipeline + bool push_frames(std::vector frames); + }; +} \ No newline at end of file diff --git a/nodes/vp_des_node.cpp b/nodes/vp_des_node.cpp new file mode 100755 index 0000000..9aaec21 --- /dev/null +++ b/nodes/vp_des_node.cpp @@ -0,0 +1,60 @@ + +#include "vp_des_node.h" + + +namespace vp_nodes { + + + vp_des_node::vp_des_node(std::string node_name, int channel_index): + vp_node(node_name), channel_index(channel_index) { + stream_status.channel_index = channel_index; + } + + vp_des_node::~vp_des_node() { + + } + + // do nothing in des nodes + void vp_des_node::dispatch_run() { + // dispatch thread terminates immediately in all des nodes + } + + // it is the end point of stream(frame meta) in pipe, this method MUST be called at the end of handle_frame_meta in derived class. + // using 'return vp_des_node::handle_frame_meta(...);' + std::shared_ptr vp_des_node::handle_frame_meta(std::shared_ptr meta) { + // update cache of stream status + stream_status.frame_index = meta->frame_index; + stream_status.width = meta->frame.size().width; + stream_status.height = meta->frame.size().height; + stream_status.direction = to_string(); + + // calculate the duration between now and the time when meta created, which is latency. + auto delta_time = std::chrono::duration_cast(std::chrono::system_clock::now() - meta->create_time); + stream_status.latency = delta_time.count(); + + // update fps if need + fps_counter++; + delta_time = std::chrono::duration_cast(std::chrono::system_clock::now() - fps_last_time); + if (delta_time.count() >= fps_epoch) { + stream_status.fps = fps_counter * 1000.0 / delta_time.count(); + fps_counter = 0; + fps_last_time = std::chrono::system_clock::now(); + } + + // activate the stream status hooker if need + invoke_stream_status_hooker(node_name, stream_status); + + return nullptr; + } + + // it is the end point of control meta in pipe, this method MUST be called at the end of handle_control_meta in derived class. + // using 'return vp_des_node::handle_control_meta(...);' + std::shared_ptr vp_des_node::handle_control_meta(std::shared_ptr meta) { + // ... + return nullptr; + } + + vp_node_type vp_des_node::node_type() { + return vp_node_type::DES; + } +} \ No newline at end of file diff --git a/nodes/vp_des_node.h b/nodes/vp_des_node.h new file mode 100755 index 0000000..081501f --- /dev/null +++ b/nodes/vp_des_node.h @@ -0,0 +1,36 @@ +#pragma once + +#include + +#include "vp_node.h" +#include "vp_stream_status_hookable.h" + +namespace vp_nodes { + // base class for des nodes, end point of meta/pipeline. + class vp_des_node: public vp_node, public vp_stream_status_hookable { + private: + // cache for stream status at current des node. + vp_stream_status stream_status; + + // period(ms) to calculate output fps + int fps_epoch = 500; + int fps_counter = 0; + std::chrono::system_clock::time_point fps_last_time; + protected: + // do nothing in des nodes + virtual void dispatch_run() override final; + // sample implementation, return nullptr in all des nodes. + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + // sample implementation, return nullptr in all des nodes. + virtual std::shared_ptr handle_control_meta(std::shared_ptr meta) override; + + // protected as it can't be instanstiated directly. + vp_des_node(std::string node_name, int channel_index); + public: + ~vp_des_node(); + + virtual vp_node_type node_type() override; + + int channel_index; + }; +} \ No newline at end of file diff --git a/nodes/vp_fake_des_node.cpp b/nodes/vp_fake_des_node.cpp new file mode 100755 index 0000000..6faf6b2 --- /dev/null +++ b/nodes/vp_fake_des_node.cpp @@ -0,0 +1,14 @@ + +#include "vp_fake_des_node.h" + +namespace vp_nodes { + + vp_fake_des_node::vp_fake_des_node(std::string node_name, + int channel_index): vp_des_node(node_name, channel_index) { + this->initialized(); + } + + vp_fake_des_node::~vp_fake_des_node() { + deinitialized(); + } +} \ No newline at end of file diff --git a/nodes/vp_fake_des_node.h b/nodes/vp_fake_des_node.h new file mode 100755 index 0000000..a5b3b90 --- /dev/null +++ b/nodes/vp_fake_des_node.h @@ -0,0 +1,17 @@ + +#pragma once + +#include "vp_des_node.h" + +namespace vp_nodes { + // fake des node, do nothing just a placeholder + class vp_fake_des_node: public vp_des_node { + private: + /* data */ + public: + vp_fake_des_node(std::string node_name, + int channel_index); + ~vp_fake_des_node(); + }; + +} \ No newline at end of file diff --git a/nodes/vp_file_des_node.cpp b/nodes/vp_file_des_node.cpp new file mode 100755 index 0000000..924b4cd --- /dev/null +++ b/nodes/vp_file_des_node.cpp @@ -0,0 +1,96 @@ + +#include "vp_file_des_node.h" + +namespace vp_nodes { + + vp_file_des_node::vp_file_des_node(std::string node_name, + int channel_index, + std::string save_dir, + std::string name_prefix, + int max_duration_for_single_file, + vp_objects::vp_size resolution_w_h, + int bite_rate, + bool osd, + std::string gst_encoder_name): + vp_des_node(node_name, channel_index), + save_dir(save_dir), + name_prefix(name_prefix), + max_duration_for_single_file(max_duration_for_single_file), + resolution_w_h(resolution_w_h), + bitrate(bitrate), + osd(osd), + gst_encoder_name(gst_encoder_name) { + // compile tips: + // remove experimental:: if gcc >= 8.0 + assert(std::experimental::filesystem::exists(save_dir)); + // max time to record + assert(max_duration_for_single_file <= 30); + VP_INFO(vp_utils::string_format("[%s] [%s]", node_name.c_str(), gst_template.c_str())); + this->initialized(); + } + + vp_file_des_node::~vp_file_des_node() { + deinitialized(); + } + + // re-implementation, return nullptr. + std::shared_ptr + vp_file_des_node::handle_frame_meta(std::shared_ptr meta) { + VP_DEBUG(vp_utils::string_format("[%s] received frame meta, channel_index=>%d, frame_index=>%d", node_name.c_str(), meta->channel_index, meta->frame_index)); + + cv::Mat resize_frame; + if (this->resolution_w_h.width != 0 && this->resolution_w_h.height != 0) { + cv::resize((osd && !meta->osd_frame.empty()) ? meta->osd_frame : meta->frame, resize_frame, cv::Size(resolution_w_h.width, resolution_w_h.height)); + } + else { + resize_frame = (osd && !meta->osd_frame.empty()) ? meta->osd_frame : meta->frame; + } + + // new video file + if (!file_writer.isOpened() || + frames_already_record >= frames_need_record) { + // total frames need to be recorded + frames_need_record = max_duration_for_single_file * 60 * meta->fps; + frames_already_record = 0; + + if (name_prefix.empty()) { + name_prefix = node_name + "_" + std::to_string(meta->channel_index); + } + + // close it first if it has opened + if (file_writer.isOpened()) { + /* code */ + file_writer.release(); + } + + auto gst_str = vp_utils::string_format(this->gst_template, gst_encoder_name.c_str(), bitrate, get_new_file_name().c_str()); + assert(file_writer.open(gst_str, cv::CAP_GSTREAMER, 0, meta->fps, {resize_frame.cols, resize_frame.rows})); + } + + file_writer.write(resize_frame); + frames_already_record++; + + // for general works defined in base class + return vp_des_node::handle_frame_meta(meta); + } + + // re-implementation, return nullptr. + std::shared_ptr + vp_file_des_node::handle_control_meta(std::shared_ptr meta) { + return nullptr; + } + + std::string vp_file_des_node::get_new_file_name() { + auto stamp = vp_utils::time_format(NOW, "--_---"); + // compile tips: + // remove experimental:: if gcc >= 8.0 + std::experimental::filesystem::path p1(save_dir); + std::experimental::filesystem::path p2(name_prefix + "_" + stamp + ".mp4"); + + // save_dir/name_prefix_stamp.mp4 + std::experimental::filesystem::path p = p1 / p2; + + assert(!std::experimental::filesystem::exists(p)); + return p.string(); + } +} \ No newline at end of file diff --git a/nodes/vp_file_des_node.h b/nodes/vp_file_des_node.h new file mode 100755 index 0000000..61d1672 --- /dev/null +++ b/nodes/vp_file_des_node.h @@ -0,0 +1,64 @@ +#pragma once + +#include +#include +#include +// compile tips: +// remove experimental/ if gcc >= 8.0 +#include +#include +#include +#include + +#include "vp_des_node.h" +#include "../objects/vp_frame_meta.h" +#include "../objects/vp_control_meta.h" +#include "../utils/vp_utils.h" + + +namespace vp_nodes { + // file des node, save stream to local file + class vp_file_des_node: public vp_des_node { + private: + /* data */ + std::string gst_template = "appsrc ! videoconvert ! %s bitrate=%d ! mp4mux ! filesink location=%s"; + cv::VideoWriter file_writer; + + int frames_already_record = -1; + int frames_need_record = 0; + + // create file name + // example: save_dir/name_prefix_stamp.mp4 + std::string get_new_file_name(); + protected: + // re-implementation, return nullptr. + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + // re-implementation, return nullptr. + virtual std::shared_ptr handle_control_meta(std::shared_ptr meta) override; + public: + vp_file_des_node(std::string node_name, + int channel_index, + std::string save_dir, + std::string name_prefix = "", + int max_duration_for_single_file = 2, + vp_objects::vp_size resolution_w_h = {}, + int bitrate = 1024, + bool osd = true, + std::string gst_encoder_name = "x264enc"); + ~vp_file_des_node(); + + // save directory + std::string save_dir; + // prefix of file name + std::string name_prefix; + // max duration for single file (minutes) + int max_duration_for_single_file; + + vp_objects::vp_size resolution_w_h; + int bitrate; + bool osd; + // set x264enc as the default encoder, we can use hardware encoder instead. + std::string gst_encoder_name = "x264enc"; + }; + +} \ No newline at end of file diff --git a/nodes/vp_file_src_node.cpp b/nodes/vp_file_src_node.cpp new file mode 100755 index 0000000..67ff445 --- /dev/null +++ b/nodes/vp_file_src_node.cpp @@ -0,0 +1,134 @@ + +#include + +#include "../utils/logger/vp_logger.h" +#include "vp_file_src_node.h" + +namespace vp_nodes { + + vp_file_src_node::vp_file_src_node(std::string node_name, + int channel_index, + std::string file_path, + float resize_ratio, + bool cycle, + std::string gst_decoder_name, + int skip_interval): + vp_src_node(node_name, channel_index, resize_ratio), + file_path(file_path), + cycle(cycle), gst_decoder_name(gst_decoder_name), skip_interval(skip_interval) { + assert(skip_interval >= 0 && skip_interval <= 9); + this->gst_template = vp_utils::string_format(this->gst_template, file_path.c_str(), gst_decoder_name.c_str()); + VP_INFO(vp_utils::string_format("[%s] [%s]", node_name.c_str(), gst_template.c_str())); + this->initialized(); + } + + vp_file_src_node::~vp_file_src_node() { + deinitialized(); + } + + // define how to read video from local file, create frame meta etc. + // please refer to the implementation of vp_node::handle_run. + void vp_file_src_node::handle_run() { + cv::Mat frame; + int video_width = 0; + int video_height = 0; + int fps = 0; + std::chrono::milliseconds delta; + int skip = 0; + + while(alive) { + // check if need work + gate.knock(); + + auto last_time = std::chrono::system_clock::now(); + // try to open capture + if (!file_capture.isOpened()) { + if(!file_capture.open(this->gst_template, cv::CAP_GSTREAMER)) { + VP_WARN(vp_utils::string_format("[%s] open file failed, try again...", node_name.c_str())); + continue; + } + } + + // video properties + if (video_width == 0 || video_height == 0 || fps == 0) { + video_width = file_capture.get(cv::CAP_PROP_FRAME_WIDTH); + video_height = file_capture.get(cv::CAP_PROP_FRAME_HEIGHT); + fps = file_capture.get(cv::CAP_PROP_FPS); + delta = std::chrono::milliseconds(1000 / fps) * (skip_interval + 1); + + original_fps = fps; + original_width = video_width; + original_height = video_height; + + // set true fps because skip some frames + fps = fps / (skip_interval + 1); + } + // stream_info_hooker activated if need + vp_stream_info stream_info {channel_index, original_fps, original_width, original_height, to_string()}; + invoke_stream_info_hooker(node_name, stream_info); + + file_capture >> frame; + if(frame.empty()) { + VP_INFO(vp_utils::string_format("[%s] reading frame complete, total frame==>%d", node_name.c_str(), frame_index)); + if (cycle) { + VP_INFO(vp_utils::string_format("[%s] cycle flag is true, continue!", node_name.c_str())); + file_capture.set(cv::CAP_PROP_POS_FRAMES, 0); + } + continue; + } + + // need skip + if (skip < skip_interval) { + skip++; + continue; + } + skip = 0; + + // need resize + cv::Mat resize_frame; + if (this->resize_ratio != 1.0f) { + cv::resize(frame, resize_frame, cv::Size(), resize_ratio, resize_ratio); + } + else { + resize_frame = frame.clone(); // clone! + } + // set true size because resize + video_width = resize_frame.cols; + video_height = resize_frame.rows; + + this->frame_index++; + // create frame meta + auto out_meta = + std::make_shared(resize_frame, this->frame_index, this->channel_index, video_width, video_height, fps); + + if (out_meta != nullptr) { + this->out_queue.push(out_meta); + + // handled hooker activated if need + if (this->meta_handled_hooker) { + meta_handled_hooker(node_name, out_queue.size(), out_meta); + } + + // important! notify consumer of out_queue in case it is waiting. + this->out_queue_semaphore.signal(); + VP_DEBUG(vp_utils::string_format("[%s] after handling meta, out_queue.size()==>%d", node_name.c_str(), out_queue.size())); + } + + // for fps + auto snap = std::chrono::system_clock::now() - last_time; + snap = std::chrono::duration_cast(snap); + if (snap < delta) { + std::this_thread::sleep_for(delta - snap); + } + } + + // send dead flag for dispatch_thread + this->out_queue.push(nullptr); + this->out_queue_semaphore.signal(); + } + + // return stream path + std::string vp_file_src_node::to_string() { + return file_path; + } +} \ No newline at end of file diff --git a/nodes/vp_file_src_node.h b/nodes/vp_file_src_node.h new file mode 100755 index 0000000..8401a3d --- /dev/null +++ b/nodes/vp_file_src_node.h @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include +#include + +#include "vp_src_node.h" + +namespace vp_nodes { + // file source node, read video from local file. + // example: + // ../video/test.mp4 + class vp_file_src_node: public vp_src_node { + private: + /* data */ + std::string gst_template = "filesrc location=%s ! qtdemux ! h264parse ! %s ! videoconvert ! appsink"; + cv::VideoCapture file_capture; + protected: + // re-implemetation + virtual void handle_run() override; + public: + vp_file_src_node(std::string node_name, + int channel_index, + std::string file_path, + float resize_ratio = 1.0, + bool cycle = true, + std::string gst_decoder_name = "avdec_h264", + int skip_interval = 0); + ~vp_file_src_node(); + + virtual std::string to_string() override; + std::string file_path; + bool cycle; + + // set avdec_h264 as the default decoder, we can use hardware decoder instead. + std::string gst_decoder_name = "avdec_h264"; + // 0 means no skip + int skip_interval = 0; + }; + +} \ No newline at end of file diff --git a/nodes/vp_image_des_node.cpp b/nodes/vp_image_des_node.cpp new file mode 100644 index 0000000..ac73a73 --- /dev/null +++ b/nodes/vp_image_des_node.cpp @@ -0,0 +1,82 @@ + + +#include "vp_image_des_node.h" + + +namespace vp_nodes { + + vp_image_des_node::vp_image_des_node(std::string node_name, + int channel_index, + std::string location, + int interval, + vp_objects::vp_size resolution_w_h, + bool osd, + std::string gst_encoder_name): + vp_des_node(node_name, channel_index), + location(location), + interval(interval), + resolution_w_h(resolution_w_h), + osd(osd), + gst_encoder_name(gst_encoder_name) { + // make sure not greater than 1 minute (too long) and not lower than 1 second (since it's too quick, use video stream instead directly) + assert(interval >= 1 && interval <= 60); + if (vp_utils::ends_with(location, ".jpeg") || vp_utils::ends_with(location, ".jpg")) { + // save to file + gst_template_file = vp_utils::string_format(gst_template_file, interval, gst_encoder_name.c_str(), location.c_str()); + to_file = true; + } + else if (location.find(":") != std::string::npos) { + // push to remote + auto parts = vp_utils::string_split(location, ':'); + assert(parts.size() == 2); + auto host = parts[0]; // ip + auto port = std::stoi(parts[1]); // try to get port + + gst_template_udp = vp_utils::string_format(gst_template_udp, interval, gst_encoder_name.c_str(), host.c_str(), port); + + to_file = false; + } + else { + // error + throw "invalid input parameter for `location`!"; + } + + auto s = to_file ? gst_template_file : gst_template_udp; + VP_INFO(vp_utils::string_format("[%s] [%s]", node_name.c_str(), s.c_str())); + this->initialized(); + } + + vp_image_des_node::~vp_image_des_node() { + deinitialized(); + } + + std::shared_ptr vp_image_des_node::handle_frame_meta(std::shared_ptr meta) { + VP_DEBUG(vp_utils::string_format("[%s] received frame meta, channel_index=>%d, frame_index=>%d", node_name.c_str(), meta->channel_index, meta->frame_index)); + + cv::Mat resize_frame; + if (this->resolution_w_h.width != 0 && this->resolution_w_h.height != 0) { + cv::resize((osd && !meta->osd_frame.empty()) ? meta->osd_frame : meta->frame, resize_frame, cv::Size(resolution_w_h.width, resolution_w_h.height)); + } + else { + resize_frame = (osd && !meta->osd_frame.empty()) ? meta->osd_frame : meta->frame; + } + + if (!image_writer.isOpened()) { + if (to_file) { + assert(image_writer.open(this->gst_template_file, cv::CAP_GSTREAMER, 0, meta->fps, {resize_frame.cols, resize_frame.rows})); + } + else { + assert(image_writer.open(this->gst_template_udp, cv::CAP_GSTREAMER, 0, meta->fps, {resize_frame.cols, resize_frame.rows})); + } + } + + image_writer.write(resize_frame); + + // for general works defined in base class + return vp_des_node::handle_frame_meta(meta); + } + + std::string vp_image_des_node::to_string() { + return to_file ? location : "udp://" + location + "/jpg"; + } +} \ No newline at end of file diff --git a/nodes/vp_image_des_node.h b/nodes/vp_image_des_node.h new file mode 100644 index 0000000..d625543 --- /dev/null +++ b/nodes/vp_image_des_node.h @@ -0,0 +1,46 @@ +#pragma once + +#include "vp_des_node.h" + +namespace vp_nodes { + // image des node, save image to local file or push image to remote via udp. + class vp_image_des_node: public vp_des_node + { + private: + // gstreamer template for saving image to file (jpeg encoding only, filename MUST end with 'jpg/jpeg') + std::string gst_template_file = "appsrc ! videoconvert ! videoscale ! videorate ! video/x-raw,framerate=1/%d ! %s ! multifilesink location=%s"; + // gstreamer template for pushing image to remote via udp （jpeg encoding only) + std::string gst_template_udp = "appsrc ! videoconvert ! videoscale ! videorate ! video/x-raw,format=I420,framerate=1/%d ! %s ! rtpjpegpay ! udpsink host=%s port=%d"; + + // gstreamer wrapper by opencv + cv::VideoWriter image_writer; + + // save/push ONE image every `interval` seconds + int interval; + // save/push image to where + // for example, `./%d.jpg` for file (end with `jpg/jpeg`), `192.168.1.90:8000` for udp (split by `:`) + std::string location = "./%d.jpg"; + + bool osd = true; + vp_objects::vp_size resolution_w_h; + + bool to_file = true; + // set jpegenc as the default encoder, we can use hardware encoder instead. + std::string gst_encoder_name = "jpegenc"; + protected: + // re-implementation, return nullptr. + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + + public: + vp_image_des_node(std::string node_name, + int channel_index, + std::string location = "./%d.jpg", + int interval = 5, + vp_objects::vp_size resolution_w_h = {}, + bool osd = true, + std::string gst_encoder_name = "jpegenc"); + ~vp_image_des_node(); + virtual std::string to_string() override; + }; + +} \ No newline at end of file diff --git a/nodes/vp_image_src_node.cpp b/nodes/vp_image_src_node.cpp new file mode 100644 index 0000000..2044950 --- /dev/null +++ b/nodes/vp_image_src_node.cpp @@ -0,0 +1,132 @@ + +#include "vp_image_src_node.h" + +namespace vp_nodes { + vp_image_src_node::vp_image_src_node(std::string node_name, + int channel_index, + std::string port_or_location, + int interval, + float resize_ratio, + bool cycle, + std::string gst_decoder_name): + vp_src_node(node_name, channel_index, resize_ratio), + port_or_location(port_or_location), + interval(interval), + cycle(cycle), gst_decoder_name(gst_decoder_name) { + // make sure not greater than 1 minute (too long) and not lower than 1 second (since it's too quick, use video stream instead directly) + assert(interval >= 1 && interval <= 60); + if (vp_utils::ends_with(port_or_location, "jpeg") || vp_utils::ends_with(port_or_location, "jpg")) { + // read from file + gst_template_file = vp_utils::string_format(gst_template_file, port_or_location.c_str(), cycle ? std::string("true").c_str() : std::string("false").c_str(), gst_decoder_name.c_str(), interval); + from_file = true; + } + else if (port_or_location.find_first_not_of("0123456789") == std::string::npos) { + // receive from remote via udp + auto port = std::stoi(port_or_location); // try to get port + gst_template_udp = vp_utils::string_format(gst_template_udp, port, gst_decoder_name.c_str(), interval); + from_file = false; + } + else { + throw "invalid input parameter for `port_or_location`!"; + } + + auto s = from_file ? gst_template_file : gst_template_udp; + VP_INFO(vp_utils::string_format("[%s] [%s]", node_name.c_str(), s.c_str())); + this->initialized(); + } + + vp_image_src_node::~vp_image_src_node() { + deinitialized(); + } + + void vp_image_src_node::handle_run() { + cv::Mat frame; + int video_width = 0; + int video_height = 0; + int fps = 0; + std::chrono::milliseconds delta; + + while(alive) { + // check if need work + gate.knock(); + + auto last_time = std::chrono::system_clock::now(); + // try to open capture + if (!image_capture.isOpened()) { + auto gst_launch_str = from_file ? gst_template_file : gst_template_udp; + if(!image_capture.open(gst_launch_str, cv::CAP_GSTREAMER)) { + VP_WARN(vp_utils::string_format("[%s] open image capture failed, try again...", node_name.c_str())); + continue; + } + } + + // video properties + if (video_width == 0 || video_height == 0 || fps == 0) { + video_width = image_capture.get(cv::CAP_PROP_FRAME_WIDTH); + video_height = image_capture.get(cv::CAP_PROP_FRAME_HEIGHT); + fps = image_capture.get(cv::CAP_PROP_FPS); + fps = fps < 1 ? 1 : fps; // since fps is too small (0.1 means 1 frame every 10 seconds) + + delta = std::chrono::milliseconds(1000 / fps); + + original_fps = fps; + original_width = video_width; + original_height = video_height; + } + // stream_info_hooker activated if need + vp_stream_info stream_info {channel_index, original_fps, original_width, original_height, to_string()}; + invoke_stream_info_hooker(node_name, stream_info); + + image_capture >> frame; + if(frame.empty()) { + VP_WARN(vp_utils::string_format("[%s] reading frame empty, total frame==>%d", node_name.c_str(), frame_index)); + continue; + } + + // need resize + cv::Mat resize_frame; + if (this->resize_ratio != 1.0f) { + cv::resize(frame, resize_frame, cv::Size(), resize_ratio, resize_ratio); + } + else { + resize_frame = frame.clone(); // clone! + } + // set true size because resize + video_width = resize_frame.cols; + video_height = resize_frame.rows; + + this->frame_index++; + // create frame meta + auto out_meta = + std::make_shared(resize_frame, this->frame_index, this->channel_index, video_width, video_height, fps); + + if (out_meta != nullptr) { + this->out_queue.push(out_meta); + + // handled hooker activated if need + if (this->meta_handled_hooker) { + meta_handled_hooker(node_name, out_queue.size(), out_meta); + } + + // important! notify consumer of out_queue in case it is waiting. + this->out_queue_semaphore.signal(); + VP_DEBUG(vp_utils::string_format("[%s] after handling meta, out_queue.size()==>%d", node_name.c_str(), out_queue.size())); + } + + // for fps + auto snap = std::chrono::system_clock::now() - last_time; + snap = std::chrono::duration_cast(snap); + if (snap < delta) { + std::this_thread::sleep_for(delta - snap); + } + } + + // send dead flag for dispatch_thread + this->out_queue.push(nullptr); + this->out_queue_semaphore.signal(); + } + + std::string vp_image_src_node::to_string() { + return from_file ? port_or_location : "udp://127.0.0.1:" + port_or_location + "/jpg"; + } +} \ No newline at end of file diff --git a/nodes/vp_image_src_node.h b/nodes/vp_image_src_node.h new file mode 100644 index 0000000..b449407 --- /dev/null +++ b/nodes/vp_image_src_node.h @@ -0,0 +1,46 @@ + +#pragma once + +#include "vp_src_node.h" + +namespace vp_nodes { + // image src node, read image from local files or receive image from remote via udp. + class vp_image_src_node: public vp_src_node + { + private: + // gstreamer template for reading image from file (jpeg encoding only, filename MUST end with 'jpg/jpeg') + std::string gst_template_file = "multifilesrc location=%s loop=%s ! jpegparse ! %s ! videorate ! video/x-raw,framerate=1/%d ! videoconvert ! appsink"; + // gstreamer template for receiving image from remote via udp （jpeg encoding only) + std::string gst_template_udp = "udpsrc port=%d ! application/x-rtp,encoding-name=jpeg ! rtpjpegdepay ! jpegparse ! %s ! videorate ! video/x-raw,framerate=1/%d ! videoconvert ! appsink"; + + cv::VideoCapture image_capture; + + // `port number` for udp mode, `directory path` for file mode. auto-detect which mode should apply according to this value. + std::string port_or_location = "./%d.jpg"; + // frequency to read/receive image + int interval = 1; + + // restart reading if images read completely (only used for file mode) + bool cycle = true; + + bool from_file = true; + protected: + // re-implemetation + virtual void handle_run() override; + + public: + vp_image_src_node(std::string node_name, + int channel_index, + std::string port_or_location, + int interval = 1, + float resize_ratio = 1.0, + bool cycle = true, + std::string gst_decoder_name = "jpegdec"); + ~vp_image_src_node(); + virtual std::string to_string() override; + + // set jpegdec as the default decoder, we can use hardware decoder instead. + std::string gst_decoder_name = "jpegdec"; + }; + +} \ No newline at end of file diff --git a/nodes/vp_infer_node.cpp b/nodes/vp_infer_node.cpp new file mode 100755 index 0000000..cc86d8e --- /dev/null +++ b/nodes/vp_infer_node.cpp @@ -0,0 +1,258 @@ + +#include + +#include "vp_infer_node.h" + +namespace vp_nodes { + vp_infer_node::vp_infer_node(std::string node_name, + vp_infer_type infer_type, + std::string model_path, + std::string model_config_path, + std::string labels_path, + int input_width, + int input_height, + int batch_size, + float scale, + cv::Scalar mean, + cv::Scalar std, + bool swap_rb, + bool swap_chn): + vp_node(node_name), + infer_type(infer_type), + model_path(model_path), + model_config_path(model_config_path), + labels_path(labels_path), + input_width(input_width), + input_height(input_height), + batch_size(batch_size), + scale(scale), + mean(mean), + std(std), + swap_rb(swap_rb), + swap_chn(swap_chn) { + // try to load network from file, + // failing means maybe it has a custom implementation for model loading in derived class such as using other backends other than opencv::dnn. + try { + net = cv::dnn::readNet(model_path, model_config_path); + #ifdef VP_WITH_CUDA + net.setPreferableBackend(cv::dnn::DNN_BACKEND_CUDA); + net.setPreferableTarget(cv::dnn::DNN_TARGET_CUDA); + #endif + } + catch(const std::exception& e) { + VP_WARN(vp_utils::string_format("[%s] cv::dnn::readNet load network failed!", node_name.c_str())); + } + + // load labels if labels_path is specified + if (labels_path != "") { + load_labels(); + } + + assert(batch_size > 0); + // primary infer nodes can handle frame meta batch by batch(whole frame), + // others can handle multi batchs ONLY inside a single frame(small croped image). + if (infer_type == vp_infer_type::PRIMARY && batch_size > 1) { + frame_meta_handle_batch = batch_size; + } + } + + vp_infer_node::~vp_infer_node() { + + } + + // handle frame meta one by one + std::shared_ptr vp_infer_node::handle_frame_meta(std::shared_ptr meta) { + std::vector> frame_meta_with_batch {meta}; + run_infer_combinations(frame_meta_with_batch); + return meta; + } + + // handle frame meta batch by batch + void vp_infer_node::handle_frame_meta(const std::vector>& meta_with_batch) { + const auto& frame_meta_with_batch = meta_with_batch; + run_infer_combinations(frame_meta_with_batch); + // no return + } + + // default implementation + // infer batch by batch + void vp_infer_node::infer(const cv::Mat& blob_to_infer, std::vector& raw_outputs) { + // blob_to_infer is a 4D matrix + // the first dim is number of batch + assert(blob_to_infer.dims == 4); + assert(!net.empty()); + + auto number_of_batch = blob_to_infer.size[0]; + if (number_of_batch <= batch_size) { + // infer one time directly + net.setInput(blob_to_infer); + net.forward(raw_outputs, net.getUnconnectedOutLayersNames()); + } + else { + // infer more times + int b_size[] = {batch_size, blob_to_infer.size[1], blob_to_infer.size[2], blob_to_infer.size[3]}; + auto times = (number_of_batch % batch_size) == 0 ? (number_of_batch / batch_size) : (number_of_batch / batch_size + 1); + for (int i = 0; i < times; i++) { + // split to small piece + int i_hwc[] = {i * batch_size, 0, 0, 0}; // 4D + auto ptr = blob_to_infer.ptr(i_hwc); + cv::Mat b_blob(4, b_size, CV_32F, (void*)ptr); + std::vector b_outputs; + + net.setInput(b_blob); + net.forward(b_outputs, net.getUnconnectedOutLayersNames()); + + // first time, initialize it + if (raw_outputs.size() == 0) { + // scan multi heads of output + for (int j = 0; j < b_outputs.size(); j++) { + if (batch_size == 1) { + // keep dims as usual, but change size[0] == number_of_batch + if (b_outputs[j].dims <= 2 && b_outputs[j].rows == 1) { + raw_outputs.push_back(cv::Mat(2, std::vector{number_of_batch, b_outputs[j].cols}.data(), CV_32F)); + } + else { + // dims add 1, and set size[0] == number_of_batch + std::vector t_size; + t_size.push_back(number_of_batch); + for (int s = 0; s < b_outputs[j].dims; s++) { + t_size.push_back(b_outputs[j].size[s]); + } + raw_outputs.push_back(cv::Mat(b_outputs[j].dims + 1, t_size.data(), CV_32F)); + } + } + else { + // kepp dims as usual, but change size[0] == number_of_batch + std::vector t_size; + t_size.push_back(number_of_batch); + + // start from 1 + for (int s = 1; s < b_outputs[j].dims; s++) { + /* code */ + t_size.push_back(b_outputs[j].size[s]); + } + raw_outputs.push_back(cv::Mat(b_outputs[j].dims, t_size.data(), CV_32F)); + } + } + } + + assert(raw_outputs.size() == b_outputs.size()); + // merge data directly + for (int j = 0; j < b_outputs.size(); j++) { + auto& des = raw_outputs[j]; + auto& src = b_outputs[j]; + + std::vector t_size; + auto s_dims_n = src.dims <= 2 ? 2 : src.dims; + for (int s = 0; s < s_dims_n; s++) { + t_size.push_back(src.size[s]); + } + + auto ptr = des.ptr(i * batch_size); + cv::Mat tmp(s_dims_n, t_size.data(), CV_32F, (void*)ptr); + + src.copyTo(tmp); + } + } + } + } + + // default implementation + // create a 4D matrix(n, c, h, w) + void vp_infer_node::preprocess(const std::vector& mats_to_infer, cv::Mat& blob_to_infer) { + cv::dnn::blobFromImages(mats_to_infer, blob_to_infer, scale, cv::Size(input_width, input_height), mean, swap_rb); + if (std != cv::Scalar(1)) { + // divide by std + } + + // NCHW -> NHWC + if (swap_chn) { + cv::Mat blob_to_infer_tmp; + cv::transposeND(blob_to_infer, {0, 2, 3, 1}, blob_to_infer_tmp); + blob_to_infer_tmp.copyTo(blob_to_infer); + } + } + + void vp_infer_node::run_infer_combinations(const std::vector>& frame_meta_with_batch) { + /* + * call logic by default: + * frame_meta_with_batch -> mats_to_infer -> blob_to_infer -> raw_outputs -> frame_meta_with_batch + */ + std::vector mats_to_infer; + // 4D matrix + cv::Mat blob_to_infer; + // multi heads of output in network, raw matrix output which need to be parsed by users. + std::vector raw_outputs; + + // start + auto start_time = std::chrono::system_clock::now(); + // 1st step, prepare + prepare(frame_meta_with_batch, mats_to_infer); + auto prepare_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + // nothing to infer + if (mats_to_infer.size() == 0) { + return; + } + + start_time = std::chrono::system_clock::now(); + // 2nd step, preprocess + preprocess(mats_to_infer, blob_to_infer); + auto preprocess_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + start_time = std::chrono::system_clock::now(); + // 3rd step, infer + infer(blob_to_infer, raw_outputs); + auto infer_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + start_time = std::chrono::system_clock::now(); + // 4th step, postprocess + postprocess(raw_outputs, frame_meta_with_batch); + auto postprocess_time = std::chrono::duration_cast(std::chrono::system_clock::now() - start_time); + + // end + infer_combinations_time_cost(mats_to_infer.size(), prepare_time.count(), preprocess_time.count(), infer_time.count(), postprocess_time.count()); + } + + // print all by default + void vp_infer_node::infer_combinations_time_cost(int data_size, int prepare_time, int preprocess_time, int infer_time, int postprocess_time) { + /* + std::cout << "########## infer combinations summary ##########" << std::endl; + std::cout << " node_name:" << node_name << std::endl; + std::cout << " data_size:" << data_size << std::endl; + std::cout << " prepare_time:" << prepare_time << "ms" << std::endl; + std::cout << " preprocess_time:" << preprocess_time << "ms" << std::endl; + std::cout << " infer_time:" << infer_time << "ms" << std::endl; + std::cout << " postprocess_time:" << postprocess_time << "ms" << std::endl; + std::cout << "########## infer combinations summary ##########" << std::endl; + */ + + std::ostringstream s_stream; + s_stream << "\n########## infer combinations summary ##########\n"; + s_stream << " node_name:" << node_name << "\n"; + s_stream << " data_size:" << data_size << "\n"; + s_stream << " prepare_time:" << prepare_time << "ms\n"; + s_stream << " preprocess_time:" << preprocess_time << "ms\n"; + s_stream << " infer_time:" << infer_time << "ms\n"; + s_stream << " postprocess_time:" << postprocess_time << "ms\n"; + s_stream << "########## infer combinations summary ##########\n"; + + // to log + VP_DEBUG(s_stream.str()); + } + + void vp_infer_node::load_labels() { + try { + std::ifstream label_stream(labels_path); + for (std::string line; std::getline(label_stream, line); ) { + if (!line.empty() && line[line.length() - 1] == '\r') { + line.erase(line.length() - 1); + } + labels.push_back(line); + } + } + catch(const std::exception& e) { + + } + } +} \ No newline at end of file diff --git a/nodes/vp_infer_node.h b/nodes/vp_infer_node.h new file mode 100755 index 0000000..a05c21b --- /dev/null +++ b/nodes/vp_infer_node.h @@ -0,0 +1,92 @@ + + +#pragma once +#include +#include +#include "vp_node.h" + +namespace vp_nodes { + + // infer type + // infer on the whole frame or small cropped image? + enum vp_infer_type { + PRIMARY, // infer on the whole frame, like detector, pose estimatation + SECONDARY // infer on small cropped image, like classifier, feature extractor and secondary detector which need detect on small cropped images. + }; + + // base class for infer node, can't be instanstiated directly. + // note: + // the class is based on opencv::dnn module which is the default way for all deep learning inference in code, + // we can implement it using other backends such as tensorrt with cuda acceleration, see vp_ppocr_text_detector_node which is based on PaddlePaddle dl framework from BaiDu corporation. + class vp_infer_node: public vp_node { + private: + // load labels if need + void load_labels(); + protected: + vp_infer_type infer_type; + std::string model_path; + std::string model_config_path; + std::string labels_path; + int input_width; + int input_height; + int batch_size; + cv::Scalar mean; + cv::Scalar std; + float scale; + bool swap_rb; + + // transpose channel or not, NCHW -> NHWC + bool swap_chn; + + // protected as it can't be instanstiated directly. + vp_infer_node(std::string node_name, + vp_infer_type infer_type, + std::string model_path, + std::string model_config_path = "", + std::string labels_path = "", + int input_width = 128, + int input_height = 128, + int batch_size = 1, + float scale = 1.0, + cv::Scalar mean = cv::Scalar(123.675, 116.28, 103.53), // imagenet dataset + cv::Scalar std = cv::Scalar(1), + bool swap_rb = true, + bool swap_chn = false); + + // the 1st step, MUST implement in specific derived class. + // prepare data for infer, fetch frames from frame meta. + virtual void prepare(const std::vector>& frame_meta_with_batch, std::vector& mats_to_infer) = 0; + + // the 2nd step, has a default implementation. + // preprocess data, such as normalization, mean substract. + virtual void preprocess(const std::vector& mats_to_infer, cv::Mat& blob_to_infer); + + // the 3rd step, has a default implementation. + // infer and retrive raw outputs. + virtual void infer(const cv::Mat& blob_to_infer, std::vector& raw_outputs); + + // the 4th step, MUST implement in specific derived class. + // postprocess on raw outputs and create/update something back to frame meta again. + virtual void postprocess(const std::vector& raw_outputs, const std::vector>& frame_meta_with_batch) = 0; + + // debug purpose(ms) + virtual void infer_combinations_time_cost(int data_size, int prepare_time, int preprocess_time, int infer_time, int postprocess_time); + + // infer operations(call prepare/preprocess/infer/postprocess by default) + // we can define new logic for infer operations by overriding it. + virtual void run_infer_combinations(const std::vector>& frame_meta_with_batch); + + // labels as text format + std::vector labels; + + // opencv::dnn as backend + cv::dnn::Net net; + + // re-implementation for one by one mode, marked as 'final' as we need not override any more in specific derived classes. + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override final; + // re-implementation for batch by batch mode, marked as 'final' as we need not override any more in specific derived classes. + virtual void handle_frame_meta(const std::vector>& meta_with_batch) override final; + public: + ~vp_infer_node(); + }; +} \ No newline at end of file diff --git a/nodes/vp_message_broker_node.cpp b/nodes/vp_message_broker_node.cpp new file mode 100755 index 0000000..d83a7d1 --- /dev/null +++ b/nodes/vp_message_broker_node.cpp @@ -0,0 +1,28 @@ + +#include "vp_message_broker_node.h" + +namespace vp_nodes { + + vp_message_broker_node::vp_message_broker_node(std::string node_name): vp_node(node_name) { + this->initialized(); + } + + vp_message_broker_node::~vp_message_broker_node() { + + } + + std::shared_ptr vp_message_broker_node::handle_control_meta(std::shared_ptr meta) { + return meta; + } + + std::shared_ptr vp_message_broker_node::handle_frame_meta(std::shared_ptr meta) { + /* + if (meta->frame_index % 15 == 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(28)); + } + if (meta->frame_index % 73 == 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(64)); + }*/ + return meta; + } +} \ No newline at end of file diff --git a/nodes/vp_message_broker_node.h b/nodes/vp_message_broker_node.h new file mode 100755 index 0000000..9ed7f0e --- /dev/null +++ b/nodes/vp_message_broker_node.h @@ -0,0 +1,18 @@ + +#pragma once + +#include "vp_node.h" + +namespace vp_nodes { + class vp_message_broker_node : public vp_node + { + private: + /* data */ + protected: + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + virtual std::shared_ptr handle_control_meta(std::shared_ptr meta) override; + public: + vp_message_broker_node(std::string node_name); + ~vp_message_broker_node(); + }; +} \ No newline at end of file diff --git a/nodes/vp_meta_hookable.h b/nodes/vp_meta_hookable.h new file mode 100755 index 0000000..f764da2 --- /dev/null +++ b/nodes/vp_meta_hookable.h @@ -0,0 +1,82 @@ +#pragma once +#include +#include +#include +#include + +#include "../objects/vp_meta.h" + +namespace vp_nodes { + // callback when meta flowing through the whole pipe, MUST NOT be blocked. + // we can do more work based on this callback, such as calculating fps/latency at each port of node, please refer to vp_analysis_board for details. + typedef std::function)> vp_meta_hooker; + + // allow hookers attached to the pipe (nodes), get notified when meta flow through each port of node (total 4 ports in node). + // this class is inherited by vp_node only. + class vp_meta_hookable { + protected: + std::mutex meta_arriving_hooker_lock; + std::mutex meta_handling_hooker_lock; + std::mutex meta_handled_hooker_lock; + std::mutex meta_leaving_hooker_lock; + // hooker activated when meta is arriving at node (pushed to in_queue of vp_node, the 1st port in node). + vp_meta_hooker meta_arriving_hooker; + // hooker activated when meta is to be handled inside node (poped from in_queue of vp_node, the 2nd port in node). + vp_meta_hooker meta_handling_hooker; + // hooker activated when meta is handled inside node (pushed to out_queue of vp_node, the 3rd port in node). + vp_meta_hooker meta_handled_hooker; + // hooker activated when meta is leaving from node (poped from out_queue of vp_node, the 4th port in node). + vp_meta_hooker meta_leaving_hooker; + public: + vp_meta_hookable(/* args */) {} + ~vp_meta_hookable() {} + + void set_meta_arriving_hooker(vp_meta_hooker meta_arriving_hooker) { + std::lock_guard guard(meta_arriving_hooker_lock); + this->meta_arriving_hooker = meta_arriving_hooker; + } + + void set_meta_handling_hooker(vp_meta_hooker meta_handling_hooker) { + std::lock_guard guard(meta_handling_hooker_lock); + this->meta_handling_hooker = meta_handling_hooker; + } + + void set_meta_handled_hooker(vp_meta_hooker meta_handled_hooker) { + std::lock_guard guard(meta_handled_hooker_lock); + this->meta_handled_hooker = meta_handled_hooker; + } + + void set_meta_leaving_hooker(vp_meta_hooker meta_leaving_hooker) { + std::lock_guard guard(meta_leaving_hooker_lock); + this->meta_leaving_hooker = meta_leaving_hooker; + } + + void invoke_meta_arriving_hooker(std::string node_name, int queue_size, std::shared_ptr meta) { + std::lock_guard guard(meta_arriving_hooker_lock); + if (this->meta_arriving_hooker) { + this->meta_arriving_hooker(node_name, queue_size, meta); + } + } + + void invoke_meta_handling_hooker(std::string node_name, int queue_size, std::shared_ptr meta) { + std::lock_guard guard(meta_handling_hooker_lock); + if (this->meta_handling_hooker) { + this->meta_handling_hooker(node_name, queue_size, meta); + } + } + + void invoke_meta_handled_hooker(std::string node_name, int queue_size, std::shared_ptr meta) { + std::lock_guard guard(meta_handled_hooker_lock); + if (this->meta_handled_hooker) { + this->meta_handled_hooker(node_name, queue_size, meta); + } + } + + void invoke_meta_leaving_hooker(std::string node_name, int queue_size, std::shared_ptr meta) { + std::lock_guard guard(meta_leaving_hooker_lock); + if (this->meta_leaving_hooker) { + this->meta_leaving_hooker(node_name, queue_size, meta); + } + } + }; +} \ No newline at end of file diff --git a/nodes/vp_meta_publisher.cpp b/nodes/vp_meta_publisher.cpp new file mode 100755 index 0000000..8dbc57f --- /dev/null +++ b/nodes/vp_meta_publisher.cpp @@ -0,0 +1,39 @@ + + +#include "vp_meta_publisher.h" + +namespace vp_nodes { + vp_meta_publisher::vp_meta_publisher(/* args */) { + + } + + vp_meta_publisher::~vp_meta_publisher() { + + } + + void vp_meta_publisher::add_subscriber(std::shared_ptr subscriber) { + std::lock_guard guard(this->subscribers_lock); + this->subscribers.push_back(subscriber); + } + + void vp_meta_publisher::remove_subscriber(std::shared_ptr subscriber) { + std::lock_guard guard(this->subscribers_lock); + for (auto i = this->subscribers.begin(); i != this->subscribers.end();) { + if(*i == subscriber) { + i = this->subscribers.erase(i); + } + else { + i++; + } + } + } + + // by default, we push meta to next nodes indiscriminately, each next node has the same meta pointer. + // in some situations, we need push meta depend on condition, refer to vp_split_node which would push meta by channel index or push a deep copy pf meta(new pointer to new memory). + void vp_meta_publisher::push_meta(std::shared_ptr meta) { + std::lock_guard guard(this->subscribers_lock); + for (auto i = this->subscribers.begin(); i != this->subscribers.end(); i++) { + (*i)->meta_flow(meta); + } + } +} \ No newline at end of file diff --git a/nodes/vp_meta_publisher.h b/nodes/vp_meta_publisher.h new file mode 100755 index 0000000..54f8ea7 --- /dev/null +++ b/nodes/vp_meta_publisher.h @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include +#include + +#include "vp_meta_subscriber.h" + +namespace vp_nodes { + class vp_meta_publisher { + private: + protected: + // push meta to next nodes + virtual void push_meta(std::shared_ptr meta); + + // non-copyable for all child class + std::mutex subscribers_lock; + // next nodes as subscribers + std::vector> subscribers; + public: + vp_meta_publisher(/* args */); + ~vp_meta_publisher(); + + // add next node + void add_subscriber(std::shared_ptr subscriber); + // remove next node + void remove_subscriber(std::shared_ptr subscriber); + }; + +} + diff --git a/nodes/vp_meta_subscriber.cpp b/nodes/vp_meta_subscriber.cpp new file mode 100755 index 0000000..d9be1b8 --- /dev/null +++ b/nodes/vp_meta_subscriber.cpp @@ -0,0 +1,13 @@ + +#include "vp_meta_subscriber.h" + +namespace vp_nodes { + + vp_meta_subscriber::vp_meta_subscriber(/* args */) { + + } + + vp_meta_subscriber::~vp_meta_subscriber() { + + } +} \ No newline at end of file diff --git a/nodes/vp_meta_subscriber.h b/nodes/vp_meta_subscriber.h new file mode 100755 index 0000000..26b71dd --- /dev/null +++ b/nodes/vp_meta_subscriber.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include "../objects/vp_meta.h" + +namespace vp_nodes { + class vp_meta_subscriber { + private: + /* data */ + public: + vp_meta_subscriber(/* args */); + ~vp_meta_subscriber(); + + // non-copyable for all child class + vp_meta_subscriber(const vp_meta_subscriber&) = delete; + vp_meta_subscriber& operator=(const vp_meta_subscriber&) = delete; + + // receive meta from previous nodes + virtual void meta_flow(std::shared_ptr meta) = 0; + }; + +} diff --git a/nodes/vp_node.cpp b/nodes/vp_node.cpp new file mode 100755 index 0000000..6f9f8fd --- /dev/null +++ b/nodes/vp_node.cpp @@ -0,0 +1,248 @@ + +#include "vp_node.h" + +namespace vp_nodes { + + vp_node::vp_node(std::string node_name): node_name(node_name) { + } + + vp_node::~vp_node() { + + } + + // there is only one thread poping data from the in_queue, we don't use lock here when poping. + // there is only one thread pushing data to the out_queue, we don't use lock here when pushing. + void vp_node::handle_run() { + // cache for batch handling if need + std::vector> frame_meta_batch_cache; + while (alive) { + // wait for producer, make sure in_queue is not empty. + this->in_queue_semaphore.wait(); + + VP_DEBUG(vp_utils::string_format("[%s] before handling meta, in_queue.size()==>%d", node_name.c_str(), in_queue.size())); + auto in_meta = this->in_queue.front(); + + // dead flag + if (in_meta == nullptr) { + continue; + } + + // handling hooker activated if need + invoke_meta_handling_hooker(node_name, in_queue.size(), in_meta); + + std::shared_ptr out_meta; + auto batch_complete = false; + + // call handlers + if (in_meta->meta_type == vp_objects::vp_meta_type::CONTROL) { + auto meta_2_handle = std::dynamic_pointer_cast(in_meta); + out_meta = this->handle_control_meta(meta_2_handle); + } + else if (in_meta->meta_type == vp_objects::vp_meta_type::FRAME) { + auto meta_2_handle = std::dynamic_pointer_cast(in_meta); + // one by one + if (frame_meta_handle_batch == 1) { + out_meta = this->handle_frame_meta(meta_2_handle); + } + else { + // batch by batch + frame_meta_batch_cache.push_back(meta_2_handle); + if (frame_meta_batch_cache.size() >= frame_meta_handle_batch) { + // cache complete + this->handle_frame_meta(frame_meta_batch_cache); + batch_complete = true; + } + else { + // cache not complete, do nothing + VP_DEBUG(vp_utils::string_format("[%s] handle meta with batch, frame_meta_batch_cache.size()==>%d", node_name.c_str(), frame_meta_batch_cache.size())); + } + } + } + else { + throw "invalid meta type!"; + } + this->in_queue.pop(); + VP_DEBUG(vp_utils::string_format("[%s] after handling meta, in_queue.size()==>%d", node_name.c_str(), in_queue.size())); + + // one by one mode + // return nullptr means do not push it to next nodes(such as in des nodes). + if (out_meta != nullptr && node_type() != vp_node_type::DES) { + VP_DEBUG(vp_utils::string_format("[%s] before handling meta, out_queue.size()==>%d", node_name.c_str(), out_queue.size())); + this->out_queue.push(out_meta); + + // handled hooker activated if need + invoke_meta_handled_hooker(node_name, out_queue.size(), out_meta); + + // notify consumer of out_queue + this->out_queue_semaphore.signal(); + VP_DEBUG(vp_utils::string_format("[%s] after handling meta, out_queue.size()==>%d", node_name.c_str(), out_queue.size())); + } + + // batch by batch mode + if (batch_complete && node_type() != vp_node_type::DES) { + // push to out_queue one by one + for (auto& i: frame_meta_batch_cache) { + VP_DEBUG(vp_utils::string_format("[%s] before handling meta, out_queue.size()==>%d", node_name.c_str(), out_queue.size())); + this->out_queue.push(i); + + // handled hooker activated if need + invoke_meta_handled_hooker(node_name, out_queue.size(), i); + + // notify consumer of out_queue + this->out_queue_semaphore.signal(); + VP_DEBUG(vp_utils::string_format("[%s] after handling meta, out_queue.size()==>%d", node_name.c_str(), out_queue.size())); + } + // clean cache for the next batch + frame_meta_batch_cache.clear(); + } + } + // send dead flag for dispatch_thread + this->out_queue.push(nullptr); + this->out_queue_semaphore.signal(); + } + + // there is only one thread poping from the out_queue, we don't use lock here when poping. + void vp_node::dispatch_run() { + while (alive) { + // wait for producer, make sure out_queue is not empty. + this->out_queue_semaphore.wait(); + + VP_DEBUG(vp_utils::string_format("[%s] before dispatching meta, out_queue.size()==>%d", node_name.c_str(), out_queue.size())); + auto out_meta = this->out_queue.front(); + // dead flag + if (out_meta == nullptr) { + continue; + } + + // leaving hooker activated if need + invoke_meta_leaving_hooker(node_name, out_queue.size(), out_meta); + + // do something.. + this->push_meta(out_meta); + this->out_queue.pop(); + VP_DEBUG(vp_utils::string_format("[%s] after dispatching meta, out_queue.size()==>%d", node_name.c_str(), out_queue.size())); + } + } + + std::shared_ptr vp_node::handle_frame_meta(std::shared_ptr meta) { + return meta; + } + + std::shared_ptr vp_node::handle_control_meta(std::shared_ptr meta) { + return meta; + } + + void vp_node::handle_frame_meta(const std::vector>& meta_with_batch) { + + } + + void vp_node::meta_flow(std::shared_ptr meta) { + if (meta == nullptr) { + return; + } + + std::lock_guard guard(this->in_queue_lock); + VP_DEBUG(vp_utils::string_format("[%s] before meta flow, in_queue.size()==>%d", node_name.c_str(), in_queue.size())); + this->in_queue.push(meta); + + // arriving hooker activated if need + invoke_meta_arriving_hooker(node_name, in_queue.size(), meta); + + // notify consumer of in_queue + this->in_queue_semaphore.signal(); + VP_DEBUG(vp_utils::string_format("[%s] after meta flow, in_queue.size()==>%d", node_name.c_str(), in_queue.size())); + } + + void vp_node::detach() { + for(auto i : this->pre_nodes) { + i->remove_subscriber(shared_from_this()); + } + this->pre_nodes.clear(); + } + + void vp_node::detach_from(std::vector pre_node_names) { + for (auto i = this->pre_nodes.begin(); i != this->pre_nodes.end();) { + if (std::find(pre_node_names.begin(), pre_node_names.end(), (*i)->node_name) != pre_node_names.end()) { + (*i)->remove_subscriber(shared_from_this()); + i = this->pre_nodes.erase(i); + } + else { + i++; + } + } + } + + void vp_node::detach_recursively() { + detach(); + auto nodes = next_nodes(); + for (auto& n: nodes) { + n->detach_recursively(); + } + } + + void vp_node::attach_to(std::vector> pre_nodes) { + // can not attach src node to any previous nodes + if (this->node_type() == vp_node_type::SRC) { + throw vp_excepts::vp_invalid_calling_error("SRC nodes must not have any previous nodes!"); + } + // can not attach any nodes to des node + for(auto i : pre_nodes) { + if (i->node_type() == vp_node_type::DES) { + throw vp_excepts::vp_invalid_calling_error("DES nodes must not have any next nodes!"); + } + i->add_subscriber(shared_from_this()); + this->pre_nodes.push_back(i); + } + } + + void vp_node::initialized() { + // start threads since all resources have been initialized + this->handle_thread = std::thread(&vp_node::handle_run, this); + this->dispatch_thread = std::thread(&vp_node::dispatch_run, this); + } + + void vp_node::deinitialized() { + // send dead flag + alive = false; + { + std::lock_guard guard(this->in_queue_lock); + this->in_queue.push(nullptr); + this->in_queue_semaphore.signal(); + } + // wait for threads exits in vp_node + if (handle_thread.joinable()) { + handle_thread.join(); + } + if (dispatch_thread.joinable()) { + dispatch_thread.join(); + } + } + + vp_node_type vp_node::node_type() { + // return vp_node_type::MID by default + // need override in child class + return vp_node_type::MID; + } + + std::vector> vp_node::next_nodes() { + std::vector> next_nodes; + std::lock_guard guard(this->subscribers_lock); + for(auto & i: this->subscribers) { + next_nodes.push_back(std::dynamic_pointer_cast(i)); + } + return next_nodes; + } + + std::string vp_node::to_string() { + // return node_name by default + return node_name; + } + + void vp_node::pendding_meta(std::shared_ptr meta) { + this->out_queue.push(meta); + // handled hooker activated if need + invoke_meta_handled_hooker(node_name, out_queue.size(), meta); + // notify consumer of out_queue + this->out_queue_semaphore.signal(); + } +} \ No newline at end of file diff --git a/nodes/vp_node.h b/nodes/vp_node.h new file mode 100755 index 0000000..daca17c --- /dev/null +++ b/nodes/vp_node.h @@ -0,0 +1,120 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "../utils/vp_semaphore.h" +#include "../utils/vp_utils.h" +#include "../utils/logger/vp_logger.h" +#include "vp_meta_publisher.h" +#include "vp_meta_hookable.h" +#include "../objects/vp_control_meta.h" +#include "../objects/vp_frame_meta.h" +#include "../excepts/vp_invalid_calling_error.h" + +namespace vp_nodes { + // node type + enum vp_node_type { + SRC, // src node, must not have input branchs + DES, // des node, must not have output branchs + MID // middle node, can have input and output branchs + }; + + // base class for all nodes + class vp_node: public vp_meta_publisher, + public vp_meta_subscriber, + public vp_meta_hookable, + public std::enable_shared_from_this { + private: + // previous nodes + std::vector> pre_nodes; + + // handle thread + std::thread handle_thread; + // dispatch thread + std::thread dispatch_thread; + + protected: + // alive or not for node + bool alive = true; + + // by default we handle frame meta one by one, in some situations we need handle them batch by batch(such as vp_infer_node). + // setting this member greater than 1 means the node will handle frame meta with batch, and vp_node::handle_frame_meta_by_batch(...) will be called other than vp_node::handle_frame_meta(...). + // note: control meta is not allowed like above, only one by one supported. + int frame_meta_handle_batch = 1; + + // cache input meta from previous nodes + std::queue> in_queue; + // + std::mutex in_queue_lock; + // cache output meta to next nodes + std::queue> out_queue; + + // synchronize for in_queue + vp_utils::vp_semaphore in_queue_semaphore; + // synchronize for out_queue + vp_utils::vp_semaphore out_queue_semaphore; + + // get meta from in_queue, handle meta and put them into out_queue looply. + // we need re-implement(define how to create meta and put it into out_queue) in src nodes since they have no previous nodes. + virtual void handle_run(); + // get meta from out_queue and push them to next nodes looply. + // we need re-implement(just do nothing) in des nodes since they have no next nodes. + virtual void dispatch_run(); + + // define how to handle frame meta [one by one], ignored in src nodes. + // return nullptr means do not push it to next nodes, such as des nodes which have no next nodes. + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta); + // define how to handle control meta, ignored in src nodes. + // return nullptr means do not push it to next nodes, such as des nodes which have no next nodes. + virtual std::shared_ptr handle_control_meta(std::shared_ptr meta); + + // define how to handle frame meta [batch by batch], ignored in src nodes. + virtual void handle_frame_meta(const std::vector>& meta_with_batch); + + // called by child classes after all resources have been initialized (in the last constructor of chain). + void initialized(); + // called by child classes before all resources going to be destroyed (in the last destructor of chain)。 + virtual void deinitialized(); + + // push meta to the back of out_queue, then it will be pushed to next nodes in order. + // take care it's different from vp_node::push_meta(meta) which will push meta to next nodes directly. + // the method can be called ONLY in handle thread inside node. + void pendding_meta(std::shared_ptr meta); + + // protected as it can't be instanstiated directly. + vp_node(std::string node_name); + public: + virtual ~vp_node(); + // clear meaningful string, such as 'file_src_0' stands for file source node at channel 0. + std::string node_name; + + // receive meta from previous nodes, + // we can hook(such as modifying meta) in child class but do not forget calling vp_node.meta_flow(...) followly. + virtual void meta_flow(std::shared_ptr meta) override; + + // retrive current node type + virtual vp_node_type node_type(); + + // detach myself from all previous nodes + void detach(); + // detach myself from specific previous nodes + void detach_from(std::vector pre_node_names); + // detach myself from all previous nodes AND the same action on all next nodes(recursively), can be used to split the whole pipeline into single nodes before process exits. + void detach_recursively(); + // attach myself to previous nodes(can be a list) + void attach_to(std::vector> pre_nodes); + + // get next nodes + std::vector> next_nodes(); + + // get description of node + virtual std::string to_string(); + }; + +} \ No newline at end of file diff --git a/nodes/vp_placeholder_node.cpp b/nodes/vp_placeholder_node.cpp new file mode 100644 index 0000000..435f1e4 --- /dev/null +++ b/nodes/vp_placeholder_node.cpp @@ -0,0 +1,13 @@ + +#include "vp_placeholder_node.h" + +namespace vp_nodes { + + vp_placeholder_node::vp_placeholder_node(std::string node_name): vp_node(node_name) { + this->initialized(); + } + + vp_placeholder_node::~vp_placeholder_node() { + deinitialized(); + } +} \ No newline at end of file diff --git a/nodes/vp_placeholder_node.h b/nodes/vp_placeholder_node.h new file mode 100644 index 0000000..5e47ebb --- /dev/null +++ b/nodes/vp_placeholder_node.h @@ -0,0 +1,16 @@ + +#pragma once + +#include "vp_node.h" + +namespace vp_nodes { + // placeholder node, do nothing just a placeholder in the middle of pipeline + class vp_placeholder_node: public vp_node { + private: + /* data */ + public: + vp_placeholder_node(std::string node_name); + ~vp_placeholder_node(); + }; + +} \ No newline at end of file diff --git a/nodes/vp_primary_infer_node.cpp b/nodes/vp_primary_infer_node.cpp new file mode 100755 index 0000000..cc94043 --- /dev/null +++ b/nodes/vp_primary_infer_node.cpp @@ -0,0 +1,45 @@ + +#include "vp_primary_infer_node.h" + +namespace vp_nodes { + + vp_primary_infer_node::vp_primary_infer_node(std::string node_name, + std::string model_path, + std::string model_config_path, + std::string labels_path, + int input_width, + int input_height, + int batch_size, + int class_id_offset, + float scale, + cv::Scalar mean, + cv::Scalar std, + bool swap_rb, + bool swap_chn): + vp_infer_node(node_name, + vp_infer_type::PRIMARY, + model_path, + model_config_path, + labels_path, + input_width, + input_height, + batch_size, + scale, + mean, + std, + swap_rb, + swap_chn), + class_id_offset(class_id_offset) { + } + + vp_primary_infer_node::~vp_primary_infer_node() { + + } + + void vp_primary_infer_node::prepare(const std::vector>& frame_meta_with_batch, std::vector& mats_to_infer) { + // fetch the whole frame, can batch by batch + for (auto& i: frame_meta_with_batch) { + mats_to_infer.push_back(i->frame); + } + } +} \ No newline at end of file diff --git a/nodes/vp_primary_infer_node.h b/nodes/vp_primary_infer_node.h new file mode 100755 index 0000000..7e0409c --- /dev/null +++ b/nodes/vp_primary_infer_node.h @@ -0,0 +1,35 @@ + + +#pragma once + +#include "vp_infer_node.h" + +namespace vp_nodes { + // primary infer node, it is the base class of infer node which MUST infer on the whole frame. + class vp_primary_infer_node: public vp_infer_node { + private: + protected: + // define how to prepare data + virtual void prepare(const std::vector>& frame_meta_with_batch, std::vector& mats_to_infer) override; + public: + vp_primary_infer_node(std::string node_name, + std::string model_path, + std::string model_config_path = "", + std::string labels_path = "", + int input_width = 640, + int input_height = 640, + int batch_size = 1, + int class_id_offset = 0, + float scale = 1.0, + cv::Scalar mean = cv::Scalar(123.675, 116.28, 103.53), // imagenet dataset + cv::Scalar std = cv::Scalar(1), + bool swap_rb = true, + bool swap_chn = false); + ~vp_primary_infer_node(); + + // because maybe multi primary infer nodes exist in one pipeline, + // we need an offset value to ensure each class id is unique and has not conflict. + // the class_id_offset should be the sum of class ids in previous vp_primary_infer_node of pipe. + int class_id_offset; + }; +} \ No newline at end of file diff --git a/nodes/vp_rtmp_des_node.cpp b/nodes/vp_rtmp_des_node.cpp new file mode 100755 index 0000000..acffb75 --- /dev/null +++ b/nodes/vp_rtmp_des_node.cpp @@ -0,0 +1,68 @@ + +#include +#include "vp_rtmp_des_node.h" +#include "../utils/vp_utils.h" + +namespace vp_nodes { + + vp_rtmp_des_node::vp_rtmp_des_node(std::string node_name, + int channel_index, + std::string rtmp_url, + vp_objects::vp_size resolution_w_h, + int bitrate, + bool osd, + std::string gst_encoder_name): + vp_des_node(node_name, channel_index), + rtmp_url(rtmp_url), + resolution_w_h(resolution_w_h), + bitrate(bitrate), + osd(osd), + gst_encoder_name(gst_encoder_name) { + // append channel_index to the end of rtmp_url. + // if original rtmp_url is rtmp://192.168.77.105/live/10000 and channel_index is 10 + // then the modified rtmp_url is rtmp://192.168.77.105/live/10000_10 + this->rtmp_url = this->rtmp_url + "_" + std::to_string(channel_index); + this->gst_template = vp_utils::string_format(this->gst_template, gst_encoder_name.c_str(), bitrate, this->rtmp_url.c_str()); + VP_INFO(vp_utils::string_format("[%s] [%s]", node_name.c_str(), gst_template.c_str())); + this->initialized(); + } + + vp_rtmp_des_node::~vp_rtmp_des_node() { + deinitialized(); + } + + // re-implementation, return nullptr. + std::shared_ptr + vp_rtmp_des_node::handle_frame_meta(std::shared_ptr meta) { + VP_DEBUG(vp_utils::string_format("[%s] received frame meta, channel_index=>%d, frame_index=>%d", node_name.c_str(), meta->channel_index, meta->frame_index)); + + cv::Mat resize_frame; + if (this->resolution_w_h.width != 0 && this->resolution_w_h.height != 0) { + cv::resize((osd && !meta->osd_frame.empty()) ? meta->osd_frame : meta->frame, resize_frame, cv::Size(resolution_w_h.width, resolution_w_h.height)); + } + else { + resize_frame = (osd && !meta->osd_frame.empty()) ? meta->osd_frame : meta->frame; + } + + if (!rtmp_writer.isOpened()) { + assert(rtmp_writer.open(this->gst_template, cv::CAP_GSTREAMER, 0, meta->fps, {resize_frame.cols, resize_frame.rows})); + } + + rtmp_writer.write(resize_frame); + + // for general works defined in base class + return vp_des_node::handle_frame_meta(meta); + } + + // re-implementation, return nullptr. + std::shared_ptr + vp_rtmp_des_node::handle_control_meta(std::shared_ptr meta) { + // for general works defined in base class + return vp_des_node::handle_control_meta(meta); + } + + std::string vp_rtmp_des_node::to_string() { + // just return rtmp url + return rtmp_url; + } +} \ No newline at end of file diff --git a/nodes/vp_rtmp_des_node.h b/nodes/vp_rtmp_des_node.h new file mode 100755 index 0000000..932a571 --- /dev/null +++ b/nodes/vp_rtmp_des_node.h @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include +#include "vp_des_node.h" + +namespace vp_nodes { + // rtmp des node, push video stream via rtmp protocal. + // example: + // rtmp://192.168.77.105/live/10000 + class vp_rtmp_des_node: public vp_des_node + { + private: + /* data */ + std::string gst_template = "appsrc ! videoconvert ! %s bitrate=%d ! h264parse ! flvmux ! rtmpsink location=%s"; + cv::VideoWriter rtmp_writer; + protected: + // re-implementation, return nullptr. + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + // re-implementation, return nullptr. + virtual std::shared_ptr handle_control_meta(std::shared_ptr meta) override; + public: + vp_rtmp_des_node(std::string node_name, + int channel_index, + std::string rtmp_url, + vp_objects::vp_size resolution_w_h = {}, + int bitrate = 1024, + bool osd = true, + std::string gst_encoder_name = "x264enc"); + ~vp_rtmp_des_node(); + + virtual std::string to_string() override; + + std::string rtmp_url; + // resolution for rtmp stream + vp_objects::vp_size resolution_w_h; + int bitrate; + + // for osd frame + bool osd; + // set x264enc as the default encoder, we can use hardware encoder instead. + std::string gst_encoder_name = "x264enc"; + }; +} \ No newline at end of file diff --git a/nodes/vp_rtmp_src_node.cpp b/nodes/vp_rtmp_src_node.cpp new file mode 100755 index 0000000..7d8351e --- /dev/null +++ b/nodes/vp_rtmp_src_node.cpp @@ -0,0 +1,121 @@ + + +#include +#include +#include + +#include "vp_rtmp_src_node.h" +#include "../utils/vp_utils.h" + +namespace vp_nodes { + + vp_rtmp_src_node::vp_rtmp_src_node(std::string node_name, + int channel_index, + std::string rtmp_url, + float resize_ratio, + std::string gst_decoder_name, + int skip_interval): + vp_src_node(node_name, channel_index, resize_ratio), + rtmp_url(rtmp_url), gst_decoder_name(gst_decoder_name), skip_interval(skip_interval) { + assert(skip_interval >= 0 && skip_interval <= 9); + this->gst_template = vp_utils::string_format(this->gst_template, rtmp_url.c_str(), gst_decoder_name.c_str()); + VP_INFO(vp_utils::string_format("[%s] [%s]", node_name.c_str(), gst_template.c_str())); + this->initialized(); + } + + vp_rtmp_src_node::~vp_rtmp_src_node() { + deinitialized(); + } + + // define how to read video from rtmp stream, create frame meta etc. + // please refer to the implementation of vp_node::handle_run. + void vp_rtmp_src_node::handle_run() { + cv::Mat frame; + int video_width = 0; + int video_height = 0; + int fps = 0; + int skip = 0; + while(alive) { + // check if need work + gate.knock(); + + // try to open capture + if (!rtmp_capture.isOpened()) { + video_width = video_height = fps = 0; + original_width = original_height = original_fps = 0; + if (!rtmp_capture.open(this->gst_template, cv::CAP_GSTREAMER)) { + VP_WARN(vp_utils::string_format("[%s] open rtmp failed, try again...", node_name.c_str())); + continue; + } + } + + // video properties + if (video_width == 0 || video_height == 0 || fps == 0) { + video_width = rtmp_capture.get(cv::CAP_PROP_FRAME_WIDTH); + video_height = rtmp_capture.get(cv::CAP_PROP_FRAME_HEIGHT); + fps = rtmp_capture.get(cv::CAP_PROP_FPS); + + original_fps = fps; + original_width = video_width; + original_height = video_height; + + // set true fps because skip some frames + fps = fps / (skip_interval + 1); + } + // stream_info_hooker activated if need + vp_stream_info stream_info {channel_index, original_fps, original_width, original_height, to_string()}; + invoke_stream_info_hooker(node_name, stream_info); + + rtmp_capture >> frame; + if(frame.empty()) { + VP_WARN(vp_utils::string_format("[%s] reading frame empty, total frame==>%d", node_name.c_str(), frame_index)); + continue; + } + + // need skip + if (skip < skip_interval) { + skip++; + continue; + } + skip = 0; + + cv::Mat resize_frame; + if (this->resize_ratio != 1.0f) { + cv::resize(frame, resize_frame, cv::Size(), resize_ratio, resize_ratio); + } + else { + resize_frame = frame.clone(); // clone!; + } + // set true size because resize + video_width = resize_frame.cols; + video_height = resize_frame.rows; + + this->frame_index++; + // create frame meta + auto out_meta = + std::make_shared(resize_frame, this->frame_index, this->channel_index, video_width, video_height, fps); + + if (out_meta != nullptr) { + this->out_queue.push(out_meta); + + // handled hooker activated if need + if (this->meta_handled_hooker) { + meta_handled_hooker(node_name, out_queue.size(), out_meta); + } + + // important! notify consumer of out_queue in case it is waiting. + this->out_queue_semaphore.signal(); + VP_DEBUG(vp_utils::string_format("[%s] after handling meta, out_queue.size()==>%d", node_name.c_str(), out_queue.size())); + } + } + + // send dead flag for dispatch_thread + this->out_queue.push(nullptr); + this->out_queue_semaphore.signal(); + } + + // return stream url + std::string vp_rtmp_src_node::to_string() { + return rtmp_url; + } +} \ No newline at end of file diff --git a/nodes/vp_rtmp_src_node.h b/nodes/vp_rtmp_src_node.h new file mode 100755 index 0000000..2048afd --- /dev/null +++ b/nodes/vp_rtmp_src_node.h @@ -0,0 +1,36 @@ +#pragma once + +#include + +#include "vp_src_node.h" + +namespace vp_nodes { + // rtmp source node, receive live video stream via rtmp protocal. + // example: + // rtmp://your-rtmp-server/live/streamname + class vp_rtmp_src_node: public vp_src_node { + private: + /* data */ + std::string gst_template = "rtmpsrc location=%s ! flvdemux ! h264parse ! %s ! videoconvert ! appsink"; + cv::VideoCapture rtmp_capture; + protected: + // re-implemetation + virtual void handle_run() override; + public: + vp_rtmp_src_node(std::string node_name, + int channel_index, + std::string rtmp_url, + float resize_ratio = 1.0, + std::string gst_decoder_name = "avdec_h264", + int skip_interval = 0); + ~vp_rtmp_src_node(); + + virtual std::string to_string() override; + + std::string rtmp_url; + // set avdec_h264 as the default decoder, we can use hardware decoder instead. + std::string gst_decoder_name = "avdec_h264"; + // 0 means no skip + int skip_interval = 0; + }; +} \ No newline at end of file diff --git a/nodes/vp_rtsp_des_node.cpp b/nodes/vp_rtsp_des_node.cpp new file mode 100644 index 0000000..4a1e164 --- /dev/null +++ b/nodes/vp_rtsp_des_node.cpp @@ -0,0 +1,95 @@ + + +#include "vp_rtsp_des_node.h" + + +namespace vp_nodes { + + vp_rtsp_des_node::vp_rtsp_des_node(std::string node_name, + int channel_index, + int rtsp_port, + std::string rtsp_name, + vp_objects::vp_size resolution_w_h, + int bitrate, + bool osd, + std::string gst_encoder_name): + vp_des_node(node_name, channel_index), + rtsp_port(rtsp_port), + rtsp_name(rtsp_name), + resolution_w_h(resolution_w_h), + bitrate(bitrate), + osd(osd), + gst_encoder_name(gst_encoder_name) { + if (rtsp_name.empty()) { + // use channel index as rtsp name + this->rtsp_name = std::to_string(channel_index); + } + gst_template = vp_utils::string_format(gst_template, gst_encoder_name.c_str(), bitrate, base_udp_port + channel_index); + VP_INFO(vp_utils::string_format("[%s] [%s]", node_name.c_str(), gst_template.c_str())); + + // start rtsp server asynchronously + start_rtsp_streaming(); + + this->initialized(); + } + + vp_rtsp_des_node::~vp_rtsp_des_node() { + deinitialized(); + } + GstRTSPServer* vp_rtsp_des_node::rtsp_server = NULL; + std::shared_ptr vp_rtsp_des_node::handle_frame_meta(std::shared_ptr meta) { + VP_DEBUG(vp_utils::string_format("[%s] received frame meta, channel_index=>%d, frame_index=>%d", node_name.c_str(), meta->channel_index, meta->frame_index)); + + cv::Mat resize_frame; + if (this->resolution_w_h.width != 0 && this->resolution_w_h.height != 0) { + cv::resize((osd && !meta->osd_frame.empty()) ? meta->osd_frame : meta->frame, resize_frame, cv::Size(resolution_w_h.width, resolution_w_h.height)); + } + else { + resize_frame = (osd && !meta->osd_frame.empty()) ? meta->osd_frame : meta->frame; + } + + if (!rtsp_writer.isOpened()) { + assert(rtsp_writer.open(this->gst_template, cv::CAP_GSTREAMER, 0, meta->fps, {resize_frame.cols, resize_frame.rows})); + } + + rtsp_writer.write(resize_frame); + + // for general works defined in base class + return vp_des_node::handle_frame_meta(meta); + } + + void vp_rtsp_des_node::start_rtsp_streaming () { + // create a rtsp server using gst-rtsp-server + if (rtsp_server == NULL) { + char port_num_Str[64] = { 0 }; + sprintf (port_num_Str, "%d", rtsp_port); + rtsp_server = gst_rtsp_server_new (); + g_object_set (rtsp_server, "service", port_num_Str, NULL); + gst_rtsp_server_attach (rtsp_server, NULL); + } + + char udpsrc_pipeline[512]; + if (udp_buffer_size == 0) + udp_buffer_size = 512 * 1024; + + // receive stream data from udpsrc internally and push it via rtsp + sprintf (udpsrc_pipeline, + "( udpsrc name=pay0 port=%d buffer-size=%lu caps=\"application/x-rtp, media=video, " + "clock-rate=90000, encoding-name=H264, payload=96 \" )", + base_udp_port + channel_index, udp_buffer_size); + + auto mounts = gst_rtsp_server_get_mount_points (rtsp_server); + + auto factory = gst_rtsp_media_factory_new (); + gst_rtsp_media_factory_set_launch (factory, udpsrc_pipeline); + gst_rtsp_media_factory_set_shared (factory, TRUE); + gst_rtsp_mount_points_add_factory (mounts, ("/" + rtsp_name).c_str(), factory); + g_object_unref (mounts); + + VP_INFO(vp_utils::string_format("[%s] is going to push rtsp stream, please visit:[%s]", node_name.c_str(), to_string().c_str())); + } + + std::string vp_rtsp_des_node::to_string() { + return "rtsp://localhost:" + std::to_string(rtsp_port) + "/" + rtsp_name; + } +} \ No newline at end of file diff --git a/nodes/vp_rtsp_des_node.h b/nodes/vp_rtsp_des_node.h new file mode 100644 index 0000000..7c3e6a7 --- /dev/null +++ b/nodes/vp_rtsp_des_node.h @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include "vp_des_node.h" + +/* +* ##### Compile Tips ##### +* install additional packages for this node before compiling: +* sudo apt-get install libgstrtspserver-1.0-dev gstreamer1.0-rtsp +* https://github.com/GStreamer/gst-rtsp-server +*/ + +namespace vp_nodes { + // rtsp des node, push video stream via rtsp. + // example: rtsp://localhost:7890/test + // note, no specialized rtsp server needed since this node itself is a rtsp server, which can be pulled directly by video players such as VLC. + class vp_rtsp_des_node: public vp_des_node { + private: + /* data */ + std::string gst_template = "appsrc ! videoconvert ! %s bitrate=%d ! h264parse ! rtph264pay ! udpsink host=localhost port=%d"; + cv::VideoWriter rtsp_writer; + + // start rtsp server + void start_rtsp_streaming (); + + // resolution for rtsp stream + vp_objects::vp_size resolution_w_h; + int bitrate; + // for osd frame + bool osd; + + int udp_buffer_size = 0; + // base udp port for stream data exchange internally, base_udp_port + channel_index would be used for each channel, MUST not occupied by others. + const int base_udp_port = 7890; + + int rtsp_port = 9000; + std::string rtsp_name = ""; + + /* gst-rtsp-server variables shared between instances for different channels */ + static GstRTSPServer* rtsp_server; + + // set x264enc as the default encoder, we can use hardware encoder instead. + std::string gst_encoder_name = "x264enc"; + protected: + // re-implementation, return nullptr. + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + public: + vp_rtsp_des_node(std::string node_name, + int channel_index, + int rtsp_port = 9000, + std::string rtsp_name = "", + vp_objects::vp_size resolution_w_h = {}, + int bitrate = 512, + bool osd = true, + std::string gst_encoder_name = "x264enc"); + ~vp_rtsp_des_node(); + + virtual std::string to_string() override; + }; +} \ No newline at end of file diff --git a/nodes/vp_rtsp_src_node.cpp b/nodes/vp_rtsp_src_node.cpp new file mode 100755 index 0000000..68c6bc0 --- /dev/null +++ b/nodes/vp_rtsp_src_node.cpp @@ -0,0 +1,121 @@ + + +#include +#include +#include + +#include "vp_rtsp_src_node.h" +#include "../utils/vp_utils.h" + +namespace vp_nodes { + + vp_rtsp_src_node::vp_rtsp_src_node(std::string node_name, + int channel_index, + std::string rtsp_url, + float resize_ratio, + std::string gst_decoder_name, + int skip_interval): + vp_src_node(node_name, channel_index, resize_ratio), + rtsp_url(rtsp_url), gst_decoder_name(gst_decoder_name), skip_interval(skip_interval) { + assert(skip_interval >= 0 && skip_interval <= 9); + this->gst_template = vp_utils::string_format(this->gst_template, rtsp_url.c_str(), gst_decoder_name.c_str()); + VP_INFO(vp_utils::string_format("[%s] [%s]", node_name.c_str(), gst_template.c_str())); + this->initialized(); + } + + vp_rtsp_src_node::~vp_rtsp_src_node() { + deinitialized(); + } + + // define how to read video from rtsp stream, create frame meta etc. + // please refer to the implementation of vp_node::handle_run. + void vp_rtsp_src_node::handle_run() { + cv::Mat frame; + int video_width = 0; + int video_height = 0; + int fps = 0; + int skip = 0; + while(alive) { + // check if need work + gate.knock(); + + // try to open capture + if (!rtsp_capture.isOpened()) { + video_width = video_height = fps = 0; + original_width = original_height = original_fps = 0; + if (!rtsp_capture.open(this->gst_template, cv::CAP_GSTREAMER)) { + VP_WARN(vp_utils::string_format("[%s] open rtsp failed, try again...", node_name.c_str())); + continue; + } + } + + // video properties + if (video_width == 0 || video_height == 0 || fps == 0) { + video_width = rtsp_capture.get(cv::CAP_PROP_FRAME_WIDTH); + video_height = rtsp_capture.get(cv::CAP_PROP_FRAME_HEIGHT); + fps = rtsp_capture.get(cv::CAP_PROP_FPS); + + original_fps = fps; + original_width = video_width; + original_height = video_height; + + // set true fps because skip some frames + fps = fps / (skip_interval + 1); + } + // stream_info_hooker activated if need + vp_stream_info stream_info {channel_index, original_fps, original_width, original_height, to_string()}; + invoke_stream_info_hooker(node_name, stream_info); + + rtsp_capture >> frame; + if(frame.empty()) { + VP_WARN(vp_utils::string_format("[%s] reading frame empty, total frame==>%d", node_name.c_str(), frame_index)); + continue; + } + + // need skip + if (skip < skip_interval) { + skip++; + continue; + } + skip = 0; + + cv::Mat resize_frame; + if (this->resize_ratio != 1.0f) { + cv::resize(frame, resize_frame, cv::Size(), resize_ratio, resize_ratio); + } + else { + resize_frame = frame.clone(); // clone!; + } + // set true size because resize + video_width = resize_frame.cols; + video_height = resize_frame.rows; + + this->frame_index++; + // create frame meta + auto out_meta = + std::make_shared(resize_frame, this->frame_index, this->channel_index, video_width, video_height, fps); + + if (out_meta != nullptr) { + this->out_queue.push(out_meta); + + // handled hooker activated if need + if (this->meta_handled_hooker) { + meta_handled_hooker(node_name, out_queue.size(), out_meta); + } + + // important! notify consumer of out_queue in case it is waiting. + this->out_queue_semaphore.signal(); + VP_DEBUG(vp_utils::string_format("[%s] after handling meta, out_queue.size()==>%d", node_name.c_str(), out_queue.size())); + } + } + + // send dead flag for dispatch_thread + this->out_queue.push(nullptr); + this->out_queue_semaphore.signal(); + } + + // return stream url + std::string vp_rtsp_src_node::to_string() { + return rtsp_url; + } +} \ No newline at end of file diff --git a/nodes/vp_rtsp_src_node.h b/nodes/vp_rtsp_src_node.h new file mode 100755 index 0000000..91bd05f --- /dev/null +++ b/nodes/vp_rtsp_src_node.h @@ -0,0 +1,36 @@ +#pragma once + +#include + +#include "vp_src_node.h" + +namespace vp_nodes { + // rtsp source node, receive video stream via rtsp protocal. + // example: + // rtsp://admin:admin12345@192.168.77.110:554/ + class vp_rtsp_src_node: public vp_src_node { + private: + /* data */ + std::string gst_template = "rtspsrc location=%s ! application/x-rtp,media=video ! rtph264depay ! h264parse ! %s ! videoconvert ! appsink"; + cv::VideoCapture rtsp_capture; + protected: + // re-implemetation + virtual void handle_run() override; + public: + vp_rtsp_src_node(std::string node_name, + int channel_index, + std::string rtsp_url, + float resize_ratio = 1.0, + std::string gst_decoder_name = "avdec_h264", + int skip_interval = 0); + ~vp_rtsp_src_node(); + + virtual std::string to_string() override; + + std::string rtsp_url; + // set avdec_h264 as the default decoder, we can use hardware decoder instead. + std::string gst_decoder_name = "avdec_h264"; + // 0 means no skip + int skip_interval = 0; + }; +} \ No newline at end of file diff --git a/nodes/vp_screen_des_node.cpp b/nodes/vp_screen_des_node.cpp new file mode 100755 index 0000000..14c1ee2 --- /dev/null +++ b/nodes/vp_screen_des_node.cpp @@ -0,0 +1,50 @@ + +#include "vp_screen_des_node.h" +#include "../utils/vp_utils.h" + +namespace vp_nodes { + vp_screen_des_node::vp_screen_des_node(std::string node_name, + int channel_index, + bool osd, + vp_objects::vp_size display_w_h): + vp_des_node(node_name, channel_index), + osd(osd), + display_w_h(display_w_h) { + this->gst_template = vp_utils::string_format(this->gst_template, node_name.c_str()); + VP_INFO(vp_utils::string_format("[%s] [%s]", node_name.c_str(), gst_template.c_str())); + this->initialized(); + } + + vp_screen_des_node::~vp_screen_des_node() { + deinitialized(); + } + + // re-implementation, return nullptr. + std::shared_ptr + vp_screen_des_node::handle_frame_meta(std::shared_ptr meta) { + VP_DEBUG(vp_utils::string_format("[%s] received frame meta, channel_index=>%d, frame_index=>%d", node_name.c_str(), meta->channel_index, meta->frame_index)); + + cv::Mat resize_frame; + if (this->display_w_h.width != 0 && this->display_w_h.height != 0) { + cv::resize((osd && !meta->osd_frame.empty()) ? meta->osd_frame : meta->frame, resize_frame, cv::Size(display_w_h.width, display_w_h.height)); + } + else { + resize_frame = (osd && !meta->osd_frame.empty()) ? meta->osd_frame : meta->frame; + } + + if (!screen_writer.isOpened()) { + assert(screen_writer.open(this->gst_template, cv::CAP_GSTREAMER, 0, meta->fps, {resize_frame.cols, resize_frame.rows})); + } + screen_writer.write(resize_frame); + + // for general works defined in base class + return vp_des_node::handle_frame_meta(meta); + } + + // re-implementation, return nullptr. + std::shared_ptr + vp_screen_des_node::handle_control_meta(std::shared_ptr meta) { + // for general works defined in base class + return vp_des_node::handle_control_meta(meta); + } +} \ No newline at end of file diff --git a/nodes/vp_screen_des_node.h b/nodes/vp_screen_des_node.h new file mode 100755 index 0000000..ef71372 --- /dev/null +++ b/nodes/vp_screen_des_node.h @@ -0,0 +1,35 @@ + + +#pragma once + +#include +#include +#include +#include "vp_des_node.h" + +namespace vp_nodes { + // screen des node, display video on local window. + class vp_screen_des_node: public vp_des_node + { + private: + /* data */ + std::string gst_template = "appsrc ! videoconvert ! videoscale ! textoverlay text=%s halignment=left valignment=top font-desc='Sans,16' shaded-background=true ! timeoverlay halignment=right valignment=top font-desc='Sans,16' shaded-background=true ! queue ! fpsdisplaysink video-sink=ximagesink sync=false"; + cv::VideoWriter screen_writer; + protected: + // re-implementation, return nullptr. + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + // re-implementation, return nullptr. + virtual std::shared_ptr handle_control_meta(std::shared_ptr meta) override; + public: + vp_screen_des_node(std::string node_name, + int channel_index, + bool osd = true, + vp_objects::vp_size display_w_h = {}); + ~vp_screen_des_node(); + + // for osd frame + bool osd; + // display size + vp_objects::vp_size display_w_h; + }; +} \ No newline at end of file diff --git a/nodes/vp_secondary_infer_node.cpp b/nodes/vp_secondary_infer_node.cpp new file mode 100755 index 0000000..adaf68b --- /dev/null +++ b/nodes/vp_secondary_infer_node.cpp @@ -0,0 +1,85 @@ + +#include "vp_secondary_infer_node.h" + +namespace vp_nodes { + + vp_secondary_infer_node::vp_secondary_infer_node(std::string node_name, + std::string model_path, + std::string model_config_path, + std::string labels_path, + int input_width, + int input_height, + int batch_size, + std::vector p_class_ids_applied_to, + int min_width_applied_to, + int min_height_applied_to, + int crop_padding, + float scale, + cv::Scalar mean, + cv::Scalar std, + bool swap_rb, + bool swap_chn): + vp_infer_node(node_name, + vp_nodes::vp_infer_type::SECONDARY, + model_path, + model_config_path, + labels_path, + input_width, + input_height, + batch_size, + scale, + mean, + std, + swap_rb, + swap_chn), + p_class_ids_applied_to(p_class_ids_applied_to), + min_width_applied_to(min_width_applied_to), + min_height_applied_to(min_height_applied_to), + crop_padding(crop_padding) { + } + + vp_secondary_infer_node::~vp_secondary_infer_node() { + } + + void vp_secondary_infer_node::prepare(const std::vector>& frame_meta_with_batch, std::vector& mats_to_infer) { + // only one by one for secondary infer node + assert(frame_meta_with_batch.size() == 1); + + // crop to get small images in frame + auto& frame_meta = frame_meta_with_batch[0]; + + // batch by batch inside single frame + for (auto& i : frame_meta->targets) { + // check if we need infer on the target + if (!need_apply(i->primary_class_id, i->width, i->height)) { + continue; + } + + // simulate croping operations, no data copyed here + auto box = cv::Rect(i->x, i->y, i->width, i->height); + + // add a padding when crop, check value range + if (crop_padding != 0) { + box = cv::Rect(box.x - crop_padding, box.y - crop_padding, box.width + crop_padding * 2, box.height + crop_padding * 2); + box.x = std::max(box.x, 0); + box.y = std::max(box.y, 0); + box.width = std::min(box.width, frame_meta->frame.cols - box.x); + box.height = std::min(box.height, frame_meta->frame.rows - box.y); + } + + mats_to_infer.push_back(frame_meta->frame(box)); + } + } + + bool vp_secondary_infer_node::need_apply(int primary_class_id, int target_width, int target_height) { + if (target_width < min_width_applied_to || target_height < min_height_applied_to) { + return false; + } + + if (p_class_ids_applied_to.size() == 0) { + return true; + } + + return std::find(p_class_ids_applied_to.begin(), p_class_ids_applied_to.end(), primary_class_id) != p_class_ids_applied_to.end(); + } +} \ No newline at end of file diff --git a/nodes/vp_secondary_infer_node.h b/nodes/vp_secondary_infer_node.h new file mode 100755 index 0000000..cffc973 --- /dev/null +++ b/nodes/vp_secondary_infer_node.h @@ -0,0 +1,45 @@ + + +#pragma once + +#include "vp_infer_node.h" + +namespace vp_nodes { + // secondary infer node, it is the base class of infer node which MUST infer on small cropped image. + // note: detector such as yolo can be also applied on small cropped images. + class vp_secondary_infer_node: public vp_infer_node { + private: + protected: + // define how to prepare data + virtual void prepare(const std::vector>& frame_meta_with_batch, std::vector& mats_to_infer) override; + bool need_apply(int primary_class_id, int target_width, int target_height); + public: + vp_secondary_infer_node(std::string node_name, + std::string model_path, + std::string model_config_path = "", + std::string labels_path = "", + int input_width = 640, + int input_height = 640, + int batch_size = 1, + std::vector p_class_ids_applied_to = std::vector(), + int min_width_applied_to = 0, + int min_height_applied_to = 0, + int crop_padding = 10, + float scale = 1.0, + cv::Scalar mean = cv::Scalar(123.675, 116.28, 103.53), // imagenet dataset + cv::Scalar std = cv::Scalar(1), + bool swap_rb = true, + bool swap_chn = false); + ~vp_secondary_infer_node(); + + // we do secondary infer logic only if the primary class-id of target is in this vector. + // not all targets in frame should be handled by vp_secondary_infer_node. + std::vector p_class_ids_applied_to; + int crop_padding; + // min height of target to be handled by vp_secondary_infer_node, 0 means no restriction + int min_height_applied_to = 0; + // min width of target to be handled by vp_secondary_infer_node，0 means no restriction + int min_width_applied_to = 0; + }; + +} \ No newline at end of file diff --git a/nodes/vp_skip_node.cpp b/nodes/vp_skip_node.cpp new file mode 100755 index 0000000..e69de29 diff --git a/nodes/vp_skip_node.h b/nodes/vp_skip_node.h new file mode 100755 index 0000000..e69de29 diff --git a/nodes/vp_split_node.cpp b/nodes/vp_split_node.cpp new file mode 100755 index 0000000..b41ce36 --- /dev/null +++ b/nodes/vp_split_node.cpp @@ -0,0 +1,47 @@ + +#include "vp_split_node.h" +#include + +namespace vp_nodes { + + vp_split_node::vp_split_node(std::string node_name, + bool split_with_channel_index, + bool split_with_deep_copy): + vp_node(node_name), + split_with_channel_index(split_with_channel_index), + split_with_deep_copy(split_with_deep_copy) { + this->initialized(); + } + + vp_split_node::~vp_split_node() { + deinitialized(); + } + + // override vp_meta_publisher::push_meta + // total 2*2 mode to push meta + void vp_split_node::push_meta(std::shared_ptr meta) { + if (this->split_with_channel_index) { + std::lock_guard guard(this->subscribers_lock); + auto & i = meta->channel_index; + // assume array index stands for channel index + // make sure the next nodes attached to current node in the order of channel index. + if (this->subscribers.size() > i) { + auto & next = this->subscribers.at(i); // get the right next node + if (this->split_with_deep_copy) { + meta = meta->clone(); // new pointer to new memory allocation in heap, old pointer is ignored. + } + next->meta_flow(meta); + } + } + else { + std::lock_guard guard(this->subscribers_lock); + // see vp_meta_publisher::push_meta + for (auto i = this->subscribers.begin(); i != this->subscribers.end(); i++) { + if (this->split_with_deep_copy) { + meta = meta->clone(); // each next node has a new pointer to new memory allocation in heap + } + (*i)->meta_flow(meta); + } + } + } +} \ No newline at end of file diff --git a/nodes/vp_split_node.h b/nodes/vp_split_node.h new file mode 100755 index 0000000..351b206 --- /dev/null +++ b/nodes/vp_split_node.h @@ -0,0 +1,31 @@ + +#include "vp_node.h" + +namespace vp_nodes { + + // split pipeline into multi branches. although all other non-Des nodes have the ability to split pipeline, but vp_split_node has more parameters and flexible behaviour. + // by default, other non-Des nodes split pipeline by simple copying, which just copy pointer of meta and push to next nodes, each next node handle the same meta allocated in heap(not thread-safe). + // in addition, other non-Des nodes push meta to next nodes without difference, each next node receive equal number of meta to previous node. + // + // in vp_split_node, we have below parameters to set: + // split_with_channel_index (false by default): if true, push meta according to its channel index, only those next nodes having the same channel index can receive meta. + // split_with_deep_copy (false by default) : if true, copy meta in heap and create new pointer, then push the new pointer to next nodes, next nodes have a totally different meta with previous. + // + // note: split_with_deep_copy = true will affect the performance of pipeline. + // above paramters can be set as true at the same time. + class vp_split_node: public vp_node + { + private: + /* data */ + protected: + // re-implement how to push meta to next nodes. + virtual void push_meta(std::shared_ptr meta) override; + public: + vp_split_node(std::string node_name, bool split_with_channel_index = false, bool split_with_deep_copy = false); + ~vp_split_node(); + + bool split_with_channel_index; + bool split_with_deep_copy; + }; + +} \ No newline at end of file diff --git a/nodes/vp_src_node.cpp b/nodes/vp_src_node.cpp new file mode 100755 index 0000000..47d5782 --- /dev/null +++ b/nodes/vp_src_node.cpp @@ -0,0 +1,99 @@ + +#include +#include "vp_src_node.h" +#include "../objects/vp_control_meta.h" +#include "../objects/vp_image_record_control_meta.h" +#include "../objects/vp_video_record_control_meta.h" + +namespace vp_nodes { + + vp_src_node::vp_src_node(std::string node_name, + int channel_index, + float resize_ratio): + vp_node(node_name), + channel_index(channel_index), + resize_ratio(resize_ratio), + frame_index(-1) { + assert(resize_ratio > 0 && resize_ratio <= 1.0f); + } + + vp_src_node::~vp_src_node() { + + } + + void vp_src_node::deinitialized() { + alive = false; + gate.open(); + vp_node::deinitialized(); + } + + void vp_src_node::handle_run() { + throw vp_excepts::vp_not_implemented_error("must have re-implementaion for 'handle_run' method in src nodes!"); + } + + std::shared_ptr + vp_src_node::handle_frame_meta(std::shared_ptr meta) { + throw vp_excepts::vp_invalid_calling_error("'handle_frame_meta' method could not be called in src nodes!"); + } + + std::shared_ptr + vp_src_node::handle_control_meta(std::shared_ptr meta) { + throw vp_excepts::vp_invalid_calling_error("'handle_control_meta' method could not be called in src nodes!"); + } + + void vp_src_node::start() { + gate.open(); + } + + void vp_src_node::stop() { + gate.close(); + } + + void vp_src_node::speak() { + auto speak_control_meta = std::make_shared(vp_objects::vp_control_type::SPEAK, this->channel_index); + this->push_meta(speak_control_meta); + } + + vp_node_type vp_src_node::node_type() { + return vp_node_type::SRC; + } + + int vp_src_node::get_original_fps() const { + return original_fps; + } + + int vp_src_node::get_original_width() const { + return original_width; + } + + int vp_src_node::get_original_height() const { + return original_height; + } + + void vp_src_node::record_video_manually(bool osd, int video_duration) { + // make sure file is not too long + assert(video_duration <= 60 && video_duration >= 5); + + // generate file_name_without_ext + // MUST be unique + auto file_name_without_ext = vp_utils::time_format(NOW, "manually_record_video_"); + + // create control meta + auto video_record_control_meta = std::make_shared(channel_index, file_name_without_ext, video_duration, osd); + + // push meta to pipe + push_meta(video_record_control_meta); + } + + void vp_src_node::record_image_manually(bool osd) { + // generate file_name_without_ext + // MUST be unique + auto file_name_without_ext = vp_utils::time_format(NOW, "manually_record_image_"); + + // create control meta + auto image_record_control_meta = std::make_shared(channel_index, file_name_without_ext, osd); + + // push meta to pipe + push_meta(image_record_control_meta); + } +} diff --git a/nodes/vp_src_node.h b/nodes/vp_src_node.h new file mode 100755 index 0000000..652b9e4 --- /dev/null +++ b/nodes/vp_src_node.h @@ -0,0 +1,70 @@ +#pragma once + +#include "vp_node.h" +#include "vp_stream_info_hookable.h" +#include "../excepts/vp_not_implemented_error.h" +#include "../excepts/vp_invalid_calling_error.h" +#include "../utils/vp_gate.h" + +namespace vp_nodes { + // base class for src nodes, start point of meta/pipeline + class vp_src_node: public vp_node, public vp_stream_info_hookable { + private: + /* data */ + + protected: + // force re-implemetation in child class + virtual void handle_run() override; + // force ignored in child class + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + // force ignored in child class + virtual std::shared_ptr handle_control_meta(std::shared_ptr meta) override; + + // protected as it can't be instanstiated directly. + vp_src_node(std::string node_name, + int channel_index, + float resize_ratio = 1.0); + + // basic stream info + int original_fps = -1; + int original_width = 0; + int original_height = 0; + + // basic channel info + int frame_index; + int channel_index; + float resize_ratio; + + // control to work or not + // all derived class need depend on the value to check if work or not (start/stop) + vp_utils::vp_gate gate; + + // new logic for sending dead flag + virtual void deinitialized() override; + public: + ~vp_src_node(); + + virtual vp_node_type node_type() override; + + // start signal to pipeline + void start(); + // stop signal to pipeline + void stop(); + // speak signal to the pipeline (each node print some message such as current status) + void speak(); + + /* Debug API */ + // record video signal to pipeline, start recording video when vp_record_node(if exists) receive the signal. + // it is a debug api since record signal is usually generated automatically inside pipe(not by users outside pipe). + void record_video_manually(bool osd = false, int video_duration = 10); + /* Debug API */ + // record image signal to pipeline, start recording image when vp_record_node(if exists) receive the signal. + // it is a debug api since record signal is usually generated automatically inside pipe(not by userd outside pipe). + void record_image_manually(bool osd = false); + + int get_original_fps() const; + int get_original_width() const; + int get_original_height() const; + }; + +} \ No newline at end of file diff --git a/nodes/vp_stream_info_hookable.h b/nodes/vp_stream_info_hookable.h new file mode 100755 index 0000000..a71bfe9 --- /dev/null +++ b/nodes/vp_stream_info_hookable.h @@ -0,0 +1,46 @@ +#pragma once + +#include +#include +#include +#include + +namespace vp_nodes { + // stream info created by src nodes. + struct vp_stream_info { + int channel_index = -1; // channel index + int original_fps = 0; // original fps of stream + int original_width = 0; // width of resolution + int original_height = 0; // height of resolution + std::string uri = ""; // uri of stream + }; + + // callback when stream info changed, happens in src nodes. MUST not be blocked. + typedef std::function vp_stream_info_hooker; + + // allow hookers attached to the pipe (src nodes specifically), hookers get notified when stream info changed. + // this class is inherited by vp_src_node only. + class vp_stream_info_hookable + { + private: + /* data */ + protected: + std::mutex stream_info_hooker_lock; + vp_stream_info_hooker stream_info_hooker; + public: + vp_stream_info_hookable(/* args */) {} + ~vp_stream_info_hookable() {} + + void set_stream_info_hooker(vp_stream_info_hooker stream_info_hooker) { + std::lock_guard guard(stream_info_hooker_lock); + this->stream_info_hooker = stream_info_hooker; + } + + void invoke_stream_info_hooker(std::string node_name, vp_stream_info stream_info) { + std::lock_guard guard(stream_info_hooker_lock); + if (this->stream_info_hooker) { + this->stream_info_hooker(node_name, stream_info); + } + } + }; +} \ No newline at end of file diff --git a/nodes/vp_stream_status_hookable.h b/nodes/vp_stream_status_hookable.h new file mode 100755 index 0000000..d9f44f5 --- /dev/null +++ b/nodes/vp_stream_status_hookable.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include +#include +#include + +namespace vp_nodes { + // stream status created by des nodes. + struct vp_stream_status { + int channel_index = -1; // channel index + int frame_index = -1; // latest frame index + int latency = 0; // latency(ms) relative to src node + + float fps = 0; // output fps of stream, maybe it is not equal to original_fps + int width = 0; // output width of stream, maybe it is not equal to original_width + int height = 0; // output height of stream, maybe it is not equal to original_height + std::string direction; // where the stream goes to + }; + + // callback when stream is going out of pipe, happens in des nodes. MUST not be blocked. + typedef std::function vp_stream_status_hooker; + + // allow hookers attached to the pipe (des nodes specifically), hookers get notified when stream is going out of pipe. + // this class is inherited by vp_des_node only. + class vp_stream_status_hookable + { + private: + /* data */ + protected: + std::mutex stream_status_hooker_lock; + vp_stream_status_hooker stream_status_hooker; + public: + vp_stream_status_hookable(/* args */) {} + ~vp_stream_status_hookable() {} + + void set_stream_status_hooker(vp_stream_status_hooker stream_status_hooker) { + std::lock_guard guard(stream_status_hooker_lock); + this->stream_status_hooker = stream_status_hooker; + } + + void invoke_stream_status_hooker(std::string node_name, vp_stream_status stream_status) { + std::lock_guard guard(stream_status_hooker_lock); + if (this->stream_status_hooker) { + this->stream_status_hooker(node_name, stream_status); + } + } + }; +} \ No newline at end of file diff --git a/nodes/vp_sync_node.cpp b/nodes/vp_sync_node.cpp new file mode 100644 index 0000000..fda11c0 --- /dev/null +++ b/nodes/vp_sync_node.cpp @@ -0,0 +1,189 @@ + +#include "vp_sync_node.h" +#include + +namespace vp_nodes { + + vp_sync_node::vp_sync_node(std::string node_name, vp_sync_mode mode, int timeout): + vp_node(node_name), + mode(mode), + timeout(timeout) { + assert(timeout > 0); + this->initialized(); + } + + vp_sync_node::~vp_sync_node() { + deinitialized(); + } + + std::shared_ptr vp_sync_node::handle_frame_meta(std::shared_ptr meta) { + if (all_indexs_last_syned.count(meta->channel_index) == 0) { + all_indexs_last_syned[meta->channel_index] = -1; // initialize by -1 + } + auto& index_last_synced = all_indexs_last_syned[meta->channel_index]; + auto& meta_waiting_for_sync = all_meta_waiting_for_sync[meta->channel_index]; + + // discard because it has been already synced or timeout before + if (meta->frame_index <= index_last_synced) { + // warn log for discard + VP_WARN(vp_utils::string_format("[%s][channel%d] discard for frame_index:[%d], because it has been synced or timeout before...", node_name.c_str(), meta->channel_index, meta->frame_index)); + return nullptr; + } + + bool need_wait = true; + bool start_of_queue = true; + for (auto i = meta_waiting_for_sync.begin(); i != meta_waiting_for_sync.end();) { + auto meta_type = (*i)->meta_type; + + // it is control meta AND at the start of queue, push to downstream directly + if (meta_type == vp_objects::vp_meta_type::CONTROL) { + if (start_of_queue) { + auto waiting_control_meta = dynamic_pointer_cast((*i)); + pendding_meta(waiting_control_meta); + i = meta_waiting_for_sync.erase(i); + VP_DEBUG(vp_utils::string_format("[%s][channel%d] sync control meta for control_uid:[%s], now cache size of sync is:[%d]", node_name.c_str(), meta->channel_index, waiting_control_meta->control_uid.c_str(), meta_waiting_for_sync.size())); + } + else { + // skip + i++; + } + } + else { + // it is frame meta + auto waiting_meta = dynamic_pointer_cast((*i)); + auto frame_index = waiting_meta->frame_index; + + auto frame_period = 1000.0 / meta->fps; + auto delta = (meta->frame_index - frame_index) * frame_period; + + // timeout logic + if (delta >= timeout) { + pendding_meta(waiting_meta); + index_last_synced = frame_index; + i = meta_waiting_for_sync.erase(i); + + // warn log for timeout + VP_WARN(vp_utils::string_format("[%s][channel%d] timeout for frame_index:[%d], push to downstream directly...", node_name.c_str(), meta->channel_index, waiting_meta->frame_index)); + } + else { + // sync logic + if (frame_index == meta->frame_index) { + sync(waiting_meta, meta); + pendding_meta(waiting_meta); + index_last_synced = frame_index; + i = meta_waiting_for_sync.erase(i); + // do not need wait for sync + need_wait = false; + VP_DEBUG(vp_utils::string_format("[%s][channel%d] sync frame meta for frame_index:[%d], now cache size of sync is:[%d]", node_name.c_str(), meta->channel_index, waiting_meta->frame_index, meta_waiting_for_sync.size())); + break; + } + // skip + i++; + start_of_queue = false; + } + } + } + + if (need_wait) { + meta_waiting_for_sync.push_back(meta); + } + return nullptr; // important + } + + std::shared_ptr vp_sync_node::handle_control_meta(std::shared_ptr meta) { + if (all_control_uids_last_synced.count(meta->channel_index) == 0) { + all_control_uids_last_synced[meta->channel_index] = ""; // initialize by empty + } + auto& control_uid_last_synced = all_control_uids_last_synced[meta->channel_index]; + auto& meta_waiting_for_sync = all_meta_waiting_for_sync[meta->channel_index]; + + if (control_uid_last_synced != meta->control_uid) { + meta_waiting_for_sync.push_back(meta); + control_uid_last_synced = meta->control_uid; + } + + return nullptr; // important + } + + void vp_sync_node::sync(std::shared_ptr des, std::shared_ptr src) { + // point to the same vp_frame_meta object + if (des == src) { + return; + } + + /* + * normal data to sync: + * 1. osd frame + * 2. mask + */ + if (des->osd_frame.empty() && !src->osd_frame.empty()) { + des->osd_frame = src->osd_frame; + } + if (des->mask.empty() && !src->mask.empty()) { + des->mask = src->mask; + } + + /* + * infer data to sync: + * 1. targets + * 2. face targets + * 3. text targets + * 4. pose targets + */ + if (mode == vp_sync_mode::MERGE) { + // merge target lists between 2 vp_frame_meta objects + // insert pointers directly + des->targets.insert(des->targets.end(), src->targets.begin(), src->targets.end()); + des->face_targets.insert(des->face_targets.end(), src->face_targets.begin(), src->face_targets.end()); + des->text_targets.insert(des->text_targets.end(), src->text_targets.begin(), src->text_targets.end()); + des->pose_targets.insert(des->pose_targets.end(), src->pose_targets.begin(), src->pose_targets.end()); + } + else { + // update properties of targets + // first make sure the size of targets are equal + assert(des->targets.size() == src->targets.size()); + assert(des->face_targets.size() == src->face_targets.size()); + assert(des->text_targets.size() == src->text_targets.size()); + assert(des->pose_targets.size() == src->pose_targets.size()); + + for (int i = 0; i < src->targets.size(); i++) { + auto& des_target = des->targets[i]; + auto& src_target = src->targets[i]; + + if (des_target->track_id == -1 && src_target->track_id != -1) { + des_target->track_id = src_target->track_id; + } + if (des_target->tracks.size() == 0 && src_target->tracks.size() != 0) { + des_target->tracks = src_target->tracks; + } + if (des_target->mask.empty() && !src_target->mask.empty()) { + des_target->mask = src_target->mask; + } + if (des_target->embeddings.size() == 0 && src_target->embeddings.size() != 0) { + des_target->embeddings = src_target->embeddings; + } + + des_target->secondary_class_ids.insert(des_target->secondary_class_ids.end(), src_target->secondary_class_ids.begin(), src_target->secondary_class_ids.end()); + des_target->secondary_labels.insert(des_target->secondary_labels.end(), src_target->secondary_labels.begin(), src_target->secondary_labels.end()); + des_target->secondary_scores.insert(des_target->secondary_scores.end(), src_target->secondary_scores.begin(), src_target->secondary_scores.end()); + + des_target->sub_targets.insert(des_target->sub_targets.end(), src_target->sub_targets.begin(), src_target->sub_targets.end()); + } + + for (int i = 0; i < src->face_targets.size(); i++) { + auto& des_face_target = des->face_targets[i]; + auto& src_face_target = src->face_targets[i]; + + if (des_face_target->track_id == -1 && src_face_target->track_id != -1) { + des_face_target->track_id = src_face_target->track_id; + } + if (des_face_target->tracks.size() == 0 && src_face_target->tracks.size() != 0) { + des_face_target->tracks = src_face_target->tracks; + } + if (des_face_target->embeddings.size() == 0 && src_face_target->embeddings.size() != 0) { + des_face_target->embeddings = src_face_target->embeddings; + } + } + } + } +} \ No newline at end of file diff --git a/nodes/vp_sync_node.h b/nodes/vp_sync_node.h new file mode 100644 index 0000000..9ebb2a3 --- /dev/null +++ b/nodes/vp_sync_node.h @@ -0,0 +1,41 @@ + +#include "vp_node.h" + +namespace vp_nodes { + // sync mode + enum class vp_sync_mode { + // merge target (vp_frame_target/vp_frame_face_target/...) collections between the 1st vp_frame_meta and the 2nd vp_frame_meta + // mainly used for sync primary infer data which created by those derived from vp_primary_infer_node + MERGE = 0, + // update properties of target(vp_frame_target/vp_frame_face_target/...) from the 1st vp_frame_meta to 2nd vp_frame_meta + // mainly used for sync secondary infer data which generated by those derived from vp_secondary_infer_node， could also be used for sync other properties like tracking ids. + UPDATE = 1 + }; + + // sync data(hold by vp_frame_meta) for the same frame in the same channel + // mainly be used to sync infer results between parallel nodes which have been splited by vp_split_node before in the same channel + // note: it could also sync vp_control_meta, just filter the same control meta by control uid. + class vp_sync_node: public vp_node + { + private: + // do sync work + void sync(std::shared_ptr des, std::shared_ptr src); + // sync mode + vp_sync_mode mode = vp_sync_mode::MERGE; + // push to downstream directly after waiting a period of time(milliseconds), because we can not wait infinitely + int timeout = 40; // wait for 40ms for each frame since 25fps is normal for video stream + + /* multi-channel supported*/ + std::map all_indexs_last_syned; + std::map all_control_uids_last_synced; + std::map>> all_meta_waiting_for_sync; + protected: + // re-implementation + virtual std::shared_ptr handle_frame_meta(std::shared_ptr meta) override; + // re-implementation + virtual std::shared_ptr handle_control_meta(std::shared_ptr meta) override; + public: + vp_sync_node(std::string node_name, vp_sync_mode mode = vp_sync_mode::MERGE, int timeout = 40); + ~vp_sync_node(); + }; +} \ No newline at end of file diff --git a/nodes/vp_udp_src_node.cpp b/nodes/vp_udp_src_node.cpp new file mode 100755 index 0000000..ead0394 --- /dev/null +++ b/nodes/vp_udp_src_node.cpp @@ -0,0 +1,118 @@ + + +#include "vp_udp_src_node.h" +#include "../utils/vp_utils.h" + +namespace vp_nodes { + + vp_udp_src_node::vp_udp_src_node(std::string node_name, + int channel_index, + int port, + float resize_ratio, + std::string gst_decoder_name, + int skip_interval): + vp_src_node(node_name, channel_index, resize_ratio), + port(port), gst_decoder_name(gst_decoder_name), skip_interval(skip_interval) { + assert(skip_interval >= 0 && skip_interval <= 9); + this->gst_template = vp_utils::string_format(this->gst_template, port, gst_decoder_name.c_str()); + VP_INFO(vp_utils::string_format("[%s] [%s]", node_name.c_str(), gst_template.c_str())); + this->initialized(); + } + + vp_udp_src_node::~vp_udp_src_node() { + deinitialized(); + } + + // define how to receive video stream via udp, create frame meta etc. + // please refer to the implementation of vp_node::handle_run. + void vp_udp_src_node::handle_run() { + cv::Mat frame; + int video_width = 0; + int video_height = 0; + int fps = 0; + int skip = 0; + while(alive) { + // check if need work + gate.knock(); + + // try to open capture + if (!udp_capture.isOpened()) { + video_width = video_height = fps = 0; + original_width = original_height = original_fps = 0; + if (!udp_capture.open(this->gst_template, cv::CAP_GSTREAMER)) { + VP_WARN(vp_utils::string_format("[%s] open udp failed, try again...", node_name.c_str())); + continue; + } + } + + // video properties + if (video_width == 0 || video_height == 0 || fps == 0) { + video_width = udp_capture.get(cv::CAP_PROP_FRAME_WIDTH); + video_height = udp_capture.get(cv::CAP_PROP_FRAME_HEIGHT); + fps = udp_capture.get(cv::CAP_PROP_FPS); + + original_fps = fps; + original_width = video_width; + original_height = video_height; + + // set true fps because skip some frames + fps = fps / (skip_interval + 1); + } + // stream_info_hooker activated if need + vp_stream_info stream_info {channel_index, original_fps, original_width, original_height, to_string()}; + invoke_stream_info_hooker(node_name, stream_info); + + udp_capture >> frame; + if(frame.empty()) { + VP_WARN(vp_utils::string_format("[%s] reading frame empty, total frame==>%d", node_name.c_str(), frame_index)); + continue; + } + + // need skip + if (skip < skip_interval) { + skip++; + continue; + } + skip = 0; + + // need resize + cv::Mat resize_frame; + if (this->resize_ratio != 1.0f) { + cv::resize(frame, resize_frame, cv::Size(), resize_ratio, resize_ratio); + } + else { + resize_frame = frame.clone(); // clone!; + } + // set true size because resize + video_width = resize_frame.cols; + video_height = resize_frame.rows; + + this->frame_index++; + // create frame meta + auto out_meta = + std::make_shared(resize_frame, this->frame_index, this->channel_index, video_width, video_height, fps); + + if (out_meta != nullptr) { + this->out_queue.push(out_meta); + + // handled hooker activated if need + if (this->meta_handled_hooker) { + meta_handled_hooker(node_name, out_queue.size(), out_meta); + } + + // important! notify consumer of out_queue in case it is waiting. + this->out_queue_semaphore.signal(); + VP_DEBUG(vp_utils::string_format("[%s] after handling meta, out_queue.size()==>%d", node_name.c_str(), out_queue.size())); + } + } + + // send dead flag for dispatch_thread + this->out_queue.push(nullptr); + this->out_queue_semaphore.signal(); + } + + // return stream uri + std::string vp_udp_src_node::to_string() { + return "udp://127.0.0.1:" + std::to_string(port); + } +} \ No newline at end of file diff --git a/nodes/vp_udp_src_node.h b/nodes/vp_udp_src_node.h new file mode 100755 index 0000000..fe12c32 --- /dev/null +++ b/nodes/vp_udp_src_node.h @@ -0,0 +1,39 @@ + + +#include +#include +#include +#include +#include "vp_src_node.h" + +namespace vp_nodes { + // udp source node, receive video stream via udp(rtp) protocal. + // example: + // udp://127.0.0.1:6000 + class vp_udp_src_node: public vp_src_node + { + private: + std::string gst_template = "udpsrc port=%d ! application/x-rtp,mdeia=video ! rtph264depay ! h264parse ! %s ! videoconvert ! appsink"; + cv::VideoCapture udp_capture; + protected: + // re-implemetation + virtual void handle_run() override; + public: + vp_udp_src_node(std::string node_name, + int channel_index, + int port, + float resize_ratio = 1.0, + std::string gst_decoder_name = "avdec_h264", + int skip_interval = 0); + ~vp_udp_src_node(); + + virtual std::string to_string() override; + + // port to listen + int port; + // set avdec_h264 as the default decoder, we can use hardware decoder instead. + std::string gst_decoder_name = "avdec_h264"; + // 0 means no skip + int skip_interval = 0; + }; +} \ No newline at end of file diff --git a/objects/ba/vp_ba_result.cpp b/objects/ba/vp_ba_result.cpp new file mode 100644 index 0000000..f03a275 --- /dev/null +++ b/objects/ba/vp_ba_result.cpp @@ -0,0 +1,34 @@ +#include "vp_ba_result.h" + + +namespace vp_objects { + + vp_ba_result::vp_ba_result(vp_ba_type type, + int channel_index, + int frame_index, + std::vector involve_target_ids_in_frame, + std::vector involve_region_in_frame, + std::string ba_label, + std::string record_image_name, + std::string record_video_name): + type(type), channel_index(channel_index), frame_index(frame_index), + involve_target_ids_in_frame(involve_target_ids_in_frame), + involve_region_in_frame(involve_region_in_frame), + ba_label(ba_label), + record_image_name(record_image_name), + record_video_name(record_video_name) { + + } + + vp_ba_result::~vp_ba_result() { + + } + + std::string vp_ba_result::to_string() { + return ""; + } + + std::shared_ptr vp_ba_result::clone() { + return std::make_shared(*this); + } +} \ No newline at end of file diff --git a/objects/ba/vp_ba_result.h b/objects/ba/vp_ba_result.h new file mode 100644 index 0000000..d1a8640 --- /dev/null +++ b/objects/ba/vp_ba_result.h @@ -0,0 +1,65 @@ +#pragma once + +#include +#include +#include +#include "../shapes/vp_point.h" + + +namespace vp_objects { + // type of behaviour analysis + enum class vp_ba_type { + NONE = 0b00000000, // none + CROSSLINE = 0b00000001, // cross line + STOP = 0b00000010, // enter stop status + UNSTOP = 0b00000100, // leave stop status + JAM = 0b00001000, // enter jam status + UNJAM = 0b00010000 // leave jam status + /* more */ + }; + + // result of behaviour analysis + // BA logic can ONLY works on vp_frame_target + class vp_ba_result + { + private: + /* data */ + public: + // type + vp_ba_type type; + // target ids which involved for this ba result, empty allowed. + std::vector involve_target_ids_in_frame; + // region (or single line) involved for this ba result, empty allowed. + std::vector involve_region_in_frame; + + // channel index of this ba result + int channel_index; + // frame index of this ba result + int frame_index; + + // name of ba + std::string ba_label = "not specified"; + + // record image name if exist + std::string record_image_name = ""; + // record video name if exist + std::string record_video_name = ""; + + vp_ba_result(vp_ba_type type, + int channel_index, + int frame_index, + std::vector involve_target_ids_in_frame, + std::vector involve_region_in_frame, + std::string ba_label = "not specified", + std::string record_image_name = "", + std::string record_video_name = ""); + ~vp_ba_result(); + + // get description for ba result + virtual std::string to_string(); + + // clone myself + std::shared_ptr clone(); + }; + +} \ No newline at end of file diff --git a/objects/shapes/vp_line.cpp b/objects/shapes/vp_line.cpp new file mode 100755 index 0000000..eaaca2b --- /dev/null +++ b/objects/shapes/vp_line.cpp @@ -0,0 +1,17 @@ + + +#include "vp_line.h" + +namespace vp_objects { + vp_line::vp_line(vp_point start, vp_point end): start(start), end(end) { + + } + + vp_line::~vp_line() { + + } + + float vp_line::length() { + return start.distance_with(end); + } +} \ No newline at end of file diff --git a/objects/shapes/vp_line.h b/objects/shapes/vp_line.h new file mode 100755 index 0000000..6661571 --- /dev/null +++ b/objects/shapes/vp_line.h @@ -0,0 +1,24 @@ + + +#pragma once + +#include "vp_point.h" + +namespace vp_objects { + // line in 2-dims coordinate system + class vp_line { + private: + /* data */ + public: + vp_line() = default; + vp_line(vp_point start, vp_point end); + ~vp_line(); + + vp_point start; + vp_point end; + + // distance between start and end point + float length(); + }; + +} \ No newline at end of file diff --git a/objects/shapes/vp_point.cpp b/objects/shapes/vp_point.cpp new file mode 100755 index 0000000..97742a6 --- /dev/null +++ b/objects/shapes/vp_point.cpp @@ -0,0 +1,18 @@ + + +#include "vp_point.h" + +namespace vp_objects { + + vp_point::vp_point(int x, int y): x(x), y(y) { + + } + + vp_point::~vp_point() { + + } + + float vp_point::distance_with(const vp_point & p) { + return std::sqrt(std::pow(x-p.x, 2) + std::pow(y-p.y, 2)); + } +} \ No newline at end of file diff --git a/objects/shapes/vp_point.h b/objects/shapes/vp_point.h new file mode 100755 index 0000000..b059b76 --- /dev/null +++ b/objects/shapes/vp_point.h @@ -0,0 +1,23 @@ + +#pragma once + +#include +#include + +namespace vp_objects { + // point in 2-dims coordinate system + class vp_point + { + private: + /* data */ + public: + vp_point(int x = 0, int y = 0); + ~vp_point(); + + int x; + int y; + + // distance between 2 points + float distance_with(const vp_point & p); + }; +} \ No newline at end of file diff --git a/objects/shapes/vp_polygon.cpp b/objects/shapes/vp_polygon.cpp new file mode 100755 index 0000000..5a37865 --- /dev/null +++ b/objects/shapes/vp_polygon.cpp @@ -0,0 +1,18 @@ + +#include +#include "vp_polygon.h" + +namespace vp_objects { + + vp_polygon::vp_polygon(std::vector vertexs): vertexs(vertexs) { + assert(vertexs.size() > 2); + } + + vp_polygon::~vp_polygon() { + + } + + bool vp_polygon::contains(const vp_point & p) { + return true; + } +} \ No newline at end of file diff --git a/objects/shapes/vp_polygon.h b/objects/shapes/vp_polygon.h new file mode 100755 index 0000000..598c358 --- /dev/null +++ b/objects/shapes/vp_polygon.h @@ -0,0 +1,25 @@ + + +#pragma once + +#include +#include "vp_point.h" + +namespace vp_objects { + class vp_polygon + { + private: + /* data */ + public: + vp_polygon() = default; + vp_polygon(std::vector vertexs); + ~vp_polygon(); + + // vertexs of the polygon + std::vector vertexs; + + // check if the polygon contains a point + bool contains(const vp_point & p); + }; + +} \ No newline at end of file diff --git a/objects/shapes/vp_rect.cpp b/objects/shapes/vp_rect.cpp new file mode 100755 index 0000000..f8ce4c1 --- /dev/null +++ b/objects/shapes/vp_rect.cpp @@ -0,0 +1,41 @@ + + +#include "vp_rect.h" + +namespace vp_objects { + + vp_rect::vp_rect(int x, int y, int width, int height): + x(x), + y(y), + width(width), + height(height) { + + } + + vp_rect::vp_rect(vp_point left_top, vp_size wh): + x(left_top.x), y(left_top.y), width(wh.width), height(wh.height) { + + } + + + vp_rect::~vp_rect() { + + } + + vp_point vp_rect::center() { + return vp_point(x + width / 2, y + height / 2); + } + + float vp_rect::iou_with(const vp_rect & rect) { + return 1.0; + } + + bool vp_rect::contains(const vp_point & p) { + return true; + } + + vp_point vp_rect::track_point() { + // by default the center point of bottom is tracking point. + return {x + width / 2, y + height}; + } +} \ No newline at end of file diff --git a/objects/shapes/vp_rect.h b/objects/shapes/vp_rect.h new file mode 100755 index 0000000..dcade83 --- /dev/null +++ b/objects/shapes/vp_rect.h @@ -0,0 +1,38 @@ +#pragma once + +#include + +#include "vp_point.h" +#include "vp_size.h" + +namespace vp_objects { + // rect in 2-dims coordinate system + class vp_rect { + private: + /* data */ + public: + vp_rect() = default; + vp_rect(int x, int y, int width, int height); + vp_rect(vp_point left_top, vp_size wh); + ~vp_rect(); + + int x; + int y; + int width; + int height; + + // get center point of the rect + vp_point center(); + + // get track point of the rect + // track point is used to locate the target(represented by the rect) + vp_point track_point(); + + // calculate the iou with another rect + float iou_with(const vp_rect & rect); + + // check if the rect contains a point + bool contains(const vp_point & p); + }; + +} \ No newline at end of file diff --git a/objects/shapes/vp_size.cpp b/objects/shapes/vp_size.cpp new file mode 100755 index 0000000..a242658 --- /dev/null +++ b/objects/shapes/vp_size.cpp @@ -0,0 +1,13 @@ + +#include "vp_size.h" + +namespace vp_objects { + + vp_size::vp_size(int width, int height): width(width), height(height) { + + } + vp_size::~vp_size() { + + } + +} \ No newline at end of file diff --git a/objects/shapes/vp_size.h b/objects/shapes/vp_size.h new file mode 100755 index 0000000..64f1153 --- /dev/null +++ b/objects/shapes/vp_size.h @@ -0,0 +1,20 @@ + +#pragma once + +#include + +namespace vp_objects { + // size(width and height) in 2-dims coordinate system + class vp_size { + private: + /* data */ + public: + vp_size(int width = 0, int height = 0); + ~vp_size(); + + + int width; + int height; + }; + +} \ No newline at end of file diff --git a/objects/vp_control_meta.cpp b/objects/vp_control_meta.cpp new file mode 100755 index 0000000..18f7bf4 --- /dev/null +++ b/objects/vp_control_meta.cpp @@ -0,0 +1,31 @@ + +#include "vp_control_meta.h" + +namespace vp_objects { + + vp_control_meta::vp_control_meta(vp_control_type control_type, int channel_index, std::string control_uid): + vp_meta(vp_meta_type::CONTROL, channel_index), control_type(control_type), control_uid(control_uid) { + if (control_uid.empty()){ + generate_uid(); + } + } + + vp_control_meta::~vp_control_meta() { + + } + + std::shared_ptr vp_control_meta::clone() { + // just call copy constructor and return new pointer + return std::make_shared(*this); + } + + void vp_control_meta::generate_uid() { + auto now = std::chrono::system_clock::now(); + auto period = now.time_since_epoch(); + + // milliseconds since 1970-1-1 00:00:00 + auto timestamp = std::chrono::duration_cast(period).count(); + + control_uid = "vp_control_" + std::to_string(timestamp); + } +} \ No newline at end of file diff --git a/objects/vp_control_meta.h b/objects/vp_control_meta.h new file mode 100755 index 0000000..f6e6a24 --- /dev/null +++ b/objects/vp_control_meta.h @@ -0,0 +1,32 @@ +#pragma once + +#include +#include "vp_meta.h" + + +namespace vp_objects { + // type of control meta + enum vp_control_type { + SPEAK, + VIDEO_RECORD, + IMAGE_RECORD + }; + + // control meta, which contains control data. + class vp_control_meta: public vp_meta { + private: + // help to generate control uid if need + void generate_uid(); + public: + vp_control_meta(vp_control_type control_type, int channel_index, std::string control_uid = ""); + ~vp_control_meta(); + + vp_control_type control_type; + // unique id to identify control meta (caould be generated in random) + std::string control_uid; + + // copy myself + virtual std::shared_ptr clone() override; + }; + +} \ No newline at end of file diff --git a/objects/vp_frame_face_target.cpp b/objects/vp_frame_face_target.cpp new file mode 100644 index 0000000..2455298 --- /dev/null +++ b/objects/vp_frame_face_target.cpp @@ -0,0 +1,32 @@ +#include "vp_frame_face_target.h" + +namespace vp_objects { + + vp_frame_face_target::vp_frame_face_target(int x, + int y, + int width, + int height, + float score, + std::vector> key_points, + std::vector embeddings): + x(x), + y(y), + width(width), + height(height), + score(score), + key_points(key_points), + embeddings(embeddings) { + + } + + vp_frame_face_target::~vp_frame_face_target() { + } + + std::shared_ptr vp_frame_face_target::clone() { + return std::make_shared(*this); + } + + vp_rect vp_frame_face_target::get_rect() const{ + return vp_rect(x, y, width, height); + } +} \ No newline at end of file diff --git a/objects/vp_frame_face_target.h b/objects/vp_frame_face_target.h new file mode 100644 index 0000000..37bf643 --- /dev/null +++ b/objects/vp_frame_face_target.h @@ -0,0 +1,57 @@ + +#pragma once +#include +#include +#include "shapes/vp_rect.h" + + +namespace vp_objects { + // target in frame detected by face detectors such as yunet. + // note: we can define new target type like vp_frame_xxx_target... if need (see vp_frame_pose_target also) + class vp_frame_face_target + { + private: + /* data */ + public: + vp_frame_face_target(int x, + int y, + int width, + int height, + float score, + std::vector> key_points = std::vector>(), + std::vector embeddings = std::vector()); + ~vp_frame_face_target(); + + // x of top left + int x; + // y of top left + int y; + // width of rect + int width; + // height of rect + int height; + + // confidence + float score; + + // feature vector created by infer nodes such as vp_sface_feature_encoder_node. + // embeddings can be used for face recognize or other reid works. + std::vector embeddings; + + // key points (5 points or more) + std::vector> key_points; + + // track id filled by vp_track_node (child class) if it exists. + int track_id = -1; + // cache of track rects in the previous frames, filled by track node if it exists. + // we can draw / analyse depend on these track rects later. + std::vector tracks; + + // clone myself + std::shared_ptr clone(); + + // rect area of target + vp_rect get_rect() const; + }; + +} \ No newline at end of file diff --git a/objects/vp_frame_meta.cpp b/objects/vp_frame_meta.cpp new file mode 100755 index 0000000..2f96106 --- /dev/null +++ b/objects/vp_frame_meta.cpp @@ -0,0 +1,72 @@ +#include + +#include "vp_frame_meta.h" + +namespace vp_objects { + + vp_frame_meta::vp_frame_meta(cv::Mat frame, int frame_index, int channel_index, int original_width, int original_height, int fps): + vp_meta(vp_meta_type::FRAME, channel_index), + frame_index(frame_index), + original_width(original_width), + original_height(original_height), + fps(fps), + frame(frame) { + assert(!frame.empty()); + } + + // copy constructor of vp_frame_meta would NOT be called at most time. + // only when it flow through vp_split_node with vp_split_node::split_with_deep_copy==true. + // in fact, all kinds of meta would NOT be copyed in its lifecycle, we just pass them by poniter most time. + vp_frame_meta::vp_frame_meta(const vp_frame_meta& meta): + vp_meta(meta), + frame_index(meta.frame_index), + original_width(meta.original_width), + original_height(meta.original_height), + description(meta.description), + fps(meta.fps) { + // deep copy frame data + this->frame = meta.frame.clone(); + this->osd_frame = meta.osd_frame.clone(); + this->mask = meta.mask.clone(); + + // deep copy targets + for(auto& i: meta.targets) { + this->targets.push_back(i->clone()); + } + // deep copy pose targets + for(auto& i: meta.pose_targets) { + this->pose_targets.push_back(i->clone()); + } + // deep copy face targets + for(auto& i: meta.face_targets) { + this->face_targets.push_back(i->clone()); + } + // deep copy text targets + for(auto& i: meta.text_targets) { + this->text_targets.push_back(i->clone()); + } + // deep copy ba results + for(auto& i: meta.ba_results) { + this->ba_results.push_back(i->clone()); + } + } + + vp_frame_meta::~vp_frame_meta() { + + } + + std::shared_ptr vp_frame_meta::clone() { + // just call copy constructor and return new pointer + return std::make_shared(*this); + } + + std::vector> vp_frame_meta::get_targets_by_ids(const std::vector& ids) { + std::vector> results; + for(auto& t: targets) { + if (std::find(ids.begin(), ids.end(), t->track_id) != ids.end()) { + results.push_back(t); + } + } + return results; + } +} \ No newline at end of file diff --git a/objects/vp_frame_meta.h b/objects/vp_frame_meta.h new file mode 100755 index 0000000..8d80967 --- /dev/null +++ b/objects/vp_frame_meta.h @@ -0,0 +1,93 @@ +#pragma once + +#include +#include +#include +#include + +#include "vp_meta.h" +#include "vp_frame_target.h" +#include "vp_frame_pose_target.h" +#include "vp_frame_face_target.h" +#include "vp_frame_text_target.h" +#include "ba/vp_ba_result.h" +/* +* ########################################## +* how does frame meta work? +* ########################################## +* frame meta, holding all data(targets/elements/...) of current frame in the video scene. frame meta are independent and don't know about each other, neither its previous frames nor next frames. +* the data in frame meta is just telling us what **current frame** is so we can not get something like 'state-switch' from a single frame meta. +* if you need know when the 'state-switch' happen, for example, you want to notify to cloud via restful api if state changed(ignore if it's keeping), +* you need cache previous frame meta(maybe partial data) in your custom node first and then compare with each other to figure out if it has changed. +* +* frame meta works like our eyes, by taking a glance at the frame in video we can see what the picture is and how many targets are there. +* but if you want to know something like state-switch, for example, a person was walking and then stop or it stop for a while and then start to walk, you have to see(cache) more frames. +* +* see more implementation of 'vp_track_node' and 'vp_message_broker_node' which saved history frame meta data and then work based on them. +* 1. vp_track_node : save previous locations of targets and then do tracking based on them, we need see more frames to track targets in video. +* 2. vp_message_broker_node : save previous ba_flags and then do notifying based on them, we need see more frames to check if state-switch has happened. +* ########################################## +*/ +namespace vp_objects { + // frame meta, which contains frame-related data. it is kind of important meta in pipeline. + class vp_frame_meta: public vp_meta { + private: + /* data */ + public: + vp_frame_meta(cv::Mat frame, int frame_index = -1, int channel_index = -1, int original_width = 0, int original_height = 0, int fps = 0); + ~vp_frame_meta(); + + // define copy constructor since we need deep copy operation. + vp_frame_meta(const vp_frame_meta& meta); + + // frame the meta belongs to, filled by src nodes. + int frame_index; + + // fps for current video. + int fps; + + // orignal frame width, fiiled by src nodes. + int original_width; + // original frame height, filled by src nodes. + int original_height; + + // image data the meta holds, filled by src nodes. + // deep copy needed here for this member. + cv::Mat frame; + + // osd image data the meta holds, filled by osd node if exists. + // deep copy needed here for this member. + cv::Mat osd_frame; + + // mask for the WHOLE frame, filled by Semantic Segmentation nodes if exists. + // deep copy needed here for this member. + cv::Mat mask; + + // text description for frame (output from LLM) + std::string description; + + // targets created/appended by primary infer nodes, and then updated by secondary infer nodes if exist. + // it is shared_ptr<...> type just to keep same as elements. + // deep copy needed here for this member. + std::vector> targets; + + // pose targets created/appened by primary infer nodes. + std::vector> pose_targets; + + // face targets created/appened by primary infer nodes. + std::vector> face_targets; + + // text targets created/appened by primary infer nodes. + std::vector> text_targets; + + // ba results created/appened by ba nodes. + std::vector> ba_results; + + // get target ptrs by target ids in current frame, ONLY supports vp_frame_target + std::vector> get_targets_by_ids(const std::vector& ids); + + // copy myself + virtual std::shared_ptr clone() override; + }; + +} \ No newline at end of file diff --git a/objects/vp_frame_pose_target.cpp b/objects/vp_frame_pose_target.cpp new file mode 100644 index 0000000..c8a6210 --- /dev/null +++ b/objects/vp_frame_pose_target.cpp @@ -0,0 +1,19 @@ + +#include "vp_frame_pose_target.h" + +namespace vp_objects { + + vp_frame_pose_target::vp_frame_pose_target(vp_pose_type type, + std::vector key_points): + type(type), + key_points(key_points) { + + } + + vp_frame_pose_target::~vp_frame_pose_target() { + } + + std::shared_ptr vp_frame_pose_target::clone() { + return std::make_shared(*this); + } +} \ No newline at end of file diff --git a/objects/vp_frame_pose_target.h b/objects/vp_frame_pose_target.h new file mode 100644 index 0000000..45dccf6 --- /dev/null +++ b/objects/vp_frame_pose_target.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include + +namespace vp_objects { + + // different types of datasets used to train openpose model. + enum vp_pose_type { + body_25, + coco, + mpi_15, + face, + hand, + yolov8_pose_17 + }; + + struct vp_pose_keypoint { + int point_type; // point type (index), nose, neck or left_eye + int x; // x in 2D image + int y; // y in 2D image + float score; // probability + }; + + // target in frame detected by openpose(or other similar models), which mainly contains point collections. + // note: we can define new target type like vp_frame_xxx_target... if need (see vp_frame_face_target also) + class vp_frame_pose_target + { + private: + /* data */ + public: + vp_frame_pose_target(vp_pose_type type, std::vector key_points); + ~vp_frame_pose_target(); + + // target type, different models create different outputs which need specific parsing. + vp_pose_type type; + // keypoints array + std::vector key_points; + + // clone myself + std::shared_ptr clone(); + }; +} \ No newline at end of file diff --git a/objects/vp_frame_target.cpp b/objects/vp_frame_target.cpp new file mode 100755 index 0000000..7a7fcd0 --- /dev/null +++ b/objects/vp_frame_target.cpp @@ -0,0 +1,56 @@ + +#include "vp_frame_target.h" + +namespace vp_objects { + + vp_frame_target::vp_frame_target(int x, + int y, + int width, + int height, + int primary_class_id, + float primary_score, + int frame_index, + int channel_index, + std::string primary_label): + x(x), + y(y), + width(width), + height(height), + primary_class_id(primary_class_id), + primary_score(primary_score), + primary_label(primary_label), + frame_index(frame_index), + channel_index(channel_index), + ba_flags(0) { + } + vp_frame_target::vp_frame_target(vp_rect rect, + int primary_class_id, + float primary_score, + int frame_index, + int channel_index, + std::string primary_label): + vp_frame_target(rect.x, + rect.y, + rect.width, + rect.height, + primary_class_id, + primary_score, + frame_index, + channel_index, + primary_label) { + + } + + + vp_frame_target::~vp_frame_target() { + + } + + std::shared_ptr vp_frame_target::clone() { + return std::make_shared(*this); + } + + vp_rect vp_frame_target::get_rect() const{ + return vp_rect(x, y, width, height); + } +} \ No newline at end of file diff --git a/objects/vp_frame_target.h b/objects/vp_frame_target.h new file mode 100755 index 0000000..507239e --- /dev/null +++ b/objects/vp_frame_target.h @@ -0,0 +1,106 @@ +#pragma once + +#include +#include +#include +#include + +#include "shapes/vp_rect.h" +#include "vp_sub_target.h" + +/* +* ################################################## +* what is frame target? +* ################################################## +* frame target are those detected by deep learning models(detectors) and then updated by other classifiers. +* we can detect vehicles, pedestrain, traffic lights, firesmoke and so on using vp_primary_infer_node, and then figure out what color the vehicles are, if the pedstrain wear a hat or not using vp_secondary_infer_node. +* vehicles, pedstrain are frame targets detected in current frame. +* +* note: +* frame target is an important concept and it contains a lot of data which would be updated/filled by vp_node when flowing through the piepline. +* see vp_frame_meta also. +* ################################################## +*/ + +namespace vp_objects { + // target in frame, detected by detectors(such as yolo/ssd). + class vp_frame_target { + private: + /* data */ + public: + // x of top left + int x; + // y of top left + int y; + // width of rect + int width; + // height of rect + int height; + + // class id created by primary infer nodes. + // allow multi primary infer nodes to exist in a pipeline, the class id is unique because we apply class_id_offset in each primary infer node. + int primary_class_id; + // score created by primary infer nodes + float primary_score; + // label created by primary infer nodes + std::string primary_label; + + // frame the target belongs to + int frame_index; + // channel the target belongs to + int channel_index; + + // track id created by track node if it exists + int track_id = -1; + // cache of track rects in the previous frames, filled by track node if it exists. + // we can draw / analyse depend on these track rects later. + std::vector tracks; + + // mask of the target, used for Instance Segmentation like mask rcnn network (ignore for other situations). + cv::Mat mask; + + // class ids filled/appended by multi secondary infer nodes. + std::vector secondary_class_ids; + // scores filled/appended by multi secondary infer nodes. + std::vector secondary_scores; + // labels filled/appended by multi secondary infer nodes. + std::vector secondary_labels; + + // sub targets inside current target. + // in case detectors applied on small cropped image. + std::vector> sub_targets; + + // feature vector(for example, 128 or 256-dims array) created by infer nodes such as vp_feature_encoder_node. + // each target has only one feature vector, the value will be override if multi vp_feature_encoder_node exist. + // embeddings can be used for reid related works. + std::vector embeddings; + + // ba flags of the target, hold by this value (created/updated by ba nodes). + // for example, 0001/0010/0100/1000 stands for 4 different flags, 1110 means 3 flags are on and another one is off, using ^|& operators to update and read. + // if 0100 stands for 'Stop' flag of target, 'ba_flags|=0100' means set 'Stop' flag as On, '(ba_flags & 0100) == 0100' means 'Stop' flag is already On. + int ba_flags; + + vp_frame_target(int x, + int y, + int width, + int height, + int primary_class_id, + float primary_score, + int frame_index, + int channel_index, + std::string primary_label = ""); + vp_frame_target(vp_rect rect, + int primary_class_id, + float primary_score, + int frame_index, + int channel_index, + std::string primary_label = ""); + ~vp_frame_target(); + + // clone myself + std::shared_ptr clone(); + + // rect area of target + vp_rect get_rect() const; + }; +} \ No newline at end of file diff --git a/objects/vp_frame_text_target.cpp b/objects/vp_frame_text_target.cpp new file mode 100644 index 0000000..6607ced --- /dev/null +++ b/objects/vp_frame_text_target.cpp @@ -0,0 +1,22 @@ + +#include "vp_frame_text_target.h" + +namespace vp_objects { + + vp_frame_text_target::vp_frame_text_target(std::vector> region_vertexes, + std::string text, + float score): + region_vertexes(region_vertexes), + text(text), + score(score) { + + } + + vp_frame_text_target::~vp_frame_text_target() { + + } + + std::shared_ptr vp_frame_text_target::clone() { + return std::make_shared(*this); + } +} \ No newline at end of file diff --git a/objects/vp_frame_text_target.h b/objects/vp_frame_text_target.h new file mode 100644 index 0000000..42151ec --- /dev/null +++ b/objects/vp_frame_text_target.h @@ -0,0 +1,26 @@ + +#pragma once + +#include +#include +#include + +namespace vp_objects { + class vp_frame_text_target + { + private: + /* data */ + public: + vp_frame_text_target(std::vector> region_vertexes, std::string text, float score); + ~vp_frame_text_target(); + + std::vector> region_vertexes; + std::string text; + float score; + + // flags for text + std::string flags = ""; + // clone myself + std::shared_ptr clone(); + }; +} \ No newline at end of file diff --git a/objects/vp_image_record_control_meta.cpp b/objects/vp_image_record_control_meta.cpp new file mode 100755 index 0000000..80fb58f --- /dev/null +++ b/objects/vp_image_record_control_meta.cpp @@ -0,0 +1,19 @@ +#include "vp_image_record_control_meta.h" + +namespace vp_objects { + + vp_image_record_control_meta::vp_image_record_control_meta(int channel_index, std::string image_file_name_without_ext, bool osd): + vp_control_meta(vp_control_type::IMAGE_RECORD, channel_index), + image_file_name_without_ext(image_file_name_without_ext), + osd(osd) { + } + + vp_image_record_control_meta::~vp_image_record_control_meta() { + + } + + std::shared_ptr vp_image_record_control_meta::clone() { + // just call copy constructor and return new pointer + return std::make_shared(*this); + } +} \ No newline at end of file diff --git a/objects/vp_image_record_control_meta.h b/objects/vp_image_record_control_meta.h new file mode 100755 index 0000000..24a9218 --- /dev/null +++ b/objects/vp_image_record_control_meta.h @@ -0,0 +1,30 @@ +#pragma once + +#include "vp_control_meta.h" + +namespace vp_objects { + // control meta for image recording, it is a specific type of vp_control_meta. + // when vp_record_node handle this control meta, the node will save the Latest Next frame in pipeline to disk. + // refer to ./nodes/record/README.md for more details + class vp_image_record_control_meta: public vp_control_meta + { + private: + /* data */ + public: + vp_image_record_control_meta(int channel_index, + std::string image_file_name_without_ext, + bool osd = false); + ~vp_image_record_control_meta(); + + // copy myself + virtual std::shared_ptr clone() override; + + // file name without extension, like 2022-10-20_22-30-20_aaa_bbb, which should be a meaningful and unique name. + // generated by sender who want to record image + std::string image_file_name_without_ext; + + // record type (osd frame or not) + bool osd = false; + }; + +} \ No newline at end of file diff --git a/objects/vp_meta.cpp b/objects/vp_meta.cpp new file mode 100755 index 0000000..1fcaace --- /dev/null +++ b/objects/vp_meta.cpp @@ -0,0 +1,70 @@ + +#include "vp_meta.h" +#include "../excepts/vp_invalid_argument_error.h" + +namespace vp_objects { + + vp_meta::vp_meta(vp_meta_type meta_type, int channel_index): + meta_type(meta_type), + channel_index(channel_index) { + create_time = std::chrono::system_clock::now(); + } + + vp_meta::~vp_meta() { + + } + + std::string vp_meta::get_traces_str() { + return ""; + } + + std::string vp_meta::get_meta_str() { + return ""; + } + +/* + void vp_meta::attach_trace(std::string node_name) { + if (trace_table.count(node_name)) { + return; + } + + std::map new_trace_record { + {vp_meta_trace_field::SEQUENCE, trace_table.size()}, + {vp_meta_trace_field::NODE_NAME, node_name}, + {vp_meta_trace_field::IN_TIME, -1}, + {vp_meta_trace_field::OUT_TIME, -1}, + {vp_meta_trace_field::TEXT_INFO, std::vector{}} + }; + + // append to the end of table + trace_table[node_name] = new_trace_record; + } + + void vp_meta::update_trace(std::string node_name, vp_meta_trace_field trace_key, std::any trace_value) { + if (trace_table.count(node_name)) { + auto & trace_record = trace_table[node_name]; + assert(trace_record.count(trace_key)); + + switch (trace_key) { + case vp_meta_trace_field::SEQUENCE: + case vp_meta_trace_field::NODE_NAME: + case vp_meta_trace_field::IN_TIME: + case vp_meta_trace_field::OUT_TIME: { + // replace directly + trace_record[trace_key] = trace_value; + break; + } + case vp_meta_trace_field::TEXT_INFO: { + // append to the end of vector + auto & trace_desc = std::any_cast&>(trace_record[trace_key]); + trace_desc.push_back(std::any_cast(trace_value)); + break; + } + default: { + throw vp_excepts::vp_invalid_argument_error("invalid trace_key for meta!"); + break; + } + } + } + }*/ +} \ No newline at end of file diff --git a/objects/vp_meta.h b/objects/vp_meta.h new file mode 100755 index 0000000..651d30b --- /dev/null +++ b/objects/vp_meta.h @@ -0,0 +1,111 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace vp_objects { + + // meta type + enum vp_meta_type { + FRAME, + CONTROL + }; + + // meta trace field + // 1. sequence ->int ,sequence number the meta flowing through pipeline + // 2. node_name ->string ,name of current node the meta flow through + // 3. in_time ->long ,time when the meta arrive current node + // 4. out_time ->long ,time when the meta leave current node + // 5. text_info ->vector ,text info while the meta inside node + enum vp_meta_trace_field { + SEQUENCE, + NODE_NAME, + IN_TIME, + OUT_TIME, + TEXT_INFO + }; + + // base class for meta + class vp_meta { + private: + + protected: + /* + trace table, single record describe how meta flows in each node. + { + "file_src_0": { + "sequence": 0, + "node_name": "file_src_0", + "in_time": 1692384, + "out_time": 1692384, + "trace_desc": ["create frame meta at time 1692384", "...", "..."] + }, + "primary_infer": { + "sequence": 1, + "node_name": "file_src_0", + "in_time": 1692384, + "out_time": 1692384, + "trace_desc": ["add 6 targets in frame meta at time 1692384", "...", "..."] + }, + "secondary_infer": { + "sequence": 2, + "node_name": "file_src_0", + "in_time": 1692384, + "out_time": 1692384, + "trace_desc": ["updated 6 targets in target_list at 1692384", "...", "..."] + }, + "osd": { + "sequence": 3, + "node_name": "file_src_0", + "in_time": 1692384, + "out_time": 1692384, + "trace_desc": ["draw 6 targets in on frame at time 1692384", "...", "..."] + }, + "file_des_0": { + "sequence": 4, + "node_name": "file_src_0", + "in_time": 1692384, + "out_time": 1692384, + "trace_desc": ["balabala", "...", "..."] + } + } + */ + //std::map> trace_table; + public: + vp_meta(vp_meta_type meta_type, int channel_index); + ~vp_meta(); + + // the time when meta created + std::chrono::system_clock::time_point create_time; + + vp_meta_type meta_type; + + // channel the meta belongs to + int channel_index; + + // write trace record or not + bool trace_on = false; + + // format traces string + virtual std::string get_traces_str(); + + // format meta string + virtual std::string get_meta_str(); + + // virtual clone method since we do not know what specific meta we need copy in some situations, return a new pointer pointting to new memory allocation in heap. + // note: every child class need implement its own clone() method. + virtual std::shared_ptr clone() = 0; + + // attach a new trace record for specific node (initialize key-value for current node) + //void attach_trace(std::string node_name); + + // update trace record + //void update_trace(std::string node_name, vp_meta_trace_field trace_key, std::any trace_value); + }; + +} \ No newline at end of file diff --git a/objects/vp_sub_target.cpp b/objects/vp_sub_target.cpp new file mode 100644 index 0000000..59e7f57 --- /dev/null +++ b/objects/vp_sub_target.cpp @@ -0,0 +1,36 @@ + +#include "vp_sub_target.h" + +namespace vp_objects { + + vp_sub_target::vp_sub_target(int x, + int y, + int width, + int height, + int class_id, + float score, + std::string label, + int frame_index, + int channel_index): + x(x), + y(y), + width(width), + height(height), + class_id(class_id), + score(score), + label(label), + frame_index(frame_index), + channel_index(channel_index) { + } + + vp_sub_target::~vp_sub_target() { + } + + std::shared_ptr vp_sub_target::clone() { + return std::make_shared(*this); + } + + vp_rect vp_sub_target::get_rect() const { + return vp_rect(x, y, width, height); + } +} \ No newline at end of file diff --git a/objects/vp_sub_target.h b/objects/vp_sub_target.h new file mode 100644 index 0000000..4aba724 --- /dev/null +++ b/objects/vp_sub_target.h @@ -0,0 +1,60 @@ +#pragma once + +#include +#include +#include + +#include "shapes/vp_rect.h" + +namespace vp_objects { + // sub target inside vp_frame_target, created by detectors which MUST infer on small cropped images (detectors are derived from vp_secondary_infer_node). + // this class has less properties/functions than vp_frame_target. + // see vp_frame_target also. + class vp_sub_target + { + private: + /* data */ + public: + vp_sub_target(int x, + int y, + int width, + int height, + int class_id, + float score, + std::string label, + int frame_index, + int channel_index); + ~vp_sub_target(); + + // x of top left + int x; + // y of top left + int y; + // width of rect + int width; + // height of rect + int height; + + // class id + int class_id; + // score + float score; + // label + std::string label; + + // frame the sub target belongs to + int frame_index; + // channel the sub target belongs to + int channel_index; + + // save some info + std::vector attachments; + + // clone myself + std::shared_ptr clone(); + + // rect area of target + vp_rect get_rect() const; + }; + +} \ No newline at end of file diff --git a/objects/vp_video_record_control_meta.cpp b/objects/vp_video_record_control_meta.cpp new file mode 100755 index 0000000..c81e03b --- /dev/null +++ b/objects/vp_video_record_control_meta.cpp @@ -0,0 +1,27 @@ + + +#include "vp_video_record_control_meta.h" + + +namespace vp_objects { + + vp_video_record_control_meta::vp_video_record_control_meta(int channel_index, + std::string video_file_name_without_ext, + int record_video_duration, + bool osd): + vp_control_meta(vp_objects::vp_control_type::VIDEO_RECORD, channel_index), + video_file_name_without_ext(video_file_name_without_ext), + record_video_duration(record_video_duration), + osd(osd) { + + } + + vp_video_record_control_meta::~vp_video_record_control_meta() { + + } + + std::shared_ptr vp_video_record_control_meta::clone() { + // just call copy constructor and return new pointer + return std::make_shared(*this); + } +} \ No newline at end of file diff --git a/objects/vp_video_record_control_meta.h b/objects/vp_video_record_control_meta.h new file mode 100755 index 0000000..d2798b9 --- /dev/null +++ b/objects/vp_video_record_control_meta.h @@ -0,0 +1,35 @@ + +#pragma once + +#include "vp_control_meta.h" + +namespace vp_objects { + // control meta for video recording, it is a specific type of vp_control_meta. + // when vp_record_node handle this control meta, the node will start recording video asynchronously, begin with the Latest Next frame in pipeline (pre-record frames excluded). + // refer to ./nodes/record/README.md for more details + class vp_video_record_control_meta: public vp_control_meta { + private: + /* data */ + public: + vp_video_record_control_meta(int channel_index, + std::string video_file_name_without_ext, + int record_video_duration = 0, + bool osd = false); + ~vp_video_record_control_meta(); + + // copy myself + virtual std::shared_ptr clone() override; + + // file name without extension, like 2022-10-20_22-30-20_aaa_bbb, which should be a meaningful and unique name. + // generated by sender who want to record video + std::string video_file_name_without_ext; + + // record video duration (seconds), not including pre-record time. + // 0 means using the value setted by vp_record_node. + int record_video_duration = 0; + + // record type (osd frame or not) + bool osd = false; + }; + +} \ No newline at end of file diff --git a/samples/1-1-1_sample.cpp b/samples/1-1-1_sample.cpp new file mode 100644 index 0000000..7ce9c4e --- /dev/null +++ b/samples/1-1-1_sample.cpp @@ -0,0 +1,42 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yunet_face_detector_node.h" +#include "../nodes/infers/vp_sface_feature_encoder_node.h" +#include "../nodes/osd/vp_face_osd_node_v2.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtmp_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## 1-1-1 sample ## +* 1 video input, 1 infer task, and 1 output. +*/ + +int main() { + VP_SET_LOG_INCLUDE_CODE_LOCATION(false); + VP_SET_LOG_INCLUDE_THREAD_ID(false); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/face.mp4", 0.6); + auto yunet_face_detector_0 = std::make_shared("yunet_face_detector_0", "./vp_data/models/face/face_detection_yunet_2022mar.onnx"); + auto sface_face_encoder_0 = std::make_shared("sface_face_encoder_0", "./vp_data/models/face/face_recognition_sface_2021dec.onnx"); + auto osd_0 = std::make_shared("osd_0"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + // construct pipeline + yunet_face_detector_0->attach_to({file_src_0}); + sface_face_encoder_0->attach_to({yunet_face_detector_0}); + osd_0->attach_to({sface_face_encoder_0}); + screen_des_0->attach_to({osd_0}); + + file_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0}); + board.display(1, false); + + std::string wait; + std::getline(std::cin, wait); + file_src_0->detach_recursively(); +} \ No newline at end of file diff --git a/samples/1-1-N_sample.cpp b/samples/1-1-N_sample.cpp new file mode 100644 index 0000000..01c89c7 --- /dev/null +++ b/samples/1-1-N_sample.cpp @@ -0,0 +1,46 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yunet_face_detector_node.h" +#include "../nodes/infers/vp_sface_feature_encoder_node.h" +#include "../nodes/osd/vp_face_osd_node_v2.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtmp_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## 1-1-N sample ## +* 1 video input, 1 infer task, and 2 outputs. +*/ + +int main() { + VP_SET_LOG_INCLUDE_CODE_LOCATION(false); + VP_SET_LOG_INCLUDE_THREAD_ID(false); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/face.mp4", 0.8); + auto yunet_face_detector_0 = std::make_shared("yunet_face_detector_0", "./vp_data/models/face/face_detection_yunet_2022mar.onnx"); + auto sface_face_encoder_0 = std::make_shared("sface_face_encoder_0", "./vp_data/models/face/face_recognition_sface_2021dec.onnx"); + auto osd_0 = std::make_shared("osd_0"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + auto rtmp_des_0 = std::make_shared("rtmp_des_0", 0, "rtmp://192.168.77.60/live/10000"); + + // construct pipeline + yunet_face_detector_0->attach_to({file_src_0}); + sface_face_encoder_0->attach_to({yunet_face_detector_0}); + osd_0->attach_to({sface_face_encoder_0}); + + // auto split + screen_des_0->attach_to({osd_0}); + rtmp_des_0->attach_to({osd_0}); + + file_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0}); + board.display(1, false); + + std::string wait; + std::getline(std::cin, wait); + file_src_0->detach_recursively(); +} \ No newline at end of file diff --git a/samples/1-N-1_sample.cpp b/samples/1-N-1_sample.cpp new file mode 100644 index 0000000..b8addd9 --- /dev/null +++ b/samples/1-N-1_sample.cpp @@ -0,0 +1,53 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_trt_vehicle_detector.h" +#include "../nodes/infers/vp_trt_vehicle_color_classifier.h" +#include "../nodes/infers/vp_trt_vehicle_type_classifier.h" +#include "../nodes/infers/vp_trt_vehicle_feature_encoder.h" +#include "../nodes/track/vp_sort_track_node.h" +#include "../nodes/vp_sync_node.h" +#include "../nodes/vp_split_node.h" +#include "../nodes/osd/vp_osd_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## 1-N-1_sample_sample ## +* 1 input video, detect vehicles first，classify by colors and types in parallel, then sync data and output on screen. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes for 1-N-1 pipeline + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/vehicle_count.mp4", 0.6); + auto trt_vehicle_detector_0 = std::make_shared("trt_detector_0", "./vp_data/models/trt/vehicle/vehicle_v8.5.trt"); + auto split = std::make_shared("split", false, true); // split by deep-copy not by channel! + auto trt_vehicle_color_classifier_0 = std::make_shared("trt_color_cls_0", "./vp_data/models/trt/vehicle/vehicle_color_v8.5.trt", std::vector{0, 1, 2}); + auto trt_vehicle_type_classifier_0 = std::make_shared("trt_type_cls_0", "./vp_data/models/trt/vehicle/vehicle_type_v8.5.trt", std::vector{0, 1, 2}); + auto sync = std::make_shared("sync", vp_nodes::vp_sync_mode::UPDATE, 160); + auto osd_0 = std::make_shared("osd_0"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + // construct 1-N-1 pipeline + trt_vehicle_detector_0->attach_to({file_src_0}); + split->attach_to({trt_vehicle_detector_0}); + trt_vehicle_color_classifier_0->attach_to({split}); + trt_vehicle_type_classifier_0->attach_to({split}); + sync->attach_to({trt_vehicle_color_classifier_0, trt_vehicle_type_classifier_0}); + osd_0->attach_to({sync}); + screen_des_0->attach_to({osd_0}); + /* + the format of pipeline: + / trt_vehicle_color_classifier_0 \ + file_src_0 -> trt_vehicle_detector_0 -> split -> -> sync -> osd_0 -> screen_des_0 + \ trt_vehicle_type_classifier_0 / + */ + + // start 1-N-1 pipeline + file_src_0->start(); + + // visualize pipelines for debug + vp_utils::vp_analysis_board board({file_src_0}); + board.display(); +} \ No newline at end of file diff --git a/samples/1-N-1_sample2.cpp b/samples/1-N-1_sample2.cpp new file mode 100644 index 0000000..aed55d8 --- /dev/null +++ b/samples/1-N-1_sample2.cpp @@ -0,0 +1,54 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_trt_vehicle_detector.h" +#include "../nodes/infers/vp_trt_vehicle_color_classifier.h" +#include "../nodes/infers/vp_trt_vehicle_type_classifier.h" +#include "../nodes/infers/vp_trt_vehicle_feature_encoder.h" +#include "../nodes/track/vp_sort_track_node.h" +#include "../nodes/vp_sync_node.h" +#include "../nodes/vp_split_node.h" +#include "../nodes/osd/vp_osd_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## 1-N-1_sample_sample ## +* 1 input video, detect vehicles first，classify by colors and tracking vehicles in parallel, then sync data and output on screen. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::DEBUG); + VP_SET_LOG_KEYWORDS_FOR_DEBUG({"sync"}); // only write debug log for sync node + VP_LOGGER_INIT(); + + // create nodes for 1-N-1 pipeline + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/jam2.mp4"); + auto trt_vehicle_detector_0 = std::make_shared("trt_detector_0", "./vp_data/models/trt/vehicle/vehicle_v8.5.trt"); + auto split = std::make_shared("split", false, true); // split by deep-copy not by channel! + auto trt_vehicle_color_classifier_0 = std::make_shared("trt_color_cls_0", "./vp_data/models/trt/vehicle/vehicle_color_v8.5.trt", std::vector{0, 1, 2}); + auto track_0 = std::make_shared("track_0"); + auto sync = std::make_shared("sync", vp_nodes::vp_sync_mode::UPDATE, 160); + auto osd_0 = std::make_shared("osd_0"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + // construct 1-N-1 pipeline + trt_vehicle_detector_0->attach_to({file_src_0}); + split->attach_to({trt_vehicle_detector_0}); + trt_vehicle_color_classifier_0->attach_to({split}); + track_0->attach_to({split}); + sync->attach_to({trt_vehicle_color_classifier_0, track_0}); + osd_0->attach_to({sync}); + screen_des_0->attach_to({osd_0}); + /* + the format of pipeline: + / trt_vehicle_color_classifier_0 \ + file_src_0 -> trt_vehicle_detector_0 -> split -> -> sync -> osd_0 -> screen_des_0 + \ track_0 / + */ + + // start 1-N-1 pipeline + file_src_0->start(); + + // visualize pipelines for debug + vp_utils::vp_analysis_board board({file_src_0}); + board.display(); +} \ No newline at end of file diff --git a/samples/1-N-1_sample3.cpp b/samples/1-N-1_sample3.cpp new file mode 100644 index 0000000..ffc77b8 --- /dev/null +++ b/samples/1-N-1_sample3.cpp @@ -0,0 +1,55 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_trt_vehicle_detector.h" +#include "../nodes/infers/vp_trt_vehicle_color_classifier.h" +#include "../nodes/infers/vp_trt_vehicle_plate_detector_v2.h" +#include "../nodes/track/vp_sort_track_node.h" +#include "../nodes/vp_sync_node.h" +#include "../nodes/vp_split_node.h" +#include "../nodes/osd/vp_osd_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## 1-N-1_sample_sample ## +* 1 input video, detect vehicles + classify by colors, detect license plate + track by sort, 2 tasks in parallel，then sync data and output on screen. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::DEBUG); + VP_SET_LOG_KEYWORDS_FOR_DEBUG({"sync"}); // only write debug log for sync node + VP_LOGGER_INIT(); + + // create nodes for 1-N-1 pipeline + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/plate2.mp4"); + auto split = std::make_shared("split", false, true); // split by deep-copy not by channel! + auto vehicle_detector = std::make_shared("vehicle_detector", "./vp_data/models/trt/vehicle/vehicle_v8.5.trt"); + auto vehicle_color_cls = std::make_shared("vehicle_color_cls", "./vp_data/models/trt/vehicle/vehicle_color_v8.5.trt", std::vector{0, 1, 2}); + auto plate_detector = std::make_shared("plate_detector", "./vp_data/models/trt/plate/det_v8.5.trt", "./vp_data/models/trt/plate/rec_v8.5.trt"); + auto plate_tracker = std::make_shared("plate_tracker"); + auto sync = std::make_shared("sync"); + auto osd = std::make_shared("osd", "./vp_data/font/NotoSansCJKsc-Medium.otf"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + // construct 1-N-1 pipeline + split->attach_to({file_src_0}); + vehicle_detector->attach_to({split}); + plate_detector->attach_to({split}); + vehicle_color_cls->attach_to({vehicle_detector}); + plate_tracker->attach_to({plate_detector}); + sync->attach_to({vehicle_color_cls, plate_tracker}); + osd->attach_to({sync}); + screen_des_0->attach_to({osd}); + /* + the format of pipeline: + / vehicle_detector -> vehicle_color_cls \ + file_src_0 -> split -> -> sync -> osd -> screen_des_0 + \ plate_detector -> plate_tracker / + */ + + // start 1-N-1 pipeline + file_src_0->start(); + + // visualize pipelines for debug + vp_utils::vp_analysis_board board({file_src_0}); + board.display(); +} \ No newline at end of file diff --git a/samples/1-N-N_sample.cpp b/samples/1-N-N_sample.cpp new file mode 100644 index 0000000..b6da13f --- /dev/null +++ b/samples/1-N-N_sample.cpp @@ -0,0 +1,60 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yunet_face_detector_node.h" +#include "../nodes/infers/vp_sface_feature_encoder_node.h" +#include "../nodes/osd/vp_face_osd_node_v2.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtmp_des_node.h" +#include "../nodes/vp_split_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## 1-N-N sample ## +* 1 video input and then split into 2 branches for different infer tasks, then 2 total outputs(no need to sync in such situations). +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/face.mp4", 0.6); + auto split = std::make_shared("split", false, true); // split by deep-copy not by channel! + + // branch a + auto yunet_face_detector_a = std::make_shared("yunet_face_detector_a", "./vp_data/models/face/face_detection_yunet_2022mar.onnx"); + auto sface_face_encoder_a = std::make_shared("sface_face_encoder_a", "./vp_data/models/face/face_recognition_sface_2021dec.onnx"); + auto osd_a = std::make_shared("osd_a"); + auto screen_des_a = std::make_shared("screen_des_a", 0); + + // branch b + auto yunet_face_detector_b = std::make_shared("yunet_face_detector_b", "./vp_data/models/face/face_detection_yunet_2022mar.onnx"); + auto sface_face_encoder_b = std::make_shared("sface_face_encoder_b", "./vp_data/models/face/face_recognition_sface_2021dec.onnx"); + auto osd_b = std::make_shared("osd_b"); + auto screen_des_b = std::make_shared("screen_des_b", 0); + + // construct pipeline + split->attach_to({file_src_0}); + + // branch a + yunet_face_detector_a->attach_to({split}); + sface_face_encoder_a->attach_to({yunet_face_detector_a}); + osd_a->attach_to({sface_face_encoder_a}); + screen_des_a->attach_to({osd_a}); + + // branch b + yunet_face_detector_b->attach_to({split}); + sface_face_encoder_b->attach_to({yunet_face_detector_b}); + osd_b->attach_to({sface_face_encoder_b}); + screen_des_b->attach_to({osd_b}); + + file_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0}); + board.display(1, false); + + std::string wait; + std::getline(std::cin, wait); + file_src_0->detach_recursively(); +} \ No newline at end of file diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt new file mode 100644 index 0000000..0bd17b7 --- /dev/null +++ b/samples/CMakeLists.txt @@ -0,0 +1,240 @@ +# can ONLY be called by parent CMakeLists.txt, because no environment settings for this script.# +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + message(FATAL_ERROR "can ONLY be called by parent CMakeLists.txt.") +endif() + +message("start build for simple samples...") +# save all exe to 'build/bin' +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +add_executable(1-1-1_sample "1-1-1_sample.cpp") +target_link_libraries(1-1-1_sample ${PROJECT_NAME}) + +add_executable(1-1-N_sample "1-1-N_sample.cpp") +target_link_libraries(1-1-N_sample ${PROJECT_NAME}) + +add_executable(1-N-N_sample "1-N-N_sample.cpp") +target_link_libraries(1-N-N_sample ${PROJECT_NAME}) + +add_executable(ba_crossline_sample "ba_crossline_sample.cpp") +target_link_libraries(ba_crossline_sample ${PROJECT_NAME}) + +add_executable(enet_seg_sample "enet_seg_sample.cpp") +target_link_libraries(enet_seg_sample ${PROJECT_NAME}) + +add_executable(face_tracking_sample "face_tracking_sample.cpp") +target_link_libraries(face_tracking_sample ${PROJECT_NAME}) + +add_executable(image_des_sample "image_des_sample.cpp") +target_link_libraries(image_des_sample ${PROJECT_NAME}) + +add_executable(image_src_sample "image_src_sample.cpp") +target_link_libraries(image_src_sample ${PROJECT_NAME}) + +add_executable(interaction_with_pipe_sample "interaction_with_pipe_sample.cpp") +target_link_libraries(interaction_with_pipe_sample ${PROJECT_NAME}) + +add_executable(mask_rcnn_sample "mask_rcnn_sample.cpp") +target_link_libraries(mask_rcnn_sample ${PROJECT_NAME}) + +add_executable(message_broker_sample "message_broker_sample.cpp") +target_link_libraries(message_broker_sample ${PROJECT_NAME}) + +add_executable(multi_detectors_and_classifiers_sample "multi_detectors_and_classifiers_sample.cpp") +target_link_libraries(multi_detectors_and_classifiers_sample ${PROJECT_NAME}) + +add_executable(N-1-N_sample "N-1-N_sample.cpp") +target_link_libraries(N-1-N_sample ${PROJECT_NAME}) + +add_executable(N-N_sample "N-N_sample.cpp") +target_link_libraries(N-N_sample ${PROJECT_NAME}) + +add_executable(openpose_sample "openpose_sample.cpp") +target_link_libraries(openpose_sample ${PROJECT_NAME}) + +add_executable(record_sample "record_sample.cpp") +target_link_libraries(record_sample ${PROJECT_NAME}) + +add_executable(rtsp_des_sample "rtsp_des_sample.cpp") +target_link_libraries(rtsp_des_sample ${PROJECT_NAME}) + +add_executable(rtsp_src_sample "rtsp_src_sample.cpp") +target_link_libraries(rtsp_src_sample ${PROJECT_NAME}) + +add_executable(vp_logger_sample "vp_logger_sample.cpp") +target_link_libraries(vp_logger_sample ${PROJECT_NAME}) + +add_executable(skip_sample "skip_sample.cpp") +target_link_libraries(skip_sample ${PROJECT_NAME}) + +add_executable(obstacle_detect_sample "obstacle_detect_sample.cpp") +target_link_libraries(obstacle_detect_sample ${PROJECT_NAME}) + +add_executable(multi_detectors_sample "multi_detectors_sample.cpp") +target_link_libraries(multi_detectors_sample ${PROJECT_NAME}) + +add_executable(firesmoke_detect_sample "firesmoke_detect_sample.cpp") +target_link_libraries(firesmoke_detect_sample ${PROJECT_NAME}) + +add_executable(face_swap_sample "face_swap_sample.cpp") +target_link_libraries(face_swap_sample ${PROJECT_NAME}) + +add_executable(video_restoration_sample "video_restoration_sample.cpp") +target_link_libraries(video_restoration_sample ${PROJECT_NAME}) + +add_executable(app_des_sample "app_des_sample.cpp") +target_link_libraries(app_des_sample ${PROJECT_NAME}) + +add_executable(app_src_des_sample "app_src_des_sample.cpp") +target_link_libraries(app_src_des_sample ${PROJECT_NAME}) + +add_executable(lane_detect_sample "lane_detect_sample.cpp") +target_link_libraries(lane_detect_sample ${PROJECT_NAME}) + +add_executable(frame_fusion_sample "frame_fusion_sample.cpp") +target_link_libraries(frame_fusion_sample ${PROJECT_NAME}) + +add_executable(yolov5_seg_sample "yolov5_seg_sample.cpp") +target_link_libraries(yolov5_seg_sample ${PROJECT_NAME}) + +add_executable(vp_test "vp_test.cpp") +target_link_libraries(vp_test ${PROJECT_NAME}) + +# samples depend on Kafka +if(VP_WITH_KAFKA) + add_executable(message_broker_kafka_sample "message_broker_kafka_sample.cpp") + target_link_libraries(message_broker_kafka_sample ${PROJECT_NAME}) +endif() + +# samples depend on LLM +if(VP_WITH_LLM) + add_executable(mllm_analyse_sample "mllm_analyse_sample.cpp") + target_link_libraries(mllm_analyse_sample ${PROJECT_NAME}) + + add_executable(mllm_analyse_sample_openai "mllm_analyse_sample_openai.cpp") + target_link_libraries(mllm_analyse_sample_openai ${PROJECT_NAME}) +endif() + +# samples depend on FFmpeg +if(VP_WITH_FFMPEG) + add_executable(ffmpeg_src_des_sample "ffmpeg_src_des_sample.cpp") + target_link_libraries(ffmpeg_src_des_sample ${PROJECT_NAME}) + + add_executable(ffmpeg_transcode_sample "ffmpeg_transcode_sample.cpp") + target_link_libraries(ffmpeg_transcode_sample ${PROJECT_NAME}) +endif() + +# samples depend on PaddlePaddle +if(VP_WITH_PADDLE) + add_executable(app_src_sample "app_src_sample.cpp") + target_link_libraries(app_src_sample ${PROJECT_NAME}) + + add_executable(paddle_infer_sample "paddle_infer_sample.cpp") + target_link_libraries(paddle_infer_sample ${PROJECT_NAME}) +endif() + +# samples depend on CUDA(maybe other dependency required too) +if(VP_WITH_CUDA) + # nvdec & nvenc plugins required from DeepStream + add_executable(nv_hard_codec_sample "nv_hard_codec_sample.cpp") + target_link_libraries(nv_hard_codec_sample ${PROJECT_NAME}) +endif() + +# samples depend on TensorRT +if(VP_WITH_TRT) + add_executable(ba_jam_sample "ba_jam_sample.cpp") + target_link_libraries(ba_jam_sample ${PROJECT_NAME}) + + add_executable(ba_stop_sample "ba_stop_sample.cpp") + target_link_libraries(ba_stop_sample ${PROJECT_NAME}) + + add_executable(body_scan_and_plate_detect_sample "body_scan_and_plate_detect_sample.cpp") + target_link_libraries(body_scan_and_plate_detect_sample ${PROJECT_NAME}) + + add_executable(dynamic_pipeline_sample "dynamic_pipeline_sample.cpp") + target_link_libraries(dynamic_pipeline_sample ${PROJECT_NAME}) + + add_executable(dynamic_pipeline_sample2 "dynamic_pipeline_sample2.cpp") + target_link_libraries(dynamic_pipeline_sample2 ${PROJECT_NAME}) + + add_executable(message_broker_sample2 "message_broker_sample2.cpp") + target_link_libraries(message_broker_sample2 ${PROJECT_NAME}) + + add_executable(multi_trt_infer_nodes_sample "multi_trt_infer_nodes_sample.cpp") + target_link_libraries(multi_trt_infer_nodes_sample ${PROJECT_NAME}) + + add_executable(plate_recognize_sample "plate_recognize_sample.cpp") + target_link_libraries(plate_recognize_sample ${PROJECT_NAME}) + + add_executable(src_des_sample "src_des_sample.cpp") + target_link_libraries(src_des_sample ${PROJECT_NAME}) + + add_executable(trt_infer_sample "trt_infer_sample.cpp") + target_link_libraries(trt_infer_sample ${PROJECT_NAME}) + + add_executable(vehicle_body_scan_sample "vehicle_body_scan_sample.cpp") + target_link_libraries(vehicle_body_scan_sample ${PROJECT_NAME}) + + add_executable(vehicle_cluster_based_on_classify_encoding_sample "vehicle_cluster_based_on_classify_encoding_sample.cpp") + target_link_libraries(vehicle_cluster_based_on_classify_encoding_sample ${PROJECT_NAME}) + + add_executable(vehicle_tracking_sample "vehicle_tracking_sample.cpp") + target_link_libraries(vehicle_tracking_sample ${PROJECT_NAME}) + + add_executable(1-N-1_sample "1-N-1_sample.cpp") + target_link_libraries(1-N-1_sample ${PROJECT_NAME}) + + add_executable(1-N-1_sample2 "1-N-1_sample2.cpp") + target_link_libraries(1-N-1_sample2 ${PROJECT_NAME}) + + add_executable(1-N-1_sample3 "1-N-1_sample3.cpp") + target_link_libraries(1-N-1_sample3 ${PROJECT_NAME}) + + add_executable(trt_yolov8_sample "trt_yolov8_sample.cpp") + target_link_libraries(trt_yolov8_sample ${PROJECT_NAME}) + + add_executable(trt_yolov8_sample2 "trt_yolov8_sample2.cpp") + target_link_libraries(trt_yolov8_sample2 ${PROJECT_NAME}) + + add_executable(rtmp_src_sample "rtmp_src_sample.cpp") + target_link_libraries(rtmp_src_sample ${PROJECT_NAME}) +endif() + +if(VP_BUILD_COMPLEX_SAMPLES) + message("start build for complex sampels...") + # save exe to 'face_recognize' + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/face_recognize) + add_executable(face_recognize_pipeline "face_recognize/face_recognize_pipeline.cpp") + target_link_libraries(face_recognize_pipeline ${PROJECT_NAME}) + + if(VP_WITH_TRT) # samples depend on TensorRT + # save exe to 'similiarity_search' + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/similiarity_search) + add_executable(face_encoding_pipeline "similiarity_search/face_encoding_pipeline.cpp") + target_link_libraries(face_encoding_pipeline ${PROJECT_NAME}) + add_executable(vehicle_encoding_pipeline "similiarity_search/vehicle_encoding_pipeline.cpp") + target_link_libraries(vehicle_encoding_pipeline ${PROJECT_NAME}) + + # save exe to 'vehicle_behaviour_analysis' + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/vehicle_behaviour_analysis) + add_executable(vehicle_ba_pipeline "vehicle_behaviour_analysis/vehicle_ba_pipeline.cpp") + target_link_libraries(vehicle_ba_pipeline ${PROJECT_NAME}) + + # save exe to 'vehicle_property_and_similiarity_search' + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/vehicle_property_and_similiarity_search) + add_executable(vehicle_encoding_classify_pipeline "vehicle_property_and_similiarity_search/vehicle_encoding_classify_pipeline.cpp") + target_link_libraries(vehicle_encoding_classify_pipeline ${PROJECT_NAME}) + + # save exe to 'lpr_camera' + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/lpr_camera) + add_executable(plate_recognize_pipeline "lpr_camera/plate_recognize_pipeline.cpp") + target_link_libraries(plate_recognize_pipeline ${PROJECT_NAME}) + endif() + + if(VP_WITH_PADDLE) + # save exe to 'math_review' + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/math_review) + add_executable(math_expression_check_pipeline "math_review/math_expression_check_pipeline.cpp") + target_link_libraries(math_expression_check_pipeline ${PROJECT_NAME}) + endif() + +endif() \ No newline at end of file diff --git a/samples/N-1-N_sample.cpp b/samples/N-1-N_sample.cpp new file mode 100644 index 0000000..95a4561 --- /dev/null +++ b/samples/N-1-N_sample.cpp @@ -0,0 +1,61 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yunet_face_detector_node.h" +#include "../nodes/infers/vp_sface_feature_encoder_node.h" +#include "../nodes/osd/vp_face_osd_node_v2.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtmp_des_node.h" +#include "../nodes/vp_split_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## N-1-N sample ## +* 2 video input and merge into 1 branch automatically for 1 infer task, then resume to 2 branches for outputs again. +*/ + +int main() { + VP_SET_LOG_INCLUDE_THREAD_ID(false); + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::WARN); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/face.mp4", 0.6); + auto file_src_1 = std::make_shared("file_src_1", 1, "./vp_data/test_video/face2.mp4", 0.6); + auto yunet_face_detector = std::make_shared("yunet_face_detector_0", "./vp_data/models/face/face_detection_yunet_2022mar.onnx"); + auto sface_face_encoder = std::make_shared("sface_face_encoder_0", "./vp_data/models/face/face_recognition_sface_2021dec.onnx"); + + auto split = std::make_shared("split", true); // split by channel index + + auto osd_0 = std::make_shared("osd_0"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + auto rtmp_des_0 = std::make_shared("rtmp_des_0", 0, "rtmp://192.168.77.60/live/10000"); + + auto osd_1 = std::make_shared("osd_1"); + auto screen_des_1 = std::make_shared("screen_des_1", 1); + auto rtmp_des_1 = std::make_shared("rtmp_des_1", 1, "rtmp://192.168.77.60/live/10000"); + + // construct pipeline + yunet_face_detector->attach_to({file_src_0, file_src_1}); + sface_face_encoder->attach_to({yunet_face_detector}); + + split->attach_to({sface_face_encoder}); + + // split by vp_split_node + osd_0->attach_to({split}); + osd_1->attach_to({split}); + + // auto split again on channel 0 + screen_des_0->attach_to({osd_0}); + rtmp_des_0->attach_to({osd_0}); + + // auto split again on channel 1 + screen_des_1->attach_to({osd_1}); + rtmp_des_1->attach_to({osd_1}); + + file_src_0->start(); + file_src_1->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0, file_src_1}); + board.display(); +} \ No newline at end of file diff --git a/samples/N-N_sample.cpp b/samples/N-N_sample.cpp new file mode 100644 index 0000000..a1409e9 --- /dev/null +++ b/samples/N-N_sample.cpp @@ -0,0 +1,52 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yunet_face_detector_node.h" +#include "../nodes/infers/vp_sface_feature_encoder_node.h" +#include "../nodes/osd/vp_face_osd_node_v2.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtmp_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## N-N sample ## +* multi pipe exist separately and each pipe is 1-1-1 (can be any structure like 1-1-N, 1-N-N) +*/ + +int main() { + VP_SET_LOG_INCLUDE_CODE_LOCATION(false); + VP_SET_LOG_INCLUDE_THREAD_ID(false); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/face.mp4", 0.6); + auto yunet_face_detector_0 = std::make_shared("yunet_face_detector_0", "./vp_data/models/face/face_detection_yunet_2022mar.onnx"); + auto sface_face_encoder_0 = std::make_shared("sface_face_encoder_0", "./vp_data/models/face/face_recognition_sface_2021dec.onnx"); + auto osd_0 = std::make_shared("osd_0"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + auto file_src_1 = std::make_shared("file_src_1", 0, "./vp_data/test_video/face2.mp4"); + auto yunet_face_detector_1 = std::make_shared("yunet_face_detector_1", "./vp_data/models/face/face_detection_yunet_2022mar.onnx"); + auto sface_face_encoder_1 = std::make_shared("sface_face_encoder_1", "./vp_data/models/face/face_recognition_sface_2021dec.onnx"); + auto osd_1 = std::make_shared("osd_1"); + auto screen_des_1 = std::make_shared("screen_des_1", 0); + + // construct pipeline + // pipe 0 + yunet_face_detector_0->attach_to({file_src_0}); + sface_face_encoder_0->attach_to({yunet_face_detector_0}); + osd_0->attach_to({sface_face_encoder_0}); + screen_des_0->attach_to({osd_0}); + + // pipe 1 + yunet_face_detector_1->attach_to({file_src_1}); + sface_face_encoder_1->attach_to({yunet_face_detector_1}); + osd_1->attach_to({sface_face_encoder_1}); + screen_des_1->attach_to({osd_1}); + + file_src_0->start(); + file_src_1->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0, file_src_1}); + board.display(); +} \ No newline at end of file diff --git a/samples/README.md b/samples/README.md new file mode 100644 index 0000000..1d2c566 --- /dev/null +++ b/samples/README.md @@ -0,0 +1,144 @@ + +## 1-1-1_sample ## +1 video input, 1 infer task, and 1 output. +![](../doc/p10.png) + +## 1-1-N_sample ## +1 video input, 1 infer task, and 2 outputs. +![](../doc/p11.png) + + +## 1-N-N_sample ## +1 video input and then split into 2 branches for different infer tasks, then 2 total outputs. +![](../doc/p12.png) + + +## N-1-N_sample ## +2 video input and merge into 1 branch automatically for 1 infer task, then resume to 2 branches for outputs again. +![](../doc/p13.png) + + +## N-N_sample ## +multi pipe exist separately and each pipe is 1-1-1 (can be any structure like 1-1-N, 1-N-N) +![](../doc/p14.png) + + +## paddle_infer_sample ## +ocr based on paddle (install paddle_inference first!), 1 video input and 2 outputs (screen, rtmp) +![](../doc/p15.png) + + +## src_des_sample ## +show how src nodes and des nodes work. +3 (file, rtsp, udp) input and merge into 1 infer task, then resume to 3 branches for outputs (screen, rtmp, fake) +![](../doc/p16.png) + + +## trt_infer_sample ## +vehicle and plate detector based on tensorrt (install tensorrt first!), 1 video input and 3 outputs (screen, file, rtmp) +![](../doc/p17.png) + + +## vp_logger_sample ## +show how `vp_logger` works. + +## face_tracking_sample ## +tracking for multi faces. +![](../doc/p18.png) + +## vehicle_tracking_sample ## +tracking for multi vehicles. +![](../doc/p22.png) + +## interaction_with_pipe_sample ## +show how to interact with pipe, such as start/stop channel by calling api. + +## record_sample ## +show how `vp_record_node` works. + +## message_broker_sample & message_broker_sample2 ## +show how message broker nodes work. +![](../doc/p20.png) +![](../doc/p21.png) + +## mask_rcnn_sample ## +show image segmentation by mask-rcnn. +![](../doc/p30.png) + +## openpose_sample ## +show pose estimation by openpose network. +![](../doc/p31.png) + +## enet_seg_sample ## +show semantic segmentation by enet network. +![](../doc/p32.png) + +## multi_detectors_and_classifiers_sample ## +show multi infer node work together. +![](../doc/p33.png) + +## image_des_sample ## +show save/push image to local file or remote via udp. +![](../doc/p34.png) + +## image_src_sample ## +show read/receive image from local file or remote via udp. +![](../doc/p35.png) + +## rtsp_des_sample ## +show push video stream via rtsp, no rtsp server needed, you can visit it directly. +![](../doc/p36.png) + +## ba_crossline_sample ## +count for vehicle based on tracking, the simplest one of behaviour analysis. +![](../doc/p37.png) + +## plate_recognize_sample ## +vehicle plate detect and recognize on the whole frame (no need to detect vechile first) +![](../doc/p38.png) + +## vehicle_body_scan_sample ## +detect parts of vehicle based on side view of body +![](../doc/p40.png) + +## body_scan_and_plate_detect_sample ## +2 channels to detect parts of vehicle and detect vehicle plate, you can do something like data fusion later +![](../doc/p39.png) + +## app_src_sample ## +send data to pipeline from host coda using app_src_node +![](../doc/p41.png) + +## vehicle_cluster_based_on_classify_encoding_sample ## +vehicle cluster based on labels(classify) and encoding(feature extract), pipeline would display 3 windows (cluster by t-SNE, cluster by labels, detect result) +![](../doc/p42.png) + +## ba_stop_sample ## +vehicle stop behaviour analysis +![](../doc/p49.png) + +## similiarity search ## +flask demo for vehicle and face similiarity search
+![](../doc/p44.png)![](../doc/p43.png)![](../doc/p45.png) + +## behaviour analysis ## +flask demo for crossline and stop
+![](../doc/p48.png) + +## property and similiarity search ## +flask demo for vehicle search by similiarity and properties
+![](../doc/p46.png)![](../doc/p47.png) + +## ba_jam_sample ## +traffic jam behaviour analysis
+![](../doc/p50.png) + +## face recognize ## +flask demo for face recognize
+![](../doc/p51.png) + +## license plate recognize ## +flask demo for license plate recognize
+![](../doc/p52.png) + +[for more samples](../SAMPLES.md) \ No newline at end of file diff --git a/samples/app_des_sample.cpp b/samples/app_des_sample.cpp new file mode 100644 index 0000000..2541760 --- /dev/null +++ b/samples/app_des_sample.cpp @@ -0,0 +1,55 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yunet_face_detector_node.h" +#include "../nodes/osd/vp_face_osd_node.h" +#include "../nodes/vp_app_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## app_des_sample ## +* 1. reading video from file +* 2. detect faces and draw results +* 3. display on screen in host code using cv::imshow(...) +* using vp_app_des_node INSTEAD OF vp_screen_des_node for displaying. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/face.mp4"); + auto yunet_face_detector_0 = std::make_shared("yunet_face_detector_0", "./vp_data/models/face/face_detection_yunet_2022mar.onnx"); + auto osd_0 = std::make_shared("osd_0"); + auto app_des_0 = std::make_shared("app_des_0", 0); + + // register callback to display result + std::string ori_win_title = "original frame using cv::imshow(...)"; + std::string osd_win_title = "osd frame using cv::imshow(...)"; + cv::namedWindow(ori_win_title,cv::WindowFlags::WINDOW_NORMAL); + cv::namedWindow(osd_win_title,cv::WindowFlags::WINDOW_NORMAL); + app_des_0->set_app_des_result_hooker([&](std::string node_name, std::shared_ptr meta) { + // only deal with frame meta + if (meta->meta_type == vp_objects::vp_meta_type::FRAME) { + auto frame_meta = std::dynamic_pointer_cast(meta); + cv::imshow(ori_win_title, frame_meta->frame); + + // osd frame may be empty + if (!frame_meta->osd_frame.empty()) { + cv::imshow(osd_win_title, frame_meta->osd_frame); + } + } + }); + + // construct pipeline + yunet_face_detector_0->attach_to({file_src_0}); + osd_0->attach_to({yunet_face_detector_0}); + app_des_0->attach_to({osd_0}); + + // start pipeline + file_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0}); + board.display(); +} \ No newline at end of file diff --git a/samples/app_src_des_sample.cpp b/samples/app_src_des_sample.cpp new file mode 100644 index 0000000..6ddb619 --- /dev/null +++ b/samples/app_src_des_sample.cpp @@ -0,0 +1,75 @@ +#include "../nodes/vp_app_src_node.h" +#include "../nodes/infers/vp_yunet_face_detector_node.h" +#include "../nodes/osd/vp_face_osd_node.h" +#include "../nodes/vp_app_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## app_src_des_sample ## +* 1. receive images from host code， based on vp_app_src_node +* 2. detect faces and draw results +* 3. display on screen in host code again using cv::imshow(...) and print rectangles and keypoints of face, based on vp_app_des_node +* we treat VideoPipe(pipeline) as a simple face detector tool in this sample. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto app_src_0 = std::make_shared("app_src_0", 0); + auto yunet_face_detector_0 = std::make_shared("yunet_face_detector_0", "./vp_data/models/face/face_detection_yunet_2022mar.onnx"); + auto osd_0 = std::make_shared("osd_0"); + auto app_des_0 = std::make_shared("app_des_0", 0); + + // register callback to display result + std::string ori_win_title = "original frame using cv::imshow(...)"; + std::string osd_win_title = "osd frame using cv::imshow(...)"; + cv::namedWindow(ori_win_title,cv::WindowFlags::WINDOW_NORMAL); + cv::namedWindow(osd_win_title,cv::WindowFlags::WINDOW_NORMAL); + app_des_0->set_app_des_result_hooker([&](std::string node_name, std::shared_ptr meta) { + // only deal with frame meta + if (meta->meta_type == vp_objects::vp_meta_type::FRAME) { + auto frame_meta = std::dynamic_pointer_cast(meta); + cv::imshow(ori_win_title, frame_meta->frame); + + // osd frame may be empty + if (!frame_meta->osd_frame.empty()) { + cv::imshow(osd_win_title, frame_meta->osd_frame); + } + + // print rectangle and keypoints of face + std::cout << "detected " << frame_meta->face_targets.size() << " faces:(print from host code)" << std::endl; + for (int i = 0; i < frame_meta->face_targets.size(); ++i) { + auto f = frame_meta->face_targets[i]; + std::cout << i << ": [" << f->x << "," << f->y << "," << f->width << "," << f->height << "] ["; + for (auto k: f->key_points) { + std::cout << "(" << k.first << "," << k.second << ")"; + } + std::cout << "]" << std::endl; + } + } + }); + + // construct pipeline + yunet_face_detector_0->attach_to({app_src_0}); + osd_0->attach_to({yunet_face_detector_0}); + app_des_0->attach_to({osd_0}); + + // start pipeline + app_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({app_src_0}); + board.display(1, false); // no block + + // read image in host code manually + auto img = cv::imread("./vp_data/test_images/faces/swap/2mans.jpg"); + // push image to pipeline in host code manually + app_src_0->push_frames({img}); + + // pipeline will process image asynchronously and display result here... + std::string wait; + std:getline(std::cin, wait); +} \ No newline at end of file diff --git a/samples/app_src_sample.cpp b/samples/app_src_sample.cpp new file mode 100644 index 0000000..60589ce --- /dev/null +++ b/samples/app_src_sample.cpp @@ -0,0 +1,80 @@ +#include "../nodes/vp_app_src_node.h" +#include "../nodes/infers/vp_ppocr_text_detector_node.h" +#include "../nodes/osd/vp_text_osd_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## app src sample ## +* receive frames(cv::Mat) from host code +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto app_src_0 = std::make_shared("app_src_0", 0); + auto ppocr_text_detector = std::make_shared("ppocr_text_detector", + "./vp_data/models/text/ppocr/ch_PP-OCRv3_det_infer", + "./vp_data/models/text/ppocr/ch_ppocr_mobile_v2.0_cls_infer", + "./vp_data/models/text/ppocr/ch_PP-OCRv3_rec_infer", + "./vp_data/models/text/ppocr/ppocr_keys_v1.txt"); + auto osd_0 = std::make_shared("osd_0", "./vp_data/font/NotoSansCJKsc-Medium.otf"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + // construct pipeline + ppocr_text_detector->attach_to({app_src_0}); + osd_0->attach_to({ppocr_text_detector}); + screen_des_0->attach_to({osd_0}); + + // start pipeline + app_src_0->start(); + + // visualize pipeline for debug + vp_utils::vp_analysis_board board({app_src_0}); + board.display(1, false); // no block since we need interactions from console later + + // simulate push frame to pipeline regularly in a separate thread + bool exit = false; + auto simulate_run = [&]() { + auto index = 0; + auto count = 0; + auto path = "./vp_data/test_images/text/"; + while (!exit) { + auto frame = cv::imread(path + std::to_string(index) + ".jpg"); + assert(!frame.empty()); + + app_src_0->push_frames({frame}); // push frame to pipeline, return false means failed + count++; + std::cout << "main thread has pushed [" << count << "] frames into pipeline..." << std::endl; + + index++; + index = index % 3; + std::this_thread::sleep_for(std::chrono::milliseconds(2000)); // sleep for 2 seconds + } + }; + std::thread simulate_thread(simulate_run); + + /* interact from console, start or stop the pipeline */ + /* no except check */ + std::string input; + std::getline(std::cin, input); + while (input != "quit") { + if (input == "start") { + app_src_0->start(); + } + else if (input == "stop") { + app_src_0->stop(); // app_src_0->push_frames(...) will print Warn message since it has stopped working + } + else { + std::cout << "invalid command!" << std::endl; + } + std::getline(std::cin, input); + } + + std::cout << "app_src_sample sample exits..." << std::endl; + exit = true; + simulate_thread.join(); + app_src_0->detach_recursively(); +} \ No newline at end of file diff --git a/samples/ba_crossline_sample.cpp b/samples/ba_crossline_sample.cpp new file mode 100644 index 0000000..b6f178d --- /dev/null +++ b/samples/ba_crossline_sample.cpp @@ -0,0 +1,51 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yolo_detector_node.h" +#include "../nodes/track/vp_sort_track_node.h" +#include "../nodes/ba/vp_ba_crossline_node.h" +#include "../nodes/osd/vp_ba_crossline_osd_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtmp_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## ba crossline sample ## +* behaviour analysis for crossline. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/vehicle_count.mp4", 0.4); + auto yolo_detector = std::make_shared("yolo_detector", "./vp_data/models/det_cls/yolov3-tiny-2022-0721_best.weights", "./vp_data/models/det_cls/yolov3-tiny-2022-0721.cfg", "./vp_data/models/det_cls/yolov3_tiny_5classes.txt"); + auto tracker = std::make_shared("sort_tracker"); + + // define a line in frame for every channel (value MUST in the scope of frame'size) + vp_objects::vp_point start(0, 250); // change to proper value + vp_objects::vp_point end(700, 220); // change to proper value + std::map lines = {{0, vp_objects::vp_line(start, end)}}; // channel0 -> line + auto ba_crossline = std::make_shared("ba_crossline", lines); + auto osd = std::make_shared("osd"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + auto rtmp_des_0 = std::make_shared("rtmp_des_0", 0, "rtmp://192.168.77.60/live/9000"); + + // construct pipeline + yolo_detector->attach_to({file_src_0}); + tracker->attach_to({yolo_detector}); + ba_crossline->attach_to({tracker}); + osd->attach_to({ba_crossline}); + screen_des_0->attach_to({osd}); + rtmp_des_0->attach_to({osd}); + + file_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0}); + board.display(1, false); + + std::string wait; + std::getline(std::cin, wait); + file_src_0->detach_recursively(); +} \ No newline at end of file diff --git a/samples/ba_jam_sample.cpp b/samples/ba_jam_sample.cpp new file mode 100644 index 0000000..e7c7c93 --- /dev/null +++ b/samples/ba_jam_sample.cpp @@ -0,0 +1,67 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_trt_vehicle_detector.h" +#include "../nodes/track/vp_sort_track_node.h" +#include "../nodes/ba/vp_ba_jam_node.h" +#include "../nodes/osd/vp_ba_jam_osd_node.h" +#include "../nodes/record/vp_record_node.h" +#include "../nodes/vp_split_node.h" +#include "../nodes/vp_screen_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## ba jam sample ## +* behaviour analysis for jam, single instance of ba node work on 2 channels. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/jam.mp4", 0.5); + auto file_src_1 = std::make_shared("file_src_1", 1, "./vp_data/test_video/jam2.mp4"); + auto trt_vehicle_detector = std::make_shared("vehicle_detector", "./vp_data/models/trt/vehicle/vehicle_v8.5.trt"); + auto tracker = std::make_shared("sort_tracker"); + + // define a region in frame for every channel (value MUST in the scope of frame'size) + std::map> regions = { + {0, std::vector{vp_objects::vp_point(20, 360), + vp_objects::vp_point(400, 250), + vp_objects::vp_point(535, 250), + vp_objects::vp_point(555, 560), + vp_objects::vp_point(30, 550)}}, // channel0 -> region + {1, std::vector{vp_objects::vp_point(20*0.6, 588*0.6), + vp_objects::vp_point(786*0.6, 180*0.6), + vp_objects::vp_point(968*0.6, 166*0.6), + vp_objects::vp_point(1220*0.6, 665*0.6)}} // channel1 -> region + }; + auto ba_jam = std::make_shared("ba_jam", regions); + auto osd = std::make_shared("jam_osd"); + auto recorder = std::make_shared("recorder", "./record", "./record"); + auto split = std::make_shared("split", true); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + auto screen_des_1 = std::make_shared("screen_des_1", 1); + + // construct pipeline + trt_vehicle_detector->attach_to({file_src_0, file_src_1}); + tracker->attach_to({trt_vehicle_detector}); + ba_jam->attach_to({tracker}); + osd->attach_to({ba_jam}); + recorder->attach_to({osd}); + split->attach_to({recorder}); + screen_des_0->attach_to({split}); + screen_des_1->attach_to({split}); + + file_src_0->start(); + file_src_1->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0, file_src_1}); + board.display(1, false); + + std::string wait; + std::getline(std::cin, wait); + file_src_0->detach_recursively(); + file_src_1->detach_recursively(); +} \ No newline at end of file diff --git a/samples/ba_stop_sample.cpp b/samples/ba_stop_sample.cpp new file mode 100644 index 0000000..bf39ab3 --- /dev/null +++ b/samples/ba_stop_sample.cpp @@ -0,0 +1,52 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_trt_vehicle_detector.h" +#include "../nodes/track/vp_sort_track_node.h" +#include "../nodes/ba/vp_ba_stop_node.h" +#include "../nodes/osd/vp_ba_stop_osd_node.h" +#include "../nodes/vp_split_node.h" +#include "../nodes/vp_screen_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## ba stop sample ## +* behaviour analysis for stop, single instance of ba node work on 2 channels. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/vehicle_stop.mp4", 0.6); + auto file_src_1 = std::make_shared("file_src_1", 1, "./vp_data/test_video/vehicle_stop.mp4", 0.6); + auto trt_vehicle_detector = std::make_shared("vehicle_detector", "./vp_data//models/trt/vehicle/vehicle_v8.5.trt"); + auto tracker = std::make_shared("sort_tracker"); + + // define a region in frame for every channel (value MUST in the scope of frame'size) + std::map> regions = { + {0, std::vector{vp_objects::vp_point(20, 30), vp_objects::vp_point(600, 40), vp_objects::vp_point(600, 300), vp_objects::vp_point(10, 300)}}, // channel0 -> region + {1, std::vector{vp_objects::vp_point(20, 30), vp_objects::vp_point(1000, 40), vp_objects::vp_point(1000, 600), vp_objects::vp_point(10, 600)}} // channel1 -> region + }; + auto ba_stop = std::make_shared("ba_stop", regions); + auto osd = std::make_shared("osd"); + auto split = std::make_shared("split", true); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + auto screen_des_1 = std::make_shared("screen_des_1", 1); + + // construct pipeline + trt_vehicle_detector->attach_to({file_src_0, file_src_1}); + tracker->attach_to({trt_vehicle_detector}); + ba_stop->attach_to({tracker}); + osd->attach_to({ba_stop}); + split->attach_to({osd}); + screen_des_0->attach_to({split}); + screen_des_1->attach_to({split}); + + file_src_0->start(); + file_src_1->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0, file_src_1}); + board.display(); +} \ No newline at end of file diff --git a/samples/body_scan_and_plate_detect_sample.cpp b/samples/body_scan_and_plate_detect_sample.cpp new file mode 100644 index 0000000..7000451 --- /dev/null +++ b/samples/body_scan_and_plate_detect_sample.cpp @@ -0,0 +1,52 @@ +#include "../nodes/vp_image_src_node.h" +#include "../nodes/infers/vp_trt_vehicle_scanner.h" +#include "../nodes/infers/vp_trt_vehicle_plate_detector_v2.h" +#include "../nodes/osd/vp_osd_node.h" +#include "../nodes/osd/vp_plate_osd_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## body_scan_and_plate_detect_sample ## +* first channel detects wheels and vehicle type based on side view of vehicle +* second channel detects plate based on head view of vehicle +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto image_src_0 = std::make_shared("image_src_0", 0, "./vp_data/test_images/body/%d.jpg"); + auto image_src_1 = std::make_shared("image_src_1", 1, "./vp_data/test_images/plates/%d.jpg"); + auto vehicle_scanner = std::make_shared("vehicle_scanner", + "./vp_data/models/trt/vehicle/vehicle_scan_v8.5.trt"); + auto plate_detector = std::make_shared("plate_detector", + "./vp_data/models/trt/plate/det_v8.5.trt", + "./vp_data/models/trt/plate/rec_v8.5.trt"); + auto osd_0 = std::make_shared("osd_0"); + auto osd_1 = std::make_shared("osd_1", "./vp_data/font/NotoSansCJKsc-Medium.otf"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + auto screen_des_1 = std::make_shared("screen_des_1", 1); + + // construct first pipeline + vehicle_scanner->attach_to({image_src_0}); + osd_0->attach_to({vehicle_scanner}); + screen_des_0->attach_to({osd_0}); + // construct second pipeline + plate_detector->attach_to({image_src_1}); + osd_1->attach_to({plate_detector}); + screen_des_1->attach_to({osd_1}); + + image_src_0->start(); + image_src_1->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({image_src_0, image_src_1}); + board.display(1, false); + + std::string wait; + std::getline(std::cin, wait); + image_src_0->detach_recursively(); + image_src_1->detach_recursively(); +} \ No newline at end of file diff --git a/samples/dynamic_pipeline_sample.cpp b/samples/dynamic_pipeline_sample.cpp new file mode 100644 index 0000000..33f44f4 --- /dev/null +++ b/samples/dynamic_pipeline_sample.cpp @@ -0,0 +1,175 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_trt_vehicle_detector.h" +#include "../nodes/infers/vp_trt_vehicle_color_classifier.h" +#include "../nodes/track/vp_sort_track_node.h" +#include "../nodes/osd/vp_osd_node.h" +#include "../nodes/broker/vp_json_console_broker_node.h" +#include "../nodes/vp_split_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## dynamic_pipeline_sample ## +* 1. insert and remove channels(SRC & DES nodes) to/from existing pipeline. +* 2. insert and remove MID nodes to/from existing pipeline. +* 3. all nodes destroy and process exit normally after user press enter from console. +* 4. no need to stop the pipeline and all operations in multi-threads, it works with hot-plug mode. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/vehicle_stop.mp4", 0.4); + auto trt_vehicle_detector = std::make_shared("trt_vehicle_detector", "./vp_data/models/trt/vehicle/vehicle_v8.5.trt"); + auto osd = std::make_shared("osd", "./vp_data/font/NotoSansCJKsc-Medium.otf"); + auto split = std::make_shared("split", true); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + // construct pipeline the first time + trt_vehicle_detector->attach_to({file_src_0}); + osd->attach_to({trt_vehicle_detector}); + split->attach_to({osd}); + screen_des_0->attach_to({split}); + + // start pipeline + file_src_0->start(); + // visualize pipeline for debug + std::vector> src_nodes = {file_src_0}; + vp_utils::vp_analysis_board board(src_nodes); + board.display(1, false); // no block + /* the original format of pipeline is: + file_src_0 -> trt_vehicle_detector -> osd -> split -> screen_des_0 + */ + + // simulation function for dynamically operating on piepline + bool exit = false; + auto simulate_run = [&]() { + while (!exit) { + // 1. wait for 5 seconds then insert the 2nd channel(input and output) to pipeline + std::this_thread::sleep_for(std::chrono::seconds(5)); + auto file_src_1 = std::make_shared("file_src_1", 1, "./vp_data/test_video/vehicle_stop.mp4", 0.4); + auto screen_des_1 = std::make_shared("screen_des_1", 1); + trt_vehicle_detector->attach_to({file_src_1}); + screen_des_1->attach_to({split}); + // start 2nd channel + file_src_1->start(); + // reload board using 2 SRC nodes + src_nodes.push_back(file_src_1); + board.reload(src_nodes); + /* now the format of pipeline is: + file_src_0 \ / screen_des_0 + -> trt_vehicle_detector -> osd -> split -> + file_src_1 / \ screen_des_1 + */ + + // 2. wait for 5 seconds then insert the 3rd channel(input and output) to pipeline + std::this_thread::sleep_for(std::chrono::seconds(5)); + auto file_src_2 = std::make_shared("file_src_2", 2, "./vp_data/test_video/vehicle_stop.mp4", 0.4); + auto screen_des_2 = std::make_shared("screen_des_2", 2); + trt_vehicle_detector->attach_to({file_src_2}); + screen_des_2->attach_to({split}); + // start 3rd channel + file_src_2->start(); + // reload board using 3 SRC nodes + src_nodes.push_back(file_src_2); + board.reload(src_nodes); + /* now the format of pipeline is: + file_src_0 \ / screen_des_0 + file_src_1 -> trt_vehicle_detector -> osd -> split -> screen_des_1 + file_src_2 / \ screen_des_2 + */ + + // 3. wait for 5 seconds then remove the DES node(output) of 3rd channel from pipeline + std::this_thread::sleep_for(std::chrono::seconds(5)); + screen_des_2->detach(); // call detach() since there is only 1 previous node for screen_des_2 + screen_des_2 = nullptr; // force call destructor immediately + // reload board using previous SRC nodes + board.reload(); + /* now the format of pipeline is: + file_src_0 \ / screen_des_0 + file_src_1 -> trt_vehicle_detector -> osd -> split -> screen_des_1 + file_src_2 / + */ + + // 4. wait for 5 seconds then remove the SRC node(input) of 3rd channel from pipeline + std::this_thread::sleep_for(std::chrono::seconds(5)); + trt_vehicle_detector->detach_from({"file_src_2"}); // call detach_from(...) since there are many previous nodes for trt_vehicle_detector + file_src_2->stop(); // call stop() on SRC node which is not reused later + file_src_2 = nullptr; // force call destructor immediately + // reload board using 2 SRC nodes + src_nodes.pop_back(); + board.reload(src_nodes); + /* now the format of pipeline is: + file_src_0 \ / screen_des_0 + -> trt_vehicle_detector -> osd -> split -> + file_src_1 / \ screen_des_1 + */ + + // 5. wait for 5 seconds then insert a secondary classifier node, track node and broker node into pipeline + std::this_thread::sleep_for(std::chrono::seconds(5)); + auto trt_color_cls = std::make_shared("trt_color_cls", "./vp_data/models/trt/vehicle/vehicle_color_v8.5.trt", std::vector{0, 1, 2}); + auto track = std::make_shared("track"); + auto console_broker = std::make_shared("console_broker"); + osd->detach(); // first detach osd node from pipeline + osd->attach_to({console_broker}); // then attach osd node to broker node + console_broker->attach_to({track}); // attach broker node to track node + track->attach_to({trt_color_cls}); // attach track node to classifier node + trt_color_cls->attach_to({trt_vehicle_detector}); // attach classifier node into pipeline + // reload board using previous SRC nodes + board.reload(); + /* now the format of pipeline is: + file_src_0 \ / screen_des_0 + -> trt_vehicle_detector -> trt_color_cls -> track -> console_broker -> osd -> split -> + file_src_1 / \ screen_des_1 + */ + + // 6. wait for 10 seconds then remove the 2nd channel (both SRC node and DES node) from pipeline + std::this_thread::sleep_for(std::chrono::seconds(10)); + screen_des_1->detach(); // call detach() since there is only 1 previous node for screen_des_1 + trt_vehicle_detector->detach_from({"file_src_1"}); // call detach_from(...) since there are many previous nodes for trt_vehicle_detector + file_src_1->stop(); // call stop() on SRC node which is not reused later + screen_des_1 = nullptr; // force call destructor immediately + file_src_1 = nullptr; // force call destructor immediately + // reload board using 1 SRC nodes + src_nodes.pop_back(); + board.reload(src_nodes); + /* now the format of pipeline is: + file_src_0 -> trt_vehicle_detector -> trt_color_cls -> track -> console_broker -> osd -> split -> screen_des_0 + */ + + // 7. wait for 10 seconds then remove classifier node, track node and broker node from pipeline + std::this_thread::sleep_for(std::chrono::seconds(10)); + osd->detach(); + console_broker->detach(); + track->detach(); + trt_color_cls->detach(); + osd->attach_to({trt_vehicle_detector}); // relink osd and trt_vehicle_detector + + console_broker = nullptr; + track = nullptr; + trt_color_cls = nullptr; + // reload board using previous SRC nodes + board.reload(); + /* now the format of pipeline is: + file_src_0 -> trt_vehicle_detector -> osd -> split -> screen_des_0 + */ + } + }; + + // start simulation thread + std::thread simulator(simulate_run); + + // enter to exit + std::string input; + std::getline(std::cin, input); + exit = true; + simulator.join(); + + // split pipeline into single nodes before process exit + for (auto& n: src_nodes) { + n->detach_recursively(); + } + // pipeline destroyed and process exit normally +} \ No newline at end of file diff --git a/samples/dynamic_pipeline_sample2.cpp b/samples/dynamic_pipeline_sample2.cpp new file mode 100644 index 0000000..f135f13 --- /dev/null +++ b/samples/dynamic_pipeline_sample2.cpp @@ -0,0 +1,73 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_trt_vehicle_detector.h" +#include "../nodes/osd/vp_osd_node.h" +#include "../nodes/vp_split_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../utils/analysis_board/vp_analysis_board.h" +#include "../nodes/record/vp_record_node.h" +#include "../nodes/broker/vp_json_console_broker_node.h" +#include "../nodes/vp_rtsp_src_node.h" +#include "../nodes/vp_rtmp_des_node.h" +#include "../nodes/infers/vp_trt_vehicle_type_classifier.h" +/* +* ## dynamic_pipeline_sample2 ## +* insert/remove nodes to/from pipeline step by step, then process exits normally. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/vehicle_stop.mp4", 0.4); + + auto trt_vehicle_detector = std::make_shared("trt_vehicle_detector", "./vp_data/models/trt/vehicle/vehicle_v8.5.trt"); + auto osd = std::make_shared("osd", "./vp_data/font/NotoSansCJKsc-Medium.otf"); + auto console_broker = std::make_shared("console_broker"); + auto split = std::make_shared("split", true); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + trt_vehicle_detector->attach_to({file_src_0}); + osd->attach_to({trt_vehicle_detector}); + console_broker->attach_to({osd}); + split->attach_to({console_broker}); + screen_des_0->attach_to({split}); + file_src_0->start(); + + std::vector> src_nodes = {file_src_0}; + std::string wait; + + std::cin >> wait; // display pipeline + vp_utils::vp_analysis_board board(src_nodes); + board.display(1, false); // no block + + std::cin >> wait; // add rtsp input + auto rtsp_src_1 = std::make_shared("rtsp_src_1", 1, "rtsp://admin:admin12345@192.168.77.203", 0.4); + trt_vehicle_detector->attach_to({rtsp_src_1}); + rtsp_src_1->start(); + src_nodes.push_back(rtsp_src_1); + board.reload(src_nodes); + + std::cin >> wait; // add rtmp output + auto rtmp_des_1 = std::make_shared("rtmp_des_1", 1, "rtmp://192.168.77.60/live/dynamic_pipeline_sample2"); + rtmp_des_1->attach_to({split}); + board.reload(); + + std::cin >> wait; // add classifier + auto trt_vehicle_type_classifier = std::make_shared("trt_type_cls", "./vp_data/models/trt/vehicle/vehicle_type_v8.5.trt", std::vector{0, 1, 2}); + osd->detach(); + osd->attach_to({trt_vehicle_type_classifier}); + trt_vehicle_type_classifier->attach_to({trt_vehicle_detector}); + board.reload(); + + std::cin >> wait; // remove broker + console_broker->detach(); + split->detach(); + split->attach_to({osd}); + board.reload(); + + std::cin >> wait; // destroy pipeline and process exit + for(auto& n: src_nodes) { + n->detach_recursively(); + } +} \ No newline at end of file diff --git a/samples/enet_seg_sample.cpp b/samples/enet_seg_sample.cpp new file mode 100644 index 0000000..eba9720 --- /dev/null +++ b/samples/enet_seg_sample.cpp @@ -0,0 +1,36 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_enet_seg_node.h" +#include "../nodes/osd/vp_seg_osd_node.h" +#include "../nodes/vp_screen_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## enet seg sample ## +* semantic segmentation based on ENet. +* 1 input, 2 outputs including orignal frame and mask frame. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/enet_seg.mp4"); + auto enet_seg = std::make_shared("enet_seg", "./vp_data/models/enet-cityscapes/enet-model.net"); + auto seg_osd_0 = std::make_shared("seg_osd_0", "./vp_data/models/enet-cityscapes/enet-classes.txt", "./vp_data/models/enet-cityscapes/enet-colors.txt"); + auto screen_des_mask = std::make_shared("screen_des_mask", 0, true, vp_objects::vp_size(400, 225)); + auto screen_des_original = std::make_shared("screen_des_original", 0, false, vp_objects::vp_size(400, 225)); + + // construct pipeline + enet_seg->attach_to({file_src_0}); + seg_osd_0->attach_to({enet_seg}); + screen_des_mask->attach_to({seg_osd_0}); + screen_des_original->attach_to({seg_osd_0}); + + file_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0}); + board.display(); +} \ No newline at end of file diff --git a/samples/face_swap_sample.cpp b/samples/face_swap_sample.cpp new file mode 100644 index 0000000..e64d44c --- /dev/null +++ b/samples/face_swap_sample.cpp @@ -0,0 +1,46 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yunet_face_detector_node.h" +#include "../nodes/infers/vp_face_swap_node.h" +#include "../nodes/osd/vp_face_osd_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_file_des_node.h" +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## face_swap_sample ## +* swap face for any video/images, no training need before running. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_SET_LOG_INCLUDE_CODE_LOCATION(false); + VP_SET_LOG_INCLUDE_THREAD_ID(false); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/face.mp4", 1.0, true, "avdec_h264", 4); + auto yunet_face_detector = std::make_shared("yunet_face_detector", "./vp_data/models/face/face_detection_yunet_2022mar.onnx"); + auto face_swap = std::make_shared("face_swap", "./vp_data/models/face/face_detection_yunet_2022mar.onnx", "./vp_data/models/face/swap/w600k_r50.onnx", "./vp_data/models/face/swap/emap.txt", "./vp_data/models/face/swap/inswapper_128.onnx", "./github/inswapper/data/mans1.jpeg"); + //auto osd = std::make_shared("osd"); + auto screen_des_0_ori = std::make_shared("screen_des_0_ori", 0, false); + auto screen_des_0_osd = std::make_shared("screen_des_0_osd", 0); + auto file_des_node_0 = std::make_shared("file_des_0", 0, ".", "", 1); + + // construct pipeline + yunet_face_detector->attach_to({file_src_0}); + face_swap->attach_to({yunet_face_detector}); + //osd->attach_to({face_swap}); + screen_des_0_ori->attach_to({face_swap}); + screen_des_0_osd->attach_to({face_swap}); + file_des_node_0->attach_to({face_swap}); // save swap result to file + + file_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0}); + board.display(1, false); + + std::string wait; + std::getline(std::cin, wait); + file_src_0->detach_recursively(); +} \ No newline at end of file diff --git a/samples/face_tracking_sample.cpp b/samples/face_tracking_sample.cpp new file mode 100644 index 0000000..4b58ce1 --- /dev/null +++ b/samples/face_tracking_sample.cpp @@ -0,0 +1,41 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yunet_face_detector_node.h" +#include "../nodes/infers/vp_sface_feature_encoder_node.h" +#include "../nodes/track/vp_sort_track_node.h" +#include "../nodes/osd/vp_face_osd_node.h" +#include "../nodes/vp_screen_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## face tracking sample ## +* track for multi faces using vp_sort_track_node. +*/ + +int main() { + VP_SET_LOG_INCLUDE_CODE_LOCATION(false); + VP_SET_LOG_INCLUDE_THREAD_ID(false); + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/face.mp4"); + auto yunet_face_detector_0 = std::make_shared("yunet_face_detector_0", "./vp_data/models/face/face_detection_yunet_2022mar.onnx"); + auto sface_face_encoder_0 = std::make_shared("sface_face_encoder_0", "./vp_data/models/face/face_recognition_sface_2021dec.onnx"); + auto track_0 = std::make_shared("track_0", vp_nodes::vp_track_for::FACE); // track for face + auto osd_0 = std::make_shared("osd_0"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + // construct pipeline + yunet_face_detector_0->attach_to({file_src_0}); + sface_face_encoder_0->attach_to({yunet_face_detector_0}); + track_0->attach_to({sface_face_encoder_0}); + osd_0->attach_to({track_0}); + screen_des_0->attach_to({osd_0}); + + file_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0}); + board.display(); +} \ No newline at end of file diff --git a/samples/ffmpeg_src_des_sample.cpp b/samples/ffmpeg_src_des_sample.cpp new file mode 100644 index 0000000..f547f0f --- /dev/null +++ b/samples/ffmpeg_src_des_sample.cpp @@ -0,0 +1,56 @@ +#include "../nodes/ffio/vp_ff_src_node.h" +#include "../nodes/infers/vp_yunet_face_detector_node.h" +#include "../nodes/infers/vp_sface_feature_encoder_node.h" +#include "../nodes/osd/vp_face_osd_node_v2.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/ffio/vp_ff_des_node.h" +#include "../nodes/vp_rtmp_des_node.h" +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## ffmpeg_src_des_sample ## +* reading & pushing stream based on ffmpeg (soft decode and encode using CPUs). +*/ + +int main() { + VP_SET_LOG_INCLUDE_CODE_LOCATION(false); + VP_SET_LOG_INCLUDE_THREAD_ID(false); + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + /* + uri for vp_ff_src_node: + 1. rtmp://192.168.77.196/live/1000 --> reading rtmp live stream + 2. rtsp://192.168.77.213/live/mainstream --> reading rtsp live stream + 3. ./vp_data/test_video/face.mp4 --> reading video file + + uri for vp_ff_des_node: + 1. rtmp://192.168.77.196/live/10000 --> pushing rtmp live stream + 2. ./output/records.mp4 --> saving to video file + */ + auto ff_src_0 = std::make_shared("ff_src_0", 0, "./vp_data/test_video/face.mp4", "h264", 0.6); + auto yunet_face_detector_0 = std::make_shared("yunet_face_detector_0", "./vp_data/models/face/face_detection_yunet_2022mar.onnx"); + auto sface_face_encoder_0 = std::make_shared("sface_face_encoder_0", "./vp_data/models/face/face_recognition_sface_2021dec.onnx"); + auto osd_0 = std::make_shared("osd_0"); + auto ff_des_0 = std::make_shared("ff_des_0", 0, "rtmp://192.168.77.60/live/20000"); + //auto rtmp_des_0 = std::make_shared("rtmp_des_0", 0, "rtmp://192.168.77.196/live/20000"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + // construct pipeline + yunet_face_detector_0->attach_to({ff_src_0}); + sface_face_encoder_0->attach_to({yunet_face_detector_0}); + osd_0->attach_to({sface_face_encoder_0}); + ff_des_0->attach_to({osd_0}); + screen_des_0->attach_to({osd_0}); + + ff_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({ff_src_0}); + board.display(1, false); + + std::string wait; + std::getline(std::cin, wait); + ff_src_0->detach_recursively(); +} \ No newline at end of file diff --git a/samples/ffmpeg_transcode_sample.cpp b/samples/ffmpeg_transcode_sample.cpp new file mode 100644 index 0000000..58841f6 --- /dev/null +++ b/samples/ffmpeg_transcode_sample.cpp @@ -0,0 +1,56 @@ +#include "../nodes/ffio/vp_ff_src_node.h" +#include "../nodes/infers/vp_yunet_face_detector_node.h" +#include "../nodes/infers/vp_sface_feature_encoder_node.h" +#include "../nodes/osd/vp_face_osd_node_v2.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/ffio/vp_ff_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## ffmpeg_transcode_sample ## +* transcoding in parallel, modify codec & resolution & bitrate & add watermask in picture. +*/ + +int main() { + VP_SET_LOG_INCLUDE_CODE_LOCATION(false); + VP_SET_LOG_INCLUDE_THREAD_ID(false); + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto ff_src_0 = std::make_shared("ff_src_0", 0, "rtmp://192.168.77.196/live/1000", "h264", 0.5); + auto ff_des_0 = std::make_shared("ff_des_0", 0, "rtmp://192.168.77.196/live/0"); + + auto ff_src_1 = std::make_shared("ff_src_1", 0, "rtmp://192.168.77.196/live/1000", "h264", 0.5); + auto ff_des_1 = std::make_shared("ff_des_1", 0, "rtmp://192.168.77.196/live/1"); + + auto ff_src_2 = std::make_shared("ff_src_2", 0, "rtsp://192.168.77.213/live/mainstream", "h264", 0.5); + auto ff_des_2 = std::make_shared("ff_des_2", 0, "rtmp://192.168.77.196/live/2"); + + auto ff_src_3 = std::make_shared("ff_src_3", 0, "rtsp://192.168.77.213/live/mainstream", "h264", 0.5); + auto ff_des_3 = std::make_shared("ff_des_3", 0, "rtmp://192.168.77.196/live/3"); + + // construct pipeline + ff_des_0->attach_to({ff_src_0}); + ff_des_1->attach_to({ff_src_1}); + ff_des_2->attach_to({ff_src_2}); + ff_des_3->attach_to({ff_src_3}); + + // start pipelines + ff_src_0->start(); + ff_src_1->start(); + ff_src_2->start(); + ff_src_3->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({ff_src_0, ff_src_1, ff_src_2, ff_src_3}); + board.display(1, false); + + std::string wait; + std::getline(std::cin, wait); + ff_src_0->detach_recursively(); + ff_src_1->detach_recursively(); + ff_src_2->detach_recursively(); + ff_src_3->detach_recursively(); +} \ No newline at end of file diff --git a/samples/firesmoke_detect_sample.cpp b/samples/firesmoke_detect_sample.cpp new file mode 100644 index 0000000..61da69c --- /dev/null +++ b/samples/firesmoke_detect_sample.cpp @@ -0,0 +1,46 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yolo_detector_node.h" +#include "../nodes/osd/vp_osd_node.h" +#include "../nodes/vp_split_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtsp_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## firesmoke_detect_sample ## +* detect firesmoke using yolo. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/smoke2.mp4", 0.5); + auto file_src_1 = std::make_shared("file_src_1", 1, "./vp_data/test_video/fire.mp4", 0.5); + auto yolo_detector = std::make_shared("firesmoke_detector", "./vp_data/models/det_cls/firesmoke_yolov5s.onnx", "", "./vp_data/models/det_cls/firesmoke_3classes.txt", 640, 384); + auto osd = std::make_shared("osd"); + auto split = std::make_shared("split_by_channel", true); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + auto srceen_des_1 = std::make_shared("screen_des_1", 1); + + // construct pipeline + yolo_detector->attach_to({file_src_0, file_src_1}); + osd->attach_to({yolo_detector}); + split->attach_to({osd}); + screen_des_0->attach_to({split}); + srceen_des_1->attach_to({split}); + + file_src_0->start(); + file_src_1->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0, file_src_1}); + board.display(1, false); + + std::string wait; + std::getline(std::cin, wait); + file_src_0->detach_recursively(); + file_src_1->detach_recursively(); +} \ No newline at end of file diff --git a/samples/frame_fusion_sample.cpp b/samples/frame_fusion_sample.cpp new file mode 100644 index 0000000..5add942 --- /dev/null +++ b/samples/frame_fusion_sample.cpp @@ -0,0 +1,47 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/proc/vp_frame_fusion_node.h" +#include "../nodes/vp_split_node.h" +#include "../nodes/vp_placeholder_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## frame_fusion_sample ## +* fuse frames of 2 channels, just merge adjacent frames from 2 channels directly without considering timestamp synchronization. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_SET_LOG_INCLUDE_CODE_LOCATION(false); + VP_SET_LOG_INCLUDE_THREAD_ID(false); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/infrared.mp4"); // source of fusion + auto file_src_1 = std::make_shared("file_src_1", 1, "./vp_data/test_video/rgb.mp4"); // destination of fusion + // initialize calibration points manually + std::vector src_cali_points = {vp_objects::vp_point(133, 111), vp_objects::vp_point(338, 110), vp_objects::vp_point(15, 330), vp_objects::vp_point(14, 214)}; + std::vector des_cali_points = {vp_objects::vp_point(1219, 365), vp_objects::vp_point(1787, 367), vp_objects::vp_point(891, 982), vp_objects::vp_point(892, 659)}; + auto fusion = std::make_shared("fusion", src_cali_points, des_cali_points); + auto split = std::make_shared("split", true); + auto screen_des_0_ori = std::make_shared("screen_des_0_ori", 0, false); // original frame for the first channel + auto screen_des_1_osd = std::make_shared("screen_des_1_osd", 1); // fusion result(osd frame) for the second channel + + // construct pipeline + fusion->attach_to({file_src_0, file_src_1}); + split->attach_to({fusion}); + screen_des_0_ori->attach_to({split}); + screen_des_1_osd->attach_to({split}); + + file_src_0->start(); + file_src_1->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0, file_src_1}); + board.display(1, false); + + std::string wait; + std::getline(std::cin, wait); + file_src_0->detach_recursively(); + file_src_1->detach_recursively(); +} \ No newline at end of file diff --git a/samples/image_des_sample.cpp b/samples/image_des_sample.cpp new file mode 100644 index 0000000..f2528a7 --- /dev/null +++ b/samples/image_des_sample.cpp @@ -0,0 +1,44 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yunet_face_detector_node.h" +#include "../nodes/infers/vp_sface_feature_encoder_node.h" +#include "../nodes/osd/vp_face_osd_node_v2.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_image_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## image_des_sample ## +* show how vp_image_des_node works, save image to local file or push image to remote via udp. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/face.mp4", 0.6); + auto yunet_face_detector_0 = std::make_shared("yunet_face_detector_0", "./vp_data/models/face/face_detection_yunet_2022mar.onnx"); + auto sface_face_encoder_0 = std::make_shared("sface_face_encoder_0", "./vp_data/models/face/face_recognition_sface_2021dec.onnx"); + auto osd_0 = std::make_shared("osd_0"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + /* save to file, `%d` is placeholder for filename index */ + //auto image_des_0 = std::make_shared("image_file_des_0", 0, "./images/%d.jpg", 3, vp_objects::vp_size(), false); + + /* push via udp, receiving command for test: `gst-launch-1.0 udpsrc port=6000 ! application/x-rtp,encoding-name=jpeg ! rtpjpegdepay ! jpegparse ! jpegdec ! queue ! videoconvert ! autovideosink` */ + auto image_des_0 = std::make_shared("image_udp_des_0", 0, "192.168.77.248:6000", 2, vp_objects::vp_size(600, 300)); + + // construct pipeline + yunet_face_detector_0->attach_to({file_src_0}); + sface_face_encoder_0->attach_to({yunet_face_detector_0}); + osd_0->attach_to({sface_face_encoder_0}); + screen_des_0->attach_to({osd_0}); + image_des_0->attach_to({osd_0}); + + file_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0}); + board.display(); +} \ No newline at end of file diff --git a/samples/image_src_sample.cpp b/samples/image_src_sample.cpp new file mode 100644 index 0000000..6220d16 --- /dev/null +++ b/samples/image_src_sample.cpp @@ -0,0 +1,41 @@ +#include "../nodes/vp_image_src_node.h" +#include "../nodes/infers/vp_yolo_detector_node.h" +#include "../nodes/osd/vp_osd_node.h" +#include "../nodes/vp_split_node.h" +#include "../nodes/vp_screen_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## image_des_sample ## +* show how vp_image_src_node works, read image from local file or receive image from remote via udp. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto image_src_0 = std::make_shared("image_file_src_0", 0, "./vp_data/test_images/vehicle/%d.jpg", 1, 0.4); // read 1 image EVERY 1 second from local files, such as test_0.jpg,test_1.jpg + /* sending command for test: `gst-launch-1.0 filesrc location=16.mp4 ! qtdemux ! avdec_h264 ! videoconvert ! videoscale ! video/x-raw,width=416,height=416 ! videorate ! video/x-raw,framerate=1/1 ! jpegenc ! rtpjpegpay ! udpsink host=ip port=6000` */ + auto image_src_1 = std::make_shared("image_udp_src_1", 1, "6000", 3); // receive 1 image EVERY 3 seconds from remote via udp , such as 127.0.0.1:6000 + auto yolo_detector = std::make_shared("yolo_detector", "./vp_data/models/det_cls/yolov3-tiny-2022-0721_best.weights", "./vp_data/models/det_cls/yolov3-tiny-2022-0721.cfg", "./vp_data/models/det_cls/yolov3_tiny_5classes.txt"); + auto osd = std::make_shared("osd"); + auto split = std::make_shared("split_by_channel", true); // split by channel index (important!) + auto screen_des_0 = std::make_shared("screen_des_0", 0); + auto screen_des_1 = std::make_shared("screen_des_1", 1); + + // construct pipeline + yolo_detector->attach_to({image_src_0, image_src_1}); + osd->attach_to({yolo_detector}); + split->attach_to({osd}); + screen_des_0->attach_to({split}); + screen_des_1->attach_to({split}); + + image_src_0->start(); // start read from local file + image_src_1->start(); // start receive from remote via udp + + // for debug purpose + vp_utils::vp_analysis_board board({image_src_0, image_src_1}); + board.display(); +} \ No newline at end of file diff --git a/samples/interaction_with_pipe_sample.cpp b/samples/interaction_with_pipe_sample.cpp new file mode 100644 index 0000000..d1b700b --- /dev/null +++ b/samples/interaction_with_pipe_sample.cpp @@ -0,0 +1,94 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yunet_face_detector_node.h" +#include "../nodes/infers/vp_sface_feature_encoder_node.h" +#include "../nodes/osd/vp_face_osd_node_v2.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtmp_des_node.h" +#include "../nodes/vp_split_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## interaction_with_pipe sample ## +* show how to interact with pipe, start/stop/speak on src nodes independently. +*/ + +int main() { + VP_LOGGER_INIT(); + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/face.mp4", 0.6); + auto file_src_1 = std::make_shared("file_src_1", 1, "./vp_data/test_video/face2.mp4", 0.6); + auto yunet_face_detector = std::make_shared("yunet_face_detector_0", "./vp_data/models/face/face_detection_yunet_2022mar.onnx"); + auto sface_face_encoder = std::make_shared("sface_face_encoder_0", "./vp_data/models/face/face_recognition_sface_2021dec.onnx"); + + auto split = std::make_shared("split", true); // split by channel index + + auto osd_0 = std::make_shared("osd_0"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + auto rtmp_des_0 = std::make_shared("rtmp_des_0", 0, "rtmp://192.168.77.60/live/10000"); + + auto osd_1 = std::make_shared("osd_1"); + auto screen_des_1 = std::make_shared("screen_des_1", 1); + auto rtmp_des_1 = std::make_shared("rtmp_des_1", 1, "rtmp://192.168.77.60/live/10000"); + + // construct pipeline + yunet_face_detector->attach_to({file_src_0, file_src_1}); + sface_face_encoder->attach_to({yunet_face_detector}); + + split->attach_to({sface_face_encoder}); + + // split by vp_split_node + osd_0->attach_to({split}); + osd_1->attach_to({split}); + + // auto split again on channel 0 + screen_des_0->attach_to({osd_0}); + rtmp_des_0->attach_to({osd_0}); + + // auto split again on channel 1 + screen_des_1->attach_to({osd_1}); + rtmp_des_1->attach_to({osd_1}); + + // for debug purpose + std::vector> src_nodes_in_pipe{file_src_0, file_src_1}; + vp_utils::vp_analysis_board board(src_nodes_in_pipe); + board.display(1, false); // no block since we need interactions from console later + + + /* interact from console */ + /* no except check */ + std::string input; + std::getline(std::cin, input); + // input format: `start channel`, like `start 0` means start channel 0 + auto inputs = vp_utils::string_split(input, ' '); + while (inputs[0] != "quit") { + // no except check + auto command = inputs[0]; + auto index = std::stoi(inputs[1]); + auto src_by_channel = std::dynamic_pointer_cast(src_nodes_in_pipe[index]); + if (command == "start") { + src_by_channel->start(); + } + else if (command == "stop") { + src_by_channel->stop(); + } + else if (command == "speak") { + src_by_channel->speak(); + } + else { + std::cout << "invalid command!" << std::endl; + } + std::getline(std::cin, input); + inputs = vp_utils::string_split(input, ' '); + if (inputs.size() != 2) { + std::cout << "invalid input!" << std::endl; + break; + } + } + + std::cout << "interaction_with_pipe sample exits..." << std::endl; + file_src_0->detach_recursively(); + file_src_1->detach_recursively(); +} \ No newline at end of file diff --git a/samples/lane_detect_sample.cpp b/samples/lane_detect_sample.cpp new file mode 100644 index 0000000..e7d5ce9 --- /dev/null +++ b/samples/lane_detect_sample.cpp @@ -0,0 +1,40 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_lane_detector_node.h" +#include "../nodes/osd/vp_lane_osd_node.h" +#include "../nodes/vp_screen_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## lane_detect_sample ## +* detect lanes on road. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/vehicle_count.mp4", 0.6, true, "avdec_h264", 4); + auto lane_detector = std::make_shared("lane_detector", "./vp_data/models/lane/lane_det.onnx"); + auto lane_osd = std::make_shared("lane_osd"); + auto screen_des_0_osd = std::make_shared("screen_des_0_osd", 0); + auto srceen_des_0_ori = std::make_shared("srceen_des_0_ori", 0, false); + + // construct pipeline + lane_detector->attach_to({file_src_0}); + lane_osd->attach_to({lane_detector}); + screen_des_0_osd->attach_to({lane_osd}); + srceen_des_0_ori->attach_to({lane_osd}); + + // start pipeline + file_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0}); + board.display(1, false); + + std::string wait; + std::getline(std::cin, wait); + file_src_0->detach_recursively(); +} \ No newline at end of file diff --git a/samples/mask_rcnn_sample.cpp b/samples/mask_rcnn_sample.cpp new file mode 100644 index 0000000..bdf8b80 --- /dev/null +++ b/samples/mask_rcnn_sample.cpp @@ -0,0 +1,36 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_mask_rcnn_detector_node.h" +#include "../nodes/track/vp_sort_track_node.h" +#include "../nodes/osd/vp_osd_node_v3.h" +#include "../nodes/vp_screen_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## mask rcnn sample ## +* image segmentation using mask rcnn. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/mask_rcnn.mp4", 0.6); + auto mask_rcnn_detector = std::make_shared("mask_rcnn_detector", "./vp_data/models/mask_rcnn/frozen_inference_graph.pb", "./vp_data/models/mask_rcnn/mask_rcnn_inception_v2_coco_2018_01_28.pbtxt", "./vp_data/models/coco_80classes.txt"); + auto track_0 = std::make_shared("sort_track_0"); + auto osd_v3_0 = std::make_shared("osd_v3_0", "./vp_data/font/NotoSansCJKsc-Medium.otf"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + // construct pipeline + mask_rcnn_detector->attach_to({file_src_0}); + track_0->attach_to({mask_rcnn_detector}); + osd_v3_0->attach_to({track_0}); + screen_des_0->attach_to({osd_v3_0}); + + file_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0}); + board.display(); +} \ No newline at end of file diff --git a/samples/message_broker_kafka_sample.cpp b/samples/message_broker_kafka_sample.cpp new file mode 100644 index 0000000..2d35b28 --- /dev/null +++ b/samples/message_broker_kafka_sample.cpp @@ -0,0 +1,48 @@ + +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yunet_face_detector_node.h" +#include "../nodes/infers/vp_sface_feature_encoder_node.h" +#include "../nodes/broker/vp_json_kafka_broker_node.h" +#include "../nodes/osd/vp_face_osd_node_v2.h" +#include "../nodes/vp_screen_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## message_broker_kafka_sample ## +* show how message broker node works. +* serialize vp_frame_face_target objects to json and broke to kafka. +*/ + +int main() { + VP_SET_LOG_INCLUDE_CODE_LOCATION(false); + VP_SET_LOG_INCLUDE_THREAD_ID(false); + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/face.mp4", 0.6); + auto yunet_face_detector_0 = std::make_shared("yunet_face_detector_0", "./vp_data/models/face/face_detection_yunet_2022mar.onnx"); + auto sface_face_encoder_0 = std::make_shared("sface_face_encoder_0", "./vp_data/models/face/face_recognition_sface_2021dec.onnx"); + auto json_kafka_broker_0 = std::make_shared("json_kafka_broker_0", "192.168.77.87:9092", "videopipe_topic", vp_nodes::vp_broke_for::FACE); + auto osd_0 = std::make_shared("osd_0"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + // construct pipeline + yunet_face_detector_0->attach_to({file_src_0}); + sface_face_encoder_0->attach_to({yunet_face_detector_0}); + json_kafka_broker_0->attach_to({sface_face_encoder_0}); + osd_0->attach_to({json_kafka_broker_0}); + screen_des_0->attach_to({osd_0}); + + // start pipeline + file_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0}); + board.display(1, false); + + std::string wait; + std::getline(std::cin, wait); + file_src_0->detach_recursively(); +} \ No newline at end of file diff --git a/samples/message_broker_sample.cpp b/samples/message_broker_sample.cpp new file mode 100644 index 0000000..0e71430 --- /dev/null +++ b/samples/message_broker_sample.cpp @@ -0,0 +1,44 @@ + +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yunet_face_detector_node.h" +#include "../nodes/infers/vp_sface_feature_encoder_node.h" +#include "../nodes/broker/vp_json_console_broker_node.h" +#include "../nodes/broker/vp_xml_file_broker_node.h" +#include "../nodes/osd/vp_face_osd_node_v2.h" +#include "../nodes/vp_screen_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## message broker sample ## +* show how message broker node works. +* serialize vp_frame_face_target objects to json and broke to console. +*/ + +int main() { + VP_SET_LOG_INCLUDE_CODE_LOCATION(false); + VP_SET_LOG_INCLUDE_THREAD_ID(false); + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/face.mp4", 0.6); + auto yunet_face_detector_0 = std::make_shared("yunet_face_detector_0", "./vp_data/models/face/face_detection_yunet_2022mar.onnx"); + auto sface_face_encoder_0 = std::make_shared("sface_face_encoder_0", "./vp_data/models/face/face_recognition_sface_2021dec.onnx"); + auto json_console_broker_0 = std::make_shared("json_console_broker_0", vp_nodes::vp_broke_for::FACE); + auto osd_0 = std::make_shared("osd_0"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + // construct pipeline + yunet_face_detector_0->attach_to({file_src_0}); + sface_face_encoder_0->attach_to({yunet_face_detector_0}); + json_console_broker_0->attach_to({sface_face_encoder_0}); + osd_0->attach_to({json_console_broker_0}); + screen_des_0->attach_to({osd_0}); + + file_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0}); + board.display(); +} \ No newline at end of file diff --git a/samples/message_broker_sample2.cpp b/samples/message_broker_sample2.cpp new file mode 100644 index 0000000..f934169 --- /dev/null +++ b/samples/message_broker_sample2.cpp @@ -0,0 +1,41 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_trt_vehicle_detector.h" +#include "../nodes/infers/vp_trt_vehicle_plate_detector.h" +#include "../nodes/osd/vp_osd_node_v2.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/broker/vp_xml_socket_broker_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## message broker sample2 ## +* show how message broker node works. +* serialize vp_frame_target (vp_sub_target) objects to xml and broke to socket via udp. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/plate.mp4"); + auto trt_vehicle_detector = std::make_shared("vehicle_detector", "./vp_data/models/trt/vehicle/vehicle_v8.5.trt"); + auto trt_vehicle_plate_detector = std::make_shared("vehicle_plate_detector", "./vp_data/models/trt/plate/det_v8.5.trt", "./vp_data/models/trt/plate/rec_v8.5.trt"); + auto xml_socket_broker_0 = std::make_shared("xml_socket_broker_0", "192.168.77.68", 6666); + auto osd_0 = std::make_shared("osd_0", "./vp_data/font/NotoSansCJKsc-Medium.otf"); + auto screen_des_0 = std::make_shared("screen_des_0", 0, true, vp_objects::vp_size{640, 360}); + + // construct pipeline + trt_vehicle_detector->attach_to({file_src_0}); + trt_vehicle_plate_detector->attach_to({trt_vehicle_detector}); + xml_socket_broker_0->attach_to({trt_vehicle_plate_detector}); + osd_0->attach_to({xml_socket_broker_0}); + screen_des_0->attach_to({osd_0}); + + // start pipeline + file_src_0->start(); + + // visualize pipeline for debug + vp_utils::vp_analysis_board board({file_src_0}); + board.display(); +} \ No newline at end of file diff --git a/samples/mllm_analyse_sample.cpp b/samples/mllm_analyse_sample.cpp new file mode 100644 index 0000000..0d78db0 --- /dev/null +++ b/samples/mllm_analyse_sample.cpp @@ -0,0 +1,56 @@ +#include "../nodes/vp_image_src_node.h" +#include "../nodes/infers/vp_mllm_analyser_node.h" +#include "../nodes/osd/vp_mllm_osd_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtmp_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## mllm_analyse_sample ## +* image(frame) analyse based on Multimodal Large Language Model(from Ollama). +* read images from disk and analyse the image using MLLM using the prepared prompt. +*/ +int main() { + VP_SET_LOG_INCLUDE_CODE_LOCATION(false); + VP_SET_LOG_INCLUDE_THREAD_ID(false); + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto image_src_0 = std::make_shared("image_file_src_0", 0, "./vp_data/test_images/llm/writing/%d.jpg", 2, 0.5); + auto understanding_prompt = "一句话描述图片内容，要求包含：\n" + "1. 对天气的描述\n" + "2. 对环境的描述\n" + "3. 对位置的描述（如果可以从图片上的文字信息得出）\n" + "4. 字数不超过50字" + "你的输出结果是："; + auto writing_prompt = "根据图片写一段故事，要求包含：\n" + "1. 完整的故事结构\n" + "2. 故事内容要包含时间、地点、人物等元素" + "3. 字数不超过50字" + "你的输出结果是："; + auto mllm_analyser_0 = std::make_shared("mllm_analyser_0", // node name + "minicpm-v:8b", // mllm model name (support image as input) + writing_prompt, // prompt + "http://192.168.77.219:11434", // api base url + "", // api key (not required by Ollama) + llmlib::LLMBackendType::Ollama); // backend type, make sure Ollama is installed at 192.168.77.219 + auto mllm_osd_0 = std::make_shared("osd_0", "./vp_data/font/NotoSansCJKsc-Medium.otf"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + // construct pipeline + mllm_analyser_0->attach_to({image_src_0}); + mllm_osd_0->attach_to({mllm_analyser_0}); + screen_des_0->attach_to({mllm_osd_0}); + + image_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({image_src_0}); + board.display(1, false); + + std::string wait; + std::getline(std::cin, wait); + image_src_0->detach_recursively(); +} \ No newline at end of file diff --git a/samples/mllm_analyse_sample_openai.cpp b/samples/mllm_analyse_sample_openai.cpp new file mode 100644 index 0000000..8380554 --- /dev/null +++ b/samples/mllm_analyse_sample_openai.cpp @@ -0,0 +1,50 @@ +#include "../nodes/vp_image_src_node.h" +#include "../nodes/infers/vp_mllm_analyser_node.h" +#include "../nodes/osd/vp_mllm_osd_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtmp_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## mllm_analyse_sample_openai ## +* image(frame) analyse based on Multimodal Large Language Model(from aliyun or other OpenAI-compatible api services). +* read images from disk and analyse the image using MLLM using the prepared prompt. +*/ +int main() { + VP_SET_LOG_INCLUDE_CODE_LOCATION(false); + VP_SET_LOG_INCLUDE_THREAD_ID(false); + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto image_src_0 = std::make_shared("image_file_src_0", 0, "./vp_data/test_images/llm/understanding/%d.jpg", 2, 0.5); + auto writing_prompt = "给图片打标签，要求包含：\n" + "1. 先仔细观察图片内容，为图片赋予适合的标签\n" + "2. 给出的标签最多不超过5个\n" + "3. 输出按以下格式：\n" + "通过仔细观察图片，可以为图片赋予这些标签：['标签1', '标签2', '标签3']。"; + auto mllm_analyser_0 = std::make_shared("mllm_analyser_0", // node name + "qwen-vl-max", // mllm model name (from aliyun, support image as input) + writing_prompt, // prompt + "https://dashscope.aliyuncs.com/compatible-mode/v1", // api base url + "sk-XXX", // api key (from aliyun) + llmlib::LLMBackendType::OpenAI); // backend type + auto mllm_osd_0 = std::make_shared("osd_0", "./vp_data/font/NotoSansCJKsc-Medium.otf"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + // construct pipeline + mllm_analyser_0->attach_to({image_src_0}); + mllm_osd_0->attach_to({mllm_analyser_0}); + screen_des_0->attach_to({mllm_osd_0}); + + image_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({image_src_0}); + board.display(1, false); + + std::string wait; + std::getline(std::cin, wait); + image_src_0->detach_recursively(); +} \ No newline at end of file diff --git a/samples/multi_detectors_and_classifiers_sample.cpp b/samples/multi_detectors_and_classifiers_sample.cpp new file mode 100644 index 0000000..ce2494d --- /dev/null +++ b/samples/multi_detectors_and_classifiers_sample.cpp @@ -0,0 +1,48 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yolo_detector_node.h" +#include "../nodes/infers/vp_classifier_node.h" +#include "../nodes/osd/vp_osd_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## multi detectors and classifiers sample ## +* show multi infer nodes work together. +* 1 detector and 2 classifiers applied on primary class ids(1/2/3). +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/vehicle_stop.mp4", 0.6); + /* primary detector */ + // labels for detector model + // 0 - person + // 1 - car + // 2 - bus + // 3 - truck + // 4 - 2wheel + auto primary_detector = std::make_shared("primary_detector", "./vp_data/models/det_cls/yolov3-tiny-2022-0721_best.weights", "./vp_data/models/det_cls/yolov3-tiny-2022-0721.cfg", "./vp_data/models/det_cls/yolov3_tiny_5classes.txt", 416, 416, 1); + /* secondary classifier 1, applied to car(1)/bus(2)/truck(3) only */ + auto _1st_classifier = std::make_shared("1st_classifier", "./vp_data/models/det_cls/vehicle/resnet18-batch=N-type_view_0322_nhwc.onnx", "", "./vp_data/models/det_cls/vehicle/vehicle_types.txt", 224, 224, 1, std::vector{1, 2, 3}, 20, 20, 10, false, 1, cv::Scalar(), cv::Scalar(), true, true); + /* secondary classifier 2, applied to car(1)/bus(2)/truck(3) only */ + auto _2nd_classifier = std::make_shared("2nd_classifier", "./vp_data/models/det_cls/vehicle/resnet18-batch=N-color_view_0322_nhwc.onnx", "", "./vp_data/models/det_cls/vehicle/vehicle_colors.txt", 224, 224, 1, std::vector{1, 2, 3}, 20, 20, 10, false, 1, cv::Scalar(), cv::Scalar(), true, true); + auto osd_0 = std::make_shared("osd_0"); + auto screen_des_0 = std::make_shared("screen_des_o", 0); + + // construct pipeline + primary_detector->attach_to({file_src_0}); + _1st_classifier->attach_to({primary_detector}); + _2nd_classifier->attach_to({_1st_classifier}); + osd_0->attach_to({_2nd_classifier}); + screen_des_0->attach_to({osd_0}); + + // start + file_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0}); + board.display(); +} \ No newline at end of file diff --git a/samples/multi_detectors_sample.cpp b/samples/multi_detectors_sample.cpp new file mode 100644 index 0000000..29317f5 --- /dev/null +++ b/samples/multi_detectors_sample.cpp @@ -0,0 +1,49 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yolo_detector_node.h" +#include "../nodes/osd/vp_osd_node.h" +#include "../nodes/vp_split_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtsp_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## multi_detectors_sample ## +* detect obstacles AND vehicles using yolo on road. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/unclear.mp4", 0.5); + auto file_src_1 = std::make_shared("file_src_1", 1, "./vp_data/test_video/roadblock.mp4", 0.6); + auto obstacle_detector = std::make_shared("obstacle_detector", "./vp_data/models/det_cls/obstacles_yolov5s.onnx", "", "./vp_data/models/det_cls/obstacles_2classes.txt", 640, 640); + // MUST set class_id_offset for the 2nd detector which is equal with total classes of the 1st detectors + auto vehicle_detector = std::make_shared("vehicle_detector", "./vp_data/models/det_cls/yolov3-tiny-2022-0721_best.weights", "./vp_data/models/det_cls/yolov3-tiny-2022-0721.cfg", "./vp_data/models/det_cls/yolov3_tiny_5classes.txt", 416, 416, 1, 2); + auto osd = std::make_shared("osd"); + auto split = std::make_shared("split_by_channel", true); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + auto srceen_des_1 = std::make_shared("screen_des_1", 1); + + // construct pipeline + obstacle_detector->attach_to({file_src_0, file_src_1}); + vehicle_detector->attach_to({obstacle_detector}); + osd->attach_to({vehicle_detector}); + split->attach_to({osd}); + screen_des_0->attach_to({split}); + srceen_des_1->attach_to({split}); + + file_src_0->start(); + file_src_1->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0, file_src_1}); + board.display(1, false); + + std::string wait; + std::getline(std::cin, wait); + file_src_0->detach_recursively(); + file_src_1->detach_recursively(); +} \ No newline at end of file diff --git a/samples/multi_trt_infer_nodes_sample.cpp b/samples/multi_trt_infer_nodes_sample.cpp new file mode 100644 index 0000000..c46f4f2 --- /dev/null +++ b/samples/multi_trt_infer_nodes_sample.cpp @@ -0,0 +1,43 @@ +#include "../nodes/vp_image_src_node.h" +#include "../nodes/infers/vp_trt_vehicle_detector.h" +#include "../nodes/infers/vp_trt_vehicle_color_classifier.h" +#include "../nodes/infers/vp_trt_vehicle_type_classifier.h" +#include "../nodes/infers/vp_trt_vehicle_feature_encoder.h" +#include "../nodes/track/vp_sort_track_node.h" +#include "../nodes/osd/vp_osd_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## multi_trt_infer_nodes_sample ## +* detect/classify/encoding on vehicle object using tensorrt +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto image_src_0 = std::make_shared("image_src_0", 0, "./vp_data/test_images/vehicle/%d.jpg"); + auto trt_vehicle_detector = std::make_shared("trt_detector", "./vp_data/models/trt/vehicle/vehicle_v8.5.trt"); + auto trt_vehicle_color_classifier = std::make_shared("trt_color_cls", "./vp_data/models/trt/vehicle/vehicle_color_v8.5.trt", std::vector{0, 1, 2}); + auto trt_vehicle_type_classifier = std::make_shared("trt_type_cls", "./vp_data/models/trt/vehicle/vehicle_type_v8.5.trt", std::vector{0, 1, 2}); + auto trt_vehicle_feature_encoder = std::make_shared("trt_encoder", "./vp_data/models/trt/vehicle/vehicle_embedding_v8.5.trt", std::vector{0, 1, 2}); + auto osd_0 = std::make_shared("osd_0"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + // construct pipeline + trt_vehicle_detector->attach_to({image_src_0}); + trt_vehicle_color_classifier->attach_to({trt_vehicle_detector}); + trt_vehicle_type_classifier->attach_to({trt_vehicle_color_classifier}); + trt_vehicle_feature_encoder->attach_to({trt_vehicle_type_classifier}); + osd_0->attach_to({trt_vehicle_feature_encoder}); + screen_des_0->attach_to({osd_0}); + + // start pipeline + image_src_0->start(); + + // visualize pipeline for debug + vp_utils::vp_analysis_board board({image_src_0}); + board.display(); +} \ No newline at end of file diff --git a/samples/nv_hard_codec_sample.cpp b/samples/nv_hard_codec_sample.cpp new file mode 100644 index 0000000..c4614eb --- /dev/null +++ b/samples/nv_hard_codec_sample.cpp @@ -0,0 +1,55 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yunet_face_detector_node.h" +#include "../nodes/infers/vp_sface_feature_encoder_node.h" +#include "../nodes/osd/vp_face_osd_node_v2.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtmp_des_node.h" +#include "../nodes/vp_rtmp_des_node.h" +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## nv_hard_codec_sample ## +* use hardware-based `nvv4l2decoder`/`nvv4l2h264enc` gstreamer plugins (come from DeepStream 4.0+) to decode/encode video stream, which would occupy NVIDIA GPUs. +* run `nvidia-smi -a` command to watch the Utilization of GPU/Decode/Encode . +* +* for more information: https://github.com/sherlockchou86/video_pipe_c/blob/master/doc/env.md#about-hardware-acceleration +*/ + +int main() { + VP_SET_LOG_INCLUDE_CODE_LOCATION(false); + VP_SET_LOG_INCLUDE_THREAD_ID(false); + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/face.mp4", 0.6, true, "nvv4l2decoder ! nvvideoconvert"); + auto yunet_face_detector_0 = std::make_shared("yunet_face_detector_0", "./vp_data/models/face/face_detection_yunet_2022mar.onnx"); + auto sface_face_encoder_0 = std::make_shared("sface_face_encoder_0", "./vp_data/models/face/face_recognition_sface_2021dec.onnx"); + auto osd_0 = std::make_shared("osd_0"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + auto rtmp_des_0 = std::make_shared("rtmp_des_0", 0, "rtmp://192.168.77.60/live/10000", vp_objects::vp_size(), 1024 * 4000, true, "nvvideoconvert ! nvv4l2h264enc"); + + // construct pipeline + yunet_face_detector_0->attach_to({file_src_0}); + sface_face_encoder_0->attach_to({yunet_face_detector_0}); + osd_0->attach_to({sface_face_encoder_0}); + screen_des_0->attach_to({osd_0}); + rtmp_des_0->attach_to({osd_0}); + + file_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0}); + board.display(1, false); + + std::string wait; + std::getline(std::cin, wait); + file_src_0->detach_recursively(); + +/* + cv::VideoCapture cap("filesrc location=/windows2/zhzhi/vp_data/test_video/face.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! nvvideoconvert ! appsink"); + cv::Mat frame; + cap.read(frame); + return 0; +*/ +} \ No newline at end of file diff --git a/samples/obstacle_detect_sample.cpp b/samples/obstacle_detect_sample.cpp new file mode 100644 index 0000000..fd2bb7b --- /dev/null +++ b/samples/obstacle_detect_sample.cpp @@ -0,0 +1,46 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yolo_detector_node.h" +#include "../nodes/osd/vp_osd_node.h" +#include "../nodes/vp_split_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtsp_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## obstacle_detect_sample ## +* detect obstacles using yolo on road. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/unclear.mp4", 0.5); + auto file_src_1 = std::make_shared("file_src_1", 1, "./vp_data/test_video/roadblock.mp4", 0.5); + auto yolo_detector = std::make_shared("obstacle_detector", "./vp_data/models/det_cls/obstacles_yolov5s.onnx", "", "./vp_data/models/det_cls/obstacles_2classes.txt", 640, 640); + auto osd = std::make_shared("osd"); + auto split = std::make_shared("split_by_channel", true); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + auto srceen_des_1 = std::make_shared("screen_des_1", 1); + + // construct pipeline + yolo_detector->attach_to({file_src_0, file_src_1}); + osd->attach_to({yolo_detector}); + split->attach_to({osd}); + screen_des_0->attach_to({split}); + srceen_des_1->attach_to({split}); + + file_src_0->start(); + file_src_1->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0, file_src_1}); + board.display(1, false); + + std::string wait; + std::getline(std::cin, wait); + file_src_0->detach_recursively(); + file_src_1->detach_recursively(); +} \ No newline at end of file diff --git a/samples/openpose_sample.cpp b/samples/openpose_sample.cpp new file mode 100644 index 0000000..ceded4e --- /dev/null +++ b/samples/openpose_sample.cpp @@ -0,0 +1,33 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_openpose_detector_node.h" +#include "../nodes/osd/vp_pose_osd_node.h" +#include "../nodes/vp_screen_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## openpose sample ## +* pose estimation by OpenPose network. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/pose.mp4"); + auto openpose_detector = std::make_shared("openpose_detector", "./vp_data/models/openpose/pose/body_25_pose_iter_584000.caffemodel", "./vp_data/models/openpose/pose/body_25_pose_deploy.prototxt", "", 368, 368, 1, 0, 0.1, vp_objects::vp_pose_type::body_25); + auto pose_osd_0 = std::make_shared("pose_osd_0"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + // construct pipeline + openpose_detector->attach_to({file_src_0}); + pose_osd_0->attach_to({openpose_detector}); + screen_des_0->attach_to({pose_osd_0}); + + file_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0}); + board.display(); +} \ No newline at end of file diff --git a/samples/paddle_infer_sample.cpp b/samples/paddle_infer_sample.cpp new file mode 100644 index 0000000..58a724c --- /dev/null +++ b/samples/paddle_infer_sample.cpp @@ -0,0 +1,40 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_ppocr_text_detector_node.h" +#include "../nodes/osd/vp_text_osd_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtmp_des_node.h" +#include "../utils/analysis_board/vp_analysis_board.h" +#include "../nodes/vp_file_des_node.h" + +/* +* ## paddle infer sample ## +* ocr based on paddle (install paddle_inference first!) +* 1 video input and 2 outputs (screen, rtmp) +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/ocr.mp4", 0.4); + auto ppocr_text_detector = std::make_shared("ppocr_text_detector", "./vp_data/models/text/ppocr/ch_PP-OCRv3_det_infer","./vp_data/models/text/ppocr/ch_ppocr_mobile_v2.0_cls_infer","./vp_data/models/text/ppocr/ch_PP-OCRv3_rec_infer","./vp_data/models/text/ppocr/ppocr_keys_v1.txt"); + auto osd_0 = std::make_shared("osd_0", "./vp_data/font/NotoSansCJKsc-Medium.otf"); + auto screen_des_0 = std::make_shared("screen_des_0", 0, true); + auto rtmp_des_0 = std::make_shared("rtmp_des_0", 0, "rtmp://192.168.77.60/live/10000"); + + // construct pipeline + ppocr_text_detector->attach_to({file_src_0}); + osd_0->attach_to({ppocr_text_detector}); + + // split into 2 sub branches automatically + screen_des_0->attach_to({osd_0}); + rtmp_des_0->attach_to({osd_0}); + + // start pipeline + file_src_0->start(); + + // visualize pipeline for debug + vp_utils::vp_analysis_board board({file_src_0}); + board.display(); +} \ No newline at end of file diff --git a/samples/plate_recognize_sample.cpp b/samples/plate_recognize_sample.cpp new file mode 100644 index 0000000..9d62315 --- /dev/null +++ b/samples/plate_recognize_sample.cpp @@ -0,0 +1,33 @@ +#include "../nodes/vp_image_src_node.h" +#include "../nodes/infers/vp_trt_vehicle_plate_detector_v2.h" +#include "../nodes/osd/vp_plate_osd_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## plate recognize sample ## +* detect and recognize plate in the whole frame +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto image_src_0 = std::make_shared("image_src_0", 0, "./vp_data/test_images/plates/%d.jpg", 1); + auto plate_detector = std::make_shared("plate_detector", "./vp_data/models/trt/plate/det_v8.5.trt", "./vp_data/models/trt/plate/rec_v8.5.trt"); + auto osd_0 = std::make_shared("osd_0", "./vp_data/font/NotoSansCJKsc-Medium.otf"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + // construct pipeline + plate_detector->attach_to({image_src_0}); + osd_0->attach_to({plate_detector}); + screen_des_0->attach_to({osd_0}); + + // start pipeline + image_src_0->start(); + + // visualize pipeline for debug + vp_utils::vp_analysis_board board({image_src_0}); + board.display(); +} \ No newline at end of file diff --git a/samples/record_sample.cpp b/samples/record_sample.cpp new file mode 100644 index 0000000..6d6c8e5 --- /dev/null +++ b/samples/record_sample.cpp @@ -0,0 +1,109 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yunet_face_detector_node.h" +#include "../nodes/infers/vp_sface_feature_encoder_node.h" +#include "../nodes/track/vp_sort_track_node.h" +#include "../nodes/osd/vp_face_osd_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtmp_des_node.h" +#include "../nodes/vp_split_node.h" +#include "../nodes/record/vp_record_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## record sample ## +* show how to use vp_record_node to record image and video. +* NOTE: +* the recording signal in this demo is triggered by users outside pipe (via calling vp_src_node::record_video_manually or vp_src_node::record_image_manually) +* in product situations, recording signal is triggered inside pipe automatically. +*/ + +int main() { + VP_SET_LOG_INCLUDE_THREAD_ID(false); + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_SET_LOG_TO_CONSOLE(false); // need interact on console + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/face.mp4", 0.6); + auto file_src_1 = std::make_shared("file_src_1", 1, "./vp_data/test_video/face2.mp4", 0.6); + auto yunet_face_detector = std::make_shared("yunet_face_detector_0", "./vp_data/models/face/face_detection_yunet_2022mar.onnx"); + auto sface_face_encoder = std::make_shared("sface_face_encoder_0", "./vp_data/models/face/face_recognition_sface_2021dec.onnx"); + auto track = std::make_shared("track", vp_nodes::vp_track_for::FACE); + auto osd = std::make_shared("osd"); + auto recorder = std::make_shared("recorder", "./record", "./record"); + + auto split = std::make_shared("split", true); // split by channel index + auto screen_des_0 = std::make_shared("screen_des_0", 0); + auto screen_des_1 = std::make_shared("screen_des_1", 1); + + // construct pipeline + yunet_face_detector->attach_to({file_src_0, file_src_1}); + sface_face_encoder->attach_to({yunet_face_detector}); + track->attach_to({sface_face_encoder}); + osd->attach_to({track}); + recorder->attach_to({osd}); + split->attach_to({recorder}); + // split by vp_split_node + screen_des_0->attach_to({split}); + screen_des_1->attach_to({split}); + + /* + * set hookers for vp_record_node when task compeleted + */ + // define hooker + auto record_hooker = [](int channel, vp_nodes::vp_record_info record_info) { + auto record_type = record_info.record_type == vp_nodes::vp_record_type::IMAGE ? "image" : "video"; + + std::cout << "channel:[" << channel << "] [" << record_type << "]" << " record task completed! full path: " << record_info.full_record_path << std::endl; + }; + recorder->set_image_record_complete_hooker(record_hooker); + recorder->set_video_record_complete_hooker(record_hooker); + + // start channels + file_src_0->start(); + file_src_1->start(); + + // for debug purpose + std::vector> src_nodes_in_pipe{file_src_0, file_src_1}; + vp_utils::vp_analysis_board board(src_nodes_in_pipe); + board.display(1, false); // no block + + + /* interact from console */ + /* no except check */ + std::string input; + std::getline(std::cin, input); + // input format: `image channel` or `video channel`, like `video 0` means start recording video at channel 0 + auto inputs = vp_utils::string_split(input, ' '); + while (inputs[0] != "quit") { + // no except check + auto command = inputs[0]; + auto index = std::stoi(inputs[1]); + auto src_by_channel = std::dynamic_pointer_cast(src_nodes_in_pipe[index]); + if (command == "video") { + src_by_channel->record_video_manually(true); // debug api + // or + // src_by_channel->record_video_manually(true, 5); + // src_by_channel->record_video_manually(false, 20); + } + else if (command == "image") { + src_by_channel->record_image_manually(); // debug api + // or + // src_by_channel->record_image_manually(true); + } + else { + std::cout << "invalid command!" << std::endl; + } + std::getline(std::cin, input); + inputs = vp_utils::string_split(input, ' '); + if (inputs.size() != 2) { + std::cout << "invalid input!" << std::endl; + break; + } + } + + std::cout << "record sample exits..." << std::endl; + file_src_0->detach_recursively(); + file_src_1->detach_recursively(); +} \ No newline at end of file diff --git a/samples/rtmp_src_sample.cpp b/samples/rtmp_src_sample.cpp new file mode 100644 index 0000000..a6fc623 --- /dev/null +++ b/samples/rtmp_src_sample.cpp @@ -0,0 +1,44 @@ +#include "../nodes/vp_rtmp_src_node.h" +#include "../nodes/infers/vp_trt_vehicle_detector.h" +#include "../nodes/osd/vp_osd_node.h" +#include "../nodes/track/vp_sort_track_node.h" +#include "../nodes/vp_rtmp_des_node.h" +#include "../nodes/vp_screen_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## rtmp_src_sample ## +* 1 rtmp video input, 1 infer task, and 1 output. +*/ + +int main() { + VP_SET_LOG_INCLUDE_CODE_LOCATION(false); + VP_SET_LOG_INCLUDE_THREAD_ID(false); + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto rtmp_src_0 = std::make_shared("rtmp_src_0", 0, "rtmp://192.168.77.196/live/1000", 0.6); + auto trt_vehicle_detector = std::make_shared("vehicle_detector", "./vp_data/models/trt/vehicle/vehicle_v8.5.trt"); + auto track_0 = std::make_shared("track_0"); + auto osd_0 = std::make_shared("osd_0", "./vp_data/font/NotoSansCJKsc-Medium.otf"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + auto rtmp_des_0 = std::make_shared("rtmp_des_0", 0, "rtmp://192.168.77.196/live/2000", vp_objects::vp_size{1280, 720}, 1024 * 2); + + // construct pipeline + trt_vehicle_detector->attach_to({rtmp_src_0}); + track_0->attach_to({trt_vehicle_detector}); + osd_0->attach_to({track_0}); + rtmp_des_0->attach_to({osd_0}); + + rtmp_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({rtmp_src_0}); + board.display(1, false); + + std::string wait; + std::getline(std::cin, wait); + rtmp_src_0->detach_recursively(); +} \ No newline at end of file diff --git a/samples/rtsp_des_sample.cpp b/samples/rtsp_des_sample.cpp new file mode 100644 index 0000000..4d45ae2 --- /dev/null +++ b/samples/rtsp_des_sample.cpp @@ -0,0 +1,50 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yolo_detector_node.h" +#include "../nodes/osd/vp_osd_node.h" +#include "../nodes/vp_split_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtsp_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## rtsp_des_sample ## +* show how vp_rtsp_des_node works, push video stream via rtsp, no specialized rtsp server needed. +* visit `rtsp://server-ip:rtsp_port/rtsp_name directly. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/vehicle_count.mp4", 0.5); + auto file_src_1 = std::make_shared("file_src_1", 1, "./vp_data/test_video/vehicle_stop.mp4", 0.5); + auto yolo_detector = std::make_shared("yolo_detector", "./vp_data/models/det_cls/yolov3-tiny-2022-0721_best.weights", "./vp_data/models/det_cls/yolov3-tiny-2022-0721.cfg", "./vp_data/models/det_cls/yolov3_tiny_5classes.txt"); + auto osd = std::make_shared("osd"); + auto split = std::make_shared("split_by_channel", true); + //auto screen_des_0 = std::make_shared("screen_des_0", 0); + auto rtsp_des_0 = std::make_shared("rtsp_des_0", 0, 8000, "rtsp_0"); + + //auto srceen_des_1 = std::make_shared("screen_des_1", 1); + auto rtsp_des_1 = std::make_shared("rtsp_des_1", 1, 8000, "rtsp_1"); + + // construct pipeline + yolo_detector->attach_to({file_src_0, file_src_1}); + osd->attach_to({yolo_detector}); + split->attach_to({osd}); + rtsp_des_0->attach_to({split}); + rtsp_des_1->attach_to({split}); + + file_src_0->start(); + file_src_1->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0, file_src_1}); + board.display(1, false); + + std::string wait; + std::getline(std::cin, wait); + file_src_0->detach_recursively(); + file_src_1->detach_recursively(); +} \ No newline at end of file diff --git a/samples/rtsp_src_sample.cpp b/samples/rtsp_src_sample.cpp new file mode 100644 index 0000000..664cebe --- /dev/null +++ b/samples/rtsp_src_sample.cpp @@ -0,0 +1,48 @@ +#include "../nodes/vp_rtsp_src_node.h" +#include "../nodes/infers/vp_yunet_face_detector_node.h" +#include "../nodes/infers/vp_sface_feature_encoder_node.h" +#include "../nodes/osd/vp_face_osd_node_v2.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtmp_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## rtsp_src_sample ## +* 1 rtsp video input, 1 infer task, and 1 output. +* support switching/restarting input rtsp stream. +*/ + +int main() { + VP_SET_LOG_INCLUDE_CODE_LOCATION(false); + VP_SET_LOG_INCLUDE_THREAD_ID(false); + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto rtsp_src_0 = std::make_shared("rtsp_src_0", 0, "rtsp://192.168.77.213/live/mainstream", 0.6); + auto yunet_face_detector_0 = std::make_shared("yunet_face_detector_0", "./vp_data/models/face/face_detection_yunet_2022mar.onnx"); + auto sface_face_encoder_0 = std::make_shared("sface_face_encoder_0", "./vp_data/models/face/face_recognition_sface_2021dec.onnx"); + auto osd_0 = std::make_shared("osd_0"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + // construct pipeline + yunet_face_detector_0->attach_to({rtsp_src_0}); + sface_face_encoder_0->attach_to({yunet_face_detector_0}); + osd_0->attach_to({sface_face_encoder_0}); + screen_des_0->attach_to({osd_0}); + + rtsp_src_0->start(); + + /* manually switching/restarting RTSP video sources, where the video sources have different widths, heights, and frame rates. + continuously print width, height, fps. */ + while (true) { + auto width = rtsp_src_0->get_original_width(); + auto height = rtsp_src_0->get_original_height(); + auto fps = rtsp_src_0->get_original_fps(); + std::cout << "original_width: " << width << "original_height: " << height << "original_fps: " << fps << std::endl; + this_thread::sleep_for(std::chrono::milliseconds(1000)); + } + + rtsp_src_0->detach_recursively(); +} \ No newline at end of file diff --git a/samples/skip_sample.cpp b/samples/skip_sample.cpp new file mode 100644 index 0000000..06b024a --- /dev/null +++ b/samples/skip_sample.cpp @@ -0,0 +1,43 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/vp_rtsp_src_node.h" +#include "../nodes/infers/vp_yolo_detector_node.h" +#include "../nodes/osd/vp_osd_node.h" +#include "../nodes/vp_split_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtmp_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## skip_sample ## +* 2 inputs , and skip 2 frames every 3 frames for the 2nd channel. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/vehicle_count.mp4", 0.5); + auto rtsp_src_1 = std::make_shared("rtsp_src_1", 1, "rtsp://admin:admin12345@192.168.3.157", 0.4, "avdec_h264", 2); // skip 2 frames every 3 frames + auto yolo_detector = std::make_shared("yolo_detector", "./vp_data/models/det_cls/yolov3-tiny-2022-0721_best.weights", "./vp_data/models/det_cls/yolov3-tiny-2022-0721.cfg", "./vp_data/models/det_cls/yolov3_tiny_5classes.txt"); + auto split = std::make_shared("split", true); + auto osd = std::make_shared("osd", "./vp_data/font/NotoSansCJKsc-Medium.otf"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + auto rtmp_des_1 = std::make_shared("rtmp_des_0", 1, "rtmp://192.168.77.60/live/10000"); + + // construct pipeline + yolo_detector->attach_to({file_src_0, rtsp_src_1}); + osd->attach_to({yolo_detector}); + split->attach_to({osd}); + screen_des_0->attach_to({split}); + rtmp_des_1->attach_to({split}); + + // start channels + file_src_0->start(); + rtsp_src_1->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0, rtsp_src_1}); + board.display(); +} \ No newline at end of file diff --git a/samples/src_des_sample.cpp b/samples/src_des_sample.cpp new file mode 100644 index 0000000..d47e56f --- /dev/null +++ b/samples/src_des_sample.cpp @@ -0,0 +1,68 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/vp_rtsp_src_node.h" +#include "../nodes/vp_udp_src_node.h" + +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtmp_des_node.h" +#include "../nodes/vp_fake_des_node.h" +#include "../nodes/vp_file_des_node.h" + +#include "../nodes/infers/vp_trt_vehicle_detector.h" +#include "../nodes/osd/vp_osd_node.h" +#include "../nodes/vp_split_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## src des sample ## +* show how src nodes and des nodes work +* 3 (file, rtsp, udp) input and merge into 1 infer task, then resume to 3 branches for outputs (screen, rtmp, fake) +*/ + +int main() { + + // log config + // ... + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/vehicle_count.mp4", 0.5); + auto rtsp_src_1 = std::make_shared("rtsp_src_1", 1, "rtsp://admin:admin12345@192.168.77.203", 0.4); + auto udp_src_2 = std::make_shared("udp_src_2", 2, 6000, 0.3); + + auto trt_vehicle_detector = std::make_shared("trt_vehicle_detector","./vp_data/models/trt/vehicle/vehicle_v8.5.trt"); + + auto split = std::make_shared("", true); + + auto osd_0 = std::make_shared("osd_0", "./vp_data/font/NotoSansCJKsc-Medium.otf"); + auto osd_1 = std::make_shared("osd_1", "./vp_data/font/NotoSansCJKsc-Medium.otf"); + auto osd_2 = std::make_shared("osd_2", "./vp_data/font/NotoSansCJKsc-Medium.otf"); + + auto screen_des_0 = std::make_shared("screen_des_0", 0); + auto rtmp_des_1 = std::make_shared("rtmp_des_0", 1, "rtmp://192.168.77.60/live/10000"); + auto fake_des_2 = std::make_shared("fake_des_2", 2); + + // construct pipeline + // auto merge + trt_vehicle_detector->attach_to({file_src_0, rtsp_src_1, udp_src_2}); + + split->attach_to({trt_vehicle_detector}); + + // resume to 3 branches + osd_0->attach_to({split}); + osd_1->attach_to({split}); + osd_2->attach_to({split}); + + screen_des_0->attach_to({osd_0}); + rtmp_des_1->attach_to({osd_1}); + fake_des_2->attach_to({osd_2}); + + // start channels + file_src_0->start(); + rtsp_src_1->start(); + udp_src_2->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0, rtsp_src_1, udp_src_2}); + board.display(); +} \ No newline at end of file diff --git a/samples/trt_infer_sample.cpp b/samples/trt_infer_sample.cpp new file mode 100644 index 0000000..3f0ab62 --- /dev/null +++ b/samples/trt_infer_sample.cpp @@ -0,0 +1,45 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_trt_vehicle_detector.h" +#include "../nodes/infers/vp_trt_vehicle_plate_detector.h" +#include "../nodes/osd/vp_osd_node_v2.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtmp_des_node.h" +#include "../utils/analysis_board/vp_analysis_board.h" +#include "../nodes/vp_file_des_node.h" + +/* +* ## trt infer sample ## +* vehicle and plate detector based on tensorrt (install tensorrt first!) +* 1 video input and 3 outputs (screen, file, rtmp) +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/plate.mp4"); + auto trt_vehicle_detector = std::make_shared("vehicle_detector", "./vp_data/models/trt/vehicle/vehicle_v8.5.trt"); + auto trt_vehicle_plate_detector = std::make_shared("vehicle_plate_detector", "./vp_data/models/trt/plate/det_v8.5.trt", "./vp_data/models/trt/plate/rec_v8.5.trt"); + auto osd_0 = std::make_shared("osd_0", "./vp_data/font/NotoSansCJKsc-Medium.otf"); + auto screen_des_0 = std::make_shared("screen_des_0", 0, true, vp_objects::vp_size{640, 360}); + auto rtmp_des_0 = std::make_shared("rtmp_des_0", 0, "rtmp://192.168.77.60/live/10000", vp_objects::vp_size{1280, 720}); + auto file_des_0 = std::make_shared("file_des_0", 0, "."); + + // construct pipeline + trt_vehicle_detector->attach_to({file_src_0}); + trt_vehicle_plate_detector->attach_to({trt_vehicle_detector}); + osd_0->attach_to({trt_vehicle_plate_detector}); + + // split into 3 sub branches automatically + screen_des_0->attach_to({osd_0}); + rtmp_des_0->attach_to({osd_0}); + file_des_0->attach_to({osd_0}); + + // start pipeline + file_src_0->start(); + + // visualize pipeline for debug + vp_utils::vp_analysis_board board({file_src_0}); + board.display(); +} \ No newline at end of file diff --git a/samples/trt_yolov8_sample.cpp b/samples/trt_yolov8_sample.cpp new file mode 100644 index 0000000..b71448c --- /dev/null +++ b/samples/trt_yolov8_sample.cpp @@ -0,0 +1,60 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_trt_yolov8_detector.h" +#include "../nodes/infers/vp_trt_yolov8_seg_detector.h" +#include "../nodes/infers/vp_trt_yolov8_pose_detector.h" +#include "../nodes/osd/vp_osd_node_v3.h" +#include "../nodes/osd/vp_pose_osd_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## trt yolov8 sample ## +* detection/segmentation/pose_estimation using yolov8 based on tensorrt +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes for 1st pipeline + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/face2.mp4"); + auto yolo8_detector = std::make_shared("yolo8_detector", "./vp_data/models/trt/others/yolov8s_v8.5.engine", "./vp_data/models/coco_80classes.txt"); + auto osd_0 = std::make_shared("osd_0", "./vp_data/font/NotoSansCJKsc-Medium.otf"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + // create nodes for 2nd pipeline + auto file_src_1 = std::make_shared("file_src_1", 1, "./vp_data/test_video/face2.mp4"); + auto yolo8_seg_detector = std::make_shared("yolo8_seg_detector", "./vp_data/models/trt/others/yolov8s-seg_v8.5.engine", "./vp_data/models/coco_80classes.txt"); + auto osd_1 = std::make_shared("osd_1", "./vp_data/font/NotoSansCJKsc-Medium.otf"); + auto screen_des_1 = std::make_shared("screen_des_1", 1); + + // create nodes for 3rd pipeline + auto file_src_2 = std::make_shared("file_src_2", 2, "./vp_data/test_video/face2.mp4"); + auto yolo8_pose_detector = std::make_shared("yolo8_pose_detector", "./vp_data/models/trt/others/yolov8s-pose_v8.5.engine"); + auto osd_2 = std::make_shared("osd_2"); + auto screen_des_2 = std::make_shared("screen_des_2", 2); + + // construct 1st pipeline + yolo8_detector->attach_to({file_src_0}); + osd_0->attach_to({yolo8_detector}); + screen_des_0->attach_to({osd_0}); + + // construct 2nd pipeline + yolo8_seg_detector->attach_to({file_src_1}); + osd_1->attach_to({yolo8_seg_detector}); + screen_des_1->attach_to({osd_1}); + + // construct 3rd pipeline + yolo8_pose_detector->attach_to({file_src_2}); + osd_2->attach_to({yolo8_pose_detector}); + screen_des_2->attach_to({osd_2}); + + // start pipelines + file_src_0->start(); + file_src_1->start(); + file_src_2->start(); + + // visualize pipeline for debug + vp_utils::vp_analysis_board board({file_src_0, file_src_1, file_src_2}); + board.display(); +} \ No newline at end of file diff --git a/samples/trt_yolov8_sample2.cpp b/samples/trt_yolov8_sample2.cpp new file mode 100644 index 0000000..b1d22e8 --- /dev/null +++ b/samples/trt_yolov8_sample2.cpp @@ -0,0 +1,36 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_trt_yolov8_detector.h" +#include "../nodes/infers/vp_trt_yolov8_classifier.h" +#include "../nodes/osd/vp_osd_node_v3.h" +#include "../nodes/vp_screen_des_node.h" +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## trt yolov8 sample2 ## +* detection/classification using yolov8 based on tensorrt +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/mask_rcnn.mp4"); + auto yolo8_detector = std::make_shared("yolo8_detector", "./vp_data/models/trt/others/yolov8s_v8.5.engine", "./vp_data/models/coco_80classes.txt"); + auto yolo8_classifier = std::make_shared("yolo8_classifier", "./vp_data/models/trt/others/yolov8s-cls_v8.5.engine", "./vp_data/models/imagenet_1000labels1.txt"); + auto osd = std::make_shared("osd", "./vp_data/font/NotoSansCJKsc-Medium.otf"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + // construct pipeline + yolo8_detector->attach_to({file_src_0}); + yolo8_classifier->attach_to({yolo8_detector}); + osd->attach_to({yolo8_classifier}); + screen_des_0->attach_to({osd}); + + // start pipeline + file_src_0->start(); + + // visualize pipeline for debug + vp_utils::vp_analysis_board board({file_src_0}); + board.display(); +} \ No newline at end of file diff --git a/samples/vehicle_body_scan_sample.cpp b/samples/vehicle_body_scan_sample.cpp new file mode 100644 index 0000000..cf439d8 --- /dev/null +++ b/samples/vehicle_body_scan_sample.cpp @@ -0,0 +1,32 @@ +#include "../nodes/vp_image_src_node.h" +#include "../nodes/infers/vp_trt_vehicle_scanner.h" +#include "../nodes/osd/vp_osd_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## vehicle_body_scan_sample ## +* detect wheels and vehicle type based on side view of vehicle +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto image_src_0 = std::make_shared("image_src_0", 0, "./vp_data/test_images/body/%d.jpg"); + auto vehicle_scanner = std::make_shared("vehicle_scanner", "./vp_data/models/trt/vehicle/vehicle_scan_v8.5.trt"); + auto osd = std::make_shared("osd"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + // construct pipeline + vehicle_scanner->attach_to({image_src_0}); + osd->attach_to({vehicle_scanner}); + screen_des_0->attach_to({osd}); + + image_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({image_src_0}); + board.display(); +} \ No newline at end of file diff --git a/samples/vehicle_cluster_based_on_classify_encoding_sample.cpp b/samples/vehicle_cluster_based_on_classify_encoding_sample.cpp new file mode 100644 index 0000000..b334674 --- /dev/null +++ b/samples/vehicle_cluster_based_on_classify_encoding_sample.cpp @@ -0,0 +1,59 @@ +#include "../nodes/vp_image_src_node.h" +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_trt_vehicle_detector.h" +#include "../nodes/infers/vp_trt_vehicle_color_classifier.h" +#include "../nodes/infers/vp_trt_vehicle_type_classifier.h" +#include "../nodes/infers/vp_trt_vehicle_feature_encoder.h" +#include "../nodes/track/vp_sort_track_node.h" +#include "../nodes/osd/vp_osd_node.h" +#include "../nodes/osd/vp_cluster_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_fake_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## vehicle_cluster_based_on_classify_encoding_sample ## +* vehicle cluster based on classify(categories) and encoding(features), pipeline would display 3 windows (cluster by t-SNE, cluster by categories, detect osd result) +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto image_src_0 = std::make_shared("image_src_0", 0, "./vp_data/test_images/vehicle/%d.jpg"); + //auto file_src_0 = std::make_shared("file_src_0", 0, "./test_video/22.mp4"); + auto trt_vehicle_detector = std::make_shared("trt_detector", "./vp_data/models/trt/vehicle/vehicle_v8.5.trt"); + auto trt_vehicle_color_classifier = std::make_shared("trt_color_cls", "./vp_data/models/trt/vehicle/vehicle_color_v8.5.trt", std::vector{0, 1, 2}); + auto trt_vehicle_type_classifier = std::make_shared("trt_type_cls", "./vp_data/models/trt/vehicle/vehicle_type_v8.5.trt", std::vector{0, 1, 2}); + auto trt_vehicle_feature_encoder = std::make_shared("trt_encoder", "./vp_data/models/trt/vehicle/vehicle_embedding_v8.5.trt", std::vector{0, 1, 2}); + auto osd_0 = std::make_shared("osd_0"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + auto cluster_0 = std::make_shared("cluster_0", true, std::vector{"red", "white", "black", "blue", "yellow", "bus", "small_truck", "van", "tanker"}, 1000); + auto fake_des_0 = std::make_shared("fake_des_0", 0); + + // construct pipeline + trt_vehicle_detector->attach_to({image_src_0}); + //trt_vehicle_detector->attach_to({file_src_0}); + trt_vehicle_color_classifier->attach_to({trt_vehicle_detector}); + trt_vehicle_type_classifier->attach_to({trt_vehicle_color_classifier}); + trt_vehicle_feature_encoder->attach_to({trt_vehicle_type_classifier}); + + // split into 2 branches automatically + /* branch of osd -> screen des*/ + osd_0->attach_to({trt_vehicle_feature_encoder}); + screen_des_0->attach_to({osd_0}); + /* branch of cluster -> fake des*/ + cluster_0->attach_to({trt_vehicle_feature_encoder}); + fake_des_0->attach_to({cluster_0}); // to keep pipeline complete + + // start pipeline + image_src_0->start(); + //file_src_0->start(); + + // visualize pipeline for debug + //vp_utils::vp_analysis_board board({file_src_0}); + vp_utils::vp_analysis_board board({image_src_0}); + board.display(); +} \ No newline at end of file diff --git a/samples/vehicle_tracking_sample.cpp b/samples/vehicle_tracking_sample.cpp new file mode 100644 index 0000000..94098fb --- /dev/null +++ b/samples/vehicle_tracking_sample.cpp @@ -0,0 +1,32 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_trt_vehicle_detector.h" +#include "../nodes/osd/vp_osd_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/track/vp_sort_track_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/vehicle_stop.mp4", 0.5); + auto trt_vehicle_detector = std::make_shared("vehicle_detector", "./vp_data/models/trt/vehicle/vehicle_v8.5.trt"); + auto track_0 = std::make_shared("track_0"); + auto osd_0 = std::make_shared("osd_0", "./vp_data/font/NotoSansCJKsc-Medium.otf"); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + // construct pipeline + trt_vehicle_detector->attach_to({file_src_0}); + track_0->attach_to({trt_vehicle_detector}); + osd_0->attach_to({track_0}); + screen_des_0->attach_to({osd_0}); + + // start pipeline + file_src_0->start(); + + // visualize pipeline for debug + vp_utils::vp_analysis_board board({file_src_0}); + board.display(); +} \ No newline at end of file diff --git a/samples/video_restoration_sample.cpp b/samples/video_restoration_sample.cpp new file mode 100644 index 0000000..19c3160 --- /dev/null +++ b/samples/video_restoration_sample.cpp @@ -0,0 +1,42 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/vp_image_src_node.h" +#include "../nodes/infers/vp_yunet_face_detector_node.h" +#include "../nodes/infers/vp_restoration_node.h" +#include "../nodes/osd/vp_face_osd_node.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_file_des_node.h" +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## video_restoration_sample ## +* enhance for any video/images, no training need before running. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_SET_LOG_INCLUDE_CODE_LOCATION(false); + VP_SET_LOG_INCLUDE_THREAD_ID(false); + VP_LOGGER_INIT(); + + // create nodes + auto image_src_0 = std::make_shared("image_file_src_0", 0, "./vp_data/test_images/restoration/1/%d.jpg", 3); + auto restoration_node = std::make_shared("restoration_node", "./vp_data/models/restoration/realesrgan-x4.onnx"); + auto screen_des_0_ori = std::make_shared("screen_des_0_ori", 0, false); + auto screen_des_0_osd = std::make_shared("screen_des_0_osd", 0); + + // construct pipeline + restoration_node->attach_to({image_src_0}); + screen_des_0_ori->attach_to({restoration_node}); + screen_des_0_osd->attach_to({restoration_node}); + + // start pipeline + image_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({image_src_0}); + board.display(1, false); + + std::string wait; + std::getline(std::cin, wait); + image_src_0->detach_recursively(); +} \ No newline at end of file diff --git a/samples/vp_logger_sample.cpp b/samples/vp_logger_sample.cpp new file mode 100644 index 0000000..4cd3d42 --- /dev/null +++ b/samples/vp_logger_sample.cpp @@ -0,0 +1,130 @@ +#include "../utils/vp_utils.h" +#include "../utils/logger/vp_logger.h" + +#include +#include +#include + +/* +* ## sample for vp_logger ## +* show how vp_logger works +*/ + +int main() { + // config for log content + // VP_SET_LOG_INCLUDE_THREAD_ID(false); + // VP_SET_LOG_INCLUDE_CODE_LOCATION(false); + // VP_SET_LOG_INCLUDE_LEVEL(false); + + // config for output + // VP_SET_LOG_TO_CONSOLE(false); + // VP_SET_LOG_TO_FILE(false); + + // config for log folder + VP_SET_LOG_DIR("./log"); + + // config log level + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::DEBUG); + + // config for kafka, ignored automatically if not prepared for kafka + VP_SET_LOG_TO_KAFKA(true); // false by default if not set + VP_SET_LOG_KAFKA_SERVERS_AND_TOPIC("192.168.77.87:9092/vp_log"); + + // init + VP_LOGGER_INIT(); + + // 6 threads logging separately + auto func1 = []() { + while (true) { + /* code */ + auto id = std::this_thread::get_id(); + std::stringstream ss; + ss << std::hex << id; + auto thread_id = ss.str(); + VP_ERROR(vp_utils::string_format("thread id: %s", thread_id.c_str())); + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + } + + }; + + auto func2 = []() { + while (true) { + /* code */ + auto id = std::this_thread::get_id(); + std::stringstream ss; + ss << std::hex << id; + auto thread_id = ss.str(); + VP_DEBUG(vp_utils::string_format("thread id: %s", thread_id.c_str())); + std::this_thread::sleep_for(std::chrono::milliseconds(13)); + } + + }; + + auto func3 = []() { + while (true) { + /* code */ + auto id = std::this_thread::get_id(); + std::stringstream ss; + ss << std::hex << id; + auto thread_id = ss.str(); + VP_INFO(vp_utils::string_format("thread id: %s", thread_id.c_str())); + std::this_thread::sleep_for(std::chrono::milliseconds(4)); + } + + }; + + auto func4 = []() { + while (true) { + /* code */ + auto id = std::this_thread::get_id(); + std::stringstream ss; + ss << std::hex << id; + auto thread_id = ss.str(); + VP_WARN(vp_utils::string_format("thread id: %s", thread_id.c_str())); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + + }; + + auto func5 = []() { + while (true) { + /* code */ + auto id = std::this_thread::get_id(); + std::stringstream ss; + ss << std::hex << id; + auto thread_id = ss.str(); + VP_ERROR(vp_utils::string_format("thread id: %s", thread_id.c_str())); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + + }; + + auto func6 = []() { + while (true) { + /* code */ + auto id = std::this_thread::get_id(); + std::stringstream ss; + ss << std::hex << id; + auto thread_id = ss.str(); + VP_INFO(vp_utils::string_format("thread id: %s", thread_id.c_str())); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + + }; + + std::thread t1(func1); + std::thread t2(func2); + std::thread t3(func3); + std::thread t4(func4); + std::thread t5(func5); + std::thread t6(func6); + + t1.join(); + t2.join(); + t3.join(); + t4.join(); + t5.join(); + t6.join(); + + std::getchar(); +} \ No newline at end of file diff --git a/samples/vp_test.cpp b/samples/vp_test.cpp new file mode 100644 index 0000000..c126792 --- /dev/null +++ b/samples/vp_test.cpp @@ -0,0 +1,60 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/vp_rtsp_src_node.h" +#include "../nodes/infers/vp_yunet_face_detector_node.h" +#include "../nodes/infers/vp_sface_feature_encoder_node.h" +#include "../nodes/osd/vp_face_osd_node_v2.h" +#include "../nodes/vp_screen_des_node.h" +#include "../nodes/vp_rtmp_des_node.h" +#include "../nodes/vp_fake_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## vp_test ## +* test anything for videopipe in this cpp. +*/ + +int main() { + VP_SET_LOG_INCLUDE_CODE_LOCATION(false); + VP_SET_LOG_INCLUDE_THREAD_ID(false); + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto rtsp_src_0 = std::make_shared("rtsp_src_0", 0, "rtsp://192.168.77.193:8554/stream/main", 1, "avdec_h264", 1); + auto fake_des_0 = std::make_shared("fake_des_0", 0); + auto rtsp_src_1 = std::make_shared("rtsp_src_1", 1, "rtsp://192.168.77.193:8554/stream/main", 1, "avdec_h264", 1); + auto fake_des_1 = std::make_shared("fake_des_1", 1); + auto rtsp_src_2 = std::make_shared("rtsp_src_2", 2, "rtsp://192.168.77.193:8554/stream/main", 1, "avdec_h264", 1); + auto fake_des_2 = std::make_shared("fake_des_2", 2); + auto rtsp_src_3 = std::make_shared("rtsp_src_3", 3, "rtsp://192.168.77.193:8554/stream/main", 1, "avdec_h264", 1); + auto fake_des_3 = std::make_shared("fake_des_3", 3); + auto rtsp_src_4 = std::make_shared("rtsp_src_4", 4, "rtsp://192.168.77.193:8554/stream/main", 1, "avdec_h264", 1); + auto fake_des_4 = std::make_shared("fake_des_4", 4); + + // construct pipeline + fake_des_0->attach_to({rtsp_src_0}); + fake_des_1->attach_to({rtsp_src_1}); + fake_des_2->attach_to({rtsp_src_2}); + fake_des_3->attach_to({rtsp_src_3}); + fake_des_4->attach_to({rtsp_src_4}); + + // start + rtsp_src_0->start(); + rtsp_src_1->start(); + rtsp_src_2->start(); + rtsp_src_3->start(); + rtsp_src_4->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({rtsp_src_0, rtsp_src_1, rtsp_src_2, rtsp_src_3, rtsp_src_4}); + board.display(1, false); + + std::string wait; + std::getline(std::cin, wait); + rtsp_src_0->detach_recursively(); + rtsp_src_1->detach_recursively(); + rtsp_src_2->detach_recursively(); + rtsp_src_3->detach_recursively(); + rtsp_src_4->detach_recursively(); +} \ No newline at end of file diff --git a/samples/yolov5_seg_sample.cpp b/samples/yolov5_seg_sample.cpp new file mode 100644 index 0000000..633f7c1 --- /dev/null +++ b/samples/yolov5_seg_sample.cpp @@ -0,0 +1,34 @@ +#include "../nodes/vp_file_src_node.h" +#include "../nodes/infers/vp_yolo5_seg_node.h" +#include "../nodes/osd/vp_osd_node_v3.h" +#include "../nodes/vp_screen_des_node.h" + +#include "../utils/analysis_board/vp_analysis_board.h" + +/* +* ## yolov5-seg sample ## +* driving area segmentation(das) using yolov5-seg-v7.0. +*/ + +int main() { + VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO); + VP_LOGGER_INIT(); + + // create nodes + auto file_src_0 = std::make_shared("file_src_0", 0, "./vp_data/test_video/vehicle_count.mp4", 0.6); + // 2 classes using yolov5-seg + auto das_detector = std::make_shared("das_detector", "./vp_data/models/lane/das.onnx", "./vp_data/models/lane/das_2classes.txt"); + auto osd_v3_0 = std::make_shared("osd_v3_0", "./vp_data/font/NotoSansCJKsc-Medium.otf", false); + auto screen_des_0 = std::make_shared("screen_des_0", 0); + + // construct pipeline + das_detector->attach_to({file_src_0}); + osd_v3_0->attach_to({das_detector}); + screen_des_0->attach_to({osd_v3_0}); + + file_src_0->start(); + + // for debug purpose + vp_utils::vp_analysis_board board({file_src_0}); + board.display(); +} \ No newline at end of file diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100755 index 0000000..8f15291 --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +#################################### +# run release.sh +# to collect libraries & header files used out of VideoPipe workspace. +#################################### +SO_SRC_DIRS=( + "../build/libs" +) +H_SRC_DIRS=( + "../excepts" + "../nodes" + "../objects" + "../third_party" + "../utils" +) + +LIBS_DIR="./libs" +INCLUDE_DIR="./include" +CHECK_ELF_SO=true + +echo "🧹 removing existing files..." +rm -rf "$LIBS_DIR" "$INCLUDE_DIR" + +mkdir -p "$LIBS_DIR" +mkdir -p "$INCLUDE_DIR" + +echo "✅ collecting libraries & header files..." +echo "target library path: $LIBS_DIR" +echo "target header path: $INCLUDE_DIR" +echo + +collect_so_files() { + local src_dir + for src_dir in "${SO_SRC_DIRS[@]}"; do + if [ ! -d "$src_dir" ]; then + echo "⚠️ warn: .so source folder not exists, ignore: $src_dir" + continue + fi + + echo "🔍 scanning .so files: $src_dir" + find "$src_dir" -type f -name "*.so*" | while read -r so_file; do + if $CHECK_ELF_SO; then + if ! file "$so_file" 2>/dev/null | grep -q "ELF.*shared object"; then + continue + fi + fi + + rel_path="${so_file#$src_dir/}" + dest_path="$LIBS_DIR/$rel_path" + dest_dir=$(dirname "$dest_path") + + mkdir -p "$dest_dir" + cp "$so_file" "$dest_path" + echo "📦 copied .so: $so_file -> $dest_path" + done + done +} + +collect_h_files() { + local src_dir + for src_dir in "${H_SRC_DIRS[@]}"; do + if [ ! -d "$src_dir" ]; then + echo "⚠️ warn: .h source folder not exists, ignore: $src_dir" + continue + fi + + dir_name=$(basename "$src_dir") + + echo "🔍 scanning .h files: $src_dir (keep directory name: $dir_name)" + + find "$src_dir" -type f \( -name "*.h" -o -name "*.hpp" -o -name "*.hxx" \) | while read -r h_file; do + rel_path="${h_file#$src_dir/}" + + dest_path="$INCLUDE_DIR/$dir_name/$rel_path" + dest_dir=$(dirname "$dest_path") + + mkdir -p "$dest_dir" + cp "$h_file" "$dest_path" + echo "📘 copied .h: $h_file -> $dest_path" + done + done +} + +collect_so_files +echo +collect_h_files + +echo +echo "🎉 completely！" +echo "libraries saved to: $LIBS_DIR" +echo "headers saved to: $INCLUDE_DIR" \ No newline at end of file diff --git a/third_party/bhtsne/README.md b/third_party/bhtsne/README.md new file mode 100755 index 0000000..a0a86e5 --- /dev/null +++ b/third_party/bhtsne/README.md @@ -0,0 +1,13 @@ +implement of t-SNE algorithm, reduce high dims of feature to 2D/3D used for display on screen later. + +header-only style please refer to https://github.com/lvdmaaten/bhtsne + + +``` +compile test: +g++ tsne_test.cpp -o bh_tsne_test -O2 + +and run: +./bh_tsne_test + +``` \ No newline at end of file diff --git a/third_party/bhtsne/sptree.h b/third_party/bhtsne/sptree.h new file mode 100755 index 0000000..24e44c0 --- /dev/null +++ b/third_party/bhtsne/sptree.h @@ -0,0 +1,545 @@ +/* + * + * Copyright (c) 2014, Laurens van der Maaten (Delft University of Technology) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the Delft University of Technology. + * 4. Neither the name of the Delft University of Technology nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY LAURENS VAN DER MAATEN ''AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL LAURENS VAN DER MAATEN BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + */ + + +#ifndef SPTREE_H +#define SPTREE_H + +using namespace std; + + +class Cell { + + unsigned int dimension; + double* corner; + double* width; + + +public: + Cell(unsigned int inp_dimension); + Cell(unsigned int inp_dimension, double* inp_corner, double* inp_width); + ~Cell(); + + double getCorner(unsigned int d); + double getWidth(unsigned int d); + void setCorner(unsigned int d, double val); + void setWidth(unsigned int d, double val); + bool containsPoint(double point[]); +}; + + +class SPTree +{ + + // Fixed constants + static const unsigned int QT_NODE_CAPACITY = 1; + + // A buffer we use when doing force computations + double* buff; + + // Properties of this node in the tree + SPTree* parent; + unsigned int dimension; + bool is_leaf; + unsigned int size; + unsigned int cum_size; + + // Axis-aligned bounding box stored as a center with half-dimensions to represent the boundaries of this quad tree + Cell* boundary; + + // Indices in this space-partitioning tree node, corresponding center-of-mass, and list of all children + double* data; + double* center_of_mass; + unsigned int index[QT_NODE_CAPACITY]; + + // Children + SPTree** children; + unsigned int no_children; + +public: + SPTree(unsigned int D, double* inp_data, unsigned int N); + SPTree(unsigned int D, double* inp_data, double* inp_corner, double* inp_width); + SPTree(unsigned int D, double* inp_data, unsigned int N, double* inp_corner, double* inp_width); + SPTree(SPTree* inp_parent, unsigned int D, double* inp_data, unsigned int N, double* inp_corner, double* inp_width); + SPTree(SPTree* inp_parent, unsigned int D, double* inp_data, double* inp_corner, double* inp_width); + ~SPTree(); + void setData(double* inp_data); + SPTree* getParent(); + void construct(Cell boundary); + bool insert(unsigned int new_index); + void subdivide(); + bool isCorrect(); + void rebuildTree(); + void getAllIndices(unsigned int* indices); + unsigned int getDepth(); + void computeNonEdgeForces(unsigned int point_index, double theta, double neg_f[], double* sum_Q); + void computeEdgeForces(unsigned int* row_P, unsigned int* col_P, double* val_P, int N, double* pos_f); + void print(); + +private: + void init(SPTree* inp_parent, unsigned int D, double* inp_data, double* inp_corner, double* inp_width); + void fill(unsigned int N); + unsigned int getAllIndices(unsigned int* indices, unsigned int loc); + bool isChild(unsigned int test_index, unsigned int start, unsigned int end); +}; + + +/* copy from original sptree.cpp file */ +/* + * + * Copyright (c) 2014, Laurens van der Maaten (Delft University of Technology) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the Delft University of Technology. + * 4. Neither the name of the Delft University of Technology nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY LAURENS VAN DER MAATEN ''AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL LAURENS VAN DER MAATEN BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + */ + +#include +#include +#include +#include +#include +#include "sptree.h" + + + +// Constructs cell +Cell::Cell(unsigned int inp_dimension) { + dimension = inp_dimension; + corner = (double*) malloc(dimension * sizeof(double)); + width = (double*) malloc(dimension * sizeof(double)); +} + +Cell::Cell(unsigned int inp_dimension, double* inp_corner, double* inp_width) { + dimension = inp_dimension; + corner = (double*) malloc(dimension * sizeof(double)); + width = (double*) malloc(dimension * sizeof(double)); + for(int d = 0; d < dimension; d++) setCorner(d, inp_corner[d]); + for(int d = 0; d < dimension; d++) setWidth( d, inp_width[d]); +} + +// Destructs cell +Cell::~Cell() { + free(corner); + free(width); +} + +double Cell::getCorner(unsigned int d) { + return corner[d]; +} + +double Cell::getWidth(unsigned int d) { + return width[d]; +} + +void Cell::setCorner(unsigned int d, double val) { + corner[d] = val; +} + +void Cell::setWidth(unsigned int d, double val) { + width[d] = val; +} + +// Checks whether a point lies in a cell +bool Cell::containsPoint(double point[]) +{ + for(int d = 0; d < dimension; d++) { + if(corner[d] - width[d] > point[d]) return false; + if(corner[d] + width[d] < point[d]) return false; + } + return true; +} + + +// Default constructor for SPTree -- build tree, too! +SPTree::SPTree(unsigned int D, double* inp_data, unsigned int N) +{ + + // Compute mean, width, and height of current map (boundaries of SPTree) + int nD = 0; + double* mean_Y = (double*) calloc(D, sizeof(double)); + double* min_Y = (double*) malloc(D * sizeof(double)); for(unsigned int d = 0; d < D; d++) min_Y[d] = DBL_MAX; + double* max_Y = (double*) malloc(D * sizeof(double)); for(unsigned int d = 0; d < D; d++) max_Y[d] = -DBL_MAX; + for(unsigned int n = 0; n < N; n++) { + for(unsigned int d = 0; d < D; d++) { + mean_Y[d] += inp_data[n * D + d]; + if(inp_data[nD + d] < min_Y[d]) min_Y[d] = inp_data[nD + d]; + if(inp_data[nD + d] > max_Y[d]) max_Y[d] = inp_data[nD + d]; + } + nD += D; + } + for(int d = 0; d < D; d++) mean_Y[d] /= (double) N; + + // Construct SPTree + double* width = (double*) malloc(D * sizeof(double)); + for(int d = 0; d < D; d++) width[d] = fmax(max_Y[d] - mean_Y[d], mean_Y[d] - min_Y[d]) + 1e-5; + init(NULL, D, inp_data, mean_Y, width); + fill(N); + + // Clean up memory + free(mean_Y); + free(max_Y); + free(min_Y); + free(width); +} + + +// Constructor for SPTree with particular size and parent -- build the tree, too! +SPTree::SPTree(unsigned int D, double* inp_data, unsigned int N, double* inp_corner, double* inp_width) +{ + init(NULL, D, inp_data, inp_corner, inp_width); + fill(N); +} + + +// Constructor for SPTree with particular size (do not fill the tree) +SPTree::SPTree(unsigned int D, double* inp_data, double* inp_corner, double* inp_width) +{ + init(NULL, D, inp_data, inp_corner, inp_width); +} + + +// Constructor for SPTree with particular size and parent (do not fill tree) +SPTree::SPTree(SPTree* inp_parent, unsigned int D, double* inp_data, double* inp_corner, double* inp_width) { + init(inp_parent, D, inp_data, inp_corner, inp_width); +} + + +// Constructor for SPTree with particular size and parent -- build the tree, too! +SPTree::SPTree(SPTree* inp_parent, unsigned int D, double* inp_data, unsigned int N, double* inp_corner, double* inp_width) +{ + init(inp_parent, D, inp_data, inp_corner, inp_width); + fill(N); +} + + +// Main initialization function +void SPTree::init(SPTree* inp_parent, unsigned int D, double* inp_data, double* inp_corner, double* inp_width) +{ + parent = inp_parent; + dimension = D; + no_children = 2; + for(unsigned int d = 1; d < D; d++) no_children *= 2; + data = inp_data; + is_leaf = true; + size = 0; + cum_size = 0; + + boundary = new Cell(dimension); + for(unsigned int d = 0; d < D; d++) boundary->setCorner(d, inp_corner[d]); + for(unsigned int d = 0; d < D; d++) boundary->setWidth( d, inp_width[d]); + + children = (SPTree**) malloc(no_children * sizeof(SPTree*)); + for(unsigned int i = 0; i < no_children; i++) children[i] = NULL; + + center_of_mass = (double*) malloc(D * sizeof(double)); + for(unsigned int d = 0; d < D; d++) center_of_mass[d] = .0; + + buff = (double*) malloc(D * sizeof(double)); +} + + +// Destructor for SPTree +SPTree::~SPTree() +{ + for(unsigned int i = 0; i < no_children; i++) { + if(children[i] != NULL) delete children[i]; + } + free(children); + free(center_of_mass); + free(buff); + delete boundary; +} + + +// Update the data underlying this tree +void SPTree::setData(double* inp_data) +{ + data = inp_data; +} + + +// Get the parent of the current tree +SPTree* SPTree::getParent() +{ + return parent; +} + + +// Insert a point into the SPTree +bool SPTree::insert(unsigned int new_index) +{ + // Ignore objects which do not belong in this quad tree + double* point = data + new_index * dimension; + if(!boundary->containsPoint(point)) + return false; + + // Online update of cumulative size and center-of-mass + cum_size++; + double mult1 = (double) (cum_size - 1) / (double) cum_size; + double mult2 = 1.0 / (double) cum_size; + for(unsigned int d = 0; d < dimension; d++) center_of_mass[d] *= mult1; + for(unsigned int d = 0; d < dimension; d++) center_of_mass[d] += mult2 * point[d]; + + // If there is space in this quad tree and it is a leaf, add the object here + if(is_leaf && size < QT_NODE_CAPACITY) { + index[size] = new_index; + size++; + return true; + } + + // Don't add duplicates for now (this is not very nice) + bool any_duplicate = false; + for(unsigned int n = 0; n < size; n++) { + bool duplicate = true; + for(unsigned int d = 0; d < dimension; d++) { + if(point[d] != data[index[n] * dimension + d]) { duplicate = false; break; } + } + any_duplicate = any_duplicate | duplicate; + } + if(any_duplicate) return true; + + // Otherwise, we need to subdivide the current cell + if(is_leaf) subdivide(); + + // Find out where the point can be inserted + for(unsigned int i = 0; i < no_children; i++) { + if(children[i]->insert(new_index)) return true; + } + + // Otherwise, the point cannot be inserted (this should never happen) + return false; +} + + +// Create four children which fully divide this cell into four quads of equal area +void SPTree::subdivide() { + + // Create new children + double* new_corner = (double*) malloc(dimension * sizeof(double)); + double* new_width = (double*) malloc(dimension * sizeof(double)); + for(unsigned int i = 0; i < no_children; i++) { + unsigned int div = 1; + for(unsigned int d = 0; d < dimension; d++) { + new_width[d] = .5 * boundary->getWidth(d); + if((i / div) % 2 == 1) new_corner[d] = boundary->getCorner(d) - .5 * boundary->getWidth(d); + else new_corner[d] = boundary->getCorner(d) + .5 * boundary->getWidth(d); + div *= 2; + } + children[i] = new SPTree(this, dimension, data, new_corner, new_width); + } + free(new_corner); + free(new_width); + + // Move existing points to correct children + for(unsigned int i = 0; i < size; i++) { + bool success = false; + for(unsigned int j = 0; j < no_children; j++) { + if(!success) success = children[j]->insert(index[i]); + } + index[i] = -1; + } + + // Empty parent node + size = 0; + is_leaf = false; +} + + +// Build SPTree on dataset +void SPTree::fill(unsigned int N) +{ + for(unsigned int i = 0; i < N; i++) insert(i); +} + + +// Checks whether the specified tree is correct +bool SPTree::isCorrect() +{ + for(unsigned int n = 0; n < size; n++) { + double* point = data + index[n] * dimension; + if(!boundary->containsPoint(point)) return false; + } + if(!is_leaf) { + bool correct = true; + for(int i = 0; i < no_children; i++) correct = correct && children[i]->isCorrect(); + return correct; + } + else return true; +} + + + +// Build a list of all indices in SPTree +void SPTree::getAllIndices(unsigned int* indices) +{ + getAllIndices(indices, 0); +} + + +// Build a list of all indices in SPTree +unsigned int SPTree::getAllIndices(unsigned int* indices, unsigned int loc) +{ + + // Gather indices in current quadrant + for(unsigned int i = 0; i < size; i++) indices[loc + i] = index[i]; + loc += size; + + // Gather indices in children + if(!is_leaf) { + for(int i = 0; i < no_children; i++) loc = children[i]->getAllIndices(indices, loc); + } + return loc; +} + + +unsigned int SPTree::getDepth() { + if(is_leaf) return 1; + int depth = 0; + for(unsigned int i = 0; i < no_children; i++) depth = fmax(depth, children[i]->getDepth()); + return 1 + depth; +} + + +// Compute non-edge forces using Barnes-Hut algorithm +void SPTree::computeNonEdgeForces(unsigned int point_index, double theta, double neg_f[], double* sum_Q) +{ + + // Make sure that we spend no time on empty nodes or self-interactions + if(cum_size == 0 || (is_leaf && size == 1 && index[0] == point_index)) return; + + // Compute distance between point and center-of-mass + double D = .0; + unsigned int ind = point_index * dimension; + for(unsigned int d = 0; d < dimension; d++) buff[d] = data[ind + d] - center_of_mass[d]; + for(unsigned int d = 0; d < dimension; d++) D += buff[d] * buff[d]; + + // Check whether we can use this node as a "summary" + double max_width = 0.0; + double cur_width; + for(unsigned int d = 0; d < dimension; d++) { + cur_width = boundary->getWidth(d); + max_width = (max_width > cur_width) ? max_width : cur_width; + } + if(is_leaf || max_width / sqrt(D) < theta) { + + // Compute and add t-SNE force between point and current node + D = 1.0 / (1.0 + D); + double mult = cum_size * D; + *sum_Q += mult; + mult *= D; + for(unsigned int d = 0; d < dimension; d++) neg_f[d] += mult * buff[d]; + } + else { + + // Recursively apply Barnes-Hut to children + for(unsigned int i = 0; i < no_children; i++) children[i]->computeNonEdgeForces(point_index, theta, neg_f, sum_Q); + } +} + + +// Computes edge forces +void SPTree::computeEdgeForces(unsigned int* row_P, unsigned int* col_P, double* val_P, int N, double* pos_f) +{ + + // Loop over all edges in the graph + unsigned int ind1 = 0; + unsigned int ind2 = 0; + double D; + for(unsigned int n = 0; n < N; n++) { + for(unsigned int i = row_P[n]; i < row_P[n + 1]; i++) { + + // Compute pairwise distance and Q-value + D = 1.0; + ind2 = col_P[i] * dimension; + for(unsigned int d = 0; d < dimension; d++) buff[d] = data[ind1 + d] - data[ind2 + d]; + for(unsigned int d = 0; d < dimension; d++) D += buff[d] * buff[d]; + D = val_P[i] / D; + + // Sum positive force + for(unsigned int d = 0; d < dimension; d++) pos_f[ind1 + d] += D * buff[d]; + } + ind1 += dimension; + } +} + + +// Print out tree +void SPTree::print() +{ + if(cum_size == 0) { + printf("Empty node\n"); + return; + } + + if(is_leaf) { + printf("Leaf node; data = ["); + for(int i = 0; i < size; i++) { + double* point = data + index[i] * dimension; + for(int d = 0; d < dimension; d++) printf("%f, ", point[d]); + printf(" (index = %d)", index[i]); + if(i < size - 1) printf("\n"); + else printf("]\n"); + } + } + else { + printf("Intersection node with center-of-mass = ["); + for(int d = 0; d < dimension; d++) printf("%f, ", center_of_mass[d]); + printf("]; children are:\n"); + for(int i = 0; i < no_children; i++) children[i]->print(); + } +} + +#endif diff --git a/third_party/bhtsne/tsne.h b/third_party/bhtsne/tsne.h new file mode 100755 index 0000000..077fd7b --- /dev/null +++ b/third_party/bhtsne/tsne.h @@ -0,0 +1,774 @@ +/* + * + * Copyright (c) 2014, Laurens van der Maaten (Delft University of Technology) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the Delft University of Technology. + * 4. Neither the name of the Delft University of Technology nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY LAURENS VAN DER MAATEN ''AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL LAURENS VAN DER MAATEN BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + */ + + +#ifndef TSNE_H +#define TSNE_H + +#ifdef __cplusplus +extern "C" { +namespace TSNE { +#endif + void run(double* X, int N, int D, double* Y, int no_dims, double perplexity, double theta, int rand_seed, + bool skip_random_init, int max_iter, int stop_lying_iter, int mom_switch_iter); + bool load_data(double** data, int* n, int* d, int* no_dims, double* theta, double* perplexity, int* rand_seed, int* max_iter); + void save_data(double* data, int* landmarks, double* costs, int n, int d); +#ifdef __cplusplus +}; +} +#endif + + +/* copy from original tsne.cpp file */ +/* + * + * Copyright (c) 2014, Laurens van der Maaten (Delft University of Technology) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the Delft University of Technology. + * 4. Neither the name of the Delft University of Technology nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY LAURENS VAN DER MAATEN ''AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL LAURENS VAN DER MAATEN BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + */ + +#include +#include +#include +#include +#include +#include +#include "vptree.h" +#include "sptree.h" + + +using namespace std; + +static double sign(double x) { return (x == .0 ? .0 : (x < .0 ? -1.0 : 1.0)); } + +static void zeroMean(double* X, int N, int D); +static void computeGaussianPerplexity(double* X, int N, int D, double* P, double perplexity); +static void computeGaussianPerplexity(double* X, int N, int D, unsigned int** _row_P, unsigned int** _col_P, double** _val_P, double perplexity, int K); +static double randn(); +static void computeExactGradient(double* P, double* Y, int N, int D, double* dC); +static void computeGradient(unsigned int* inp_row_P, unsigned int* inp_col_P, double* inp_val_P, double* Y, int N, int D, double* dC, double theta); +static double evaluateError(double* P, double* Y, int N, int D); +static double evaluateError(unsigned int* row_P, unsigned int* col_P, double* val_P, double* Y, int N, int D, double theta); +static void computeSquaredEuclideanDistance(double* X, int N, int D, double* DD); +static void symmetrizeMatrix(unsigned int** row_P, unsigned int** col_P, double** val_P, int N); + +// Perform t-SNE +void TSNE::run(double* X, int N, int D, double* Y, int no_dims, double perplexity, double theta, int rand_seed, + bool skip_random_init, int max_iter, int stop_lying_iter, int mom_switch_iter) { + + // Set random seed + if (skip_random_init != true) { + if(rand_seed >= 0) { + printf("Using random seed: %d\n", rand_seed); + srand((unsigned int) rand_seed); + } else { + printf("Using current time as random seed...\n"); + srand(time(NULL)); + } + } + + // Determine whether we are using an exact algorithm + if(N - 1 < 3 * perplexity) { printf("Perplexity too large for the number of data points!\n"); exit(1); } + printf("Using no_dims = %d, perplexity = %f, and theta = %f\n", no_dims, perplexity, theta); + bool exact = (theta == .0) ? true : false; + + // Set learning parameters + float total_time = .0; + clock_t start, end; + double momentum = .5, final_momentum = .8; + double eta = 200.0; + + // Allocate some memory + double* dY = (double*) malloc(N * no_dims * sizeof(double)); + double* uY = (double*) malloc(N * no_dims * sizeof(double)); + double* gains = (double*) malloc(N * no_dims * sizeof(double)); + if(dY == NULL || uY == NULL || gains == NULL) { printf("Memory allocation failed!\n"); exit(1); } + for(int i = 0; i < N * no_dims; i++) uY[i] = .0; + for(int i = 0; i < N * no_dims; i++) gains[i] = 1.0; + + // Normalize input data (to prevent numerical problems) + printf("Computing input similarities...\n"); + start = clock(); + zeroMean(X, N, D); + double max_X = .0; + for(int i = 0; i < N * D; i++) { + if(fabs(X[i]) > max_X) max_X = fabs(X[i]); + } + for(int i = 0; i < N * D; i++) X[i] /= max_X; + + // Compute input similarities for exact t-SNE + double* P; unsigned int* row_P; unsigned int* col_P; double* val_P; + if(exact) { + + // Compute similarities + printf("Exact?"); + P = (double*) malloc(N * N * sizeof(double)); + if(P == NULL) { printf("Memory allocation failed!\n"); exit(1); } + computeGaussianPerplexity(X, N, D, P, perplexity); + + // Symmetrize input similarities + printf("Symmetrizing...\n"); + int nN = 0; + for(int n = 0; n < N; n++) { + int mN = (n + 1) * N; + for(int m = n + 1; m < N; m++) { + P[nN + m] += P[mN + n]; + P[mN + n] = P[nN + m]; + mN += N; + } + nN += N; + } + double sum_P = .0; + for(int i = 0; i < N * N; i++) sum_P += P[i]; + for(int i = 0; i < N * N; i++) P[i] /= sum_P; + } + + // Compute input similarities for approximate t-SNE + else { + + // Compute asymmetric pairwise input similarities + computeGaussianPerplexity(X, N, D, &row_P, &col_P, &val_P, perplexity, (int) (3 * perplexity)); + + // Symmetrize input similarities + symmetrizeMatrix(&row_P, &col_P, &val_P, N); + double sum_P = .0; + for(int i = 0; i < row_P[N]; i++) sum_P += val_P[i]; + for(int i = 0; i < row_P[N]; i++) val_P[i] /= sum_P; + } + end = clock(); + + // Lie about the P-values + if(exact) { for(int i = 0; i < N * N; i++) P[i] *= 12.0; } + else { for(int i = 0; i < row_P[N]; i++) val_P[i] *= 12.0; } + + // Initialize solution (randomly) + if (skip_random_init != true) { + for(int i = 0; i < N * no_dims; i++) Y[i] = randn() * .0001; + } + + // Perform main training loop + if(exact) printf("Input similarities computed in %4.2f seconds!\nLearning embedding...\n", (float) (end - start) / CLOCKS_PER_SEC); + else printf("Input similarities computed in %4.2f seconds (sparsity = %f)!\nLearning embedding...\n", (float) (end - start) / CLOCKS_PER_SEC, (double) row_P[N] / ((double) N * (double) N)); + start = clock(); + + for(int iter = 0; iter < max_iter; iter++) { + + // Compute (approximate) gradient + if(exact) computeExactGradient(P, Y, N, no_dims, dY); + else computeGradient(row_P, col_P, val_P, Y, N, no_dims, dY, theta); + + // Update gains + for(int i = 0; i < N * no_dims; i++) gains[i] = (sign(dY[i]) != sign(uY[i])) ? (gains[i] + .2) : (gains[i] * .8); + for(int i = 0; i < N * no_dims; i++) if(gains[i] < .01) gains[i] = .01; + + // Perform gradient update (with momentum and gains) + for(int i = 0; i < N * no_dims; i++) uY[i] = momentum * uY[i] - eta * gains[i] * dY[i]; + for(int i = 0; i < N * no_dims; i++) Y[i] = Y[i] + uY[i]; + + // Make solution zero-mean + zeroMean(Y, N, no_dims); + + // Stop lying about the P-values after a while, and switch momentum + if(iter == stop_lying_iter) { + if(exact) { for(int i = 0; i < N * N; i++) P[i] /= 12.0; } + else { for(int i = 0; i < row_P[N]; i++) val_P[i] /= 12.0; } + } + if(iter == mom_switch_iter) momentum = final_momentum; + + // Print out progress + if (iter > 0 && (iter % 50 == 0 || iter == max_iter - 1)) { + end = clock(); + double C = .0; + if(exact) C = evaluateError(P, Y, N, no_dims); + else C = evaluateError(row_P, col_P, val_P, Y, N, no_dims, theta); // doing approximate computation here! + if(iter == 0) + printf("Iteration %d: error is %f\n", iter + 1, C); + else { + total_time += (float) (end - start) / CLOCKS_PER_SEC; + printf("Iteration %d: error is %f (50 iterations in %4.2f seconds)\n", iter, C, (float) (end - start) / CLOCKS_PER_SEC); + } + start = clock(); + } + } + end = clock(); total_time += (float) (end - start) / CLOCKS_PER_SEC; + + // Clean up memory + free(dY); + free(uY); + free(gains); + if(exact) free(P); + else { + free(row_P); row_P = NULL; + free(col_P); col_P = NULL; + free(val_P); val_P = NULL; + } + printf("Fitting performed in %4.2f seconds.\n", total_time); +} + + +// Compute gradient of the t-SNE cost function (using Barnes-Hut algorithm) +static void computeGradient(unsigned int* inp_row_P, unsigned int* inp_col_P, double* inp_val_P, double* Y, int N, int D, double* dC, double theta) +{ + + // Construct space-partitioning tree on current map + SPTree* tree = new SPTree(D, Y, N); + + // Compute all terms required for t-SNE gradient + double sum_Q = .0; + double* pos_f = (double*) calloc(N * D, sizeof(double)); + double* neg_f = (double*) calloc(N * D, sizeof(double)); + if(pos_f == NULL || neg_f == NULL) { printf("Memory allocation failed!\n"); exit(1); } + tree->computeEdgeForces(inp_row_P, inp_col_P, inp_val_P, N, pos_f); + for(int n = 0; n < N; n++) tree->computeNonEdgeForces(n, theta, neg_f + n * D, &sum_Q); + + // Compute final t-SNE gradient + for(int i = 0; i < N * D; i++) { + dC[i] = pos_f[i] - (neg_f[i] / sum_Q); + } + free(pos_f); + free(neg_f); + delete tree; +} + +// Compute gradient of the t-SNE cost function (exact) +static void computeExactGradient(double* P, double* Y, int N, int D, double* dC) { + + // Make sure the current gradient contains zeros + for(int i = 0; i < N * D; i++) dC[i] = 0.0; + + // Compute the squared Euclidean distance matrix + double* DD = (double*) malloc(N * N * sizeof(double)); + if(DD == NULL) { printf("Memory allocation failed!\n"); exit(1); } + computeSquaredEuclideanDistance(Y, N, D, DD); + + // Compute Q-matrix and normalization sum + double* Q = (double*) malloc(N * N * sizeof(double)); + if(Q == NULL) { printf("Memory allocation failed!\n"); exit(1); } + double sum_Q = .0; + int nN = 0; + for(int n = 0; n < N; n++) { + for(int m = 0; m < N; m++) { + if(n != m) { + Q[nN + m] = 1 / (1 + DD[nN + m]); + sum_Q += Q[nN + m]; + } + } + nN += N; + } + + // Perform the computation of the gradient + nN = 0; + int nD = 0; + for(int n = 0; n < N; n++) { + int mD = 0; + for(int m = 0; m < N; m++) { + if(n != m) { + double mult = (P[nN + m] - (Q[nN + m] / sum_Q)) * Q[nN + m]; + for(int d = 0; d < D; d++) { + dC[nD + d] += (Y[nD + d] - Y[mD + d]) * mult; + } + } + mD += D; + } + nN += N; + nD += D; + } + + // Free memory + free(DD); DD = NULL; + free(Q); Q = NULL; +} + + +// Evaluate t-SNE cost function (exactly) +static double evaluateError(double* P, double* Y, int N, int D) { + + // Compute the squared Euclidean distance matrix + double* DD = (double*) malloc(N * N * sizeof(double)); + double* Q = (double*) malloc(N * N * sizeof(double)); + if(DD == NULL || Q == NULL) { printf("Memory allocation failed!\n"); exit(1); } + computeSquaredEuclideanDistance(Y, N, D, DD); + + // Compute Q-matrix and normalization sum + int nN = 0; + double sum_Q = DBL_MIN; + for(int n = 0; n < N; n++) { + for(int m = 0; m < N; m++) { + if(n != m) { + Q[nN + m] = 1 / (1 + DD[nN + m]); + sum_Q += Q[nN + m]; + } + else Q[nN + m] = DBL_MIN; + } + nN += N; + } + for(int i = 0; i < N * N; i++) Q[i] /= sum_Q; + + // Sum t-SNE error + double C = .0; + for(int n = 0; n < N * N; n++) { + C += P[n] * log((P[n] + FLT_MIN) / (Q[n] + FLT_MIN)); + } + + // Clean up memory + free(DD); + free(Q); + return C; +} + +// Evaluate t-SNE cost function (approximately) +static double evaluateError(unsigned int* row_P, unsigned int* col_P, double* val_P, double* Y, int N, int D, double theta) +{ + + // Get estimate of normalization term + SPTree* tree = new SPTree(D, Y, N); + double* buff = (double*) calloc(D, sizeof(double)); + double sum_Q = .0; + for(int n = 0; n < N; n++) tree->computeNonEdgeForces(n, theta, buff, &sum_Q); + + // Loop over all edges to compute t-SNE error + int ind1, ind2; + double C = .0, Q; + for(int n = 0; n < N; n++) { + ind1 = n * D; + for(int i = row_P[n]; i < row_P[n + 1]; i++) { + Q = .0; + ind2 = col_P[i] * D; + for(int d = 0; d < D; d++) buff[d] = Y[ind1 + d]; + for(int d = 0; d < D; d++) buff[d] -= Y[ind2 + d]; + for(int d = 0; d < D; d++) Q += buff[d] * buff[d]; + Q = (1.0 / (1.0 + Q)) / sum_Q; + C += val_P[i] * log((val_P[i] + FLT_MIN) / (Q + FLT_MIN)); + } + } + + // Clean up memory + free(buff); + delete tree; + return C; +} + + +// Compute input similarities with a fixed perplexity +static void computeGaussianPerplexity(double* X, int N, int D, double* P, double perplexity) { + + // Compute the squared Euclidean distance matrix + double* DD = (double*) malloc(N * N * sizeof(double)); + if(DD == NULL) { printf("Memory allocation failed!\n"); exit(1); } + computeSquaredEuclideanDistance(X, N, D, DD); + + // Compute the Gaussian kernel row by row + int nN = 0; + for(int n = 0; n < N; n++) { + + // Initialize some variables + bool found = false; + double beta = 1.0; + double min_beta = -DBL_MAX; + double max_beta = DBL_MAX; + double tol = 1e-5; + double sum_P; + + // Iterate until we found a good perplexity + int iter = 0; + while(!found && iter < 200) { + + // Compute Gaussian kernel row + for(int m = 0; m < N; m++) P[nN + m] = exp(-beta * DD[nN + m]); + P[nN + n] = DBL_MIN; + + // Compute entropy of current row + sum_P = DBL_MIN; + for(int m = 0; m < N; m++) sum_P += P[nN + m]; + double H = 0.0; + for(int m = 0; m < N; m++) H += beta * (DD[nN + m] * P[nN + m]); + H = (H / sum_P) + log(sum_P); + + // Evaluate whether the entropy is within the tolerance level + double Hdiff = H - log(perplexity); + if(Hdiff < tol && -Hdiff < tol) { + found = true; + } + else { + if(Hdiff > 0) { + min_beta = beta; + if(max_beta == DBL_MAX || max_beta == -DBL_MAX) + beta *= 2.0; + else + beta = (beta + max_beta) / 2.0; + } + else { + max_beta = beta; + if(min_beta == -DBL_MAX || min_beta == DBL_MAX){ + if (beta < 0) { + beta *= 2; + } else { + beta = beta <= 1.0 ? -0.5 : beta / 2.0; + } + } else { + beta = (beta + min_beta) / 2.0; + } + } + } + + // Update iteration counter + iter++; + } + + // Row normalize P + for(int m = 0; m < N; m++) P[nN + m] /= sum_P; + nN += N; + } + + // Clean up memory + free(DD); DD = NULL; +} + + +// Compute input similarities with a fixed perplexity using ball trees (this function allocates memory another function should free) +static void computeGaussianPerplexity(double* X, int N, int D, unsigned int** _row_P, unsigned int** _col_P, double** _val_P, double perplexity, int K) { + + if(perplexity > K) printf("Perplexity should be lower than K!\n"); + + // Allocate the memory we need + *_row_P = (unsigned int*) malloc((N + 1) * sizeof(unsigned int)); + *_col_P = (unsigned int*) calloc(N * K, sizeof(unsigned int)); + *_val_P = (double*) calloc(N * K, sizeof(double)); + if(*_row_P == NULL || *_col_P == NULL || *_val_P == NULL) { printf("Memory allocation failed!\n"); exit(1); } + unsigned int* row_P = *_row_P; + unsigned int* col_P = *_col_P; + double* val_P = *_val_P; + double* cur_P = (double*) malloc((N - 1) * sizeof(double)); + if(cur_P == NULL) { printf("Memory allocation failed!\n"); exit(1); } + row_P[0] = 0; + for(int n = 0; n < N; n++) row_P[n + 1] = row_P[n] + (unsigned int) K; + + // Build ball tree on data set + VpTree* tree = new VpTree(); + vector obj_X(N, DataPoint(D, -1, X)); + for(int n = 0; n < N; n++) obj_X[n] = DataPoint(D, n, X + n * D); + tree->create(obj_X); + + // Loop over all points to find nearest neighbors + printf("Building tree...\n"); + vector indices; + vector distances; + for(int n = 0; n < N; n++) { + + if(n % 10000 == 0) printf(" - point %d of %d\n", n, N); + + // Find nearest neighbors + indices.clear(); + distances.clear(); + tree->search(obj_X[n], K + 1, &indices, &distances); + + // Initialize some variables for binary search + bool found = false; + double beta = 1.0; + double min_beta = -DBL_MAX; + double max_beta = DBL_MAX; + double tol = 1e-5; + + // Iterate until we found a good perplexity + int iter = 0; double sum_P; + while(!found && iter < 200) { + + // Compute Gaussian kernel row + for(int m = 0; m < K; m++) cur_P[m] = exp(-beta * distances[m + 1] * distances[m + 1]); + + // Compute entropy of current row + sum_P = DBL_MIN; + for(int m = 0; m < K; m++) sum_P += cur_P[m]; + double H = .0; + for(int m = 0; m < K; m++) H += beta * (distances[m + 1] * distances[m + 1] * cur_P[m]); + H = (H / sum_P) + log(sum_P); + + // Evaluate whether the entropy is within the tolerance level + double Hdiff = H - log(perplexity); + if(Hdiff < tol && -Hdiff < tol) { + found = true; + } + else { + if(Hdiff > 0) { + min_beta = beta; + if(max_beta == DBL_MAX || max_beta == -DBL_MAX) + beta *= 2.0; + else + beta = (beta + max_beta) / 2.0; + } + else { + max_beta = beta; + if(min_beta == -DBL_MAX || min_beta == DBL_MAX) + beta /= 2.0; + else + beta = (beta + min_beta) / 2.0; + } + } + + // Update iteration counter + iter++; + } + + // Row-normalize current row of P and store in matrix + for(unsigned int m = 0; m < K; m++) cur_P[m] /= sum_P; + for(unsigned int m = 0; m < K; m++) { + col_P[row_P[n] + m] = (unsigned int) indices[m + 1].index(); + val_P[row_P[n] + m] = cur_P[m]; + } + } + + // Clean up memory + obj_X.clear(); + free(cur_P); + delete tree; +} + + +// Symmetrizes a sparse matrix +static void symmetrizeMatrix(unsigned int** _row_P, unsigned int** _col_P, double** _val_P, int N) { + + // Get sparse matrix + unsigned int* row_P = *_row_P; + unsigned int* col_P = *_col_P; + double* val_P = *_val_P; + + // Count number of elements and row counts of symmetric matrix + int* row_counts = (int*) calloc(N, sizeof(int)); + if(row_counts == NULL) { printf("Memory allocation failed!\n"); exit(1); } + for(int n = 0; n < N; n++) { + for(int i = row_P[n]; i < row_P[n + 1]; i++) { + + // Check whether element (col_P[i], n) is present + bool present = false; + for(int m = row_P[col_P[i]]; m < row_P[col_P[i] + 1]; m++) { + if(col_P[m] == n) present = true; + } + if(present) row_counts[n]++; + else { + row_counts[n]++; + row_counts[col_P[i]]++; + } + } + } + int no_elem = 0; + for(int n = 0; n < N; n++) no_elem += row_counts[n]; + + // Allocate memory for symmetrized matrix + unsigned int* sym_row_P = (unsigned int*) malloc((N + 1) * sizeof(unsigned int)); + unsigned int* sym_col_P = (unsigned int*) malloc(no_elem * sizeof(unsigned int)); + double* sym_val_P = (double*) malloc(no_elem * sizeof(double)); + if(sym_row_P == NULL || sym_col_P == NULL || sym_val_P == NULL) { printf("Memory allocation failed!\n"); exit(1); } + + // Construct new row indices for symmetric matrix + sym_row_P[0] = 0; + for(int n = 0; n < N; n++) sym_row_P[n + 1] = sym_row_P[n] + (unsigned int) row_counts[n]; + + // Fill the result matrix + int* offset = (int*) calloc(N, sizeof(int)); + if(offset == NULL) { printf("Memory allocation failed!\n"); exit(1); } + for(int n = 0; n < N; n++) { + for(unsigned int i = row_P[n]; i < row_P[n + 1]; i++) { // considering element(n, col_P[i]) + + // Check whether element (col_P[i], n) is present + bool present = false; + for(unsigned int m = row_P[col_P[i]]; m < row_P[col_P[i] + 1]; m++) { + if(col_P[m] == n) { + present = true; + if(n <= col_P[i]) { // make sure we do not add elements twice + sym_col_P[sym_row_P[n] + offset[n]] = col_P[i]; + sym_col_P[sym_row_P[col_P[i]] + offset[col_P[i]]] = n; + sym_val_P[sym_row_P[n] + offset[n]] = val_P[i] + val_P[m]; + sym_val_P[sym_row_P[col_P[i]] + offset[col_P[i]]] = val_P[i] + val_P[m]; + } + } + } + + // If (col_P[i], n) is not present, there is no addition involved + if(!present) { + sym_col_P[sym_row_P[n] + offset[n]] = col_P[i]; + sym_col_P[sym_row_P[col_P[i]] + offset[col_P[i]]] = n; + sym_val_P[sym_row_P[n] + offset[n]] = val_P[i]; + sym_val_P[sym_row_P[col_P[i]] + offset[col_P[i]]] = val_P[i]; + } + + // Update offsets + if(!present || (present && n <= col_P[i])) { + offset[n]++; + if(col_P[i] != n) offset[col_P[i]]++; + } + } + } + + // Divide the result by two + for(int i = 0; i < no_elem; i++) sym_val_P[i] /= 2.0; + + // Return symmetrized matrices + free(*_row_P); *_row_P = sym_row_P; + free(*_col_P); *_col_P = sym_col_P; + free(*_val_P); *_val_P = sym_val_P; + + // Free up some memery + free(offset); offset = NULL; + free(row_counts); row_counts = NULL; +} + +// Compute squared Euclidean distance matrix +static void computeSquaredEuclideanDistance(double* X, int N, int D, double* DD) { + const double* XnD = X; + for(int n = 0; n < N; ++n, XnD += D) { + const double* XmD = XnD + D; + double* curr_elem = &DD[n*N + n]; + *curr_elem = 0.0; + double* curr_elem_sym = curr_elem + N; + for(int m = n + 1; m < N; ++m, XmD+=D, curr_elem_sym+=N) { + *(++curr_elem) = 0.0; + for(int d = 0; d < D; ++d) { + *curr_elem += (XnD[d] - XmD[d]) * (XnD[d] - XmD[d]); + } + *curr_elem_sym = *curr_elem; + } + } +} + + +// Makes data zero-mean +static void zeroMean(double* X, int N, int D) { + + // Compute data mean + double* mean = (double*) calloc(D, sizeof(double)); + if(mean == NULL) { printf("Memory allocation failed!\n"); exit(1); } + int nD = 0; + for(int n = 0; n < N; n++) { + for(int d = 0; d < D; d++) { + mean[d] += X[nD + d]; + } + nD += D; + } + for(int d = 0; d < D; d++) { + mean[d] /= (double) N; + } + + // Subtract data mean + nD = 0; + for(int n = 0; n < N; n++) { + for(int d = 0; d < D; d++) { + X[nD + d] -= mean[d]; + } + nD += D; + } + free(mean); mean = NULL; +} + + +// Generates a Gaussian random number +static double randn() { + double x, y, radius; + do { + x = 2 * (rand() / ((double) RAND_MAX + 1)) - 1; + y = 2 * (rand() / ((double) RAND_MAX + 1)) - 1; + radius = (x * x) + (y * y); + } while((radius >= 1.0) || (radius == 0.0)); + radius = sqrt(-2 * log(radius) / radius); + x *= radius; + y *= radius; + return x; +} + +// Function that loads data from a t-SNE file +// Note: this function does a malloc that should be freed elsewhere +bool TSNE::load_data(double** data, int* n, int* d, int* no_dims, double* theta, double* perplexity, int* rand_seed, int* max_iter) { + + // Open file, read first 2 integers, allocate memory, and read the data + FILE *h; + if((h = fopen("data.dat", "r+b")) == NULL) { + printf("Error: could not open data file.\n"); + return false; + } + fread(n, sizeof(int), 1, h); // number of datapoints + fread(d, sizeof(int), 1, h); // original dimensionality + fread(theta, sizeof(double), 1, h); // gradient accuracy + fread(perplexity, sizeof(double), 1, h); // perplexity + fread(no_dims, sizeof(int), 1, h); // output dimensionality + fread(max_iter, sizeof(int),1,h); // maximum number of iterations + *data = (double*) malloc(*d * *n * sizeof(double)); + if(*data == NULL) { printf("Memory allocation failed!\n"); exit(1); } + fread(*data, sizeof(double), *n * *d, h); // the data + if(!feof(h)) fread(rand_seed, sizeof(int), 1, h); // random seed + fclose(h); + printf("Read the %i x %i data matrix successfully!\n", *n, *d); + return true; +} + +// Function that saves map to a t-SNE file +void TSNE::save_data(double* data, int* landmarks, double* costs, int n, int d) { + + // Open file, write first 2 integers and then the data + FILE *h; + if((h = fopen("result.dat", "w+b")) == NULL) { + printf("Error: could not open data file.\n"); + return; + } + fwrite(&n, sizeof(int), 1, h); + fwrite(&d, sizeof(int), 1, h); + fwrite(data, sizeof(double), n * d, h); + fwrite(landmarks, sizeof(int), n, h); + fwrite(costs, sizeof(double), n, h); + fclose(h); + printf("Wrote the %i x %i data matrix successfully!\n", n, d); +} + +#endif diff --git a/third_party/bhtsne/tsne_test.cpp b/third_party/bhtsne/tsne_test.cpp new file mode 100755 index 0000000..4751631 --- /dev/null +++ b/third_party/bhtsne/tsne_test.cpp @@ -0,0 +1,49 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "tsne.h" + +// Function that runs the Barnes-Hut implementation of t-SNE +// simulate input data from code, print result to console +// 3 dims ---> 2 dims for this test +int main() { + + // Define some variables + int origN = 8, N = 8, D = 3, no_dims = 2, max_iter = 1000; + double perplexity = 2, theta = 0.5; + int rand_seed = -1; + double data[] = {100, 23.9, 12, + 45.2, 46.9, 10, + 201, 44.8, 30, + 201, 40.2, 32, + 101, 7.8, 33, + 21, 4, 35, + 441, 4.8, 36, + 2.1, 32.4, 56}; + + N = origN; + + // Now fire up the SNE implementation + double* Y = (double*) malloc(N * no_dims * sizeof(double)); + if(Y == NULL) { printf("Memory allocation failed!\n"); exit(1); } + + // data: input high dims of features, N * D + // Y: output low dims of features, N * no_dims + TSNE::run(data, N, D, Y, no_dims, perplexity, theta, rand_seed, false, max_iter, 250, 250); + + // Print the results + for (int i = 0; i < N; i++) { + /* code */ + std::cout << Y[i] << "," << Y[i+1] << std::endl; + } + + // you can display result on 2D screen here + + // Clean up the memory + free(Y); Y = NULL; +} diff --git a/third_party/bhtsne/vptree.h b/third_party/bhtsne/vptree.h new file mode 100755 index 0000000..9e301d6 --- /dev/null +++ b/third_party/bhtsne/vptree.h @@ -0,0 +1,272 @@ +/* + * + * Copyright (c) 2014, Laurens van der Maaten (Delft University of Technology) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the Delft University of Technology. + * 4. Neither the name of the Delft University of Technology nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY LAURENS VAN DER MAATEN ''AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL LAURENS VAN DER MAATEN BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + */ + + +/* This code was adopted with minor modifications from Steve Hanov's great tutorial at http://stevehanov.ca/blog/index.php?id=130 */ + +#include +#include +#include +#include +#include +#include +#include + + +#ifndef VPTREE_H +#define VPTREE_H + +class DataPoint +{ + int _ind; + +public: + double* _x; + int _D; + DataPoint() { + _D = 1; + _ind = -1; + _x = NULL; + } + DataPoint(int D, int ind, double* x) { + _D = D; + _ind = ind; + _x = (double*) malloc(_D * sizeof(double)); + for(int d = 0; d < _D; d++) _x[d] = x[d]; + } + DataPoint(const DataPoint& other) { // this makes a deep copy -- should not free anything + if(this != &other) { + _D = other.dimensionality(); + _ind = other.index(); + _x = (double*) malloc(_D * sizeof(double)); + for(int d = 0; d < _D; d++) _x[d] = other.x(d); + } + } + ~DataPoint() { if(_x != NULL) free(_x); } + DataPoint& operator= (const DataPoint& other) { // asignment should free old object + if(this != &other) { + if(_x != NULL) free(_x); + _D = other.dimensionality(); + _ind = other.index(); + _x = (double*) malloc(_D * sizeof(double)); + for(int d = 0; d < _D; d++) _x[d] = other.x(d); + } + return *this; + } + int index() const { return _ind; } + int dimensionality() const { return _D; } + double x(int d) const { return _x[d]; } +}; + +double euclidean_distance(const DataPoint &t1, const DataPoint &t2) { + double dd = .0; + double* x1 = t1._x; + double* x2 = t2._x; + double diff; + for(int d = 0; d < t1._D; d++) { + diff = (x1[d] - x2[d]); + dd += diff * diff; + } + return sqrt(dd); +} + + +template +class VpTree +{ +public: + + // Default constructor + VpTree() : _root(0) {} + + // Destructor + ~VpTree() { + delete _root; + } + + // Function to create a new VpTree from data + void create(const std::vector& items) { + delete _root; + _items = items; + _root = buildFromPoints(0, items.size()); + } + + // Function that uses the tree to find the k nearest neighbors of target + void search(const T& target, int k, std::vector* results, std::vector* distances) + { + + // Use a priority queue to store intermediate results on + std::priority_queue heap; + + // Variable that tracks the distance to the farthest point in our results + _tau = DBL_MAX; + + // Perform the search + search(_root, target, k, heap); + + // Gather final results + results->clear(); distances->clear(); + while(!heap.empty()) { + results->push_back(_items[heap.top().index]); + distances->push_back(heap.top().dist); + heap.pop(); + } + + // Results are in reverse order + std::reverse(results->begin(), results->end()); + std::reverse(distances->begin(), distances->end()); + } + +private: + std::vector _items; + double _tau; + + // Single node of a VP tree (has a point and radius; left children are closer to point than the radius) + struct Node + { + int index; // index of point in node + double threshold; // radius(?) + Node* left; // points closer by than threshold + Node* right; // points farther away than threshold + + Node() : + index(0), threshold(0.), left(0), right(0) {} + + ~Node() { // destructor + delete left; + delete right; + } + }* _root; + + + // An item on the intermediate result queue + struct HeapItem { + HeapItem( int index, double dist) : + index(index), dist(dist) {} + int index; + double dist; + bool operator<(const HeapItem& o) const { + return dist < o.dist; + } + }; + + // Distance comparator for use in std::nth_element + struct DistanceComparator + { + const T& item; + DistanceComparator(const T& item) : item(item) {} + bool operator()(const T& a, const T& b) { + return distance(item, a) < distance(item, b); + } + }; + + // Function that (recursively) fills the tree + Node* buildFromPoints( int lower, int upper ) + { + if (upper == lower) { // indicates that we're done here! + return NULL; + } + + // Lower index is center of current node + Node* node = new Node(); + node->index = lower; + + if (upper - lower > 1) { // if we did not arrive at leaf yet + + // Choose an arbitrary point and move it to the start + int i = (int) ((double)rand() / RAND_MAX * (upper - lower - 1)) + lower; + std::swap(_items[lower], _items[i]); + + // Partition around the median distance + int median = (upper + lower) / 2; + std::nth_element(_items.begin() + lower + 1, + _items.begin() + median, + _items.begin() + upper, + DistanceComparator(_items[lower])); + + // Threshold of the new node will be the distance to the median + node->threshold = distance(_items[lower], _items[median]); + + // Recursively build tree + node->index = lower; + node->left = buildFromPoints(lower + 1, median); + node->right = buildFromPoints(median, upper); + } + + // Return result + return node; + } + + // Helper function that searches the tree + void search(Node* node, const T& target, int k, std::priority_queue& heap) + { + if(node == NULL) return; // indicates that we're done here + + // Compute distance between target and current node + double dist = distance(_items[node->index], target); + + // If current node within radius tau + if(dist < _tau) { + if(heap.size() == k) heap.pop(); // remove furthest node from result list (if we already have k results) + heap.push(HeapItem(node->index, dist)); // add current node to result list + if(heap.size() == k) _tau = heap.top().dist; // update value of tau (farthest point in result list) + } + + // Return if we arrived at a leaf + if(node->left == NULL && node->right == NULL) { + return; + } + + // If the target lies within the radius of ball + if(dist < node->threshold) { + if(dist - _tau <= node->threshold) { // if there can still be neighbors inside the ball, recursively search left child first + search(node->left, target, k, heap); + } + + if(dist + _tau >= node->threshold) { // if there can still be neighbors outside the ball, recursively search right child + search(node->right, target, k, heap); + } + + // If the target lies outsize the radius of the ball + } else { + if(dist + _tau >= node->threshold) { // if there can still be neighbors outside the ball, recursively search right child first + search(node->right, target, k, heap); + } + + if (dist - _tau <= node->threshold) { // if there can still be neighbors inside the ball, recursively search left child + search(node->left, target, k, heap); + } + } + } +}; + +#endif diff --git a/third_party/cereal/README.md b/third_party/cereal/README.md new file mode 100644 index 0000000..8157b7b --- /dev/null +++ b/third_party/cereal/README.md @@ -0,0 +1,7 @@ + +cereal project from https://github.com/USCiLab/cereal +used for serializing objects to json or xml. + +**NOTE** + +has changed `#include` from absolute path to relative path. \ No newline at end of file diff --git a/third_party/cereal/access.hpp b/third_party/cereal/access.hpp new file mode 100755 index 0000000..5debed9 --- /dev/null +++ b/third_party/cereal/access.hpp @@ -0,0 +1,351 @@ +/*! \file access.hpp + \brief Access control and default construction */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_ACCESS_HPP_ +#define CEREAL_ACCESS_HPP_ + +#include +#include +#include +#include + +#include "macros.hpp" +#include "specialize.hpp" +#include "details/helpers.hpp" + +namespace cereal +{ + // ###################################################################### + //! A class that allows cereal to load smart pointers to types that have no default constructor + /*! If your class does not have a default constructor, cereal will not be able + to load any smart pointers to it unless you overload LoadAndConstruct + for your class, and provide an appropriate load_and_construct method. You can also + choose to define a member static function instead of specializing this class. + + The specialization of LoadAndConstruct must be placed within the cereal namespace: + + @code{.cpp} + struct MyType + { + MyType( int x ); // note: no default ctor + int myX; + + // Define a serialize or load/save pair as you normally would + template + void serialize( Archive & ar ) + { + ar( myX ); + } + }; + + // Provide a specialization for LoadAndConstruct for your type + namespace cereal + { + template <> struct LoadAndConstruct + { + // load_and_construct will be passed the archive that you will be loading + // from as well as a construct object which you can use as if it were the + // constructor for your type. cereal will handle all memory management for you. + template + static void load_and_construct( Archive & ar, cereal::construct & construct ) + { + int x; + ar( x ); + construct( x ); + } + + // if you require versioning, simply add a const std::uint32_t as the final parameter, e.g.: + // load_and_construct( Archive & ar, cereal::construct & construct, std::uint32_t const version ) + }; + } // end namespace cereal + @endcode + + Please note that just as in using external serialization functions, you cannot get + access to non-public members of your class by befriending cereal::access. If you + have the ability to modify the class you wish to serialize, it is recommended that you + use member serialize functions and a static member load_and_construct function. + + load_and_construct functions, regardless of whether they are static members of your class or + whether you create one in the LoadAndConstruct specialization, have the following signature: + + @code{.cpp} + // generally Archive will be templated, but it can be specific if desired + template + static void load_and_construct( Archive & ar, cereal::construct & construct ); + // with an optional last parameter specifying the version: const std::uint32_t version + @endcode + + Versioning behaves the same way as it does for standard serialization functions. + + @tparam T The type to specialize for + @ingroup Access */ + template + struct LoadAndConstruct + { }; + + // forward decl for construct + //! @cond PRIVATE_NEVERDEFINED + namespace memory_detail{ template struct LoadAndConstructLoadWrapper; } + namespace boost_variant_detail{ template struct LoadAndConstructLoadWrapper; } + //! @endcond + + //! Used to construct types with no default constructor + /*! When serializing a type that has no default constructor, cereal + will attempt to call either the class static function load_and_construct + or the appropriate template specialization of LoadAndConstruct. cereal + will pass that function a reference to the archive as well as a reference + to a construct object which should be used to perform the allocation once + data has been appropriately loaded. + + @code{.cpp} + struct MyType + { + // note the lack of default constructor + MyType( int xx, int yy ); + + int x, y; + double notInConstructor; + + template + void serialize( Archive & ar ) + { + ar( x, y ); + ar( notInConstructor ); + } + + template + static void load_and_construct( Archive & ar, cereal::construct & construct ) + { + int x, y; + ar( x, y ); + + // use construct object to initialize with loaded data + construct( x, y ); + + // access to member variables and functions via -> operator + ar( construct->notInConstructor ); + + // could also do the above section by: + double z; + ar( z ); + construct->notInConstructor = z; + } + }; + @endcode + + @tparam T The class type being serialized + */ + template + class construct + { + public: + //! Construct and initialize the type T with the given arguments + /*! This will forward all arguments to the underlying type T, + calling an appropriate constructor. + + Calling this function more than once will result in an exception + being thrown. + + @param args The arguments to the constructor for T + @throw Exception If called more than once */ + template + void operator()( Args && ... args ); + // implementation deferred due to reliance on cereal::access + + //! Get a reference to the initialized underlying object + /*! This must be called after the object has been initialized. + + @return A reference to the initialized object + @throw Exception If called before initialization */ + T * operator->() + { + if( !itsValid ) + throw Exception("Object must be initialized prior to accessing members"); + + return itsPtr; + } + + //! Returns a raw pointer to the initialized underlying object + /*! This is mainly intended for use with passing an instance of + a constructed object to cereal::base_class. + + It is strongly recommended to avoid using this function in + any other circumstance. + + @return A raw pointer to the initialized type */ + T * ptr() + { + return operator->(); + } + + private: + template friend struct ::cereal::memory_detail::LoadAndConstructLoadWrapper; + template friend struct ::cereal::boost_variant_detail::LoadAndConstructLoadWrapper; + + construct( T * p ) : itsPtr( p ), itsEnableSharedRestoreFunction( [](){} ), itsValid( false ) {} + construct( T * p, std::function enableSharedFunc ) : // g++4.7 ice with default lambda to std func + itsPtr( p ), itsEnableSharedRestoreFunction( enableSharedFunc ), itsValid( false ) {} + construct( construct const & ) = delete; + construct & operator=( construct const & ) = delete; + + T * itsPtr; + std::function itsEnableSharedRestoreFunction; + bool itsValid; + }; + + // ###################################################################### + //! A class that can be made a friend to give cereal access to non public functions + /*! If you desire non-public serialization functions within a class, cereal can only + access these if you declare cereal::access a friend. + + @code{.cpp} + class MyClass + { + private: + friend class cereal::access; // gives access to the private serialize + + template + void serialize( Archive & ar ) + { + // some code + } + }; + @endcode + @ingroup Access */ + class access + { + public: + // ####### Standard Serialization ######################################## + template inline + static auto member_serialize(Archive & ar, T & t) -> decltype(t.CEREAL_SERIALIZE_FUNCTION_NAME(ar)) + { return t.CEREAL_SERIALIZE_FUNCTION_NAME(ar); } + + template inline + static auto member_save(Archive & ar, T const & t) -> decltype(t.CEREAL_SAVE_FUNCTION_NAME(ar)) + { return t.CEREAL_SAVE_FUNCTION_NAME(ar); } + + template inline + static auto member_save_non_const(Archive & ar, T & t) -> decltype(t.CEREAL_SAVE_FUNCTION_NAME(ar)) + { return t.CEREAL_SAVE_FUNCTION_NAME(ar); } + + template inline + static auto member_load(Archive & ar, T & t) -> decltype(t.CEREAL_LOAD_FUNCTION_NAME(ar)) + { return t.CEREAL_LOAD_FUNCTION_NAME(ar); } + + template inline + static auto member_save_minimal(Archive const & ar, T const & t) -> decltype(t.CEREAL_SAVE_MINIMAL_FUNCTION_NAME(ar)) + { return t.CEREAL_SAVE_MINIMAL_FUNCTION_NAME(ar); } + + template inline + static auto member_save_minimal_non_const(Archive const & ar, T & t) -> decltype(t.CEREAL_SAVE_MINIMAL_FUNCTION_NAME(ar)) + { return t.CEREAL_SAVE_MINIMAL_FUNCTION_NAME(ar); } + + template inline + static auto member_load_minimal(Archive const & ar, T & t, U && u) -> decltype(t.CEREAL_LOAD_MINIMAL_FUNCTION_NAME(ar, std::forward(u))) + { return t.CEREAL_LOAD_MINIMAL_FUNCTION_NAME(ar, std::forward(u)); } + + // ####### Versioned Serialization ####################################### + template inline + static auto member_serialize(Archive & ar, T & t, const std::uint32_t version ) -> decltype(t.CEREAL_SERIALIZE_FUNCTION_NAME(ar, version)) + { return t.CEREAL_SERIALIZE_FUNCTION_NAME(ar, version); } + + template inline + static auto member_save(Archive & ar, T const & t, const std::uint32_t version ) -> decltype(t.CEREAL_SAVE_FUNCTION_NAME(ar, version)) + { return t.CEREAL_SAVE_FUNCTION_NAME(ar, version); } + + template inline + static auto member_save_non_const(Archive & ar, T & t, const std::uint32_t version ) -> decltype(t.CEREAL_SAVE_FUNCTION_NAME(ar, version)) + { return t.CEREAL_SAVE_FUNCTION_NAME(ar, version); } + + template inline + static auto member_load(Archive & ar, T & t, const std::uint32_t version ) -> decltype(t.CEREAL_LOAD_FUNCTION_NAME(ar, version)) + { return t.CEREAL_LOAD_FUNCTION_NAME(ar, version); } + + template inline + static auto member_save_minimal(Archive const & ar, T const & t, const std::uint32_t version) -> decltype(t.CEREAL_SAVE_MINIMAL_FUNCTION_NAME(ar, version)) + { return t.CEREAL_SAVE_MINIMAL_FUNCTION_NAME(ar, version); } + + template inline + static auto member_save_minimal_non_const(Archive const & ar, T & t, const std::uint32_t version) -> decltype(t.CEREAL_SAVE_MINIMAL_FUNCTION_NAME(ar, version)) + { return t.CEREAL_SAVE_MINIMAL_FUNCTION_NAME(ar, version); } + + template inline + static auto member_load_minimal(Archive const & ar, T & t, U && u, const std::uint32_t version) -> decltype(t.CEREAL_LOAD_MINIMAL_FUNCTION_NAME(ar, std::forward(u), version)) + { return t.CEREAL_LOAD_MINIMAL_FUNCTION_NAME(ar, std::forward(u), version); } + + // ####### Other Functionality ########################################## + // for detecting inheritance from enable_shared_from_this + template inline + static auto shared_from_this(T & t) -> decltype(t.shared_from_this()); + + // for placement new + template inline + static void construct( T *& ptr, Args && ... args ) + { + new (ptr) T( std::forward( args )... ); + } + + // for non-placement new with a default constructor + template inline + static T * construct() + { + return new T(); + } + + template inline + static std::false_type load_and_construct(...) + { return std::false_type(); } + + template inline + static auto load_and_construct(Archive & ar, ::cereal::construct & construct) -> decltype(T::load_and_construct(ar, construct)) + { + T::load_and_construct( ar, construct ); + } + + template inline + static auto load_and_construct(Archive & ar, ::cereal::construct & construct, const std::uint32_t version) -> decltype(T::load_and_construct(ar, construct, version)) + { + T::load_and_construct( ar, construct, version ); + } + }; // end class access + + // ###################################################################### + // Deferred Implementation, see construct for more information + template template inline + void construct::operator()( Args && ... args ) + { + if( itsValid ) + throw Exception("Attempting to construct an already initialized object"); + + ::cereal::access::construct( itsPtr, std::forward( args )... ); + itsEnableSharedRestoreFunction(); + itsValid = true; + } +} // namespace cereal + +#endif // CEREAL_ACCESS_HPP_ diff --git a/third_party/cereal/archives/adapters.hpp b/third_party/cereal/archives/adapters.hpp new file mode 100755 index 0000000..a9e6825 --- /dev/null +++ b/third_party/cereal/archives/adapters.hpp @@ -0,0 +1,163 @@ +/*! \file adapters.hpp + \brief Archive adapters that provide additional functionality + on top of an existing archive */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_ARCHIVES_ADAPTERS_HPP_ +#define CEREAL_ARCHIVES_ADAPTERS_HPP_ + +#include "../details/helpers.hpp" +#include + +namespace cereal +{ + #ifdef CEREAL_FUTURE_EXPERIMENTAL + + // Forward declaration for friend access + template U & get_user_data( A & ); + + //! Wraps an archive and gives access to user data + /*! This adapter is useful if you require access to + either raw pointers or references within your + serialization functions. + + While cereal does not directly support serialization + raw pointers or references, it is sometimes the case + that you may want to supply something such as a raw + pointer or global reference to some constructor. + In this situation this adapter would likely be used + with the construct class to allow for non-default + constructors. + + @note This feature is experimental and may be altered or removed in a future release. See issue #46. + + @code{.cpp} + struct MyUserData + { + int * myRawPointer; + std::reference_wrapper myReference; + }; + + struct MyClass + { + // Note the raw pointer parameter + MyClass( int xx, int * rawP ); + + int x; + + template + void serialize( Archive & ar ) + { ar( x ); } + + template + static void load_and_construct( Archive & ar, cereal::construct & construct ) + { + int xx; + ar( xx ); + // note the need to use get_user_data to retrieve user data from the archive + construct( xx, cereal::get_user_data( ar ).myRawPointer ); + } + }; + + int main() + { + { + MyUserData md; + md.myRawPointer = &something; + md.myReference = someInstanceOfType; + + std::ifstream is( "data.xml" ); + cereal::UserDataAdapter ar( md, is ); + + std::unique_ptr sc; + ar( sc ); // use as normal + } + + return 0; + } + @endcode + + @relates get_user_data + + @tparam UserData The type to give the archive access to + @tparam Archive The archive to wrap */ + template + class UserDataAdapter : public Archive + { + public: + //! Construct the archive with some user data struct + /*! This will forward all arguments (other than the user + data) to the wrapped archive type. The UserDataAdapter + can then be used identically to the wrapped archive type + + @tparam Args The arguments to pass to the constructor of + the archive. */ + template + UserDataAdapter( UserData & ud, Args && ... args ) : + Archive( std::forward( args )... ), + userdata( ud ) + { } + + private: + //! Overload the rtti function to enable dynamic_cast + void rtti() {} + friend UserData & get_user_data( Archive & ar ); + UserData & userdata; //!< The actual user data + }; + + //! Retrieves user data from an archive wrapped by UserDataAdapter + /*! This will attempt to retrieve the user data associated with + some archive wrapped by UserDataAdapter. If this is used on + an archive that is not wrapped, a run-time exception will occur. + + @note This feature is experimental and may be altered or removed in a future release. See issue #46. + + @note The correct use of this function cannot be enforced at compile + time. + + @relates UserDataAdapter + @tparam UserData The data struct contained in the archive + @tparam Archive The archive, which should be wrapped by UserDataAdapter + @param ar The archive + @throws Exception if the archive this is used upon is not wrapped with + UserDataAdapter. */ + template + UserData & get_user_data( Archive & ar ) + { + try + { + return dynamic_cast &>( ar ).userdata; + } + catch( std::bad_cast const & ) + { + throw ::cereal::Exception("Attempting to get user data from archive not wrapped in UserDataAdapter"); + } + } + #endif // CEREAL_FUTURE_EXPERIMENTAL +} // namespace cereal + +#endif // CEREAL_ARCHIVES_ADAPTERS_HPP_ diff --git a/third_party/cereal/archives/binary.hpp b/third_party/cereal/archives/binary.hpp new file mode 100755 index 0000000..5bc966f --- /dev/null +++ b/third_party/cereal/archives/binary.hpp @@ -0,0 +1,169 @@ +/*! \file binary.hpp + \brief Binary input and output archives */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_ARCHIVES_BINARY_HPP_ +#define CEREAL_ARCHIVES_BINARY_HPP_ + +#include "../cereal.hpp" +#include + +namespace cereal +{ + // ###################################################################### + //! An output archive designed to save data in a compact binary representation + /*! This archive outputs data to a stream in an extremely compact binary + representation with as little extra metadata as possible. + + This archive does nothing to ensure that the endianness of the saved + and loaded data is the same. If you need to have portability over + architectures with different endianness, use PortableBinaryOutputArchive. + + When using a binary archive and a file stream, you must use the + std::ios::binary format flag to avoid having your data altered + inadvertently. + + \ingroup Archives */ + class BinaryOutputArchive : public OutputArchive + { + public: + //! Construct, outputting to the provided stream + /*! @param stream The stream to output to. Can be a stringstream, a file stream, or + even cout! */ + BinaryOutputArchive(std::ostream & stream) : + OutputArchive(this), + itsStream(stream) + { } + + ~BinaryOutputArchive() CEREAL_NOEXCEPT = default; + + //! Writes size bytes of data to the output stream + void saveBinary( const void * data, std::streamsize size ) + { + auto const writtenSize = itsStream.rdbuf()->sputn( reinterpret_cast( data ), size ); + + if(writtenSize != size) + throw Exception("Failed to write " + std::to_string(size) + " bytes to output stream! Wrote " + std::to_string(writtenSize)); + } + + private: + std::ostream & itsStream; + }; + + // ###################################################################### + //! An input archive designed to load data saved using BinaryOutputArchive + /* This archive does nothing to ensure that the endianness of the saved + and loaded data is the same. If you need to have portability over + architectures with different endianness, use PortableBinaryOutputArchive. + + When using a binary archive and a file stream, you must use the + std::ios::binary format flag to avoid having your data altered + inadvertently. + + \ingroup Archives */ + class BinaryInputArchive : public InputArchive + { + public: + //! Construct, loading from the provided stream + BinaryInputArchive(std::istream & stream) : + InputArchive(this), + itsStream(stream) + { } + + ~BinaryInputArchive() CEREAL_NOEXCEPT = default; + + //! Reads size bytes of data from the input stream + void loadBinary( void * const data, std::streamsize size ) + { + auto const readSize = itsStream.rdbuf()->sgetn( reinterpret_cast( data ), size ); + + if(readSize != size) + throw Exception("Failed to read " + std::to_string(size) + " bytes from input stream! Read " + std::to_string(readSize)); + } + + private: + std::istream & itsStream; + }; + + // ###################################################################### + // Common BinaryArchive serialization functions + + //! Saving for POD types to binary + template inline + typename std::enable_if::value, void>::type + CEREAL_SAVE_FUNCTION_NAME(BinaryOutputArchive & ar, T const & t) + { + ar.saveBinary(std::addressof(t), sizeof(t)); + } + + //! Loading for POD types from binary + template inline + typename std::enable_if::value, void>::type + CEREAL_LOAD_FUNCTION_NAME(BinaryInputArchive & ar, T & t) + { + ar.loadBinary(std::addressof(t), sizeof(t)); + } + + //! Serializing NVP types to binary + template inline + CEREAL_ARCHIVE_RESTRICT(BinaryInputArchive, BinaryOutputArchive) + CEREAL_SERIALIZE_FUNCTION_NAME( Archive & ar, NameValuePair & t ) + { + ar( t.value ); + } + + //! Serializing SizeTags to binary + template inline + CEREAL_ARCHIVE_RESTRICT(BinaryInputArchive, BinaryOutputArchive) + CEREAL_SERIALIZE_FUNCTION_NAME( Archive & ar, SizeTag & t ) + { + ar( t.size ); + } + + //! Saving binary data + template inline + void CEREAL_SAVE_FUNCTION_NAME(BinaryOutputArchive & ar, BinaryData const & bd) + { + ar.saveBinary( bd.data, static_cast( bd.size ) ); + } + + //! Loading binary data + template inline + void CEREAL_LOAD_FUNCTION_NAME(BinaryInputArchive & ar, BinaryData & bd) + { + ar.loadBinary(bd.data, static_cast( bd.size ) ); + } +} // namespace cereal + +// register archives for polymorphic support +CEREAL_REGISTER_ARCHIVE(cereal::BinaryOutputArchive) +CEREAL_REGISTER_ARCHIVE(cereal::BinaryInputArchive) + +// tie input and output archives together +CEREAL_SETUP_ARCHIVE_TRAITS(cereal::BinaryInputArchive, cereal::BinaryOutputArchive) + +#endif // CEREAL_ARCHIVES_BINARY_HPP_ diff --git a/third_party/cereal/archives/json.hpp b/third_party/cereal/archives/json.hpp new file mode 100755 index 0000000..8b71858 --- /dev/null +++ b/third_party/cereal/archives/json.hpp @@ -0,0 +1,1024 @@ +/*! \file json.hpp + \brief JSON input and output archives */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_ARCHIVES_JSON_HPP_ +#define CEREAL_ARCHIVES_JSON_HPP_ + +#include "../cereal.hpp" +#include "../details/util.hpp" + +namespace cereal +{ + //! An exception thrown when rapidjson fails an internal assertion + /*! @ingroup Utility */ + struct RapidJSONException : Exception + { RapidJSONException( const char * what_ ) : Exception( what_ ) {} }; +} + +// Inform rapidjson that assert will throw +#ifndef CEREAL_RAPIDJSON_ASSERT_THROWS +#define CEREAL_RAPIDJSON_ASSERT_THROWS +#endif // CEREAL_RAPIDJSON_ASSERT_THROWS + +// Override rapidjson assertions to throw exceptions by default +#ifndef CEREAL_RAPIDJSON_ASSERT +#define CEREAL_RAPIDJSON_ASSERT(x) if(!(x)){ \ + throw ::cereal::RapidJSONException("rapidjson internal assertion failure: " #x); } +#endif // RAPIDJSON_ASSERT + +// Enable support for parsing of nan, inf, -inf +#ifndef CEREAL_RAPIDJSON_WRITE_DEFAULT_FLAGS +#define CEREAL_RAPIDJSON_WRITE_DEFAULT_FLAGS kWriteNanAndInfFlag +#endif + +// Enable support for parsing of nan, inf, -inf +#ifndef CEREAL_RAPIDJSON_PARSE_DEFAULT_FLAGS +#define CEREAL_RAPIDJSON_PARSE_DEFAULT_FLAGS kParseFullPrecisionFlag | kParseNanAndInfFlag +#endif + +#include "../external/rapidjson/prettywriter.h" +#include "../external/rapidjson/ostreamwrapper.h" +#include "../external/rapidjson/istreamwrapper.h" +#include "../external/rapidjson/document.h" +#include "../external/base64.hpp" + +#include +#include +#include +#include +#include + +namespace cereal +{ + // ###################################################################### + //! An output archive designed to save data to JSON + /*! This archive uses RapidJSON to build serialize data to JSON. + + JSON archives provides a human readable output but at decreased + performance (both in time and space) compared to binary archives. + + JSON archives are only guaranteed to finish flushing their contents + upon destruction and should thus be used in an RAII fashion. + + JSON benefits greatly from name-value pairs, which if present, will + name the nodes in the output. If these are not present, each level + of the output will be given an automatically generated delimited name. + + The precision of the output archive controls the number of decimals output + for floating point numbers and should be sufficiently large (i.e. at least 20) + if there is a desire to have binary equality between the numbers output and + those read in. In general you should expect a loss of precision when going + from floating point to text and back. + + JSON archives do not output the size information for any dynamically sized structure + and instead infer it from the number of children for a node. This means that data + can be hand edited for dynamic sized structures and will still be readable. This + is accomplished through the cereal::SizeTag object, which will cause the archive + to output the data as a JSON array (e.g. marked by [] instead of {}), which indicates + that the container is variable sized and may be edited. + + \ingroup Archives */ + class JSONOutputArchive : public OutputArchive, public traits::TextArchive + { + enum class NodeType { StartObject, InObject, StartArray, InArray }; + + using WriteStream = CEREAL_RAPIDJSON_NAMESPACE::OStreamWrapper; + using JSONWriter = CEREAL_RAPIDJSON_NAMESPACE::PrettyWriter; + + public: + /*! @name Common Functionality + Common use cases for directly interacting with an JSONOutputArchive */ + //! @{ + + //! A class containing various advanced options for the JSON archive + class Options + { + public: + //! Default options + static Options Default(){ return Options(); } + + //! Default options with no indentation + static Options NoIndent(){ return Options( JSONWriter::kDefaultMaxDecimalPlaces, IndentChar::space, 0 ); } + + //! The character to use for indenting + enum class IndentChar : char + { + space = ' ', + tab = '\t', + newline = '\n', + carriage_return = '\r' + }; + + //! Specify specific options for the JSONOutputArchive + /*! @param precision The precision used for floating point numbers + @param indentChar The type of character to indent with + @param indentLength The number of indentChar to use for indentation + (0 corresponds to no indentation) */ + explicit Options( int precision = JSONWriter::kDefaultMaxDecimalPlaces, + IndentChar indentChar = IndentChar::space, + unsigned int indentLength = 4 ) : + itsPrecision( precision ), + itsIndentChar( static_cast(indentChar) ), + itsIndentLength( indentLength ) { } + + private: + friend class JSONOutputArchive; + int itsPrecision; + char itsIndentChar; + unsigned int itsIndentLength; + }; + + //! Construct, outputting to the provided stream + /*! @param stream The stream to output to. + @param options The JSON specific options to use. See the Options struct + for the values of default parameters */ + JSONOutputArchive(std::ostream & stream, Options const & options = Options::Default() ) : + OutputArchive(this), + itsWriteStream(stream), + itsWriter(itsWriteStream), + itsNextName(nullptr) + { + itsWriter.SetMaxDecimalPlaces( options.itsPrecision ); + itsWriter.SetIndent( options.itsIndentChar, options.itsIndentLength ); + itsNameCounter.push(0); + itsNodeStack.push(NodeType::StartObject); + } + + //! Destructor, flushes the JSON + ~JSONOutputArchive() CEREAL_NOEXCEPT + { + if (itsNodeStack.top() == NodeType::InObject) + itsWriter.EndObject(); + else if (itsNodeStack.top() == NodeType::InArray) + itsWriter.EndArray(); + } + + //! Saves some binary data, encoded as a base64 string, with an optional name + /*! This will create a new node, optionally named, and insert a value that consists of + the data encoded as a base64 string */ + void saveBinaryValue( const void * data, size_t size, const char * name = nullptr ) + { + setNextName( name ); + writeName(); + + auto base64string = base64::encode( reinterpret_cast( data ), size ); + saveValue( base64string ); + }; + + //! @} + /*! @name Internal Functionality + Functionality designed for use by those requiring control over the inner mechanisms of + the JSONOutputArchive */ + //! @{ + + //! Starts a new node in the JSON output + /*! The node can optionally be given a name by calling setNextName prior + to creating the node + + Nodes only need to be started for types that are themselves objects or arrays */ + void startNode() + { + writeName(); + itsNodeStack.push(NodeType::StartObject); + itsNameCounter.push(0); + } + + //! Designates the most recently added node as finished + void finishNode() + { + // if we ended up serializing an empty object or array, writeName + // will never have been called - so start and then immediately end + // the object/array. + // + // We'll also end any object/arrays we happen to be in + switch(itsNodeStack.top()) + { + case NodeType::StartArray: + itsWriter.StartArray(); + // fall through + case NodeType::InArray: + itsWriter.EndArray(); + break; + case NodeType::StartObject: + itsWriter.StartObject(); + // fall through + case NodeType::InObject: + itsWriter.EndObject(); + break; + } + + itsNodeStack.pop(); + itsNameCounter.pop(); + } + + //! Sets the name for the next node created with startNode + void setNextName( const char * name ) + { + itsNextName = name; + } + + //! Saves a bool to the current node + void saveValue(bool b) { itsWriter.Bool(b); } + //! Saves an int to the current node + void saveValue(int i) { itsWriter.Int(i); } + //! Saves a uint to the current node + void saveValue(unsigned u) { itsWriter.Uint(u); } + //! Saves an int64 to the current node + void saveValue(int64_t i64) { itsWriter.Int64(i64); } + //! Saves a uint64 to the current node + void saveValue(uint64_t u64) { itsWriter.Uint64(u64); } + //! Saves a double to the current node + void saveValue(double d) { itsWriter.Double(d); } + //! Saves a string to the current node + void saveValue(std::string const & s) { itsWriter.String(s.c_str(), static_cast( s.size() )); } + //! Saves a const char * to the current node + void saveValue(char const * s) { itsWriter.String(s); } + //! Saves a nullptr to the current node + void saveValue(std::nullptr_t) { itsWriter.Null(); } + + private: + // Some compilers/OS have difficulty disambiguating the above for various flavors of longs, so we provide + // special overloads to handle these cases. + + //! 32 bit signed long saving to current node + template ::value> = traits::sfinae> inline + void saveLong(T l){ saveValue( static_cast( l ) ); } + + //! non 32 bit signed long saving to current node + template ::value> = traits::sfinae> inline + void saveLong(T l){ saveValue( static_cast( l ) ); } + + //! 32 bit unsigned long saving to current node + template ::value> = traits::sfinae> inline + void saveLong(T lu){ saveValue( static_cast( lu ) ); } + + //! non 32 bit unsigned long saving to current node + template ::value> = traits::sfinae> inline + void saveLong(T lu){ saveValue( static_cast( lu ) ); } + + public: +#if defined(_MSC_VER) && _MSC_VER < 1916 + //! MSVC only long overload to current node + void saveValue( unsigned long lu ){ saveLong( lu ); }; +#else // _MSC_VER + //! Serialize a long if it would not be caught otherwise + template ::value, + !std::is_same::value, + !std::is_same::value> = traits::sfinae> inline + void saveValue( T t ){ saveLong( t ); } + + //! Serialize an unsigned long if it would not be caught otherwise + template ::value, + !std::is_same::value, + !std::is_same::value> = traits::sfinae> inline + void saveValue( T t ){ saveLong( t ); } +#endif // _MSC_VER + + //! Save exotic arithmetic as strings to current node + /*! Handles long long (if distinct from other types), unsigned long (if distinct), and long double */ + template ::value, + !std::is_same::value, + !std::is_same::value, + !std::is_same::value, + !std::is_same::value, + (sizeof(T) >= sizeof(long double) || sizeof(T) >= sizeof(long long))> = traits::sfinae> inline + void saveValue(T const & t) + { + std::stringstream ss; ss.precision( std::numeric_limits::max_digits10 ); + ss << t; + saveValue( ss.str() ); + } + + //! Write the name of the upcoming node and prepare object/array state + /*! Since writeName is called for every value that is output, regardless of + whether it has a name or not, it is the place where we will do a deferred + check of our node state and decide whether we are in an array or an object. + + The general workflow of saving to the JSON archive is: + + 1. (optional) Set the name for the next node to be created, usually done by an NVP + 2. Start the node + 3. (if there is data to save) Write the name of the node (this function) + 4. (if there is data to save) Save the data (with saveValue) + 5. Finish the node + */ + void writeName() + { + NodeType const & nodeType = itsNodeStack.top(); + + // Start up either an object or an array, depending on state + if(nodeType == NodeType::StartArray) + { + itsWriter.StartArray(); + itsNodeStack.top() = NodeType::InArray; + } + else if(nodeType == NodeType::StartObject) + { + itsNodeStack.top() = NodeType::InObject; + itsWriter.StartObject(); + } + + // Array types do not output names + if(nodeType == NodeType::InArray) return; + + if(itsNextName == nullptr) + { + std::string name = "value" + std::to_string( itsNameCounter.top()++ ) + "\0"; + saveValue(name); + } + else + { + saveValue(itsNextName); + itsNextName = nullptr; + } + } + + //! Designates that the current node should be output as an array, not an object + void makeArray() + { + itsNodeStack.top() = NodeType::StartArray; + } + + //! @} + + private: + WriteStream itsWriteStream; //!< Rapidjson write stream + JSONWriter itsWriter; //!< Rapidjson writer + char const * itsNextName; //!< The next name + std::stack itsNameCounter; //!< Counter for creating unique names for unnamed nodes + std::stack itsNodeStack; + }; // JSONOutputArchive + + // ###################################################################### + //! An input archive designed to load data from JSON + /*! This archive uses RapidJSON to read in a JSON archive. + + As with the output JSON archive, the preferred way to use this archive is in + an RAII fashion, ensuring its destruction after all data has been read. + + Input JSON should have been produced by the JSONOutputArchive. Data can + only be added to dynamically sized containers (marked by JSON arrays) - + the input archive will determine their size by looking at the number of child nodes. + Only JSON originating from a JSONOutputArchive is officially supported, but data + from other sources may work if properly formatted. + + The JSONInputArchive does not require that nodes are loaded in the same + order they were saved by JSONOutputArchive. Using name value pairs (NVPs), + it is possible to load in an out of order fashion or otherwise skip/select + specific nodes to load. + + The default behavior of the input archive is to read sequentially starting + with the first node and exploring its children. When a given NVP does + not match the read in name for a node, the archive will search for that + node at the current level and load it if it exists. After loading an out of + order node, the archive will then proceed back to loading sequentially from + its new position. + + Consider this simple example where loading of some data is skipped: + + @code{cpp} + // imagine the input file has someData(1-9) saved in order at the top level node + ar( someData1, someData2, someData3 ); // XML loads in the order it sees in the file + ar( cereal::make_nvp( "hello", someData6 ) ); // NVP given does not + // match expected NVP name, so we search + // for the given NVP and load that value + ar( someData7, someData8, someData9 ); // with no NVP given, loading resumes at its + // current location, proceeding sequentially + @endcode + + \ingroup Archives */ + class JSONInputArchive : public InputArchive, public traits::TextArchive + { + private: + using ReadStream = CEREAL_RAPIDJSON_NAMESPACE::IStreamWrapper; + typedef CEREAL_RAPIDJSON_NAMESPACE::GenericValue> JSONValue; + typedef JSONValue::ConstMemberIterator MemberIterator; + typedef JSONValue::ConstValueIterator ValueIterator; + typedef CEREAL_RAPIDJSON_NAMESPACE::Document::GenericValue GenericValue; + + public: + /*! @name Common Functionality + Common use cases for directly interacting with an JSONInputArchive */ + //! @{ + + //! Construct, reading from the provided stream + /*! @param stream The stream to read from */ + JSONInputArchive(std::istream & stream) : + InputArchive(this), + itsNextName( nullptr ), + itsReadStream(stream) + { + itsDocument.ParseStream<>(itsReadStream); + if (itsDocument.IsArray()) + itsIteratorStack.emplace_back(itsDocument.Begin(), itsDocument.End()); + else + itsIteratorStack.emplace_back(itsDocument.MemberBegin(), itsDocument.MemberEnd()); + } + + ~JSONInputArchive() CEREAL_NOEXCEPT = default; + + //! Loads some binary data, encoded as a base64 string + /*! This will automatically start and finish a node to load the data, and can be called directly by + users. + + Note that this follows the same ordering rules specified in the class description in regards + to loading in/out of order */ + void loadBinaryValue( void * data, size_t size, const char * name = nullptr ) + { + itsNextName = name; + + std::string encoded; + loadValue( encoded ); + auto decoded = base64::decode( encoded ); + + if( size != decoded.size() ) + throw Exception("Decoded binary data size does not match specified size"); + + std::memcpy( data, decoded.data(), decoded.size() ); + itsNextName = nullptr; + }; + + private: + //! @} + /*! @name Internal Functionality + Functionality designed for use by those requiring control over the inner mechanisms of + the JSONInputArchive */ + //! @{ + + //! An internal iterator that handles both array and object types + /*! This class is a variant and holds both types of iterators that + rapidJSON supports - one for arrays and one for objects. */ + class Iterator + { + public: + Iterator() : itsIndex( 0 ), itsType(Null_) {} + + Iterator(MemberIterator begin, MemberIterator end) : + itsMemberItBegin(begin), itsMemberItEnd(end), itsIndex(0), itsSize(std::distance(begin, end)), itsType(Member) + { + if( itsSize == 0 ) + itsType = Null_; + } + + Iterator(ValueIterator begin, ValueIterator end) : + itsValueItBegin(begin), itsIndex(0), itsSize(std::distance(begin, end)), itsType(Value) + { + if( itsSize == 0 ) + itsType = Null_; + } + + //! Advance to the next node + Iterator & operator++() + { + ++itsIndex; + return *this; + } + + //! Get the value of the current node + GenericValue const & value() + { + if( itsIndex >= itsSize ) + throw cereal::Exception("No more objects in input"); + + switch(itsType) + { + case Value : return itsValueItBegin[itsIndex]; + case Member: return itsMemberItBegin[itsIndex].value; + default: throw cereal::Exception("JSONInputArchive internal error: null or empty iterator to object or array!"); + } + } + + //! Get the name of the current node, or nullptr if it has no name + const char * name() const + { + if( itsType == Member && (itsMemberItBegin + itsIndex) != itsMemberItEnd ) + return itsMemberItBegin[itsIndex].name.GetString(); + else + return nullptr; + } + + //! Adjust our position such that we are at the node with the given name + /*! @throws Exception if no such named node exists */ + inline void search( const char * searchName ) + { + const auto len = std::strlen( searchName ); + size_t index = 0; + for( auto it = itsMemberItBegin; it != itsMemberItEnd; ++it, ++index ) + { + const auto currentName = it->name.GetString(); + if( ( std::strncmp( searchName, currentName, len ) == 0 ) && + ( std::strlen( currentName ) == len ) ) + { + itsIndex = index; + return; + } + } + + throw Exception("JSON Parsing failed - provided NVP (" + std::string(searchName) + ") not found"); + } + + private: + MemberIterator itsMemberItBegin, itsMemberItEnd; //!< The member iterator (object) + ValueIterator itsValueItBegin; //!< The value iterator (array) + size_t itsIndex, itsSize; //!< The current index of this iterator + enum Type {Value, Member, Null_} itsType; //!< Whether this holds values (array) or members (objects) or nothing + }; + + //! Searches for the expectedName node if it doesn't match the actualName + /*! This needs to be called before every load or node start occurs. This function will + check to see if an NVP has been provided (with setNextName) and if so, see if that name matches the actual + next name given. If the names do not match, it will search in the current level of the JSON for that name. + If the name is not found, an exception will be thrown. + + Resets the NVP name after called. + + @throws Exception if an expectedName is given and not found */ + inline void search() + { + // store pointer to itsNextName locally and reset to nullptr in case search() throws + auto localNextName = itsNextName; + itsNextName = nullptr; + + // The name an NVP provided with setNextName() + if( localNextName ) + { + // The actual name of the current node + auto const actualName = itsIteratorStack.back().name(); + + // Do a search if we don't see a name coming up, or if the names don't match + if( !actualName || std::strcmp( localNextName, actualName ) != 0 ) + itsIteratorStack.back().search( localNextName ); + } + } + + public: + //! Starts a new node, going into its proper iterator + /*! This places an iterator for the next node to be parsed onto the iterator stack. If the next + node is an array, this will be a value iterator, otherwise it will be a member iterator. + + By default our strategy is to start with the document root node and then recursively iterate through + all children in the order they show up in the document. + We don't need to know NVPs to do this; we'll just blindly load in the order things appear in. + + If we were given an NVP, we will search for it if it does not match our the name of the next node + that would normally be loaded. This functionality is provided by search(). */ + void startNode() + { + search(); + + if(itsIteratorStack.back().value().IsArray()) + itsIteratorStack.emplace_back(itsIteratorStack.back().value().Begin(), itsIteratorStack.back().value().End()); + else + itsIteratorStack.emplace_back(itsIteratorStack.back().value().MemberBegin(), itsIteratorStack.back().value().MemberEnd()); + } + + //! Finishes the most recently started node + void finishNode() + { + itsIteratorStack.pop_back(); + ++itsIteratorStack.back(); + } + + //! Retrieves the current node name + /*! @return nullptr if no name exists */ + const char * getNodeName() const + { + return itsIteratorStack.back().name(); + } + + //! Sets the name for the next node created with startNode + void setNextName( const char * name ) + { + itsNextName = name; + } + + //! Loads a value from the current node - small signed overload + template ::value, + sizeof(T) < sizeof(int64_t)> = traits::sfinae> inline + void loadValue(T & val) + { + search(); + + val = static_cast( itsIteratorStack.back().value().GetInt() ); + ++itsIteratorStack.back(); + } + + //! Loads a value from the current node - small unsigned overload + template ::value, + sizeof(T) < sizeof(uint64_t), + !std::is_same::value> = traits::sfinae> inline + void loadValue(T & val) + { + search(); + + val = static_cast( itsIteratorStack.back().value().GetUint() ); + ++itsIteratorStack.back(); + } + + //! Loads a value from the current node - bool overload + void loadValue(bool & val) { search(); val = itsIteratorStack.back().value().GetBool(); ++itsIteratorStack.back(); } + //! Loads a value from the current node - int64 overload + void loadValue(int64_t & val) { search(); val = itsIteratorStack.back().value().GetInt64(); ++itsIteratorStack.back(); } + //! Loads a value from the current node - uint64 overload + void loadValue(uint64_t & val) { search(); val = itsIteratorStack.back().value().GetUint64(); ++itsIteratorStack.back(); } + //! Loads a value from the current node - float overload + void loadValue(float & val) { search(); val = static_cast(itsIteratorStack.back().value().GetDouble()); ++itsIteratorStack.back(); } + //! Loads a value from the current node - double overload + void loadValue(double & val) { search(); val = itsIteratorStack.back().value().GetDouble(); ++itsIteratorStack.back(); } + //! Loads a value from the current node - string overload + void loadValue(std::string & val) { search(); val = itsIteratorStack.back().value().GetString(); ++itsIteratorStack.back(); } + //! Loads a nullptr from the current node + void loadValue(std::nullptr_t&) { search(); CEREAL_RAPIDJSON_ASSERT(itsIteratorStack.back().value().IsNull()); ++itsIteratorStack.back(); } + + // Special cases to handle various flavors of long, which tend to conflict with + // the int32_t or int64_t on various compiler/OS combinations. MSVC doesn't need any of this. + #ifndef _MSC_VER + private: + //! 32 bit signed long loading from current node + template inline + typename std::enable_if::value, void>::type + loadLong(T & l){ loadValue( reinterpret_cast( l ) ); } + + //! non 32 bit signed long loading from current node + template inline + typename std::enable_if::value, void>::type + loadLong(T & l){ loadValue( reinterpret_cast( l ) ); } + + //! 32 bit unsigned long loading from current node + template inline + typename std::enable_if::value, void>::type + loadLong(T & lu){ loadValue( reinterpret_cast( lu ) ); } + + //! non 32 bit unsigned long loading from current node + template inline + typename std::enable_if::value, void>::type + loadLong(T & lu){ loadValue( reinterpret_cast( lu ) ); } + + public: + //! Serialize a long if it would not be caught otherwise + template inline + typename std::enable_if::value && + sizeof(T) >= sizeof(std::int64_t) && + !std::is_same::value, void>::type + loadValue( T & t ){ loadLong(t); } + + //! Serialize an unsigned long if it would not be caught otherwise + template inline + typename std::enable_if::value && + sizeof(T) >= sizeof(std::uint64_t) && + !std::is_same::value, void>::type + loadValue( T & t ){ loadLong(t); } + #endif // _MSC_VER + + private: + //! Convert a string to a long long + void stringToNumber( std::string const & str, long long & val ) { val = std::stoll( str ); } + //! Convert a string to an unsigned long long + void stringToNumber( std::string const & str, unsigned long long & val ) { val = std::stoull( str ); } + //! Convert a string to a long double + void stringToNumber( std::string const & str, long double & val ) { val = std::stold( str ); } + + public: + //! Loads a value from the current node - long double and long long overloads + template ::value, + !std::is_same::value, + !std::is_same::value, + !std::is_same::value, + !std::is_same::value, + (sizeof(T) >= sizeof(long double) || sizeof(T) >= sizeof(long long))> = traits::sfinae> + inline void loadValue(T & val) + { + std::string encoded; + loadValue( encoded ); + stringToNumber( encoded, val ); + } + + //! Loads the size for a SizeTag + void loadSize(size_type & size) + { + if (itsIteratorStack.size() == 1) + size = itsDocument.Size(); + else + size = (itsIteratorStack.rbegin() + 1)->value().Size(); + } + + //! @} + + private: + const char * itsNextName; //!< Next name set by NVP + ReadStream itsReadStream; //!< Rapidjson write stream + std::vector itsIteratorStack; //!< 'Stack' of rapidJSON iterators + CEREAL_RAPIDJSON_NAMESPACE::Document itsDocument; //!< Rapidjson document + }; + + // ###################################################################### + // JSONArchive prologue and epilogue functions + // ###################################################################### + + // ###################################################################### + //! Prologue for NVPs for JSON archives + /*! NVPs do not start or finish nodes - they just set up the names */ + template inline + void prologue( JSONOutputArchive &, NameValuePair const & ) + { } + + //! Prologue for NVPs for JSON archives + template inline + void prologue( JSONInputArchive &, NameValuePair const & ) + { } + + // ###################################################################### + //! Epilogue for NVPs for JSON archives + /*! NVPs do not start or finish nodes - they just set up the names */ + template inline + void epilogue( JSONOutputArchive &, NameValuePair const & ) + { } + + //! Epilogue for NVPs for JSON archives + /*! NVPs do not start or finish nodes - they just set up the names */ + template inline + void epilogue( JSONInputArchive &, NameValuePair const & ) + { } + + // ###################################################################### + //! Prologue for deferred data for JSON archives + /*! Do nothing for the defer wrapper */ + template inline + void prologue( JSONOutputArchive &, DeferredData const & ) + { } + + //! Prologue for deferred data for JSON archives + template inline + void prologue( JSONInputArchive &, DeferredData const & ) + { } + + // ###################################################################### + //! Epilogue for deferred for JSON archives + /*! NVPs do not start or finish nodes - they just set up the names */ + template inline + void epilogue( JSONOutputArchive &, DeferredData const & ) + { } + + //! Epilogue for deferred for JSON archives + /*! Do nothing for the defer wrapper */ + template inline + void epilogue( JSONInputArchive &, DeferredData const & ) + { } + + // ###################################################################### + //! Prologue for SizeTags for JSON archives + /*! SizeTags are strictly ignored for JSON, they just indicate + that the current node should be made into an array */ + template inline + void prologue( JSONOutputArchive & ar, SizeTag const & ) + { + ar.makeArray(); + } + + //! Prologue for SizeTags for JSON archives + template inline + void prologue( JSONInputArchive &, SizeTag const & ) + { } + + // ###################################################################### + //! Epilogue for SizeTags for JSON archives + /*! SizeTags are strictly ignored for JSON */ + template inline + void epilogue( JSONOutputArchive &, SizeTag const & ) + { } + + //! Epilogue for SizeTags for JSON archives + template inline + void epilogue( JSONInputArchive &, SizeTag const & ) + { } + + // ###################################################################### + //! Prologue for all other types for JSON archives (except minimal types) + /*! Starts a new node, named either automatically or by some NVP, + that may be given data by the type about to be archived + + Minimal types do not start or finish nodes */ + template ::value, + !traits::has_minimal_base_class_serialization::value, + !traits::has_minimal_output_serialization::value> = traits::sfinae> + inline void prologue( JSONOutputArchive & ar, T const & ) + { + ar.startNode(); + } + + //! Prologue for all other types for JSON archives + template ::value, + !traits::has_minimal_base_class_serialization::value, + !traits::has_minimal_input_serialization::value> = traits::sfinae> + inline void prologue( JSONInputArchive & ar, T const & ) + { + ar.startNode(); + } + + // ###################################################################### + //! Epilogue for all other types other for JSON archives (except minimal types) + /*! Finishes the node created in the prologue + + Minimal types do not start or finish nodes */ + template ::value, + !traits::has_minimal_base_class_serialization::value, + !traits::has_minimal_output_serialization::value> = traits::sfinae> + inline void epilogue( JSONOutputArchive & ar, T const & ) + { + ar.finishNode(); + } + + //! Epilogue for all other types other for JSON archives + template ::value, + !traits::has_minimal_base_class_serialization::value, + !traits::has_minimal_input_serialization::value> = traits::sfinae> + inline void epilogue( JSONInputArchive & ar, T const & ) + { + ar.finishNode(); + } + + // ###################################################################### + //! Prologue for arithmetic types for JSON archives + inline + void prologue( JSONOutputArchive & ar, std::nullptr_t const & ) + { + ar.writeName(); + } + + //! Prologue for arithmetic types for JSON archives + inline + void prologue( JSONInputArchive &, std::nullptr_t const & ) + { } + + // ###################################################################### + //! Epilogue for arithmetic types for JSON archives + inline + void epilogue( JSONOutputArchive &, std::nullptr_t const & ) + { } + + //! Epilogue for arithmetic types for JSON archives + inline + void epilogue( JSONInputArchive &, std::nullptr_t const & ) + { } + + // ###################################################################### + //! Prologue for arithmetic types for JSON archives + template ::value> = traits::sfinae> inline + void prologue( JSONOutputArchive & ar, T const & ) + { + ar.writeName(); + } + + //! Prologue for arithmetic types for JSON archives + template ::value> = traits::sfinae> inline + void prologue( JSONInputArchive &, T const & ) + { } + + // ###################################################################### + //! Epilogue for arithmetic types for JSON archives + template ::value> = traits::sfinae> inline + void epilogue( JSONOutputArchive &, T const & ) + { } + + //! Epilogue for arithmetic types for JSON archives + template ::value> = traits::sfinae> inline + void epilogue( JSONInputArchive &, T const & ) + { } + + // ###################################################################### + //! Prologue for strings for JSON archives + template inline + void prologue(JSONOutputArchive & ar, std::basic_string const &) + { + ar.writeName(); + } + + //! Prologue for strings for JSON archives + template inline + void prologue(JSONInputArchive &, std::basic_string const &) + { } + + // ###################################################################### + //! Epilogue for strings for JSON archives + template inline + void epilogue(JSONOutputArchive &, std::basic_string const &) + { } + + //! Epilogue for strings for JSON archives + template inline + void epilogue(JSONInputArchive &, std::basic_string const &) + { } + + // ###################################################################### + // Common JSONArchive serialization functions + // ###################################################################### + //! Serializing NVP types to JSON + template inline + void CEREAL_SAVE_FUNCTION_NAME( JSONOutputArchive & ar, NameValuePair const & t ) + { + ar.setNextName( t.name ); + ar( t.value ); + } + + template inline + void CEREAL_LOAD_FUNCTION_NAME( JSONInputArchive & ar, NameValuePair & t ) + { + ar.setNextName( t.name ); + ar( t.value ); + } + + //! Saving for nullptr to JSON + inline + void CEREAL_SAVE_FUNCTION_NAME(JSONOutputArchive & ar, std::nullptr_t const & t) + { + ar.saveValue( t ); + } + + //! Loading arithmetic from JSON + inline + void CEREAL_LOAD_FUNCTION_NAME(JSONInputArchive & ar, std::nullptr_t & t) + { + ar.loadValue( t ); + } + + //! Saving for arithmetic to JSON + template ::value> = traits::sfinae> inline + void CEREAL_SAVE_FUNCTION_NAME(JSONOutputArchive & ar, T const & t) + { + ar.saveValue( t ); + } + + //! Loading arithmetic from JSON + template ::value> = traits::sfinae> inline + void CEREAL_LOAD_FUNCTION_NAME(JSONInputArchive & ar, T & t) + { + ar.loadValue( t ); + } + + //! saving string to JSON + template inline + void CEREAL_SAVE_FUNCTION_NAME(JSONOutputArchive & ar, std::basic_string const & str) + { + ar.saveValue( str ); + } + + //! loading string from JSON + template inline + void CEREAL_LOAD_FUNCTION_NAME(JSONInputArchive & ar, std::basic_string & str) + { + ar.loadValue( str ); + } + + // ###################################################################### + //! Saving SizeTags to JSON + template inline + void CEREAL_SAVE_FUNCTION_NAME( JSONOutputArchive &, SizeTag const & ) + { + // nothing to do here, we don't explicitly save the size + } + + //! Loading SizeTags from JSON + template inline + void CEREAL_LOAD_FUNCTION_NAME( JSONInputArchive & ar, SizeTag & st ) + { + ar.loadSize( st.size ); + } +} // namespace cereal + +// register archives for polymorphic support +CEREAL_REGISTER_ARCHIVE(cereal::JSONInputArchive) +CEREAL_REGISTER_ARCHIVE(cereal::JSONOutputArchive) + +// tie input and output archives together +CEREAL_SETUP_ARCHIVE_TRAITS(cereal::JSONInputArchive, cereal::JSONOutputArchive) + +#endif // CEREAL_ARCHIVES_JSON_HPP_ diff --git a/third_party/cereal/archives/portable_binary.hpp b/third_party/cereal/archives/portable_binary.hpp new file mode 100755 index 0000000..8575d83 --- /dev/null +++ b/third_party/cereal/archives/portable_binary.hpp @@ -0,0 +1,334 @@ +/*! \file binary.hpp + \brief Binary input and output archives */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_ARCHIVES_PORTABLE_BINARY_HPP_ +#define CEREAL_ARCHIVES_PORTABLE_BINARY_HPP_ + +#include "../cereal.hpp" +#include +#include + +namespace cereal +{ + namespace portable_binary_detail + { + //! Returns true if the current machine is little endian + /*! @ingroup Internal */ + inline std::uint8_t is_little_endian() + { + static std::int32_t test = 1; + return *reinterpret_cast( &test ) == 1; + } + + //! Swaps the order of bytes for some chunk of memory + /*! @param data The data as a uint8_t pointer + @tparam DataSize The true size of the data + @ingroup Internal */ + template + inline void swap_bytes( std::uint8_t * data ) + { + for( std::size_t i = 0, end = DataSize / 2; i < end; ++i ) + std::swap( data[i], data[DataSize - i - 1] ); + } + } // end namespace portable_binary_detail + + // ###################################################################### + //! An output archive designed to save data in a compact binary representation portable over different architectures + /*! This archive outputs data to a stream in an extremely compact binary + representation with as little extra metadata as possible. + + This archive will record the endianness of the data as well as the desired in/out endianness + and assuming that the user takes care of ensuring serialized types are the same size + across machines, is portable over different architectures. + + When using a binary archive and a file stream, you must use the + std::ios::binary format flag to avoid having your data altered + inadvertently. + + \warning This archive has not been thoroughly tested across different architectures. + Please report any issues, optimizations, or feature requests at + the project github. + + \ingroup Archives */ + class PortableBinaryOutputArchive : public OutputArchive + { + public: + //! A class containing various advanced options for the PortableBinaryOutput archive + class Options + { + public: + //! Represents desired endianness + enum class Endianness : std::uint8_t + { big, little }; + + //! Default options, preserve system endianness + static Options Default(){ return Options(); } + + //! Save as little endian + static Options LittleEndian(){ return Options( Endianness::little ); } + + //! Save as big endian + static Options BigEndian(){ return Options( Endianness::big ); } + + //! Specify specific options for the PortableBinaryOutputArchive + /*! @param outputEndian The desired endianness of saved (output) data */ + explicit Options( Endianness outputEndian = getEndianness() ) : + itsOutputEndianness( outputEndian ) { } + + private: + //! Gets the endianness of the system + inline static Endianness getEndianness() + { return portable_binary_detail::is_little_endian() ? Endianness::little : Endianness::big; } + + //! Checks if Options is set for little endian + inline std::uint8_t is_little_endian() const + { return itsOutputEndianness == Endianness::little; } + + friend class PortableBinaryOutputArchive; + Endianness itsOutputEndianness; + }; + + //! Construct, outputting to the provided stream + /*! @param stream The stream to output to. Should be opened with std::ios::binary flag. + @param options The PortableBinary specific options to use. See the Options struct + for the values of default parameters */ + PortableBinaryOutputArchive(std::ostream & stream, Options const & options = Options::Default()) : + OutputArchive(this), + itsStream(stream), + itsConvertEndianness( portable_binary_detail::is_little_endian() ^ options.is_little_endian() ) + { + this->operator()( options.is_little_endian() ); + } + + ~PortableBinaryOutputArchive() CEREAL_NOEXCEPT = default; + + //! Writes size bytes of data to the output stream + template inline + void saveBinary( const void * data, std::streamsize size ) + { + std::streamsize writtenSize = 0; + + if( itsConvertEndianness ) + { + for( std::streamsize i = 0; i < size; i += DataSize ) + for( std::streamsize j = 0; j < DataSize; ++j ) + writtenSize += itsStream.rdbuf()->sputn( reinterpret_cast( data ) + DataSize - j - 1 + i, 1 ); + } + else + writtenSize = itsStream.rdbuf()->sputn( reinterpret_cast( data ), size ); + + if(writtenSize != size) + throw Exception("Failed to write " + std::to_string(size) + " bytes to output stream! Wrote " + std::to_string(writtenSize)); + } + + private: + std::ostream & itsStream; + const uint8_t itsConvertEndianness; //!< If set to true, we will need to swap bytes upon saving + }; + + // ###################################################################### + //! An input archive designed to load data saved using PortableBinaryOutputArchive + /*! This archive outputs data to a stream in an extremely compact binary + representation with as little extra metadata as possible. + + This archive will load the endianness of the serialized data and + if necessary transform it to match that of the local machine. This comes + at a significant performance cost compared to non portable archives if + the transformation is necessary, and also causes a small performance hit + even if it is not necessary. + + It is recommended to use portable archives only if you know that you will + be sending binary data to machines with different endianness. + + The archive will do nothing to ensure types are the same size - that is + the responsibility of the user. + + When using a binary archive and a file stream, you must use the + std::ios::binary format flag to avoid having your data altered + inadvertently. + + \warning This archive has not been thoroughly tested across different architectures. + Please report any issues, optimizations, or feature requests at + the project github. + + \ingroup Archives */ + class PortableBinaryInputArchive : public InputArchive + { + public: + //! A class containing various advanced options for the PortableBinaryInput archive + class Options + { + public: + //! Represents desired endianness + enum class Endianness : std::uint8_t + { big, little }; + + //! Default options, preserve system endianness + static Options Default(){ return Options(); } + + //! Load into little endian + static Options LittleEndian(){ return Options( Endianness::little ); } + + //! Load into big endian + static Options BigEndian(){ return Options( Endianness::big ); } + + //! Specify specific options for the PortableBinaryInputArchive + /*! @param inputEndian The desired endianness of loaded (input) data */ + explicit Options( Endianness inputEndian = getEndianness() ) : + itsInputEndianness( inputEndian ) { } + + private: + //! Gets the endianness of the system + inline static Endianness getEndianness() + { return portable_binary_detail::is_little_endian() ? Endianness::little : Endianness::big; } + + //! Checks if Options is set for little endian + inline std::uint8_t is_little_endian() const + { return itsInputEndianness == Endianness::little; } + + friend class PortableBinaryInputArchive; + Endianness itsInputEndianness; + }; + + //! Construct, loading from the provided stream + /*! @param stream The stream to read from. Should be opened with std::ios::binary flag. + @param options The PortableBinary specific options to use. See the Options struct + for the values of default parameters */ + PortableBinaryInputArchive(std::istream & stream, Options const & options = Options::Default()) : + InputArchive(this), + itsStream(stream), + itsConvertEndianness( false ) + { + uint8_t streamLittleEndian; + this->operator()( streamLittleEndian ); + itsConvertEndianness = options.is_little_endian() ^ streamLittleEndian; + } + + ~PortableBinaryInputArchive() CEREAL_NOEXCEPT = default; + + //! Reads size bytes of data from the input stream + /*! @param data The data to save + @param size The number of bytes in the data + @tparam DataSize T The size of the actual type of the data elements being loaded */ + template inline + void loadBinary( void * const data, std::streamsize size ) + { + // load data + auto const readSize = itsStream.rdbuf()->sgetn( reinterpret_cast( data ), size ); + + if(readSize != size) + throw Exception("Failed to read " + std::to_string(size) + " bytes from input stream! Read " + std::to_string(readSize)); + + // flip bits if needed + if( itsConvertEndianness ) + { + std::uint8_t * ptr = reinterpret_cast( data ); + for( std::streamsize i = 0; i < size; i += DataSize ) + portable_binary_detail::swap_bytes( ptr + i ); + } + } + + private: + std::istream & itsStream; + uint8_t itsConvertEndianness; //!< If set to true, we will need to swap bytes upon loading + }; + + // ###################################################################### + // Common BinaryArchive serialization functions + + //! Saving for POD types to portable binary + template inline + typename std::enable_if::value, void>::type + CEREAL_SAVE_FUNCTION_NAME(PortableBinaryOutputArchive & ar, T const & t) + { + static_assert( !std::is_floating_point::value || + (std::is_floating_point::value && std::numeric_limits::is_iec559), + "Portable binary only supports IEEE 754 standardized floating point" ); + ar.template saveBinary(std::addressof(t), sizeof(t)); + } + + //! Loading for POD types from portable binary + template inline + typename std::enable_if::value, void>::type + CEREAL_LOAD_FUNCTION_NAME(PortableBinaryInputArchive & ar, T & t) + { + static_assert( !std::is_floating_point::value || + (std::is_floating_point::value && std::numeric_limits::is_iec559), + "Portable binary only supports IEEE 754 standardized floating point" ); + ar.template loadBinary(std::addressof(t), sizeof(t)); + } + + //! Serializing NVP types to portable binary + template inline + CEREAL_ARCHIVE_RESTRICT(PortableBinaryInputArchive, PortableBinaryOutputArchive) + CEREAL_SERIALIZE_FUNCTION_NAME( Archive & ar, NameValuePair & t ) + { + ar( t.value ); + } + + //! Serializing SizeTags to portable binary + template inline + CEREAL_ARCHIVE_RESTRICT(PortableBinaryInputArchive, PortableBinaryOutputArchive) + CEREAL_SERIALIZE_FUNCTION_NAME( Archive & ar, SizeTag & t ) + { + ar( t.size ); + } + + //! Saving binary data to portable binary + template inline + void CEREAL_SAVE_FUNCTION_NAME(PortableBinaryOutputArchive & ar, BinaryData const & bd) + { + typedef typename std::remove_pointer::type TT; + static_assert( !std::is_floating_point::value || + (std::is_floating_point::value && std::numeric_limits::is_iec559), + "Portable binary only supports IEEE 754 standardized floating point" ); + + ar.template saveBinary( bd.data, static_cast( bd.size ) ); + } + + //! Loading binary data from portable binary + template inline + void CEREAL_LOAD_FUNCTION_NAME(PortableBinaryInputArchive & ar, BinaryData & bd) + { + typedef typename std::remove_pointer::type TT; + static_assert( !std::is_floating_point::value || + (std::is_floating_point::value && std::numeric_limits::is_iec559), + "Portable binary only supports IEEE 754 standardized floating point" ); + + ar.template loadBinary( bd.data, static_cast( bd.size ) ); + } +} // namespace cereal + +// register archives for polymorphic support +CEREAL_REGISTER_ARCHIVE(cereal::PortableBinaryOutputArchive) +CEREAL_REGISTER_ARCHIVE(cereal::PortableBinaryInputArchive) + +// tie input and output archives together +CEREAL_SETUP_ARCHIVE_TRAITS(cereal::PortableBinaryInputArchive, cereal::PortableBinaryOutputArchive) + +#endif // CEREAL_ARCHIVES_PORTABLE_BINARY_HPP_ diff --git a/third_party/cereal/archives/xml.hpp b/third_party/cereal/archives/xml.hpp new file mode 100755 index 0000000..8622c94 --- /dev/null +++ b/third_party/cereal/archives/xml.hpp @@ -0,0 +1,956 @@ +/*! \file xml.hpp + \brief XML input and output archives */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_ARCHIVES_XML_HPP_ +#define CEREAL_ARCHIVES_XML_HPP_ +#include "../cereal.hpp" +#include "../details/util.hpp" + +#include "../external/rapidxml/rapidxml.hpp" +#include "../external/rapidxml/rapidxml_print.hpp" +#include "../external/base64.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace cereal +{ + namespace xml_detail + { + #ifndef CEREAL_XML_STRING_VALUE + //! The default name for the root node in a cereal xml archive. + /*! You can define CEREAL_XML_STRING_VALUE to be different assuming you do so + before this file is included. */ + #define CEREAL_XML_STRING_VALUE "cereal" + #endif // CEREAL_XML_STRING_VALUE + + //! The name given to the root node in a cereal xml archive + static const char * CEREAL_XML_STRING = CEREAL_XML_STRING_VALUE; + + //! Returns true if the character is whitespace + inline bool isWhitespace( char c ) + { + return c == ' ' || c == '\t' || c == '\n' || c == '\r'; + } + } + + // ###################################################################### + //! An output archive designed to save data to XML + /*! This archive uses RapidXML to build an in memory XML tree of the + data it serializes before outputting it to its stream upon destruction. + This archive should be used in an RAII fashion, letting + the automatic destruction of the object cause the flush to its stream. + + XML archives provides a human readable output but at decreased + performance (both in time and space) compared to binary archives. + + XML benefits greatly from name-value pairs, which if present, will + name the nodes in the output. If these are not present, each level + of the output tree will be given an automatically generated delimited name. + + The precision of the output archive controls the number of decimals output + for floating point numbers and should be sufficiently large (i.e. at least 20) + if there is a desire to have binary equality between the numbers output and + those read in. In general you should expect a loss of precision when going + from floating point to text and back. + + XML archives can optionally print the type of everything they serialize, which + adds an attribute to each node. + + XML archives do not output the size information for any dynamically sized structure + and instead infer it from the number of children for a node. This means that data + can be hand edited for dynamic sized structures and will still be readable. This + is accomplished through the cereal::SizeTag object, which will also add an attribute + to its parent field. + \ingroup Archives */ + class XMLOutputArchive : public OutputArchive, public traits::TextArchive + { + public: + /*! @name Common Functionality + Common use cases for directly interacting with an XMLOutputArchive */ + //! @{ + + //! A class containing various advanced options for the XML archive + /*! Options can either be directly passed to the constructor, or chained using the + modifier functions for an interface analogous to named parameters */ + class Options + { + public: + //! Default options + static Options Default(){ return Options(); } + + //! Specify specific options for the XMLOutputArchive + /*! @param precision_ The precision used for floating point numbers + @param indent_ Whether to indent each line of XML + @param outputType_ Whether to output the type of each serialized object as an attribute + @param sizeAttributes_ Whether dynamically sized containers output the size=dynamic attribute */ + explicit Options( int precision_ = std::numeric_limits::max_digits10, + bool indent_ = true, + bool outputType_ = false, + bool sizeAttributes_ = true ) : + itsPrecision( precision_ ), + itsIndent( indent_ ), + itsOutputType( outputType_ ), + itsSizeAttributes( sizeAttributes_ ) + { } + + /*! @name Option Modifiers + An interface for setting option settings analogous to named parameters. + + @code{cpp} + cereal::XMLOutputArchive ar( myStream, + cereal::XMLOutputArchive::Options() + .indent(true) + .sizeAttributes(false) ); + @endcode + */ + //! @{ + + //! Sets the precision used for floaing point numbers + Options & precision( int value ){ itsPrecision = value; return * this; } + //! Whether to indent each line of XML + Options & indent( bool enable ){ itsIndent = enable; return *this; } + //! Whether to output the type of each serialized object as an attribute + Options & outputType( bool enable ){ itsOutputType = enable; return *this; } + //! Whether dynamically sized containers (e.g. vector) output the size=dynamic attribute + Options & sizeAttributes( bool enable ){ itsSizeAttributes = enable; return *this; } + + //! @} + + private: + friend class XMLOutputArchive; + int itsPrecision; + bool itsIndent; + bool itsOutputType; + bool itsSizeAttributes; + }; + + //! Construct, outputting to the provided stream upon destruction + /*! @param stream The stream to output to. Note that XML is only guaranteed to flush + its output to the stream upon destruction. + @param options The XML specific options to use. See the Options struct + for the values of default parameters */ + XMLOutputArchive( std::ostream & stream, Options const & options = Options::Default() ) : + OutputArchive(this), + itsStream(stream), + itsOutputType( options.itsOutputType ), + itsIndent( options.itsIndent ), + itsSizeAttributes(options.itsSizeAttributes) + { + // rapidxml will delete all allocations when xml_document is cleared + auto node = itsXML.allocate_node( rapidxml::node_declaration ); + node->append_attribute( itsXML.allocate_attribute( "version", "1.0" ) ); + node->append_attribute( itsXML.allocate_attribute( "encoding", "utf-8" ) ); + itsXML.append_node( node ); + + // allocate root node + auto root = itsXML.allocate_node( rapidxml::node_element, xml_detail::CEREAL_XML_STRING ); + itsXML.append_node( root ); + itsNodes.emplace( root ); + + // set attributes on the streams + itsStream << std::boolalpha; + itsStream.precision( options.itsPrecision ); + itsOS << std::boolalpha; + itsOS.precision( options.itsPrecision ); + } + + //! Destructor, flushes the XML + ~XMLOutputArchive() CEREAL_NOEXCEPT + { + const int flags = itsIndent ? 0x0 : rapidxml::print_no_indenting; + rapidxml::print( itsStream, itsXML, flags ); + itsXML.clear(); + } + + //! Saves some binary data, encoded as a base64 string, with an optional name + /*! This can be called directly by users and it will automatically create a child node for + the current XML node, populate it with a base64 encoded string, and optionally name + it. The node will be finished after it has been populated. */ + void saveBinaryValue( const void * data, size_t size, const char * name = nullptr ) + { + itsNodes.top().name = name; + + startNode(); + + auto base64string = base64::encode( reinterpret_cast( data ), size ); + saveValue( base64string ); + + if( itsOutputType ) + itsNodes.top().node->append_attribute( itsXML.allocate_attribute( "type", "cereal binary data" ) ); + + finishNode(); + } + + //! @} + /*! @name Internal Functionality + Functionality designed for use by those requiring control over the inner mechanisms of + the XMLOutputArchive */ + //! @{ + + //! Creates a new node that is a child of the node at the top of the stack + /*! Nodes will be given a name that has either been pre-set by a name value pair, + or generated based upon a counter unique to the parent node. If you want to + give a node a specific name, use setNextName prior to calling startNode. + + The node will then be pushed onto the node stack. */ + void startNode() + { + // generate a name for this new node + const auto nameString = itsNodes.top().getValueName(); + + // allocate strings for all of the data in the XML object + auto namePtr = itsXML.allocate_string( nameString.data(), nameString.length() + 1 ); + + // insert into the XML + auto node = itsXML.allocate_node( rapidxml::node_element, namePtr, nullptr, nameString.size() ); + itsNodes.top().node->append_node( node ); + itsNodes.emplace( node ); + } + + //! Designates the most recently added node as finished + void finishNode() + { + itsNodes.pop(); + } + + //! Sets the name for the next node created with startNode + void setNextName( const char * name ) + { + itsNodes.top().name = name; + } + + //! Saves some data, encoded as a string, into the current top level node + /*! The data will be be named with the most recent name if one exists, + otherwise it will be given some default delimited value that depends upon + the parent node */ + template inline + void saveValue( T const & value ) + { + itsOS.clear(); itsOS.seekp( 0, std::ios::beg ); + itsOS << value << std::ends; + + auto strValue = itsOS.str(); + + // itsOS.str() may contain data from previous calls after the first '\0' that was just inserted + // and this data is counted in the length call. We make sure to remove that section so that the + // whitespace validation is done properly + strValue.resize(std::strlen(strValue.c_str())); + + // If the first or last character is a whitespace, add xml:space attribute + const auto len = strValue.length(); + if ( len > 0 && ( xml_detail::isWhitespace( strValue[0] ) || xml_detail::isWhitespace( strValue[len - 1] ) ) ) + { + itsNodes.top().node->append_attribute( itsXML.allocate_attribute( "xml:space", "preserve" ) ); + } + + // allocate strings for all of the data in the XML object + auto dataPtr = itsXML.allocate_string(strValue.c_str(), strValue.length() + 1 ); + + // insert into the XML + itsNodes.top().node->append_node( itsXML.allocate_node( rapidxml::node_data, nullptr, dataPtr ) ); + } + + //! Overload for uint8_t prevents them from being serialized as characters + void saveValue( uint8_t const & value ) + { + saveValue( static_cast( value ) ); + } + + //! Overload for int8_t prevents them from being serialized as characters + void saveValue( int8_t const & value ) + { + saveValue( static_cast( value ) ); + } + + //! Causes the type to be appended as an attribute to the most recently made node if output type is set to true + template inline + void insertType() + { + if( !itsOutputType ) + return; + + // generate a name for this new node + const auto nameString = util::demangledName(); + + // allocate strings for all of the data in the XML object + auto namePtr = itsXML.allocate_string( nameString.data(), nameString.length() + 1 ); + + itsNodes.top().node->append_attribute( itsXML.allocate_attribute( "type", namePtr ) ); + } + + //! Appends an attribute to the current top level node + void appendAttribute( const char * name, const char * value ) + { + auto namePtr = itsXML.allocate_string( name ); + auto valuePtr = itsXML.allocate_string( value ); + itsNodes.top().node->append_attribute( itsXML.allocate_attribute( namePtr, valuePtr ) ); + } + + bool hasSizeAttributes() const { return itsSizeAttributes; } + + protected: + //! A struct that contains metadata about a node + struct NodeInfo + { + NodeInfo( rapidxml::xml_node<> * n = nullptr, + const char * nm = nullptr ) : + node( n ), + counter( 0 ), + name( nm ) + { } + + rapidxml::xml_node<> * node; //!< A pointer to this node + size_t counter; //!< The counter for naming child nodes + const char * name; //!< The name for the next child node + + //! Gets the name for the next child node created from this node + /*! The name will be automatically generated using the counter if + a name has not been previously set. If a name has been previously + set, that name will be returned only once */ + std::string getValueName() + { + if( name ) + { + auto n = name; + name = nullptr; + return {n}; + } + else + return "value" + std::to_string( counter++ ) + "\0"; + } + }; // NodeInfo + + //! @} + + private: + std::ostream & itsStream; //!< The output stream + rapidxml::xml_document<> itsXML; //!< The XML document + std::stack itsNodes; //!< A stack of nodes added to the document + std::ostringstream itsOS; //!< Used to format strings internally + bool itsOutputType; //!< Controls whether type information is printed + bool itsIndent; //!< Controls whether indenting is used + bool itsSizeAttributes; //!< Controls whether lists have a size attribute + }; // XMLOutputArchive + + // ###################################################################### + //! An output archive designed to load data from XML + /*! This archive uses RapidXML to build an in memory XML tree of the + data in the stream it is given before loading any types serialized. + + As with the output XML archive, the preferred way to use this archive is in + an RAII fashion, ensuring its destruction after all data has been read. + + Input XML should have been produced by the XMLOutputArchive. Data can + only be added to dynamically sized containers - the input archive will + determine their size by looking at the number of child nodes. Data that + did not originate from an XMLOutputArchive is not officially supported, + but may be possible to use if properly formatted. + + The XMLInputArchive does not require that nodes are loaded in the same + order they were saved by XMLOutputArchive. Using name value pairs (NVPs), + it is possible to load in an out of order fashion or otherwise skip/select + specific nodes to load. + + The default behavior of the input archive is to read sequentially starting + with the first node and exploring its children. When a given NVP does + not match the read in name for a node, the archive will search for that + node at the current level and load it if it exists. After loading an out of + order node, the archive will then proceed back to loading sequentially from + its new position. + + Consider this simple example where loading of some data is skipped: + + @code{cpp} + // imagine the input file has someData(1-9) saved in order at the top level node + ar( someData1, someData2, someData3 ); // XML loads in the order it sees in the file + ar( cereal::make_nvp( "hello", someData6 ) ); // NVP given does not + // match expected NVP name, so we search + // for the given NVP and load that value + ar( someData7, someData8, someData9 ); // with no NVP given, loading resumes at its + // current location, proceeding sequentially + @endcode + + \ingroup Archives */ + class XMLInputArchive : public InputArchive, public traits::TextArchive + { + public: + /*! @name Common Functionality + Common use cases for directly interacting with an XMLInputArchive */ + //! @{ + + //! Construct, reading in from the provided stream + /*! Reads in an entire XML document from some stream and parses it as soon + as serialization starts + + @param stream The stream to read from. Can be a stringstream or a file. */ + XMLInputArchive( std::istream & stream ) : + InputArchive( this ), + itsData( std::istreambuf_iterator( stream ), std::istreambuf_iterator() ) + { + try + { + itsData.push_back('\0'); // rapidxml will do terrible things without the data being null terminated + itsXML.parse( reinterpret_cast( itsData.data() ) ); + } + catch( rapidxml::parse_error const & ) + { + //std::cerr << "-----Original-----" << std::endl; + //stream.seekg(0); + //std::cout << std::string( std::istreambuf_iterator( stream ), std::istreambuf_iterator() ) << std::endl; + + //std::cerr << "-----Error-----" << std::endl; + //std::cerr << e.what() << std::endl; + //std::cerr << e.where() << std::endl; + throw Exception("XML Parsing failed - likely due to invalid characters or invalid naming"); + } + + // Parse the root + auto root = itsXML.first_node( xml_detail::CEREAL_XML_STRING ); + if( root == nullptr ) + throw Exception("Could not detect cereal root node - likely due to empty or invalid input"); + else + itsNodes.emplace( root ); + } + + ~XMLInputArchive() CEREAL_NOEXCEPT = default; + + //! Loads some binary data, encoded as a base64 string, optionally specified by some name + /*! This will automatically start and finish a node to load the data, and can be called directly by + users. + + Note that this follows the same ordering rules specified in the class description in regards + to loading in/out of order */ + void loadBinaryValue( void * data, size_t size, const char * name = nullptr ) + { + setNextName( name ); + startNode(); + + std::string encoded; + loadValue( encoded ); + + auto decoded = base64::decode( encoded ); + + if( size != decoded.size() ) + throw Exception("Decoded binary data size does not match specified size"); + + std::memcpy( data, decoded.data(), decoded.size() ); + + finishNode(); + } + + //! @} + /*! @name Internal Functionality + Functionality designed for use by those requiring control over the inner mechanisms of + the XMLInputArchive */ + //! @{ + + //! Prepares to start reading the next node + /*! This places the next node to be parsed onto the nodes stack. + + By default our strategy is to start with the document root node and then + recursively iterate through all children in the order they show up in the document. + We don't need to know NVPs do to this; we'll just blindly load in the order things appear in. + + We check to see if the specified NVP matches what the next automatically loaded node is. If they + match, we just continue as normal, going in order. If they don't match, we attempt to find a node + named after the NVP that is being loaded. If that NVP does not exist, we throw an exception. */ + void startNode() + { + auto next = itsNodes.top().child; // By default we would move to the next child node + auto const expectedName = itsNodes.top().name; // this is the expected name from the NVP, if provided + + // If we were given an NVP name, look for it in the current level of the document. + // We only need to do this if either we have exhausted the siblings of the current level or + // the NVP name does not match the name of the node we would normally read next + if( expectedName && ( next == nullptr || std::strcmp( next->name(), expectedName ) != 0 ) ) + { + next = itsNodes.top().search( expectedName ); + + if( next == nullptr ) + throw Exception("XML Parsing failed - provided NVP (" + std::string(expectedName) + ") not found"); + } + + itsNodes.emplace( next ); + } + + //! Finishes reading the current node + void finishNode() + { + // remove current + itsNodes.pop(); + + // advance parent + itsNodes.top().advance(); + + // Reset name + itsNodes.top().name = nullptr; + } + + //! Retrieves the current node name + //! will return @c nullptr if the node does not have a name + const char * getNodeName() const + { + return itsNodes.top().getChildName(); + } + + //! Sets the name for the next node created with startNode + void setNextName( const char * name ) + { + itsNodes.top().name = name; + } + + //! Loads a bool from the current top node + template ::value, + std::is_same::value> = traits::sfinae> inline + void loadValue( T & value ) + { + std::istringstream is( itsNodes.top().node->value() ); + is.setf( std::ios::boolalpha ); + is >> value; + } + + //! Loads a char (signed or unsigned) from the current top node + template ::value, + !std::is_same::value, + sizeof(T) == sizeof(char)> = traits::sfinae> inline + void loadValue( T & value ) + { + value = *reinterpret_cast( itsNodes.top().node->value() ); + } + + //! Load an int8_t from the current top node (ensures we parse entire number) + void loadValue( int8_t & value ) + { + int32_t val; loadValue( val ); value = static_cast( val ); + } + + //! Load a uint8_t from the current top node (ensures we parse entire number) + void loadValue( uint8_t & value ) + { + uint32_t val; loadValue( val ); value = static_cast( val ); + } + + //! Loads a type best represented as an unsigned long from the current top node + template ::value, + !std::is_same::value, + !std::is_same::value, + !std::is_same::value, + sizeof(T) < sizeof(long long)> = traits::sfinae> inline + void loadValue( T & value ) + { + value = static_cast( std::stoul( itsNodes.top().node->value() ) ); + } + + //! Loads a type best represented as an unsigned long long from the current top node + template ::value, + !std::is_same::value, + sizeof(T) >= sizeof(long long)> = traits::sfinae> inline + void loadValue( T & value ) + { + value = static_cast( std::stoull( itsNodes.top().node->value() ) ); + } + + //! Loads a type best represented as an int from the current top node + template ::value, + !std::is_same::value, + sizeof(T) <= sizeof(int)> = traits::sfinae> inline + void loadValue( T & value ) + { + value = static_cast( std::stoi( itsNodes.top().node->value() ) ); + } + + //! Loads a type best represented as a long from the current top node + template ::value, + (sizeof(T) > sizeof(int)), + sizeof(T) <= sizeof(long)> = traits::sfinae> inline + void loadValue( T & value ) + { + value = static_cast( std::stol( itsNodes.top().node->value() ) ); + } + + //! Loads a type best represented as a long long from the current top node + template ::value, + (sizeof(T) > sizeof(long)), + sizeof(T) <= sizeof(long long)> = traits::sfinae> inline + void loadValue( T & value ) + { + value = static_cast( std::stoll( itsNodes.top().node->value() ) ); + } + + //! Loads a type best represented as a float from the current top node + void loadValue( float & value ) + { + try + { + value = std::stof( itsNodes.top().node->value() ); + } + catch( std::out_of_range const & ) + { + // special case for denormalized values + std::istringstream is( itsNodes.top().node->value() ); + is >> value; + if( std::fpclassify( value ) != FP_SUBNORMAL ) + throw; + } + } + + //! Loads a type best represented as a double from the current top node + void loadValue( double & value ) + { + try + { + value = std::stod( itsNodes.top().node->value() ); + } + catch( std::out_of_range const & ) + { + // special case for denormalized values + std::istringstream is( itsNodes.top().node->value() ); + is >> value; + if( std::fpclassify( value ) != FP_SUBNORMAL ) + throw; + } + } + + //! Loads a type best represented as a long double from the current top node + void loadValue( long double & value ) + { + try + { + value = std::stold( itsNodes.top().node->value() ); + } + catch( std::out_of_range const & ) + { + // special case for denormalized values + std::istringstream is( itsNodes.top().node->value() ); + is >> value; + if( std::fpclassify( value ) != FP_SUBNORMAL ) + throw; + } + } + + //! Loads a string from the current node from the current top node + template inline + void loadValue( std::basic_string & str ) + { + std::basic_istringstream is( itsNodes.top().node->value() ); + + str.assign( std::istreambuf_iterator( is ), + std::istreambuf_iterator() ); + } + + //! Loads the size of the current top node + template inline + void loadSize( T & value ) + { + value = getNumChildren( itsNodes.top().node ); + } + + protected: + //! Gets the number of children (usually interpreted as size) for the specified node + static size_t getNumChildren( rapidxml::xml_node<> * node ) + { + size_t size = 0; + node = node->first_node(); // get first child + + while( node != nullptr ) + { + ++size; + node = node->next_sibling(); + } + + return size; + } + + //! A struct that contains metadata about a node + /*! Keeps track of some top level node, its number of + remaining children, and the current active child node */ + struct NodeInfo + { + NodeInfo( rapidxml::xml_node<> * n = nullptr ) : + node( n ), + child( n->first_node() ), + size( XMLInputArchive::getNumChildren( n ) ), + name( nullptr ) + { } + + //! Advances to the next sibling node of the child + /*! If this is the last sibling child will be null after calling */ + void advance() + { + if( size > 0 ) + { + --size; + child = child->next_sibling(); + } + } + + //! Searches for a child with the given name in this node + /*! @param searchName The name to search for (must be null terminated) + @return The node if found, nullptr otherwise */ + rapidxml::xml_node<> * search( const char * searchName ) + { + if( searchName ) + { + size_t new_size = XMLInputArchive::getNumChildren( node ); + const size_t name_size = rapidxml::internal::measure( searchName ); + + for( auto new_child = node->first_node(); new_child != nullptr; new_child = new_child->next_sibling() ) + { + if( rapidxml::internal::compare( new_child->name(), new_child->name_size(), searchName, name_size, true ) ) + { + size = new_size; + child = new_child; + + return new_child; + } + --new_size; + } + } + + return nullptr; + } + + //! Returns the actual name of the next child node, if it exists + const char * getChildName() const + { + return child ? child->name() : nullptr; + } + + rapidxml::xml_node<> * node; //!< A pointer to this node + rapidxml::xml_node<> * child; //!< A pointer to its current child + size_t size; //!< The remaining number of children for this node + const char * name; //!< The NVP name for next child node + }; // NodeInfo + + //! @} + + private: + std::vector itsData; //!< The raw data loaded + rapidxml::xml_document<> itsXML; //!< The XML document + std::stack itsNodes; //!< A stack of nodes read from the document + }; + + // ###################################################################### + // XMLArchive prologue and epilogue functions + // ###################################################################### + + // ###################################################################### + //! Prologue for NVPs for XML output archives + /*! NVPs do not start or finish nodes - they just set up the names */ + template inline + void prologue( XMLOutputArchive &, NameValuePair const & ) + { } + + //! Prologue for NVPs for XML input archives + template inline + void prologue( XMLInputArchive &, NameValuePair const & ) + { } + + // ###################################################################### + //! Epilogue for NVPs for XML output archives + /*! NVPs do not start or finish nodes - they just set up the names */ + template inline + void epilogue( XMLOutputArchive &, NameValuePair const & ) + { } + + //! Epilogue for NVPs for XML input archives + template inline + void epilogue( XMLInputArchive &, NameValuePair const & ) + { } + + // ###################################################################### + //! Prologue for deferred data for XML archives + /*! Do nothing for the defer wrapper */ + template inline + void prologue( XMLOutputArchive &, DeferredData const & ) + { } + + //! Prologue for deferred data for XML archives + template inline + void prologue( XMLInputArchive &, DeferredData const & ) + { } + + // ###################################################################### + //! Epilogue for deferred for XML archives + /*! NVPs do not start or finish nodes - they just set up the names */ + template inline + void epilogue( XMLOutputArchive &, DeferredData const & ) + { } + + //! Epilogue for deferred for XML archives + /*! Do nothing for the defer wrapper */ + template inline + void epilogue( XMLInputArchive &, DeferredData const & ) + { } + + // ###################################################################### + //! Prologue for SizeTags for XML output archives + /*! SizeTags do not start or finish nodes */ + template inline + void prologue( XMLOutputArchive & ar, SizeTag const & ) + { + if (ar.hasSizeAttributes()) + { + ar.appendAttribute("size", "dynamic"); + } + } + + template inline + void prologue( XMLInputArchive &, SizeTag const & ) + { } + + //! Epilogue for SizeTags for XML output archives + /*! SizeTags do not start or finish nodes */ + template inline + void epilogue( XMLOutputArchive &, SizeTag const & ) + { } + + template inline + void epilogue( XMLInputArchive &, SizeTag const & ) + { } + + // ###################################################################### + //! Prologue for all other types for XML output archives (except minimal types) + /*! Starts a new node, named either automatically or by some NVP, + that may be given data by the type about to be archived + + Minimal types do not start or end nodes */ + template ::value || + traits::has_minimal_output_serialization::value> = traits::sfinae> inline + void prologue( XMLOutputArchive & ar, T const & ) + { + ar.startNode(); + ar.insertType(); + } + + //! Prologue for all other types for XML input archives (except minimal types) + template ::value || + traits::has_minimal_input_serialization::value> = traits::sfinae> inline + void prologue( XMLInputArchive & ar, T const & ) + { + ar.startNode(); + } + + // ###################################################################### + //! Epilogue for all other types other for XML output archives (except minimal types) + /*! Finishes the node created in the prologue + + Minimal types do not start or end nodes */ + template ::value || + traits::has_minimal_output_serialization::value> = traits::sfinae> inline + void epilogue( XMLOutputArchive & ar, T const & ) + { + ar.finishNode(); + } + + //! Epilogue for all other types other for XML output archives (except minimal types) + template ::value || + traits::has_minimal_input_serialization::value> = traits::sfinae> inline + void epilogue( XMLInputArchive & ar, T const & ) + { + ar.finishNode(); + } + + // ###################################################################### + // Common XMLArchive serialization functions + // ###################################################################### + + //! Saving NVP types to XML + template inline + void CEREAL_SAVE_FUNCTION_NAME( XMLOutputArchive & ar, NameValuePair const & t ) + { + ar.setNextName( t.name ); + ar( t.value ); + } + + //! Loading NVP types from XML + template inline + void CEREAL_LOAD_FUNCTION_NAME( XMLInputArchive & ar, NameValuePair & t ) + { + ar.setNextName( t.name ); + ar( t.value ); + } + + // ###################################################################### + //! Saving SizeTags to XML + template inline + void CEREAL_SAVE_FUNCTION_NAME( XMLOutputArchive &, SizeTag const & ) + { } + + //! Loading SizeTags from XML + template inline + void CEREAL_LOAD_FUNCTION_NAME( XMLInputArchive & ar, SizeTag & st ) + { + ar.loadSize( st.size ); + } + + // ###################################################################### + //! Saving for POD types to xml + template ::value> = traits::sfinae> inline + void CEREAL_SAVE_FUNCTION_NAME(XMLOutputArchive & ar, T const & t) + { + ar.saveValue( t ); + } + + //! Loading for POD types from xml + template ::value> = traits::sfinae> inline + void CEREAL_LOAD_FUNCTION_NAME(XMLInputArchive & ar, T & t) + { + ar.loadValue( t ); + } + + // ###################################################################### + //! saving string to xml + template inline + void CEREAL_SAVE_FUNCTION_NAME(XMLOutputArchive & ar, std::basic_string const & str) + { + ar.saveValue( str ); + } + + //! loading string from xml + template inline + void CEREAL_LOAD_FUNCTION_NAME(XMLInputArchive & ar, std::basic_string & str) + { + ar.loadValue( str ); + } +} // namespace cereal + +// register archives for polymorphic support +CEREAL_REGISTER_ARCHIVE(cereal::XMLOutputArchive) +CEREAL_REGISTER_ARCHIVE(cereal::XMLInputArchive) + +// tie input and output archives together +CEREAL_SETUP_ARCHIVE_TRAITS(cereal::XMLInputArchive, cereal::XMLOutputArchive) + +#endif // CEREAL_ARCHIVES_XML_HPP_ diff --git a/third_party/cereal/cereal.hpp b/third_party/cereal/cereal.hpp new file mode 100755 index 0000000..9ed1714 --- /dev/null +++ b/third_party/cereal/cereal.hpp @@ -0,0 +1,1120 @@ +/*! \file cereal.hpp + \brief Main cereal functionality */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_CEREAL_HPP_ +#define CEREAL_CEREAL_HPP_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "macros.hpp" +#include "details/traits.hpp" +#include "details/helpers.hpp" +#include "types/base_class.hpp" + +namespace cereal +{ + // ###################################################################### + //! Creates a name value pair + /*! @relates NameValuePair + @ingroup Utility */ + template inline + NameValuePair make_nvp( std::string const & name, T && value ) + { + return {name.c_str(), std::forward(value)}; + } + + //! Creates a name value pair + /*! @relates NameValuePair + @ingroup Utility */ + template inline + NameValuePair make_nvp( const char * name, T && value ) + { + return {name, std::forward(value)}; + } + + //! Creates a name value pair for the variable T with the same name as the variable + /*! @relates NameValuePair + @ingroup Utility */ + #define CEREAL_NVP(T) ::cereal::make_nvp(#T, T) + + // ###################################################################### + //! Convenience function to create binary data for both const and non const pointers + /*! @param data Pointer to beginning of the data + @param size The size in bytes of the data + @relates BinaryData + @ingroup Utility */ + template inline + BinaryData binary_data( T && data, size_t size ) + { + return {std::forward(data), size}; + } + + // ###################################################################### + //! Creates a size tag from some variable. + /*! Will normally be used to serialize size (e.g. size()) information for + variable size containers. If you have a variable sized container, + the very first thing it serializes should be its size, wrapped in + a SizeTag. + + @relates SizeTag + @ingroup Utility */ + template inline + SizeTag make_size_tag( T && sz ) + { + return {std::forward(sz)}; + } + + // ###################################################################### + //! Marks data for deferred serialization + /*! cereal performs a recursive depth-first traversal of data it serializes. When + serializing smart pointers to large, nested, or cyclical data structures, it + is possible to encounter a stack overflow from excessive recursion when following + a chain of pointers. + + Deferment can help in these situations if the data can be serialized separately from + the pointers used to traverse the structure. For example, a graph structure can have its + nodes serialized before its edges: + + @code{.cpp} + struct MyEdge + { + std::shared_ptr connection; + int some_value; + + template + void serialize(Archive & archive) + { + // when we serialize an edge, we'll defer serializing the associated node + archive( cereal::defer( connection ), + some_value ); + } + }; + + struct MyGraphStructure + { + std::vector edges; + std::vector nodes; + + template + void serialize(Archive & archive) + { + // because of the deferment, we ensure all nodes are fully serialized + // before any connection pointers to those nodes are serialized + archive( edges, nodes ); + + // we have to explicitly inform the archive when it is safe to serialize + // the deferred data + archive.serializeDeferments(); + } + }; + @endcode + + @relates DeferredData + @ingroup Utility */ + template inline + DeferredData defer( T && value ) + { + return {std::forward(value)}; + } + + // ###################################################################### + //! Called before a type is serialized to set up any special archive state + //! for processing some type + /*! If designing a serializer that needs to set up any kind of special + state or output extra information for a type, specialize this function + for the archive type and the types that require the extra information. + @ingroup Internal */ + template inline + void prologue( Archive & /* archive */, T const & /* data */) + { } + + //! Called after a type is serialized to tear down any special archive state + //! for processing some type + /*! @ingroup Internal */ + template inline + void epilogue( Archive & /* archive */, T const & /* data */) + { } + + // ###################################################################### + //! Special flags for archives + /*! AllowEmptyClassElision + This allows for empty classes to be serialized even if they do not provide + a serialization function. Classes with no data members are considered to be + empty. Be warned that if this is enabled and you attempt to serialize an + empty class with improperly formed serialize or load/save functions, no + static error will occur - the error will propogate silently and your + intended serialization functions may not be called. You can manually + ensure that your classes that have custom serialization are correct + by using the traits is_output_serializable and is_input_serializable + in cereal/details/traits.hpp. + @ingroup Internal */ + enum Flags { AllowEmptyClassElision = 1 }; + + // ###################################################################### + //! Registers a specific Archive type with cereal + /*! This registration should be done once per archive. A good place to + put this is immediately following the definition of your archive. + Archive registration is only strictly necessary if you wish to + support pointers to polymorphic data types. All archives that + come with cereal are already registered. + @ingroup Internal */ + #define CEREAL_REGISTER_ARCHIVE(Archive) \ + namespace cereal { namespace detail { \ + template \ + typename polymorphic_serialization_support::type \ + instantiate_polymorphic_binding( T*, Archive*, BindingTag, adl_tag ); \ + } } /* end namespaces */ + + //! Helper macro to omit unused warning + #if defined(__GNUC__) + // GCC / clang don't want the function + #define CEREAL_UNUSED_FUNCTION + #else + #define CEREAL_UNUSED_FUNCTION static void unused() { (void)version; } + #endif + + // ###################################################################### + //! Defines a class version for some type + /*! Versioning information is optional and adds some small amount of + overhead to serialization. This overhead will occur both in terms of + space in the archive (the version information for each class will be + stored exactly once) as well as runtime (versioned serialization functions + must check to see if they need to load or store version information). + + Versioning is useful if you plan on fundamentally changing the way some + type is serialized in the future. Versioned serialization functions + cannot be used to load non-versioned data. + + By default, all types have an assumed version value of zero. By + using this macro, you may change the version number associated with + some type. cereal will then use this value as a second parameter + to your serialization functions. + + The interface for the serialization functions is nearly identical + to non-versioned serialization with the addition of a second parameter, + const std::uint32_t version, which will be supplied with the correct + version number. Serializing the version number on a save happens + automatically. + + Versioning cannot be mixed with non-versioned serialization functions. + Having both types will result result in a compile time error. Data + serialized without versioning cannot be loaded by a serialization + function with added versioning support. + + Example interface for versioning on a non-member serialize function: + + @code{cpp} + CEREAL_CLASS_VERSION( Mytype, 77 ); // register class version + + template + void serialize( Archive & ar, Mytype & t, const std::uint32_t version ) + { + // When performing a load, the version associated with the class + // is whatever it was when that data was originally serialized + // + // When we save, we'll use the version that is defined in the macro + + if( version >= some_number ) + // do this + else + // do that + } + @endcode + + Interfaces for other forms of serialization functions is similar. This + macro should be placed at global scope. + @ingroup Utility */ + + //! On C++17, define the StaticObject as inline to merge the definitions across TUs + //! This prevents multiple definition errors when this macro appears in a header file + //! included in multiple TUs. + #ifdef CEREAL_HAS_CPP17 + #define CEREAL_CLASS_VERSION(TYPE, VERSION_NUMBER) \ + namespace cereal { namespace detail { \ + template <> struct Version \ + { \ + static std::uint32_t registerVersion() \ + { \ + ::cereal::detail::StaticObject::getInstance().mapping.emplace( \ + std::type_index(typeid(TYPE)).hash_code(), VERSION_NUMBER ); \ + return VERSION_NUMBER; \ + } \ + static inline const std::uint32_t version = registerVersion(); \ + CEREAL_UNUSED_FUNCTION \ + }; /* end Version */ \ + } } // end namespaces + #else + #define CEREAL_CLASS_VERSION(TYPE, VERSION_NUMBER) \ + namespace cereal { namespace detail { \ + template <> struct Version \ + { \ + static const std::uint32_t version; \ + static std::uint32_t registerVersion() \ + { \ + ::cereal::detail::StaticObject::getInstance().mapping.emplace( \ + std::type_index(typeid(TYPE)).hash_code(), VERSION_NUMBER ); \ + return VERSION_NUMBER; \ + } \ + CEREAL_UNUSED_FUNCTION \ + }; /* end Version */ \ + const std::uint32_t Version::version = \ + Version::registerVersion(); \ + } } // end namespaces + + #endif + + // ###################################################################### + //! The base output archive class + /*! This is the base output archive for all output archives. If you create + a custom archive class, it should derive from this, passing itself as + a template parameter for the ArchiveType. + + The base class provides all of the functionality necessary to + properly forward data to the correct serialization functions. + + Individual archives should use a combination of prologue and + epilogue functions together with specializations of serialize, save, + and load to alter the functionality of their serialization. + + @tparam ArchiveType The archive type that derives from OutputArchive + @tparam Flags Flags to control advanced functionality. See the Flags + enum for more information. + @ingroup Internal */ + template + class OutputArchive : public detail::OutputArchiveBase + { + public: + //! Construct the output archive + /*! @param derived A pointer to the derived ArchiveType (pass this from the derived archive) */ + OutputArchive(ArchiveType * const derived) : self(derived), itsCurrentPointerId(1), itsCurrentPolymorphicTypeId(1) + { } + + OutputArchive & operator=( OutputArchive const & ) = delete; + + //! Serializes all passed in data + /*! This is the primary interface for serializing data with an archive */ + template inline + ArchiveType & operator()( Types && ... args ) + { + self->process( std::forward( args )... ); + return *self; + } + + //! Serializes any data marked for deferment using defer + /*! This will cause any data wrapped in DeferredData to be immediately serialized */ + void serializeDeferments() + { + for( auto & deferment : itsDeferments ) + deferment(); + } + + /*! @name Boost Transition Layer + Functionality that mirrors the syntax for Boost. This is useful if you are transitioning + a large project from Boost to cereal. The preferred interface for cereal is using operator(). */ + //! @{ + + //! Indicates this archive is not intended for loading + /*! This ensures compatibility with boost archive types. If you are transitioning + from boost, you can check this value within a member or external serialize function + (i.e., Archive::is_loading::value) to disable behavior specific to loading, until + you can transition to split save/load or save_minimal/load_minimal functions */ + using is_loading = std::false_type; + + //! Indicates this archive is intended for saving + /*! This ensures compatibility with boost archive types. If you are transitioning + from boost, you can check this value within a member or external serialize function + (i.e., Archive::is_saving::value) to enable behavior specific to loading, until + you can transition to split save/load or save_minimal/load_minimal functions */ + using is_saving = std::true_type; + + //! Serializes passed in data + /*! This is a boost compatability layer and is not the preferred way of using + cereal. If you are transitioning from boost, use this until you can + transition to the operator() overload */ + template inline + ArchiveType & operator&( T && arg ) + { + self->process( std::forward( arg ) ); + return *self; + } + + //! Serializes passed in data + /*! This is a boost compatability layer and is not the preferred way of using + cereal. If you are transitioning from boost, use this until you can + transition to the operator() overload */ + template inline + ArchiveType & operator<<( T && arg ) + { + self->process( std::forward( arg ) ); + return *self; + } + + //! @} + + //! Registers a shared pointer with the archive + /*! This function is used to track shared pointer targets to prevent + unnecessary saves from taking place if multiple shared pointers + point to the same data. + + @internal + @param sharedPointer The shared pointer itself (the adress is taked via get()). + The archive takes a copy to prevent the memory location to be freed + as long as the address is used as id. This is needed to prevent CVE-2020-11105. + @return A key that uniquely identifies the pointer */ + inline std::uint32_t registerSharedPointer(const std::shared_ptr& sharedPointer) + { + void const * addr = sharedPointer.get(); + + // Handle null pointers by just returning 0 + if(addr == 0) return 0; + itsSharedPointerStorage.push_back(sharedPointer); + + auto id = itsSharedPointerMap.find( addr ); + if( id == itsSharedPointerMap.end() ) + { + auto ptrId = itsCurrentPointerId++; + itsSharedPointerMap.insert( {addr, ptrId} ); + return ptrId | detail::msb_32bit; // mask MSB to be 1 + } + else + return id->second; + } + + //! Registers a polymorphic type name with the archive + /*! This function is used to track polymorphic types to prevent + unnecessary saves of identifying strings used by the polymorphic + support functionality. + + @internal + @param name The name to associate with a polymorphic type + @return A key that uniquely identifies the polymorphic type name */ + inline std::uint32_t registerPolymorphicType( char const * name ) + { + auto id = itsPolymorphicTypeMap.find( name ); + if( id == itsPolymorphicTypeMap.end() ) + { + auto polyId = itsCurrentPolymorphicTypeId++; + itsPolymorphicTypeMap.insert( {name, polyId} ); + return polyId | detail::msb_32bit; // mask MSB to be 1 + } + else + return id->second; + } + + private: + //! Serializes data after calling prologue, then calls epilogue + template inline + void process( T && head ) + { + prologue( *self, head ); + self->processImpl( head ); + epilogue( *self, head ); + } + + //! Unwinds to process all data + template inline + void process( T && head, Other && ... tail ) + { + self->process( std::forward( head ) ); + self->process( std::forward( tail )... ); + } + + //! Serialization of a virtual_base_class wrapper + /*! \sa virtual_base_class */ + template inline + ArchiveType & processImpl(virtual_base_class const & b) + { + traits::detail::base_class_id id(b.base_ptr); + if(itsBaseClassSet.count(id) == 0) + { + itsBaseClassSet.insert(id); + self->processImpl( *b.base_ptr ); + } + return *self; + } + + //! Serialization of a base_class wrapper + /*! \sa base_class */ + template inline + ArchiveType & processImpl(base_class const & b) + { + self->processImpl( *b.base_ptr ); + return *self; + } + + std::vector> itsDeferments; + + template inline + ArchiveType & processImpl(DeferredData const & d) + { + std::function deferment( [this, d](){ self->process( d.value ); } ); + itsDeferments.emplace_back( std::move(deferment) ); + + return *self; + } + + //! Helper macro that expands the requirements for activating an overload + /*! Requirements: + Has the requested serialization function + Does not have version and unversioned at the same time + Is output serializable AND + is specialized for this type of function OR + has no specialization at all */ + #define PROCESS_IF(name) \ + traits::EnableIf::value, \ + !traits::has_invalid_output_versioning::value, \ + (traits::is_output_serializable::value && \ + (traits::is_specialized_##name::value || \ + !traits::is_specialized::value))> = traits::sfinae + + //! Member serialization + template inline + ArchiveType & processImpl(T const & t) + { + access::member_serialize(*self, const_cast(t)); + return *self; + } + + //! Non member serialization + template inline + ArchiveType & processImpl(T const & t) + { + CEREAL_SERIALIZE_FUNCTION_NAME(*self, const_cast(t)); + return *self; + } + + //! Member split (save) + template inline + ArchiveType & processImpl(T const & t) + { + access::member_save(*self, t); + return *self; + } + + //! Non member split (save) + template inline + ArchiveType & processImpl(T const & t) + { + CEREAL_SAVE_FUNCTION_NAME(*self, t); + return *self; + } + + //! Member split (save_minimal) + template inline + ArchiveType & processImpl(T const & t) + { + self->process( access::member_save_minimal(*self, t) ); + return *self; + } + + //! Non member split (save_minimal) + template inline + ArchiveType & processImpl(T const & t) + { + self->process( CEREAL_SAVE_MINIMAL_FUNCTION_NAME(*self, t) ); + return *self; + } + + //! Empty class specialization + template ::value, + std::is_empty::value> = traits::sfinae> inline + ArchiveType & processImpl(T const &) + { + return *self; + } + + //! No matching serialization + /*! Invalid if we have invalid output versioning or + we are not output serializable, and either + don't allow empty class ellision or allow it but are not serializing an empty class */ + template ::value || + (!traits::is_output_serializable::value && + (!(Flags & AllowEmptyClassElision) || ((Flags & AllowEmptyClassElision) && !std::is_empty::value)))> = traits::sfinae> inline + ArchiveType & processImpl(T const &) + { + static_assert(traits::detail::count_output_serializers::value != 0, + "cereal could not find any output serialization functions for the provided type and archive combination. \n\n " + "Types must either have a serialize function, load/save pair, or load_minimal/save_minimal pair (you may not mix these). \n " + "Serialize functions generally have the following signature: \n\n " + "template \n " + " void serialize(Archive & ar) \n " + " { \n " + " ar( member1, member2, member3 ); \n " + " } \n\n " ); + + static_assert(traits::detail::count_output_serializers::value < 2, + "cereal found more than one compatible output serialization function for the provided type and archive combination. \n\n " + "Types must either have a serialize function, load/save pair, or load_minimal/save_minimal pair (you may not mix these). \n " + "Use specialization (see access.hpp) if you need to disambiguate between serialize vs load/save functions. \n " + "Note that serialization functions can be inherited which may lead to the aforementioned ambiguities. \n " + "In addition, you may not mix versioned with non-versioned serialization functions. \n\n "); + + return *self; + } + + //! Registers a class version with the archive and serializes it if necessary + /*! If this is the first time this class has been serialized, we will record its + version number and serialize that. + + @tparam T The type of the class being serialized */ + template inline + std::uint32_t registerClassVersion() + { + static const auto hash = std::type_index(typeid(T)).hash_code(); + const auto insertResult = itsVersionedTypes.insert( hash ); + const auto lock = detail::StaticObject::lock(); + const auto version = + detail::StaticObject::getInstance().find( hash, detail::Version::version ); + + if( insertResult.second ) // insertion took place, serialize the version number + process( make_nvp("cereal_class_version", version) ); + + return version; + } + + //! Member serialization + /*! Versioning implementation */ + template inline + ArchiveType & processImpl(T const & t) + { + access::member_serialize(*self, const_cast(t), registerClassVersion()); + return *self; + } + + //! Non member serialization + /*! Versioning implementation */ + template inline + ArchiveType & processImpl(T const & t) + { + CEREAL_SERIALIZE_FUNCTION_NAME(*self, const_cast(t), registerClassVersion()); + return *self; + } + + //! Member split (save) + /*! Versioning implementation */ + template inline + ArchiveType & processImpl(T const & t) + { + access::member_save(*self, t, registerClassVersion()); + return *self; + } + + //! Non member split (save) + /*! Versioning implementation */ + template inline + ArchiveType & processImpl(T const & t) + { + CEREAL_SAVE_FUNCTION_NAME(*self, t, registerClassVersion()); + return *self; + } + + //! Member split (save_minimal) + /*! Versioning implementation */ + template inline + ArchiveType & processImpl(T const & t) + { + self->process( access::member_save_minimal(*self, t, registerClassVersion()) ); + return *self; + } + + //! Non member split (save_minimal) + /*! Versioning implementation */ + template inline + ArchiveType & processImpl(T const & t) + { + self->process( CEREAL_SAVE_MINIMAL_FUNCTION_NAME(*self, t, registerClassVersion()) ); + return *self; + } + + #undef PROCESS_IF + + private: + ArchiveType * const self; + + //! A set of all base classes that have been serialized + std::unordered_set itsBaseClassSet; + + //! Maps from addresses to pointer ids + std::unordered_map itsSharedPointerMap; + + //! Copy of shared pointers used in #itsSharedPointerMap to make sure they are kept alive + // during lifetime of itsSharedPointerMap to prevent CVE-2020-11105. + std::vector> itsSharedPointerStorage; + + //! The id to be given to the next pointer + std::uint32_t itsCurrentPointerId; + + //! Maps from polymorphic type name strings to ids + std::unordered_map itsPolymorphicTypeMap; + + //! The id to be given to the next polymorphic type name + std::uint32_t itsCurrentPolymorphicTypeId; + + //! Keeps track of classes that have versioning information associated with them + std::unordered_set itsVersionedTypes; + }; // class OutputArchive + + // ###################################################################### + //! The base input archive class + /*! This is the base input archive for all input archives. If you create + a custom archive class, it should derive from this, passing itself as + a template parameter for the ArchiveType. + + The base class provides all of the functionality necessary to + properly forward data to the correct serialization functions. + + Individual archives should use a combination of prologue and + epilogue functions together with specializations of serialize, save, + and load to alter the functionality of their serialization. + + @tparam ArchiveType The archive type that derives from InputArchive + @tparam Flags Flags to control advanced functionality. See the Flags + enum for more information. + @ingroup Internal */ + template + class InputArchive : public detail::InputArchiveBase + { + public: + //! Construct the output archive + /*! @param derived A pointer to the derived ArchiveType (pass this from the derived archive) */ + InputArchive(ArchiveType * const derived) : + self(derived), + itsBaseClassSet(), + itsSharedPointerMap(), + itsPolymorphicTypeMap(), + itsVersionedTypes() + { } + + InputArchive & operator=( InputArchive const & ) = delete; + + //! Serializes all passed in data + /*! This is the primary interface for serializing data with an archive */ + template inline + ArchiveType & operator()( Types && ... args ) + { + process( std::forward( args )... ); + return *self; + } + + //! Serializes any data marked for deferment using defer + /*! This will cause any data wrapped in DeferredData to be immediately serialized */ + void serializeDeferments() + { + for( auto & deferment : itsDeferments ) + deferment(); + } + + /*! @name Boost Transition Layer + Functionality that mirrors the syntax for Boost. This is useful if you are transitioning + a large project from Boost to cereal. The preferred interface for cereal is using operator(). */ + //! @{ + + //! Indicates this archive is intended for loading + /*! This ensures compatibility with boost archive types. If you are transitioning + from boost, you can check this value within a member or external serialize function + (i.e., Archive::is_loading::value) to enable behavior specific to loading, until + you can transition to split save/load or save_minimal/load_minimal functions */ + using is_loading = std::true_type; + + //! Indicates this archive is not intended for saving + /*! This ensures compatibility with boost archive types. If you are transitioning + from boost, you can check this value within a member or external serialize function + (i.e., Archive::is_saving::value) to disable behavior specific to loading, until + you can transition to split save/load or save_minimal/load_minimal functions */ + using is_saving = std::false_type; + + //! Serializes passed in data + /*! This is a boost compatability layer and is not the preferred way of using + cereal. If you are transitioning from boost, use this until you can + transition to the operator() overload */ + template inline + ArchiveType & operator&( T && arg ) + { + self->process( std::forward( arg ) ); + return *self; + } + + //! Serializes passed in data + /*! This is a boost compatability layer and is not the preferred way of using + cereal. If you are transitioning from boost, use this until you can + transition to the operator() overload */ + template inline + ArchiveType & operator>>( T && arg ) + { + self->process( std::forward( arg ) ); + return *self; + } + + //! @} + + //! Retrieves a shared pointer given a unique key for it + /*! This is used to retrieve a previously registered shared_ptr + which has already been loaded. + + @internal + @param id The unique id that was serialized for the pointer + @return A shared pointer to the data + @throw Exception if the id does not exist */ + inline std::shared_ptr getSharedPointer(std::uint32_t const id) + { + if(id == 0) return std::shared_ptr(nullptr); + + auto iter = itsSharedPointerMap.find( id ); + if(iter == itsSharedPointerMap.end()) + throw Exception("Error while trying to deserialize a smart pointer. Could not find id " + std::to_string(id)); + + return iter->second; + } + + //! Registers a shared pointer to its unique identifier + /*! After a shared pointer has been allocated for the first time, it should + be registered with its loaded id for future references to it. + + @internal + @param id The unique identifier for the shared pointer + @param ptr The actual shared pointer */ + inline void registerSharedPointer(std::uint32_t const id, std::shared_ptr ptr) + { + std::uint32_t const stripped_id = id & ~detail::msb_32bit; + itsSharedPointerMap[stripped_id] = ptr; + } + + //! Retrieves the string for a polymorphic type given a unique key for it + /*! This is used to retrieve a string previously registered during + a polymorphic load. + + @internal + @param id The unique id that was serialized for the polymorphic type + @return The string identifier for the tyep */ + inline std::string getPolymorphicName(std::uint32_t const id) + { + auto name = itsPolymorphicTypeMap.find( id ); + if(name == itsPolymorphicTypeMap.end()) + { + throw Exception("Error while trying to deserialize a polymorphic pointer. Could not find type id " + std::to_string(id)); + } + return name->second; + } + + //! Registers a polymorphic name string to its unique identifier + /*! After a polymorphic type has been loaded for the first time, it should + be registered with its loaded id for future references to it. + + @internal + @param id The unique identifier for the polymorphic type + @param name The name associated with the tyep */ + inline void registerPolymorphicName(std::uint32_t const id, std::string const & name) + { + std::uint32_t const stripped_id = id & ~detail::msb_32bit; + itsPolymorphicTypeMap.insert( {stripped_id, name} ); + } + + private: + //! Serializes data after calling prologue, then calls epilogue + template inline + void process( T && head ) + { + prologue( *self, head ); + self->processImpl( head ); + epilogue( *self, head ); + } + + //! Unwinds to process all data + template inline + void process( T && head, Other && ... tail ) + { + process( std::forward( head ) ); + process( std::forward( tail )... ); + } + + //! Serialization of a virtual_base_class wrapper + /*! \sa virtual_base_class */ + template inline + ArchiveType & processImpl(virtual_base_class & b) + { + traits::detail::base_class_id id(b.base_ptr); + if(itsBaseClassSet.count(id) == 0) + { + itsBaseClassSet.insert(id); + self->processImpl( *b.base_ptr ); + } + return *self; + } + + //! Serialization of a base_class wrapper + /*! \sa base_class */ + template inline + ArchiveType & processImpl(base_class & b) + { + self->processImpl( *b.base_ptr ); + return *self; + } + + std::vector> itsDeferments; + + template inline + ArchiveType & processImpl(DeferredData const & d) + { + std::function deferment( [this, d](){ self->process( d.value ); } ); + itsDeferments.emplace_back( std::move(deferment) ); + + return *self; + } + + //! Helper macro that expands the requirements for activating an overload + /*! Requirements: + Has the requested serialization function + Does not have version and unversioned at the same time + Is input serializable AND + is specialized for this type of function OR + has no specialization at all */ + #define PROCESS_IF(name) \ + traits::EnableIf::value, \ + !traits::has_invalid_input_versioning::value, \ + (traits::is_input_serializable::value && \ + (traits::is_specialized_##name::value || \ + !traits::is_specialized::value))> = traits::sfinae + + //! Member serialization + template inline + ArchiveType & processImpl(T & t) + { + access::member_serialize(*self, t); + return *self; + } + + //! Non member serialization + template inline + ArchiveType & processImpl(T & t) + { + CEREAL_SERIALIZE_FUNCTION_NAME(*self, t); + return *self; + } + + //! Member split (load) + template inline + ArchiveType & processImpl(T & t) + { + access::member_load(*self, t); + return *self; + } + + //! Non member split (load) + template inline + ArchiveType & processImpl(T & t) + { + CEREAL_LOAD_FUNCTION_NAME(*self, t); + return *self; + } + + //! Member split (load_minimal) + template inline + ArchiveType & processImpl(T & t) + { + using OutArchiveType = typename traits::detail::get_output_from_input::type; + typename traits::has_member_save_minimal::type value; + self->process( value ); + access::member_load_minimal(*self, t, value); + return *self; + } + + //! Non member split (load_minimal) + template inline + ArchiveType & processImpl(T & t) + { + using OutArchiveType = typename traits::detail::get_output_from_input::type; + typename traits::has_non_member_save_minimal::type value; + self->process( value ); + CEREAL_LOAD_MINIMAL_FUNCTION_NAME(*self, t, value); + return *self; + } + + //! Empty class specialization + template ::value, + std::is_empty::value> = traits::sfinae> inline + ArchiveType & processImpl(T const &) + { + return *self; + } + + //! No matching serialization + /*! Invalid if we have invalid input versioning or + we are not input serializable, and either + don't allow empty class ellision or allow it but are not serializing an empty class */ + template ::value || + (!traits::is_input_serializable::value && + (!(Flags & AllowEmptyClassElision) || ((Flags & AllowEmptyClassElision) && !std::is_empty::value)))> = traits::sfinae> inline + ArchiveType & processImpl(T const &) + { + static_assert(traits::detail::count_input_serializers::value != 0, + "cereal could not find any input serialization functions for the provided type and archive combination. \n\n " + "Types must either have a serialize function, load/save pair, or load_minimal/save_minimal pair (you may not mix these). \n " + "Serialize functions generally have the following signature: \n\n " + "template \n " + " void serialize(Archive & ar) \n " + " { \n " + " ar( member1, member2, member3 ); \n " + " } \n\n " ); + + static_assert(traits::detail::count_input_serializers::value < 2, + "cereal found more than one compatible input serialization function for the provided type and archive combination. \n\n " + "Types must either have a serialize function, load/save pair, or load_minimal/save_minimal pair (you may not mix these). \n " + "Use specialization (see access.hpp) if you need to disambiguate between serialize vs load/save functions. \n " + "Note that serialization functions can be inherited which may lead to the aforementioned ambiguities. \n " + "In addition, you may not mix versioned with non-versioned serialization functions. \n\n "); + + return *self; + } + + //! Befriend for versioning in load_and_construct + template friend struct detail::Construct; + + //! Registers a class version with the archive and serializes it if necessary + /*! If this is the first time this class has been serialized, we will record its + version number and serialize that. + + @tparam T The type of the class being serialized */ + template inline + std::uint32_t loadClassVersion() + { + static const auto hash = std::type_index(typeid(T)).hash_code(); + auto lookupResult = itsVersionedTypes.find( hash ); + + if( lookupResult != itsVersionedTypes.end() ) // already exists + return lookupResult->second; + else // need to load + { + std::uint32_t version; + + process( make_nvp("cereal_class_version", version) ); + itsVersionedTypes.emplace_hint( lookupResult, hash, version ); + + return version; + } + } + + //! Member serialization + /*! Versioning implementation */ + template inline + ArchiveType & processImpl(T & t) + { + const auto version = loadClassVersion(); + access::member_serialize(*self, t, version); + return *self; + } + + //! Non member serialization + /*! Versioning implementation */ + template inline + ArchiveType & processImpl(T & t) + { + const auto version = loadClassVersion(); + CEREAL_SERIALIZE_FUNCTION_NAME(*self, t, version); + return *self; + } + + //! Member split (load) + /*! Versioning implementation */ + template inline + ArchiveType & processImpl(T & t) + { + const auto version = loadClassVersion(); + access::member_load(*self, t, version); + return *self; + } + + //! Non member split (load) + /*! Versioning implementation */ + template inline + ArchiveType & processImpl(T & t) + { + const auto version = loadClassVersion(); + CEREAL_LOAD_FUNCTION_NAME(*self, t, version); + return *self; + } + + //! Member split (load_minimal) + /*! Versioning implementation */ + template inline + ArchiveType & processImpl(T & t) + { + using OutArchiveType = typename traits::detail::get_output_from_input::type; + const auto version = loadClassVersion(); + typename traits::has_member_versioned_save_minimal::type value; + self->process(value); + access::member_load_minimal(*self, t, value, version); + return *self; + } + + //! Non member split (load_minimal) + /*! Versioning implementation */ + template inline + ArchiveType & processImpl(T & t) + { + using OutArchiveType = typename traits::detail::get_output_from_input::type; + const auto version = loadClassVersion(); + typename traits::has_non_member_versioned_save_minimal::type value; + self->process(value); + CEREAL_LOAD_MINIMAL_FUNCTION_NAME(*self, t, value, version); + return *self; + } + + #undef PROCESS_IF + + private: + ArchiveType * const self; + + //! A set of all base classes that have been serialized + std::unordered_set itsBaseClassSet; + + //! Maps from pointer ids to metadata + std::unordered_map> itsSharedPointerMap; + + //! Maps from name ids to names + std::unordered_map itsPolymorphicTypeMap; + + //! Maps from type hash codes to version numbers + std::unordered_map itsVersionedTypes; + }; // class InputArchive +} // namespace cereal + +// This include needs to come after things such as binary_data, make_nvp, etc +#include "types/common.hpp" + +#endif // CEREAL_CEREAL_HPP_ diff --git a/third_party/cereal/details/helpers.hpp b/third_party/cereal/details/helpers.hpp new file mode 100755 index 0000000..8bf6d12 --- /dev/null +++ b/third_party/cereal/details/helpers.hpp @@ -0,0 +1,422 @@ +/*! \file helpers.hpp + \brief Internal helper functionality + \ingroup Internal */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_DETAILS_HELPERS_HPP_ +#define CEREAL_DETAILS_HELPERS_HPP_ + +#include +#include +#include +#include +#include +#include + +#include "../macros.hpp" +#include "../details/static_object.hpp" + +namespace cereal +{ + // ###################################################################### + //! An exception class thrown when things go wrong at runtime + /*! @ingroup Utility */ + struct Exception : public std::runtime_error + { + explicit Exception( const std::string & what_ ) : std::runtime_error(what_) {} + explicit Exception( const char * what_ ) : std::runtime_error(what_) {} + }; + + // ###################################################################### + //! The size type used by cereal + /*! To ensure compatability between 32, 64, etc bit machines, we need to use + a fixed size type instead of size_t, which may vary from machine to + machine. + + The default value for CEREAL_SIZE_TYPE is specified in cereal/macros.hpp */ + using size_type = CEREAL_SIZE_TYPE; + + // forward decls + class BinaryOutputArchive; + class BinaryInputArchive; + + // ###################################################################### + namespace detail + { + struct NameValuePairCore {}; //!< Traits struct for NVPs + struct DeferredDataCore {}; //!< Traits struct for DeferredData + } + + // ###################################################################### + //! For holding name value pairs + /*! This pairs a name (some string) with some value such that an archive + can potentially take advantage of the pairing. + + In serialization functions, NameValuePairs are usually created like so: + @code{.cpp} + struct MyStruct + { + int a, b, c, d, e; + + template + void serialize(Archive & archive) + { + archive( CEREAL_NVP(a), + CEREAL_NVP(b), + CEREAL_NVP(c), + CEREAL_NVP(d), + CEREAL_NVP(e) ); + } + }; + @endcode + + Alternatively, you can give you data members custom names like so: + @code{.cpp} + struct MyStruct + { + int a, b, my_embarrassing_variable_name, d, e; + + template + void serialize(Archive & archive) + { + archive( CEREAL_NVP(a), + CEREAL_NVP(b), + cereal::make_nvp("var", my_embarrassing_variable_name) ); + CEREAL_NVP(d), + CEREAL_NVP(e) ); + } + }; + @endcode + + There is a slight amount of overhead to creating NameValuePairs, so there + is a third method which will elide the names when they are not used by + the Archive: + + @code{.cpp} + struct MyStruct + { + int a, b; + + template + void serialize(Archive & archive) + { + archive( cereal::make_nvp(a), + cereal::make_nvp(b) ); + } + }; + @endcode + + This third method is generally only used when providing generic type + support. Users writing their own serialize functions will normally + explicitly control whether they want to use NVPs or not. + + @internal */ + template + class NameValuePair : detail::NameValuePairCore + { + private: + // If we get passed an array, keep the type as is, otherwise store + // a reference if we were passed an l value reference, else copy the value + using Type = typename std::conditional::type>::value, + typename std::remove_cv::type, + typename std::conditional::value, + T, + typename std::decay::type>::type>::type; + + // prevent nested nvps + static_assert( !std::is_base_of::value, + "Cannot pair a name to a NameValuePair" ); + + NameValuePair & operator=( NameValuePair const & ) = delete; + + public: + //! Constructs a new NameValuePair + /*! @param n The name of the pair + @param v The value to pair. Ideally this should be an l-value reference so that + the value can be both loaded and saved to. If you pass an r-value reference, + the NameValuePair will store a copy of it instead of a reference. Thus you should + only pass r-values in cases where this makes sense, such as the result of some + size() call. + @internal */ + NameValuePair( char const * n, T && v ) : name(n), value(std::forward(v)) {} + + char const * name; + Type value; + }; + + //! A specialization of make_nvp<> that simply forwards the value for binary archives + /*! @relates NameValuePair + @internal */ + template inline + typename + std::enable_if::value || + std::is_same::value, + T && >::type + make_nvp( const char *, T && value ) + { + return std::forward(value); + } + + //! A specialization of make_nvp<> that actually creates an nvp for non-binary archives + /*! @relates NameValuePair + @internal */ + template inline + typename + std::enable_if::value && + !std::is_same::value, + NameValuePair >::type + make_nvp( const char * name, T && value) + { + return {name, std::forward(value)}; + } + + //! Convenience for creating a templated NVP + /*! For use in internal generic typing functions which have an + Archive type declared + @internal */ + #define CEREAL_NVP_(name, value) ::cereal::make_nvp(name, value) + + // ###################################################################### + //! A wrapper around data that can be serialized in a binary fashion + /*! This class is used to demarcate data that can safely be serialized + as a binary chunk of data. Individual archives can then choose how + best represent this during serialization. + + @internal */ + template + struct BinaryData + { + //! Internally store the pointer as a void *, keeping const if created with + //! a const pointer + using PT = typename std::conditional::type>::type>::value, + const void *, + void *>::type; + + BinaryData( T && d, uint64_t s ) : data(std::forward(d)), size(s) {} + + PT data; //!< pointer to beginning of data + uint64_t size; //!< size in bytes + }; + + // ###################################################################### + //! A wrapper around data that should be serialized after all non-deferred data + /*! This class is used to demarcate data that can only be safely serialized after + any data not wrapped in this class. + + @internal */ + template + class DeferredData : detail::DeferredDataCore + { + private: + // If we get passed an array, keep the type as is, otherwise store + // a reference if we were passed an l value reference, else copy the value + using Type = typename std::conditional::type>::value, + typename std::remove_cv::type, + typename std::conditional::value, + T, + typename std::decay::type>::type>::type; + + // prevent nested nvps + static_assert( !std::is_base_of::value, + "Cannot defer DeferredData" ); + + DeferredData & operator=( DeferredData const & ) = delete; + + public: + //! Constructs a new NameValuePair + /*! @param v The value to defer. Ideally this should be an l-value reference so that + the value can be both loaded and saved to. If you pass an r-value reference, + the DeferredData will store a copy of it instead of a reference. Thus you should + only pass r-values in cases where this makes sense, such as the result of some + size() call. + @internal */ + DeferredData( T && v ) : value(std::forward(v)) {} + + Type value; + }; + + // ###################################################################### + namespace detail + { + // base classes for type checking + /* The rtti virtual function only exists to enable an archive to + be used in a polymorphic fashion, if necessary. See the + archive adapters for an example of this */ + class OutputArchiveBase + { + public: + OutputArchiveBase() = default; + OutputArchiveBase( OutputArchiveBase && ) CEREAL_NOEXCEPT {} + OutputArchiveBase & operator=( OutputArchiveBase && ) CEREAL_NOEXCEPT { return *this; } + virtual ~OutputArchiveBase() CEREAL_NOEXCEPT = default; + + private: + virtual void rtti() {} + }; + + class InputArchiveBase + { + public: + InputArchiveBase() = default; + InputArchiveBase( InputArchiveBase && ) CEREAL_NOEXCEPT {} + InputArchiveBase & operator=( InputArchiveBase && ) CEREAL_NOEXCEPT { return *this; } + virtual ~InputArchiveBase() CEREAL_NOEXCEPT = default; + + private: + virtual void rtti() {} + }; + + // forward decls for polymorphic support + template struct polymorphic_serialization_support; + struct adl_tag; + + // used during saving pointers + static const uint32_t msb_32bit = 0x80000000; + static const int32_t msb2_32bit = 0x40000000; + } + + // ###################################################################### + //! A wrapper around size metadata + /*! This class provides a way for archives to have more flexibility over how + they choose to serialize size metadata for containers. For some archive + types, the size may be implicitly encoded in the output (e.g. JSON) and + not need an explicit entry. Specializing serialize or load/save for + your archive and SizeTags allows you to choose what happens. + + @internal */ + template + class SizeTag + { + private: + // Store a reference if passed an lvalue reference, otherwise + // make a copy of the data + using Type = typename std::conditional::value, + T, + typename std::decay::type>::type; + + SizeTag & operator=( SizeTag const & ) = delete; + + public: + SizeTag( T && sz ) : size(std::forward(sz)) {} + + Type size; + }; + + // ###################################################################### + //! A wrapper around a key and value for serializing data into maps. + /*! This class just provides a grouping of keys and values into a struct for + human readable archives. For example, XML archives will use this wrapper + to write maps like so: + + @code{.xml} + + + MyFirstKey + MyFirstValue + + + MySecondKey + MySecondValue + + + @endcode + + \sa make_map_item + @internal */ + template + struct MapItem + { + using KeyType = typename std::conditional< + std::is_lvalue_reference::value, + Key, + typename std::decay::type>::type; + + using ValueType = typename std::conditional< + std::is_lvalue_reference::value, + Value, + typename std::decay::type>::type; + + //! Construct a MapItem from a key and a value + /*! @internal */ + MapItem( Key && key_, Value && value_ ) : key(std::forward(key_)), value(std::forward(value_)) {} + + MapItem & operator=( MapItem const & ) = delete; + + KeyType key; + ValueType value; + + //! Serialize the MapItem with the NVPs "key" and "value" + template inline + void CEREAL_SERIALIZE_FUNCTION_NAME(Archive & archive) + { + archive( make_nvp("key", key), + make_nvp("value", value) ); + } + }; + + //! Create a MapItem so that human readable archives will group keys and values together + /*! @internal + @relates MapItem */ + template inline + MapItem make_map_item(KeyType && key, ValueType && value) + { + return {std::forward(key), std::forward(value)}; + } + + namespace detail + { + //! Tag for Version, which due to its anonymous namespace, becomes a different + //! type in each translation unit + /*! This allows CEREAL_CLASS_VERSION to be safely called in a header file */ + namespace{ struct version_binding_tag {}; } + + // ###################################################################### + //! Version information class + /*! This is the base case for classes that have not been explicitly + registered */ + template struct Version + { + static const std::uint32_t version = 0; + // we don't need to explicitly register these types since they + // always get a version number of 0 + }; + + //! Holds all registered version information + struct Versions + { + std::unordered_map mapping; + + std::uint32_t find( std::size_t hash, std::uint32_t version ) + { + const auto result = mapping.emplace( hash, version ); + return result.first->second; + } + }; // struct Versions + } // namespace detail +} // namespace cereal + +#endif // CEREAL_DETAILS_HELPERS_HPP_ diff --git a/third_party/cereal/details/polymorphic_impl.hpp b/third_party/cereal/details/polymorphic_impl.hpp new file mode 100755 index 0000000..ca79e0b --- /dev/null +++ b/third_party/cereal/details/polymorphic_impl.hpp @@ -0,0 +1,825 @@ +/*! \file polymorphic_impl.hpp + \brief Internal polymorphism support + \ingroup Internal */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* This code is heavily inspired by the boost serialization implementation by the following authors + + (C) Copyright 2002 Robert Ramey - http://www.rrsd.com . + Use, modification and distribution is subject to the Boost Software + License, Version 1.0. (See http://www.boost.org/LICENSE_1_0.txt) + + See http://www.boost.org for updates, documentation, and revision history. + + (C) Copyright 2006 David Abrahams - http://www.boost.org. + + See /boost/serialization/export.hpp, /boost/archive/detail/register_archive.hpp, + and /boost/serialization/void_cast.hpp for their implementation. Additional details + found in other files split across serialization and archive. +*/ +#ifndef CEREAL_DETAILS_POLYMORPHIC_IMPL_HPP_ +#define CEREAL_DETAILS_POLYMORPHIC_IMPL_HPP_ + +#include "../details/polymorphic_impl_fwd.hpp" +#include "../details/static_object.hpp" +#include "../types/memory.hpp" +#include "../types/string.hpp" +#include +#include +#include +#include +#include +#include + +//! Helper macro to omit unused warning +#if defined(__GNUC__) + // GCC / clang don't want the function + #define CEREAL_BIND_TO_ARCHIVES_UNUSED_FUNCTION +#else + #define CEREAL_BIND_TO_ARCHIVES_UNUSED_FUNCTION static void unused() { (void)b; } +#endif + +//! Binds a polymorhic type to all registered archives +/*! This binds a polymorphic type to all compatible registered archives that + have been registered with CEREAL_REGISTER_ARCHIVE. This must be called + after all archives are registered (usually after the archives themselves + have been included). */ +#ifdef CEREAL_HAS_CPP17 +#define CEREAL_BIND_TO_ARCHIVES(...) \ + namespace cereal { \ + namespace detail { \ + template<> \ + struct init_binding<__VA_ARGS__> { \ + static inline bind_to_archives<__VA_ARGS__> const & b= \ + ::cereal::detail::StaticObject< \ + bind_to_archives<__VA_ARGS__> \ + >::getInstance().bind(); \ + CEREAL_BIND_TO_ARCHIVES_UNUSED_FUNCTION \ + }; \ + }} /* end namespaces */ +#else +#define CEREAL_BIND_TO_ARCHIVES(...) \ + namespace cereal { \ + namespace detail { \ + template<> \ + struct init_binding<__VA_ARGS__> { \ + static bind_to_archives<__VA_ARGS__> const& b; \ + CEREAL_BIND_TO_ARCHIVES_UNUSED_FUNCTION \ + }; \ + bind_to_archives<__VA_ARGS__> const & init_binding<__VA_ARGS__>::b = \ + ::cereal::detail::StaticObject< \ + bind_to_archives<__VA_ARGS__> \ + >::getInstance().bind(); \ + }} /* end namespaces */ +#endif + +namespace cereal +{ + /* Polymorphic casting support */ + namespace detail + { + //! Base type for polymorphic void casting + /*! Contains functions for casting between registered base and derived types. + + This is necessary so that cereal can properly cast between polymorphic types + even though void pointers are used, which normally have no type information. + Runtime type information is used instead to index a compile-time made mapping + that can perform the proper cast. In the case of multiple levels of inheritance, + cereal will attempt to find the shortest path by using registered relationships to + perform the cast. + + This class will be allocated as a StaticObject and only referenced by pointer, + allowing a templated derived version of it to define strongly typed functions + that cast between registered base and derived types. */ + struct PolymorphicCaster + { + PolymorphicCaster() = default; + PolymorphicCaster( const PolymorphicCaster & ) = default; + PolymorphicCaster & operator=( const PolymorphicCaster & ) = default; + PolymorphicCaster( PolymorphicCaster && ) CEREAL_NOEXCEPT {} + PolymorphicCaster & operator=( PolymorphicCaster && ) CEREAL_NOEXCEPT { return *this; } + virtual ~PolymorphicCaster() CEREAL_NOEXCEPT = default; + + //! Downcasts to the proper derived type + virtual void const * downcast( void const * const ptr ) const = 0; + //! Upcast to proper base type + virtual void * upcast( void * const ptr ) const = 0; + //! Upcast to proper base type, shared_ptr version + virtual std::shared_ptr upcast( std::shared_ptr const & ptr ) const = 0; + }; + + //! Holds registered mappings between base and derived types for casting + /*! This will be allocated as a StaticObject and holds a map containing + all registered mappings between base and derived types. */ + struct PolymorphicCasters + { + //! Maps from a derived type index to a set of chainable casters + using DerivedCasterMap = std::unordered_map>; + //! Maps from base type index to a map from derived type index to caster + std::unordered_map map; + + std::multimap reverseMap; + + //! Error message used for unregistered polymorphic casts + #define UNREGISTERED_POLYMORPHIC_CAST_EXCEPTION(LoadSave) \ + throw cereal::Exception("Trying to " #LoadSave " a registered polymorphic type with an unregistered polymorphic cast.\n" \ + "Could not find a path to a base class (" + util::demangle(baseInfo.name()) + ") for type: " + ::cereal::util::demangledName() + "\n" \ + "Make sure you either serialize the base class at some point via cereal::base_class or cereal::virtual_base_class.\n" \ + "Alternatively, manually register the association with CEREAL_REGISTER_POLYMORPHIC_RELATION."); + + //! Checks if the mapping object that can perform the upcast or downcast exists, and returns it if so + /*! Uses the type index from the base and derived class to find the matching + registered caster. If no matching caster exists, the bool in the pair will be false and the vector + reference should not be used. */ + static std::pair const &> + lookup_if_exists( std::type_index const & baseIndex, std::type_index const & derivedIndex ) + { + // First phase of lookup - match base type index + auto const & baseMap = StaticObject::getInstance().map; + auto baseIter = baseMap.find( baseIndex ); + if (baseIter == baseMap.end()) + return {false, {}}; + + // Second phase - find a match from base to derived + auto const & derivedMap = baseIter->second; + auto derivedIter = derivedMap.find( derivedIndex ); + if (derivedIter == derivedMap.end()) + return {false, {}}; + + return {true, derivedIter->second}; + } + + //! Gets the mapping object that can perform the upcast or downcast + /*! Uses the type index from the base and derived class to find the matching + registered caster. If no matching caster exists, calls the exception function. + + The returned PolymorphicCaster is capable of upcasting or downcasting between the two types. */ + template inline + static std::vector const & lookup( std::type_index const & baseIndex, std::type_index const & derivedIndex, F && exceptionFunc ) + { + // First phase of lookup - match base type index + auto const & baseMap = StaticObject::getInstance().map; + auto baseIter = baseMap.find( baseIndex ); + if( baseIter == baseMap.end() ) + exceptionFunc(); + + // Second phase - find a match from base to derived + auto const & derivedMap = baseIter->second; + auto derivedIter = derivedMap.find( derivedIndex ); + if( derivedIter == derivedMap.end() ) + exceptionFunc(); + + return derivedIter->second; + } + + //! Performs a downcast to the derived type using a registered mapping + template inline + static const Derived * downcast( const void * dptr, std::type_info const & baseInfo ) + { + auto const & mapping = lookup( baseInfo, typeid(Derived), [&](){ UNREGISTERED_POLYMORPHIC_CAST_EXCEPTION(save) } ); + + for( auto const * dmap : mapping ) + dptr = dmap->downcast( dptr ); + + return static_cast( dptr ); + } + + //! Performs an upcast to the registered base type using the given a derived type + /*! The return is untyped because the final casting to the base type must happen in the polymorphic + serialization function, where the type is known at compile time */ + template inline + static void * upcast( Derived * const dptr, std::type_info const & baseInfo ) + { + auto const & mapping = lookup( baseInfo, typeid(Derived), [&](){ UNREGISTERED_POLYMORPHIC_CAST_EXCEPTION(load) } ); + + void * uptr = dptr; + for( auto mIter = mapping.rbegin(), mEnd = mapping.rend(); mIter != mEnd; ++mIter ) + uptr = (*mIter)->upcast( uptr ); + + return uptr; + } + + //! Upcasts for shared pointers + template inline + static std::shared_ptr upcast( std::shared_ptr const & dptr, std::type_info const & baseInfo ) + { + auto const & mapping = lookup( baseInfo, typeid(Derived), [&](){ UNREGISTERED_POLYMORPHIC_CAST_EXCEPTION(load) } ); + + std::shared_ptr uptr = dptr; + for( auto mIter = mapping.rbegin(), mEnd = mapping.rend(); mIter != mEnd; ++mIter ) + uptr = (*mIter)->upcast( uptr ); + + return uptr; + } + + #undef UNREGISTERED_POLYMORPHIC_CAST_EXCEPTION + }; + + #ifdef CEREAL_OLDER_GCC + #define CEREAL_EMPLACE_MAP(map, key, value) \ + map.insert( std::make_pair(std::move(key), std::move(value)) ); + #else // NOT CEREAL_OLDER_GCC + #define CEREAL_EMPLACE_MAP(map, key, value) \ + map.emplace( key, value ); + #endif // NOT_CEREAL_OLDER_GCC + + //! Strongly typed derivation of PolymorphicCaster + template + struct PolymorphicVirtualCaster : PolymorphicCaster + { + //! Inserts an entry in the polymorphic casting map for this pairing + /*! Creates an explicit mapping between Base and Derived in both upwards and + downwards directions, allowing void pointers to either to be properly cast + assuming dynamic type information is available */ + PolymorphicVirtualCaster() + { + const auto baseKey = std::type_index(typeid(Base)); + const auto derivedKey = std::type_index(typeid(Derived)); + + // First insert the relation Base->Derived + const auto lock = StaticObject::lock(); + auto & baseMap = StaticObject::getInstance().map; + + { + auto & derivedMap = baseMap.insert( {baseKey, PolymorphicCasters::DerivedCasterMap{}} ).first->second; + auto & derivedVec = derivedMap.insert( {derivedKey, {}} ).first->second; + derivedVec.push_back( this ); + } + + // Insert reverse relation Derived->Base + auto & reverseMap = StaticObject::getInstance().reverseMap; + CEREAL_EMPLACE_MAP(reverseMap, derivedKey, baseKey); + + // Find all chainable unregistered relations + /* The strategy here is to process only the nodes in the class hierarchy graph that have been + affected by the new insertion. The aglorithm iteratively processes a node an ensures that it + is updated with all new shortest length paths. It then rocesses the parents of the active node, + with the knowledge that all children have already been processed. + + Note that for the following, we'll use the nomenclature of parent and child to not confuse with + the inserted base derived relationship */ + { + // Checks whether there is a path from parent->child and returns a pair + // dist is set to MAX if the path does not exist + auto checkRelation = [](std::type_index const & parentInfo, std::type_index const & childInfo) -> + std::pair const &> + { + auto result = PolymorphicCasters::lookup_if_exists( parentInfo, childInfo ); + if( result.first ) + { + auto const & path = result.second; + return {path.size(), path}; + } + else + return {(std::numeric_limits::max)(), {}}; + }; + + std::stack parentStack; // Holds the parent nodes to be processed + std::vector dirtySet; // Marks child nodes that have been changed + std::unordered_set processedParents; // Marks parent nodes that have been processed + + // Checks if a child has been marked dirty + auto isDirty = [&](std::type_index const & c) + { + auto const dirtySetSize = dirtySet.size(); + for( size_t i = 0; i < dirtySetSize; ++i ) + if( dirtySet[i] == c ) + return true; + + return false; + }; + + // Begin processing the base key and mark derived as dirty + parentStack.push( baseKey ); + dirtySet.emplace_back( derivedKey ); + + while( !parentStack.empty() ) + { + using Relations = std::unordered_multimap>>; + Relations unregisteredRelations; // Defer insertions until after main loop to prevent iterator invalidation + + const auto parent = parentStack.top(); + parentStack.pop(); + + // Update paths to all children marked dirty + for( auto const & childPair : baseMap[parent] ) + { + const auto child = childPair.first; + if( isDirty( child ) && baseMap.count( child ) ) + { + auto parentChildPath = checkRelation( parent, child ); + + // Search all paths from the child to its own children (finalChild), + // looking for a shorter parth from parent to finalChild + for( auto const & finalChildPair : baseMap[child] ) + { + const auto finalChild = finalChildPair.first; + + auto parentFinalChildPath = checkRelation( parent, finalChild ); + auto childFinalChildPath = checkRelation( child, finalChild ); + + const size_t newLength = 1u + parentChildPath.first; + + if( newLength < parentFinalChildPath.first ) + { + std::vector path = parentChildPath.second; + path.insert( path.end(), childFinalChildPath.second.begin(), childFinalChildPath.second.end() ); + + // Check to see if we have a previous uncommitted path in unregisteredRelations + // that is shorter. If so, ignore this path + auto hintRange = unregisteredRelations.equal_range( parent ); + auto hint = hintRange.first; + for( ; hint != hintRange.second; ++hint ) + if( hint->second.first == finalChild ) + break; + + const bool uncommittedExists = hint != unregisteredRelations.end(); + if( uncommittedExists && (hint->second.second.size() <= newLength) ) + continue; + + auto newPath = std::pair>{finalChild, std::move(path)}; + + // Insert the new path if it doesn't exist, otherwise this will just lookup where to do the + // replacement + #ifdef CEREAL_OLDER_GCC + auto old = unregisteredRelations.insert( hint, std::make_pair(parent, newPath) ); + #else // NOT CEREAL_OLDER_GCC + auto old = unregisteredRelations.emplace_hint( hint, parent, newPath ); + #endif // NOT CEREAL_OLDER_GCC + + // If there was an uncommitted path, we need to perform a replacement + if( uncommittedExists ) + old->second = newPath; + } + } // end loop over child's children + } // end if dirty and child has children + } // end loop over children + + // Insert chained relations + for( auto const & it : unregisteredRelations ) + { + auto & derivedMap = baseMap.find( it.first )->second; + derivedMap[it.second.first] = it.second.second; + CEREAL_EMPLACE_MAP(reverseMap, it.second.first, it.first ); + } + + // Mark current parent as modified + dirtySet.emplace_back( parent ); + + // Insert all parents of the current parent node that haven't yet been processed + auto parentRange = reverseMap.equal_range( parent ); + for( auto pIter = parentRange.first; pIter != parentRange.second; ++pIter ) + { + const auto pParent = pIter->second; + if( !processedParents.count( pParent ) ) + { + parentStack.push( pParent ); + processedParents.insert( pParent ); + } + } + } // end loop over parent stack + } // end chainable relations + } // end PolymorphicVirtualCaster() + + #undef CEREAL_EMPLACE_MAP + + //! Performs the proper downcast with the templated types + void const * downcast( void const * const ptr ) const override + { + return dynamic_cast( static_cast( ptr ) ); + } + + //! Performs the proper upcast with the templated types + void * upcast( void * const ptr ) const override + { + return dynamic_cast( static_cast( ptr ) ); + } + + //! Performs the proper upcast with the templated types (shared_ptr version) + std::shared_ptr upcast( std::shared_ptr const & ptr ) const override + { + return std::dynamic_pointer_cast( std::static_pointer_cast( ptr ) ); + } + }; + + //! Registers a polymorphic casting relation between a Base and Derived type + /*! Registering a relation allows cereal to properly cast between the two types + given runtime type information and void pointers. + + Registration happens automatically via cereal::base_class and cereal::virtual_base_class + instantiations. For cases where neither is called, see the CEREAL_REGISTER_POLYMORPHIC_RELATION + macro */ + template + struct RegisterPolymorphicCaster + { + static PolymorphicCaster const * bind( std::true_type /* is_polymorphic */) + { + return &StaticObject>::getInstance(); + } + + static PolymorphicCaster const * bind( std::false_type /* is_polymorphic */ ) + { return nullptr; } + + //! Performs registration (binding) between Base and Derived + /*! If the type is not polymorphic, nothing will happen */ + static PolymorphicCaster const * bind() + { return bind( typename std::is_polymorphic::type() ); } + }; + } + + /* General polymorphism support */ + namespace detail + { + //! Binds a compile time type with a user defined string + template + struct binding_name {}; + + //! A structure holding a map from type_indices to output serializer functions + /*! A static object of this map should be created for each registered archive + type, containing entries for every registered type that describe how to + properly cast the type to its real type in polymorphic scenarios for + shared_ptr, weak_ptr, and unique_ptr. */ + template + struct OutputBindingMap + { + //! A serializer function + /*! Serializer functions return nothing and take an archive as + their first parameter (will be cast properly inside the function, + a pointer to actual data (contents of smart_ptr's get() function) + as their second parameter, and the type info of the owning smart_ptr + as their final parameter */ + typedef std::function Serializer; + + //! Struct containing the serializer functions for all pointer types + struct Serializers + { + Serializer shared_ptr, //!< Serializer function for shared/weak pointers + unique_ptr; //!< Serializer function for unique pointers + }; + + //! A map of serializers for pointers of all registered types + std::map map; + }; + + //! An empty noop deleter + template struct EmptyDeleter { void operator()(T *) const {} }; + + //! A structure holding a map from type name strings to input serializer functions + /*! A static object of this map should be created for each registered archive + type, containing entries for every registered type that describe how to + properly cast the type to its real type in polymorphic scenarios for + shared_ptr, weak_ptr, and unique_ptr. */ + template + struct InputBindingMap + { + //! Shared ptr serializer function + /*! Serializer functions return nothing and take an archive as + their first parameter (will be cast properly inside the function, + a shared_ptr (or unique_ptr for the unique case) of any base + type, and the type id of said base type as the third parameter. + Internally it will properly be loaded and cast to the correct type. */ + typedef std::function &, std::type_info const &)> SharedSerializer; + //! Unique ptr serializer function + typedef std::function> &, std::type_info const &)> UniqueSerializer; + + //! Struct containing the serializer functions for all pointer types + struct Serializers + { + SharedSerializer shared_ptr; //!< Serializer function for shared/weak pointers + UniqueSerializer unique_ptr; //!< Serializer function for unique pointers + }; + + //! A map of serializers for pointers of all registered types + std::map map; + }; + + // forward decls for archives from cereal.hpp + class InputArchiveBase; + class OutputArchiveBase; + + //! Creates a binding (map entry) between an input archive type and a polymorphic type + /*! Bindings are made when types are registered, assuming that at least one + archive has already been registered. When this struct is created, + it will insert (at run time) an entry into a map that properly handles + casting for serializing polymorphic objects */ + template struct InputBindingCreator + { + //! Initialize the binding + InputBindingCreator() + { + auto & map = StaticObject>::getInstance().map; + auto lock = StaticObject>::lock(); + auto key = std::string(binding_name::name()); + auto lb = map.lower_bound(key); + + if (lb != map.end() && lb->first == key) + return; + + typename InputBindingMap::Serializers serializers; + + serializers.shared_ptr = + [](void * arptr, std::shared_ptr & dptr, std::type_info const & baseInfo) + { + Archive & ar = *static_cast(arptr); + std::shared_ptr ptr; + + ar( CEREAL_NVP_("ptr_wrapper", ::cereal::memory_detail::make_ptr_wrapper(ptr)) ); + + dptr = PolymorphicCasters::template upcast( ptr, baseInfo ); + }; + + serializers.unique_ptr = + [](void * arptr, std::unique_ptr> & dptr, std::type_info const & baseInfo) + { + Archive & ar = *static_cast(arptr); + std::unique_ptr ptr; + + ar( CEREAL_NVP_("ptr_wrapper", ::cereal::memory_detail::make_ptr_wrapper(ptr)) ); + + dptr.reset( PolymorphicCasters::template upcast( ptr.release(), baseInfo )); + }; + + map.insert( lb, { std::move(key), std::move(serializers) } ); + } + }; + + //! Creates a binding (map entry) between an output archive type and a polymorphic type + /*! Bindings are made when types are registered, assuming that at least one + archive has already been registered. When this struct is created, + it will insert (at run time) an entry into a map that properly handles + casting for serializing polymorphic objects */ + template struct OutputBindingCreator + { + //! Writes appropriate metadata to the archive for this polymorphic type + static void writeMetadata(Archive & ar) + { + // Register the polymorphic type name with the archive, and get the id + char const * name = binding_name::name(); + std::uint32_t id = ar.registerPolymorphicType(name); + + // Serialize the id + ar( CEREAL_NVP_("polymorphic_id", id) ); + + // If the msb of the id is 1, then the type name is new, and we should serialize it + if( id & detail::msb_32bit ) + { + std::string namestring(name); + ar( CEREAL_NVP_("polymorphic_name", namestring) ); + } + } + + //! Holds a properly typed shared_ptr to the polymorphic type + class PolymorphicSharedPointerWrapper + { + public: + /*! Wrap a raw polymorphic pointer in a shared_ptr to its true type + + The wrapped pointer will not be responsible for ownership of the held pointer + so it will not attempt to destroy it; instead the refcount of the wrapped + pointer will be tied to a fake 'ownership pointer' that will do nothing + when it ultimately goes out of scope. + + The main reason for doing this, other than not to destroy the true object + with our wrapper pointer, is to avoid meddling with the internal reference + count in a polymorphic type that inherits from std::enable_shared_from_this. + + @param dptr A void pointer to the contents of the shared_ptr to serialize */ + PolymorphicSharedPointerWrapper( T const * dptr ) : refCount(), wrappedPtr( refCount, dptr ) + { } + + //! Get the wrapped shared_ptr */ + inline std::shared_ptr const & operator()() const { return wrappedPtr; } + + private: + std::shared_ptr refCount; //!< The ownership pointer + std::shared_ptr wrappedPtr; //!< The wrapped pointer + }; + + //! Does the actual work of saving a polymorphic shared_ptr + /*! This function will properly create a shared_ptr from the void * that is passed in + before passing it to the archive for serialization. + + In addition, this will also preserve the state of any internal enable_shared_from_this mechanisms + + @param ar The archive to serialize to + @param dptr Pointer to the actual data held by the shared_ptr */ + static inline void savePolymorphicSharedPtr( Archive & ar, T const * dptr, std::true_type /* has_shared_from_this */ ) + { + ::cereal::memory_detail::EnableSharedStateHelper state( const_cast(dptr) ); + PolymorphicSharedPointerWrapper psptr( dptr ); + ar( CEREAL_NVP_("ptr_wrapper", memory_detail::make_ptr_wrapper( psptr() ) ) ); + } + + //! Does the actual work of saving a polymorphic shared_ptr + /*! This function will properly create a shared_ptr from the void * that is passed in + before passing it to the archive for serialization. + + This version is for types that do not inherit from std::enable_shared_from_this. + + @param ar The archive to serialize to + @param dptr Pointer to the actual data held by the shared_ptr */ + static inline void savePolymorphicSharedPtr( Archive & ar, T const * dptr, std::false_type /* has_shared_from_this */ ) + { + PolymorphicSharedPointerWrapper psptr( dptr ); + ar( CEREAL_NVP_("ptr_wrapper", memory_detail::make_ptr_wrapper( psptr() ) ) ); + } + + //! Initialize the binding + OutputBindingCreator() + { + auto & map = StaticObject>::getInstance().map; + auto key = std::type_index(typeid(T)); + auto lb = map.lower_bound(key); + + if (lb != map.end() && lb->first == key) + return; + + typename OutputBindingMap::Serializers serializers; + + serializers.shared_ptr = + [&](void * arptr, void const * dptr, std::type_info const & baseInfo) + { + Archive & ar = *static_cast(arptr); + writeMetadata(ar); + + auto ptr = PolymorphicCasters::template downcast( dptr, baseInfo ); + + #if defined(_MSC_VER) && _MSC_VER < 1916 && !defined(__clang__) + savePolymorphicSharedPtr( ar, ptr, ::cereal::traits::has_shared_from_this::type() ); // MSVC doesn't like typename here + #else // not _MSC_VER + savePolymorphicSharedPtr( ar, ptr, typename ::cereal::traits::has_shared_from_this::type() ); + #endif // _MSC_VER + }; + + serializers.unique_ptr = + [&](void * arptr, void const * dptr, std::type_info const & baseInfo) + { + Archive & ar = *static_cast(arptr); + writeMetadata(ar); + + std::unique_ptr> const ptr( PolymorphicCasters::template downcast( dptr, baseInfo ) ); + + ar( CEREAL_NVP_("ptr_wrapper", memory_detail::make_ptr_wrapper(ptr)) ); + }; + + map.insert( { std::move(key), std::move(serializers) } ); + } + }; + + //! Used to help out argument dependent lookup for finding potential overloads + //! of instantiate_polymorphic_binding + struct adl_tag {}; + + //! Tag for init_binding, bind_to_archives and instantiate_polymorphic_binding. + //! For C++14 and below, we must instantiate a unique StaticObject per TU that is + //! otherwise identical -- otherwise we get multiple definition problems (ODR violations). + //! To achieve this, put a tag in an anonymous namespace and use it as a template argument. + //! + //! For C++17, we can use static inline global variables to unify these definitions across + //! all TUs in the same shared object (DLL). The tag is therefore not necessary. + //! For convenience, keep it to not complicate other code, but don't put it in + //! an anonymous namespace. Now the template instantiations will correspond + //! to the same type, and since they are marked inline with C++17, they will be merged + //! across all TUs. +#ifdef CEREAL_HAS_CPP17 + struct polymorphic_binding_tag {}; +#else + namespace { struct polymorphic_binding_tag {}; } +#endif + + + //! Causes the static object bindings between an archive type and a serializable type T + template + struct create_bindings + { + static const InputBindingCreator & + load(std::true_type) + { + return cereal::detail::StaticObject>::getInstance(); + } + + static const OutputBindingCreator & + save(std::true_type) + { + return cereal::detail::StaticObject>::getInstance(); + } + + inline static void load(std::false_type) {} + inline static void save(std::false_type) {} + }; + + //! When specialized, causes the compiler to instantiate its parameter + template + struct instantiate_function {}; + + /*! This struct is used as the return type of instantiate_polymorphic_binding + for specific Archive types. When the compiler looks for overloads of + instantiate_polymorphic_binding, it will be forced to instantiate this + struct during overload resolution, even though it will not be part of a valid + overload */ + template + struct polymorphic_serialization_support + { + #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + //! Creates the appropriate bindings depending on whether the archive supports + //! saving or loading + virtual CEREAL_DLL_EXPORT void instantiate() CEREAL_USED; + #else // NOT _MSC_VER + //! Creates the appropriate bindings depending on whether the archive supports + //! saving or loading + static CEREAL_DLL_EXPORT void instantiate() CEREAL_USED; + //! This typedef causes the compiler to instantiate this static function + typedef instantiate_function unused; + #endif // _MSC_VER + }; + + // instantiate implementation + template + CEREAL_DLL_EXPORT void polymorphic_serialization_support::instantiate() + { + create_bindings::save( std::integral_constant::value && + traits::is_output_serializable::value>{} ); + + create_bindings::load( std::integral_constant::value && + traits::is_input_serializable::value>{} ); + } + + //! Begins the binding process of a type to all registered archives + /*! Archives need to be registered prior to this struct being instantiated via + the CEREAL_REGISTER_ARCHIVE macro. Overload resolution will then force + several static objects to be made that allow us to bind together all + registered archive types with the parameter type T. */ + template + struct bind_to_archives + { + //! Binding for non abstract types + void bind(std::false_type) const + { + instantiate_polymorphic_binding(static_cast(nullptr), 0, Tag{}, adl_tag{}); + } + + //! Binding for abstract types + void bind(std::true_type) const + { } + + //! Binds the type T to all registered archives + /*! If T is abstract, we will not serialize it and thus + do not need to make a binding */ + bind_to_archives const & bind() const + { + static_assert( std::is_polymorphic::value, + "Attempting to register non polymorphic type" ); + bind( std::is_abstract() ); + return *this; + } + }; + + //! Used to hide the static object used to bind T to registered archives + template + struct init_binding; + + //! Base case overload for instantiation + /*! This will end up always being the best overload due to the second + parameter always being passed as an int. All other overloads will + accept pointers to archive types and have lower precedence than int. + + Since the compiler needs to check all possible overloads, the + other overloads created via CEREAL_REGISTER_ARCHIVE, which will have + lower precedence due to requring a conversion from int to (Archive*), + will cause their return types to be instantiated through the static object + mechanisms even though they are never called. + + See the documentation for the other functions to try and understand this */ + template + void instantiate_polymorphic_binding( T*, int, BindingTag, adl_tag ) {} + } // namespace detail +} // namespace cereal + +#endif // CEREAL_DETAILS_POLYMORPHIC_IMPL_HPP_ diff --git a/third_party/cereal/details/polymorphic_impl_fwd.hpp b/third_party/cereal/details/polymorphic_impl_fwd.hpp new file mode 100755 index 0000000..83ae7fa --- /dev/null +++ b/third_party/cereal/details/polymorphic_impl_fwd.hpp @@ -0,0 +1,65 @@ +/*! \file polymorphic_impl_fwd.hpp + \brief Internal polymorphism support forward declarations + \ingroup Internal */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* This code is heavily inspired by the boost serialization implementation by the following authors + + (C) Copyright 2002 Robert Ramey - http://www.rrsd.com . + Use, modification and distribution is subject to the Boost Software + License, Version 1.0. (See http://www.boost.org/LICENSE_1_0.txt) + + See http://www.boost.org for updates, documentation, and revision history. + + (C) Copyright 2006 David Abrahams - http://www.boost.org. + + See /boost/serialization/export.hpp and /boost/archive/detail/register_archive.hpp for their + implementation. +*/ + +#ifndef CEREAL_DETAILS_POLYMORPHIC_IMPL_FWD_HPP_ +#define CEREAL_DETAILS_POLYMORPHIC_IMPL_FWD_HPP_ + +namespace cereal +{ + namespace detail + { + //! Forward declaration, see polymorphic_impl.hpp for more information + template + struct RegisterPolymorphicCaster; + + //! Forward declaration, see polymorphic_impl.hpp for more information + struct PolymorphicCasters; + + //! Forward declaration, see polymorphic_impl.hpp for more information + template + struct PolymorphicRelation; + } // namespace detail +} // namespace cereal + +#endif // CEREAL_DETAILS_POLYMORPHIC_IMPL_FWD_HPP_ diff --git a/third_party/cereal/details/static_object.hpp b/third_party/cereal/details/static_object.hpp new file mode 100755 index 0000000..def7f1a --- /dev/null +++ b/third_party/cereal/details/static_object.hpp @@ -0,0 +1,128 @@ +/*! \file static_object.hpp + \brief Internal polymorphism static object support + \ingroup Internal */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_DETAILS_STATIC_OBJECT_HPP_ +#define CEREAL_DETAILS_STATIC_OBJECT_HPP_ + +#include "../macros.hpp" + +#if CEREAL_THREAD_SAFE +#include +#endif + +//! Prevent link optimization from removing non-referenced static objects +/*! Especially for polymorphic support, we create static objects which + may not ever be explicitly referenced. Most linkers will detect this + and remove the code causing various unpleasant runtime errors. These + macros, adopted from Boost (see force_include.hpp) prevent this + (C) Copyright 2002 Robert Ramey - http://www.rrsd.com . + Use, modification and distribution is subject to the Boost Software + License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at + http://www.boost.org/LICENSE_1_0.txt) */ + +#if defined(_MSC_VER) && !defined(__clang__) +# define CEREAL_DLL_EXPORT __declspec(dllexport) +# define CEREAL_USED +#else // clang or gcc +# define CEREAL_DLL_EXPORT __attribute__ ((visibility("default"))) +# define CEREAL_USED __attribute__ ((__used__)) +#endif + +namespace cereal +{ + namespace detail + { + //! A static, pre-execution object + /*! This class will create a single copy (singleton) of some + type and ensures that merely referencing this type will + cause it to be instantiated and initialized pre-execution. + For example, this is used heavily in the polymorphic pointer + serialization mechanisms to bind various archive types with + different polymorphic classes */ + template + class CEREAL_DLL_EXPORT StaticObject + { + private: + + static T & create() + { + static T t; + //! Forces instantiation at pre-execution time + (void)instance; + return t; + } + + StaticObject( StaticObject const & /*other*/ ) {} + + public: + static T & getInstance() + { + return create(); + } + + //! A class that acts like std::lock_guard + class LockGuard + { + #if CEREAL_THREAD_SAFE + public: + LockGuard(std::mutex & m) : lock(m) {} + private: + std::unique_lock lock; + #else + public: + LockGuard() = default; + LockGuard(LockGuard const &) = default; // prevents implicit copy ctor warning + ~LockGuard() CEREAL_NOEXCEPT {} // prevents variable not used + #endif + }; + + //! Attempts to lock this static object for the current scope + /*! @note This function is a no-op if cereal is not compiled with + thread safety enabled (CEREAL_THREAD_SAFE = 1). + + This function returns an object that holds a lock for + this StaticObject that will release its lock upon destruction. This + call will block until the lock is available. */ + static LockGuard lock() + { + #if CEREAL_THREAD_SAFE + static std::mutex instanceMutex; + return LockGuard{instanceMutex}; + #else + return LockGuard{}; + #endif + } + + private: + static T & instance; + }; + + template T & StaticObject::instance = StaticObject::create(); + } // namespace detail +} // namespace cereal + +#endif // CEREAL_DETAILS_STATIC_OBJECT_HPP_ diff --git a/third_party/cereal/details/traits.hpp b/third_party/cereal/details/traits.hpp new file mode 100755 index 0000000..5a09ac2 --- /dev/null +++ b/third_party/cereal/details/traits.hpp @@ -0,0 +1,1411 @@ +/*! \file traits.hpp + \brief Internal type trait support + \ingroup Internal */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_DETAILS_TRAITS_HPP_ +#define CEREAL_DETAILS_TRAITS_HPP_ + +#ifndef __clang__ +#if (__GNUC__ == 4 && __GNUC_MINOR__ <= 7) +#define CEREAL_OLDER_GCC +#endif // gcc 4.7 or earlier +#endif // __clang__ + +#include +#include + +#include "../macros.hpp" +#include "../access.hpp" + +namespace cereal +{ + namespace traits + { + using yes = std::true_type; + using no = std::false_type; + + namespace detail + { + // ###################################################################### + //! Used to delay a static_assert until template instantiation + template + struct delay_static_assert : std::false_type {}; + + // ###################################################################### + // SFINAE Helpers + #ifdef CEREAL_OLDER_GCC // when VS supports better SFINAE, we can use this as the default + template struct Void { typedef void type; }; + #endif // CEREAL_OLDER_GCC + + //! Return type for SFINAE Enablers + enum class sfinae {}; + + // ###################################################################### + // Helper functionality for boolean integral constants and Enable/DisableIf + template struct meta_bool_and : std::integral_constant::value> {}; + template struct meta_bool_and : std::integral_constant {}; + + template struct meta_bool_or : std::integral_constant::value> {}; + template struct meta_bool_or : std::integral_constant {}; + + // workaround needed due to bug in MSVC 2013, see + // http://connect.microsoft.com/VisualStudio/feedback/details/800231/c-11-alias-template-issue + template + struct EnableIfHelper : std::enable_if::value, sfinae> {}; + + template + struct DisableIfHelper : std::enable_if::value, sfinae> {}; + } // namespace detail + + //! Used as the default value for EnableIf and DisableIf template parameters + /*! @relates EnableIf + @relates DisableIf */ + static const detail::sfinae sfinae = {}; + + // ###################################################################### + //! Provides a way to enable a function if conditions are met + /*! This is intended to be used in a near identical fashion to std::enable_if + while being significantly easier to read at the cost of not allowing for as + complicated of a condition. + + This will compile (allow the function) if every condition evaluates to true. + at compile time. This should be used with SFINAE to ensure that at least + one other candidate function works when one fails due to an EnableIf. + + This should be used as the las template parameter to a function as + an unnamed parameter with a default value of cereal::traits::sfinae: + + @code{cpp} + // using by making the last template argument variadic + template ::value> = sfinae> + void func(T t ); + @endcode + + Note that this performs a logical AND of all conditions, so you will need + to construct more complicated requirements with this fact in mind. + + @relates DisableIf + @relates sfinae + @tparam Conditions The conditions which will be logically ANDed to enable the function. */ + template + using EnableIf = typename detail::EnableIfHelper::type; + + // ###################################################################### + //! Provides a way to disable a function if conditions are met + /*! This is intended to be used in a near identical fashion to std::enable_if + while being significantly easier to read at the cost of not allowing for as + complicated of a condition. + + This will compile (allow the function) if every condition evaluates to false. + This should be used with SFINAE to ensure that at least one other candidate + function works when one fails due to a DisableIf. + + This should be used as the las template parameter to a function as + an unnamed parameter with a default value of cereal::traits::sfinae: + + @code{cpp} + // using by making the last template argument variadic + template ::value> = sfinae> + void func(T t ); + @endcode + + This is often used in conjunction with EnableIf to form an enable/disable pair of + overloads. + + Note that this performs a logical AND of all conditions, so you will need + to construct more complicated requirements with this fact in mind. If all conditions + hold, the function will be disabled. + + @relates EnableIf + @relates sfinae + @tparam Conditions The conditions which will be logically ANDed to disable the function. */ + template + using DisableIf = typename detail::DisableIfHelper::type; + + // ###################################################################### + namespace detail + { + template + struct get_output_from_input : no + { + static_assert( detail::delay_static_assert::value, + "Could not find an associated output archive for input archive." ); + }; + + template + struct get_input_from_output : no + { + static_assert( detail::delay_static_assert::value, + "Could not find an associated input archive for output archive." ); + }; + } + + //! Sets up traits that relate an input archive to an output archive + #define CEREAL_SETUP_ARCHIVE_TRAITS(InputArchive, OutputArchive) \ + namespace cereal { namespace traits { namespace detail { \ + template <> struct get_output_from_input \ + { using type = OutputArchive; }; \ + template <> struct get_input_from_output \ + { using type = InputArchive; }; } } } /* end namespaces */ + + // ###################################################################### + //! Used to convert a MAKE_HAS_XXX macro into a versioned variant + #define CEREAL_MAKE_VERSIONED_TEST ,0 + + // ###################################################################### + //! Creates a test for whether a non const member function exists + /*! This creates a class derived from std::integral_constant that will be true if + the type has the proper member function for the given archive. + + @param name The name of the function to test for (e.g. serialize, load, save) + @param test_name The name to give the test for the function being tested for (e.g. serialize, versioned_serialize) + @param versioned Either blank or the macro CEREAL_MAKE_VERSIONED_TEST */ + #ifdef CEREAL_OLDER_GCC + #define CEREAL_MAKE_HAS_MEMBER_TEST(name, test_name, versioned) \ + template \ + struct has_member_##test_name : no {}; \ + template \ + struct has_member_##test_name(), std::declval() versioned ) ) >::type> : yes {} + #else // NOT CEREAL_OLDER_GCC + #define CEREAL_MAKE_HAS_MEMBER_TEST(name, test_name, versioned) \ + namespace detail \ + { \ + template \ + struct has_member_##name##_##versioned##_impl \ + { \ + template \ + static auto test(int) -> decltype( cereal::access::member_##name( std::declval(), std::declval() versioned ), yes()); \ + template \ + static no test(...); \ + static const bool value = std::is_same(0)), yes>::value; \ + }; \ + } /* end namespace detail */ \ + template \ + struct has_member_##test_name : std::integral_constant::value> {} + #endif // NOT CEREAL_OLDER_GCC + + // ###################################################################### + //! Creates a test for whether a non const non-member function exists + /*! This creates a class derived from std::integral_constant that will be true if + the type has the proper non-member function for the given archive. */ + #define CEREAL_MAKE_HAS_NON_MEMBER_TEST(test_name, func, versioned) \ + namespace detail \ + { \ + template \ + struct has_non_member_##test_name##_impl \ + { \ + template \ + static auto test(int) -> decltype( func( std::declval(), std::declval() versioned ), yes()); \ + template \ + static no test( ... ); \ + static const bool value = std::is_same( 0 ) ), yes>::value; \ + }; \ + } /* end namespace detail */ \ + template \ + struct has_non_member_##test_name : std::integral_constant::value> {} + + // ###################################################################### + // Member Serialize + CEREAL_MAKE_HAS_MEMBER_TEST(serialize, serialize,); + + // ###################################################################### + // Member Serialize (versioned) + CEREAL_MAKE_HAS_MEMBER_TEST(serialize, versioned_serialize, CEREAL_MAKE_VERSIONED_TEST); + + // ###################################################################### + // Non Member Serialize + CEREAL_MAKE_HAS_NON_MEMBER_TEST(serialize, CEREAL_SERIALIZE_FUNCTION_NAME,); + + // ###################################################################### + // Non Member Serialize (versioned) + CEREAL_MAKE_HAS_NON_MEMBER_TEST(versioned_serialize, CEREAL_SERIALIZE_FUNCTION_NAME, CEREAL_MAKE_VERSIONED_TEST); + + // ###################################################################### + // Member Load + CEREAL_MAKE_HAS_MEMBER_TEST(load, load,); + + // ###################################################################### + // Member Load (versioned) + CEREAL_MAKE_HAS_MEMBER_TEST(load, versioned_load, CEREAL_MAKE_VERSIONED_TEST); + + // ###################################################################### + // Non Member Load + CEREAL_MAKE_HAS_NON_MEMBER_TEST(load, CEREAL_LOAD_FUNCTION_NAME,); + + // ###################################################################### + // Non Member Load (versioned) + CEREAL_MAKE_HAS_NON_MEMBER_TEST(versioned_load, CEREAL_LOAD_FUNCTION_NAME, CEREAL_MAKE_VERSIONED_TEST); + + // ###################################################################### + #undef CEREAL_MAKE_HAS_NON_MEMBER_TEST + #undef CEREAL_MAKE_HAS_MEMBER_TEST + + // ###################################################################### + //! Creates a test for whether a member save function exists + /*! This creates a class derived from std::integral_constant that will be true if + the type has the proper member function for the given archive. + + @param test_name The name to give the test (e.g. save or versioned_save) + @param versioned Either blank or the macro CEREAL_MAKE_VERSIONED_TEST */ + #ifdef CEREAL_OLDER_GCC + #define CEREAL_MAKE_HAS_MEMBER_SAVE_IMPL(test_name, versioned) \ + namespace detail \ + { \ + template \ + struct has_member_##test_name##_impl \ + { \ + template struct test : no {}; \ + template \ + struct test(), \ + std::declval() versioned ) ) >::type> : yes {}; \ + static const bool value = test(); \ + \ + template struct test2 : no {}; \ + template \ + struct test2(), \ + std::declval::type&>() versioned ) ) >::type> : yes {}; \ + static const bool not_const_type = test2(); \ + }; \ + } /* end namespace detail */ + #else /* NOT CEREAL_OLDER_GCC =================================== */ + #define CEREAL_MAKE_HAS_MEMBER_SAVE_IMPL(test_name, versioned) \ + namespace detail \ + { \ + template \ + struct has_member_##test_name##_impl \ + { \ + template \ + static auto test(int) -> decltype( cereal::access::member_save( std::declval(), \ + std::declval() versioned ), yes()); \ + template static no test(...); \ + static const bool value = std::is_same(0)), yes>::value; \ + \ + template \ + static auto test2(int) -> decltype( cereal::access::member_save_non_const( \ + std::declval(), \ + std::declval::type&>() versioned ), yes()); \ + template static no test2(...); \ + static const bool not_const_type = std::is_same(0)), yes>::value; \ + }; \ + } /* end namespace detail */ + #endif /* NOT CEREAL_OLDER_GCC */ + + // ###################################################################### + // Member Save + CEREAL_MAKE_HAS_MEMBER_SAVE_IMPL(save, ) + + template + struct has_member_save : std::integral_constant::value> + { + typedef typename detail::has_member_save_impl check; + static_assert( check::value || !check::not_const_type, + "cereal detected a non-const save. \n " + "save member functions must always be const" ); + }; + + // ###################################################################### + // Member Save (versioned) + CEREAL_MAKE_HAS_MEMBER_SAVE_IMPL(versioned_save, CEREAL_MAKE_VERSIONED_TEST) + + template + struct has_member_versioned_save : std::integral_constant::value> + { + typedef typename detail::has_member_versioned_save_impl check; + static_assert( check::value || !check::not_const_type, + "cereal detected a versioned non-const save. \n " + "save member functions must always be const" ); + }; + + // ###################################################################### + #undef CEREAL_MAKE_HAS_MEMBER_SAVE_IMPL + + // ###################################################################### + //! Creates a test for whether a non-member save function exists + /*! This creates a class derived from std::integral_constant that will be true if + the type has the proper non-member function for the given archive. + + @param test_name The name to give the test (e.g. save or versioned_save) + @param versioned Either blank or the macro CEREAL_MAKE_VERSIONED_TEST */ + #define CEREAL_MAKE_HAS_NON_MEMBER_SAVE_TEST(test_name, versioned) \ + namespace detail \ + { \ + template \ + struct has_non_member_##test_name##_impl \ + { \ + template \ + static auto test(int) -> decltype( CEREAL_SAVE_FUNCTION_NAME( \ + std::declval(), \ + std::declval() versioned ), yes()); \ + template static no test(...); \ + static const bool value = std::is_same(0)), yes>::value; \ + \ + template \ + static auto test2(int) -> decltype( CEREAL_SAVE_FUNCTION_NAME( \ + std::declval(), \ + std::declval::type&>() versioned ), yes()); \ + template static no test2(...); \ + static const bool not_const_type = std::is_same(0)), yes>::value; \ + }; \ + } /* end namespace detail */ \ + \ + template \ + struct has_non_member_##test_name : std::integral_constant::value> \ + { \ + using check = typename detail::has_non_member_##test_name##_impl; \ + static_assert( check::value || !check::not_const_type, \ + "cereal detected a non-const type parameter in non-member " #test_name ". \n " \ + #test_name " non-member functions must always pass their types as const" ); \ + }; + + // ###################################################################### + // Non Member Save + CEREAL_MAKE_HAS_NON_MEMBER_SAVE_TEST(save, ) + + // ###################################################################### + // Non Member Save (versioned) + CEREAL_MAKE_HAS_NON_MEMBER_SAVE_TEST(versioned_save, CEREAL_MAKE_VERSIONED_TEST) + + // ###################################################################### + #undef CEREAL_MAKE_HAS_NON_MEMBER_SAVE_TEST + + // ###################################################################### + // Minimal Utilities + namespace detail + { + // Determines if the provided type is an std::string + template struct is_string : std::false_type {}; + + template + struct is_string> : std::true_type {}; + } + + // Determines if the type is valid for use with a minimal serialize function + template + struct is_minimal_type : std::integral_constant::value || std::is_arithmetic::value> {}; + + // ###################################################################### + //! Creates implementation details for whether a member save_minimal function exists + /*! This creates a class derived from std::integral_constant that will be true if + the type has the proper member function for the given archive. + + @param test_name The name to give the test (e.g. save_minimal or versioned_save_minimal) + @param versioned Either blank or the macro CEREAL_MAKE_VERSIONED_TEST */ + #ifdef CEREAL_OLDER_GCC + #define CEREAL_MAKE_HAS_MEMBER_SAVE_MINIMAL_IMPL(test_name, versioned) \ + namespace detail \ + { \ + template \ + struct has_member_##test_name##_impl \ + { \ + template struct test : no {}; \ + template \ + struct test(), \ + std::declval() versioned ) ) >::type> : yes {}; \ + \ + static const bool value = test(); \ + \ + template struct test2 : no {}; \ + template \ + struct test2(), \ + std::declval::type&>() versioned ) ) >::type> : yes {}; \ + static const bool not_const_type = test2(); \ + \ + static const bool valid = value || !not_const_type; \ + }; \ + } /* end namespace detail */ + #else /* NOT CEREAL_OLDER_GCC =================================== */ + #define CEREAL_MAKE_HAS_MEMBER_SAVE_MINIMAL_IMPL(test_name, versioned) \ + namespace detail \ + { \ + template \ + struct has_member_##test_name##_impl \ + { \ + template \ + static auto test(int) -> decltype( cereal::access::member_save_minimal( \ + std::declval(), \ + std::declval() versioned ), yes()); \ + template static no test(...); \ + static const bool value = std::is_same(0)), yes>::value; \ + \ + template \ + static auto test2(int) -> decltype( cereal::access::member_save_minimal_non_const( \ + std::declval(), \ + std::declval::type&>() versioned ), yes()); \ + template static no test2(...); \ + static const bool not_const_type = std::is_same(0)), yes>::value; \ + \ + static const bool valid = value || !not_const_type; \ + }; \ + } /* end namespace detail */ + #endif // NOT CEREAL_OLDER_GCC + + // ###################################################################### + //! Creates helpers for minimal save functions + /*! The get_member_*_type structs allow access to the return type of a save_minimal, + assuming that the function actually exists. If the function does not + exist, the type will be void. + + @param test_name The name to give the test (e.g. save_minimal or versioned_save_minimal) + @param versioned Either blank or the macro CEREAL_MAKE_VERSIONED_TEST */ + #define CEREAL_MAKE_HAS_MEMBER_SAVE_MINIMAL_HELPERS_IMPL(test_name, versioned) \ + namespace detail \ + { \ + template \ + struct get_member_##test_name##_type { using type = void; }; \ + \ + template \ + struct get_member_##test_name##_type \ + { \ + using type = decltype( cereal::access::member_save_minimal( std::declval(), \ + std::declval() versioned ) ); \ + }; \ + } /* end namespace detail */ + + // ###################################################################### + //! Creates a test for whether a member save_minimal function exists + /*! This creates a class derived from std::integral_constant that will be true if + the type has the proper member function for the given archive. + + @param test_name The name to give the test (e.g. save_minimal or versioned_save_minimal) */ + #define CEREAL_MAKE_HAS_MEMBER_SAVE_MINIMAL_TEST(test_name) \ + template \ + struct has_member_##test_name : std::integral_constant::value> \ + { \ + using check = typename detail::has_member_##test_name##_impl; \ + static_assert( check::valid, \ + "cereal detected a non-const member " #test_name ". \n " \ + #test_name " member functions must always be const" ); \ + \ + using type = typename detail::get_member_##test_name##_type::type; \ + static_assert( (check::value && is_minimal_type::value) || !check::value, \ + "cereal detected a member " #test_name " with an invalid return type. \n " \ + "return type must be arithmetic or string" ); \ + }; + + // ###################################################################### + // Member Save Minimal + CEREAL_MAKE_HAS_MEMBER_SAVE_MINIMAL_IMPL(save_minimal, ) + CEREAL_MAKE_HAS_MEMBER_SAVE_MINIMAL_HELPERS_IMPL(save_minimal, ) + CEREAL_MAKE_HAS_MEMBER_SAVE_MINIMAL_TEST(save_minimal) + + // ###################################################################### + // Member Save Minimal (versioned) + CEREAL_MAKE_HAS_MEMBER_SAVE_MINIMAL_IMPL(versioned_save_minimal, CEREAL_MAKE_VERSIONED_TEST) + CEREAL_MAKE_HAS_MEMBER_SAVE_MINIMAL_HELPERS_IMPL(versioned_save_minimal, CEREAL_MAKE_VERSIONED_TEST) + CEREAL_MAKE_HAS_MEMBER_SAVE_MINIMAL_TEST(versioned_save_minimal) + + // ###################################################################### + #undef CEREAL_MAKE_HAS_MEMBER_SAVE_MINIMAL_IMPL + #undef CEREAL_MAKE_HAS_MEMBER_SAVE_MINIMAL_HELPERS_IMPL + #undef CEREAL_MAKE_HAS_MEMBER_SAVE_MINIMAL_TEST + + // ###################################################################### + //! Creates a test for whether a non-member save_minimal function exists + /*! This creates a class derived from std::integral_constant that will be true if + the type has the proper member function for the given archive. + + @param test_name The name to give the test (e.g. save_minimal or versioned_save_minimal) + @param versioned Either blank or the macro CEREAL_MAKE_VERSIONED_TEST */ + #define CEREAL_MAKE_HAS_NON_MEMBER_SAVE_MINIMAL_TEST(test_name, versioned) \ + namespace detail \ + { \ + template \ + struct has_non_member_##test_name##_impl \ + { \ + template \ + static auto test(int) -> decltype( CEREAL_SAVE_MINIMAL_FUNCTION_NAME( \ + std::declval(), \ + std::declval() versioned ), yes()); \ + template static no test(...); \ + static const bool value = std::is_same(0)), yes>::value; \ + \ + template \ + static auto test2(int) -> decltype( CEREAL_SAVE_MINIMAL_FUNCTION_NAME( \ + std::declval(), \ + std::declval::type&>() versioned ), yes()); \ + template static no test2(...); \ + static const bool not_const_type = std::is_same(0)), yes>::value; \ + \ + static const bool valid = value || !not_const_type; \ + }; \ + \ + template \ + struct get_non_member_##test_name##_type { using type = void; }; \ + \ + template \ + struct get_non_member_##test_name##_type \ + { \ + using type = decltype( CEREAL_SAVE_MINIMAL_FUNCTION_NAME( std::declval(), \ + std::declval() versioned ) ); \ + }; \ + } /* end namespace detail */ \ + \ + template \ + struct has_non_member_##test_name : std::integral_constant::value> \ + { \ + using check = typename detail::has_non_member_##test_name##_impl; \ + static_assert( check::valid, \ + "cereal detected a non-const type parameter in non-member " #test_name ". \n " \ + #test_name " non-member functions must always pass their types as const" ); \ + \ + using type = typename detail::get_non_member_##test_name##_type::type; \ + static_assert( (check::value && is_minimal_type::value) || !check::value, \ + "cereal detected a non-member " #test_name " with an invalid return type. \n " \ + "return type must be arithmetic or string" ); \ + }; + + // ###################################################################### + // Non-Member Save Minimal + CEREAL_MAKE_HAS_NON_MEMBER_SAVE_MINIMAL_TEST(save_minimal, ) + + // ###################################################################### + // Non-Member Save Minimal (versioned) + CEREAL_MAKE_HAS_NON_MEMBER_SAVE_MINIMAL_TEST(versioned_save_minimal, CEREAL_MAKE_VERSIONED_TEST) + + // ###################################################################### + #undef CEREAL_MAKE_HAS_NON_MEMBER_SAVE_MINIMAL_TEST + + // ###################################################################### + // Load Minimal Utilities + namespace detail + { + //! Used to help strip away conversion wrappers + /*! If someone writes a non-member load/save minimal function that accepts its + parameter as some generic template type and needs to perform trait checks + on that type, our NoConvert wrappers will interfere with this. Using + the struct strip_minmal, users can strip away our wrappers to get to + the underlying type, allowing traits to work properly */ + struct NoConvertBase {}; + + //! A struct that prevents implicit conversion + /*! Any type instantiated with this struct will be unable to implicitly convert + to another type. Is designed to only allow conversion to Source const &. + + @tparam Source the type of the original source */ + template + struct NoConvertConstRef : NoConvertBase + { + using type = Source; //!< Used to get underlying type easily + + template ::value>::type> + operator Dest () = delete; + + //! only allow conversion if the types are the same and we are converting into a const reference + template ::value>::type> + operator Dest const & (); + }; + + //! A struct that prevents implicit conversion + /*! Any type instantiated with this struct will be unable to implicitly convert + to another type. Is designed to only allow conversion to Source &. + + @tparam Source the type of the original source */ + template + struct NoConvertRef : NoConvertBase + { + using type = Source; //!< Used to get underlying type easily + + template ::value>::type> + operator Dest () = delete; + + #ifdef __clang__ + template ::value>::type> + operator Dest const & () = delete; + #endif // __clang__ + + //! only allow conversion if the types are the same and we are converting into a const reference + template ::value>::type> + operator Dest & (); + }; + + //! A type that can implicitly convert to anything else + struct AnyConvert + { + template + operator Dest & (); + + template + operator Dest const & () const; + }; + } // namespace detail + + // ###################################################################### + //! Creates a test for whether a member load_minimal function exists + /*! This creates a class derived from std::integral_constant that will be true if + the type has the proper member function for the given archive. + + Our strategy here is to first check if a function matching the signature more or less exists + (allow anything like load_minimal(xxx) using AnyConvert, and then secondly enforce + that it has the correct signature using NoConvertConstRef + + @param test_name The name to give the test (e.g. load_minimal or versioned_load_minimal) + @param versioned Either blank or the macro CEREAL_MAKE_VERSIONED_TEST */ + #ifdef CEREAL_OLDER_GCC + #define CEREAL_MAKE_HAS_MEMBER_LOAD_MINIMAL_IMPL(test_name, versioned) \ + namespace detail \ + { \ + template struct has_member_##test_name##_impl : no {}; \ + template \ + struct has_member_##test_name##_impl(), \ + std::declval(), AnyConvert() versioned ) ) >::type> : yes {}; \ + \ + template struct has_member_##test_name##_type_impl : no {}; \ + template \ + struct has_member_##test_name##_type_impl(), \ + std::declval(), NoConvertConstRef() versioned ) ) >::type> : yes {}; \ + } /* end namespace detail */ + #else /* NOT CEREAL_OLDER_GCC =================================== */ + #define CEREAL_MAKE_HAS_MEMBER_LOAD_MINIMAL_IMPL(test_name, versioned) \ + namespace detail \ + { \ + template \ + struct has_member_##test_name##_impl \ + { \ + template \ + static auto test(int) -> decltype( cereal::access::member_load_minimal( \ + std::declval(), \ + std::declval(), AnyConvert() versioned ), yes()); \ + template static no test(...); \ + static const bool value = std::is_same(0)), yes>::value; \ + }; \ + template \ + struct has_member_##test_name##_type_impl \ + { \ + template \ + static auto test(int) -> decltype( cereal::access::member_load_minimal( \ + std::declval(), \ + std::declval(), NoConvertConstRef() versioned ), yes()); \ + template static no test(...); \ + static const bool value = std::is_same(0)), yes>::value; \ + \ + }; \ + } /* end namespace detail */ + #endif // NOT CEREAL_OLDER_GCC + + // ###################################################################### + //! Creates helpers for minimal load functions + /*! The has_member_*_wrapper structs ensure that the load and save types for the + requested function type match appropriately. + + @param load_test_name The name to give the test (e.g. load_minimal or versioned_load_minimal) + @param save_test_name The name to give the test (e.g. save_minimal or versioned_save_minimal, + should match the load name. + @param save_test_prefix The name to give the test (e.g. save_minimal or versioned_save_minimal, + should match the load name, without the trailing "_minimal" (e.g. + save or versioned_save). Needed because the preprocessor is an abomination. + @param versioned Either blank or the macro CEREAL_MAKE_VERSIONED_TEST */ + #define CEREAL_MAKE_HAS_MEMBER_LOAD_MINIMAL_HELPERS_IMPL(load_test_name, save_test_name, save_test_prefix, versioned) \ + namespace detail \ + { \ + template \ + struct has_member_##load_test_name##_wrapper : std::false_type {}; \ + \ + template \ + struct has_member_##load_test_name##_wrapper \ + { \ + using AOut = typename detail::get_output_from_input::type; \ + \ + static_assert( has_member_##save_test_prefix##_minimal::value, \ + "cereal detected member " #load_test_name " but no valid member " #save_test_name ". \n " \ + "cannot evaluate correctness of " #load_test_name " without valid " #save_test_name "." ); \ + \ + using SaveType = typename detail::get_member_##save_test_prefix##_minimal_type::type; \ + const static bool value = has_member_##load_test_name##_impl::value; \ + const static bool valid = has_member_##load_test_name##_type_impl::value; \ + \ + static_assert( valid || !value, "cereal detected different or invalid types in corresponding member " \ + #load_test_name " and " #save_test_name " functions. \n " \ + "the paramater to " #load_test_name " must be a constant reference to the type that " \ + #save_test_name " returns." ); \ + }; \ + } /* end namespace detail */ + + // ###################################################################### + //! Creates a test for whether a member load_minimal function exists + /*! This creates a class derived from std::integral_constant that will be true if + the type has the proper member function for the given archive. + + @param load_test_name The name to give the test (e.g. load_minimal or versioned_load_minimal) + @param load_test_prefix The above parameter minus the trailing "_minimal" */ + #define CEREAL_MAKE_HAS_MEMBER_LOAD_MINIMAL_TEST(load_test_name, load_test_prefix) \ + template \ + struct has_member_##load_test_prefix##_minimal : std::integral_constant::value>::value> {}; + + // ###################################################################### + // Member Load Minimal + CEREAL_MAKE_HAS_MEMBER_LOAD_MINIMAL_IMPL(load_minimal, ) + CEREAL_MAKE_HAS_MEMBER_LOAD_MINIMAL_HELPERS_IMPL(load_minimal, save_minimal, save, ) + CEREAL_MAKE_HAS_MEMBER_LOAD_MINIMAL_TEST(load_minimal, load) + + // ###################################################################### + // Member Load Minimal (versioned) + CEREAL_MAKE_HAS_MEMBER_LOAD_MINIMAL_IMPL(versioned_load_minimal, CEREAL_MAKE_VERSIONED_TEST) + CEREAL_MAKE_HAS_MEMBER_LOAD_MINIMAL_HELPERS_IMPL(versioned_load_minimal, versioned_save_minimal, versioned_save, CEREAL_MAKE_VERSIONED_TEST) + CEREAL_MAKE_HAS_MEMBER_LOAD_MINIMAL_TEST(versioned_load_minimal, versioned_load) + + // ###################################################################### + #undef CEREAL_MAKE_HAS_MEMBER_LOAD_MINIMAL_IMPL + #undef CEREAL_MAKE_HAS_MEMBER_LOAD_MINIMAL_HELPERS_IMPL + #undef CEREAL_MAKE_HAS_MEMBER_LOAD_MINIMAL_TEST + + // ###################################################################### + // Non-Member Load Minimal + namespace detail + { + #ifdef CEREAL_OLDER_GCC + void CEREAL_LOAD_MINIMAL_FUNCTION_NAME(); // prevents nonsense complaining about not finding this + void CEREAL_SAVE_MINIMAL_FUNCTION_NAME(); + #endif // CEREAL_OLDER_GCC + } // namespace detail + + // ###################################################################### + //! Creates a test for whether a non-member load_minimal function exists + /*! This creates a class derived from std::integral_constant that will be true if + the type has the proper member function for the given archive. + + See notes from member load_minimal implementation. + + Note that there should be an additional const check on load_minimal after the valid check, + but this currently interferes with many valid uses of minimal serialization. It has been + removed (see #565 on github) and previously was: + + @code + static_assert( check::const_valid || !check::exists, + "cereal detected an invalid serialization type parameter in non-member " #test_name ". " + #test_name " non-member functions must accept their serialization type by non-const reference" ); + @endcode + + See #132, #436, #263, and #565 on https://github.com/USCiLab/cereal for more details. + + @param test_name The name to give the test (e.g. load_minimal or versioned_load_minimal) + @param save_name The corresponding name the save test would have (e.g. save_minimal or versioned_save_minimal) + @param versioned Either blank or the macro CEREAL_MAKE_VERSIONED_TEST */ + #define CEREAL_MAKE_HAS_NON_MEMBER_LOAD_MINIMAL_TEST(test_name, save_name, versioned) \ + namespace detail \ + { \ + template \ + struct has_non_member_##test_name##_impl \ + { \ + template \ + static auto test(int) -> decltype( CEREAL_LOAD_MINIMAL_FUNCTION_NAME( \ + std::declval(), std::declval(), AnyConvert() versioned ), yes() ); \ + template static no test( ... ); \ + static const bool exists = std::is_same( 0 ) ), yes>::value; \ + \ + template \ + static auto test2(int) -> decltype( CEREAL_LOAD_MINIMAL_FUNCTION_NAME( \ + std::declval(), std::declval(), NoConvertConstRef() versioned ), yes() ); \ + template static no test2( ... ); \ + static const bool valid = std::is_same( 0 ) ), yes>::value; \ + \ + template \ + static auto test3(int) -> decltype( CEREAL_LOAD_MINIMAL_FUNCTION_NAME( \ + std::declval(), NoConvertRef(), AnyConvert() versioned ), yes() ); \ + template static no test3( ... ); \ + static const bool const_valid = std::is_same( 0 ) ), yes>::value; \ + }; \ + \ + template \ + struct has_non_member_##test_name##_wrapper : std::false_type {}; \ + \ + template \ + struct has_non_member_##test_name##_wrapper \ + { \ + using AOut = typename detail::get_output_from_input::type; \ + \ + static_assert( detail::has_non_member_##save_name##_impl::valid, \ + "cereal detected non-member " #test_name " but no valid non-member " #save_name ". \n " \ + "cannot evaluate correctness of " #test_name " without valid " #save_name "." ); \ + \ + using SaveType = typename detail::get_non_member_##save_name##_type::type; \ + using check = has_non_member_##test_name##_impl; \ + static const bool value = check::exists; \ + \ + static_assert( check::valid || !check::exists, "cereal detected different types in corresponding non-member " \ + #test_name " and " #save_name " functions. \n " \ + "the paramater to " #test_name " must be a constant reference to the type that " #save_name " returns." ); \ + }; \ + } /* namespace detail */ \ + \ + template \ + struct has_non_member_##test_name : std::integral_constant::exists>::value> {}; + + // ###################################################################### + // Non-Member Load Minimal + CEREAL_MAKE_HAS_NON_MEMBER_LOAD_MINIMAL_TEST(load_minimal, save_minimal, ) + + // ###################################################################### + // Non-Member Load Minimal (versioned) + CEREAL_MAKE_HAS_NON_MEMBER_LOAD_MINIMAL_TEST(versioned_load_minimal, versioned_save_minimal, CEREAL_MAKE_VERSIONED_TEST) + + // ###################################################################### + #undef CEREAL_MAKE_HAS_NON_MEMBER_LOAD_MINIMAL_TEST + + // ###################################################################### + namespace detail + { + // const stripped away before reaching here, prevents errors on conversion from + // construct to construct + template + struct has_member_load_and_construct_impl : std::integral_constant( std::declval(), std::declval< ::cereal::construct&>() ) ), void>::value> + { }; + + template + struct has_member_versioned_load_and_construct_impl : std::integral_constant( std::declval(), std::declval< ::cereal::construct&>(), 0 ) ), void>::value> + { }; + } // namespace detail + + //! Member load and construct check + template + struct has_member_load_and_construct : detail::has_member_load_and_construct_impl::type, A> + { }; + + //! Member load and construct check (versioned) + template + struct has_member_versioned_load_and_construct : detail::has_member_versioned_load_and_construct_impl::type, A> + { }; + + // ###################################################################### + //! Creates a test for whether a non-member load_and_construct specialization exists + /*! This creates a class derived from std::integral_constant that will be true if + the type has the proper non-member function for the given archive. */ + #define CEREAL_MAKE_HAS_NON_MEMBER_LOAD_AND_CONSTRUCT_TEST(test_name, versioned) \ + namespace detail \ + { \ + template \ + struct has_non_member_##test_name##_impl \ + { \ + template \ + static auto test(int) -> decltype( LoadAndConstruct::load_and_construct( \ + std::declval(), std::declval< ::cereal::construct&>() versioned ), yes()); \ + template \ + static no test( ... ); \ + static const bool value = std::is_same( 0 ) ), yes>::value; \ + }; \ + } /* end namespace detail */ \ + template \ + struct has_non_member_##test_name : \ + std::integral_constant::type, A>::value> {}; + + // ###################################################################### + //! Non member load and construct check + CEREAL_MAKE_HAS_NON_MEMBER_LOAD_AND_CONSTRUCT_TEST(load_and_construct, ) + + // ###################################################################### + //! Non member load and construct check (versioned) + CEREAL_MAKE_HAS_NON_MEMBER_LOAD_AND_CONSTRUCT_TEST(versioned_load_and_construct, CEREAL_MAKE_VERSIONED_TEST) + + // ###################################################################### + //! Has either a member or non member load and construct + template + struct has_load_and_construct : std::integral_constant::value || has_non_member_load_and_construct::value || + has_member_versioned_load_and_construct::value || has_non_member_versioned_load_and_construct::value> + { }; + + // ###################################################################### + #undef CEREAL_MAKE_HAS_NON_MEMBER_LOAD_AND_CONSTRUCT_TEST + + // ###################################################################### + // End of serialization existence tests + #undef CEREAL_MAKE_VERSIONED_TEST + + // ###################################################################### + template + struct has_member_split : std::integral_constant::value && has_member_save::value) || + (has_member_versioned_load::value && has_member_versioned_save::value)> {}; + + // ###################################################################### + template + struct has_non_member_split : std::integral_constant::value && has_non_member_save::value) || + (has_non_member_versioned_load::value && has_non_member_versioned_save::value)> {}; + + // ###################################################################### + template + struct has_invalid_output_versioning : std::integral_constant::value && has_member_save::value) || + (has_non_member_versioned_save::value && has_non_member_save::value) || + (has_member_versioned_serialize::value && has_member_serialize::value) || + (has_non_member_versioned_serialize::value && has_non_member_serialize::value) || + (has_member_versioned_save_minimal::value && has_member_save_minimal::value) || + (has_non_member_versioned_save_minimal::value && has_non_member_save_minimal::value)> {}; + + // ###################################################################### + template + struct has_invalid_input_versioning : std::integral_constant::value && has_member_load::value) || + (has_non_member_versioned_load::value && has_non_member_load::value) || + (has_member_versioned_serialize::value && has_member_serialize::value) || + (has_non_member_versioned_serialize::value && has_non_member_serialize::value) || + (has_member_versioned_load_minimal::value && has_member_load_minimal::value) || + (has_non_member_versioned_load_minimal::value && has_non_member_load_minimal::value)> {}; + + // ###################################################################### + namespace detail + { + //! Create a test for a cereal::specialization entry + #define CEREAL_MAKE_IS_SPECIALIZED_IMPL(name) \ + template \ + struct is_specialized_##name : std::integral_constant>::value> {} + + CEREAL_MAKE_IS_SPECIALIZED_IMPL(member_serialize); + CEREAL_MAKE_IS_SPECIALIZED_IMPL(member_load_save); + CEREAL_MAKE_IS_SPECIALIZED_IMPL(member_load_save_minimal); + CEREAL_MAKE_IS_SPECIALIZED_IMPL(non_member_serialize); + CEREAL_MAKE_IS_SPECIALIZED_IMPL(non_member_load_save); + CEREAL_MAKE_IS_SPECIALIZED_IMPL(non_member_load_save_minimal); + + #undef CEREAL_MAKE_IS_SPECIALIZED_IMPL + + //! Number of specializations detected + template + struct count_specializations : std::integral_constant::value + + is_specialized_member_load_save::value + + is_specialized_member_load_save_minimal::value + + is_specialized_non_member_serialize::value + + is_specialized_non_member_load_save::value + + is_specialized_non_member_load_save_minimal::value> {}; + } // namespace detail + + //! Check if any specialization exists for a type + template + struct is_specialized : std::integral_constant::value || + detail::is_specialized_member_load_save::value || + detail::is_specialized_member_load_save_minimal::value || + detail::is_specialized_non_member_serialize::value || + detail::is_specialized_non_member_load_save::value || + detail::is_specialized_non_member_load_save_minimal::value> + { + static_assert(detail::count_specializations::value <= 1, "More than one explicit specialization detected for type."); + }; + + //! Create the static assertion for some specialization + /*! This assertion will fail if the type is indeed specialized and does not have the appropriate + type of serialization functions */ + #define CEREAL_MAKE_IS_SPECIALIZED_ASSERT(name, versioned_name, print_name, spec_name) \ + static_assert( (is_specialized::value && detail::is_specialized_##spec_name::value && \ + (has_##name::value || has_##versioned_name::value)) \ + || !(is_specialized::value && detail::is_specialized_##spec_name::value), \ + "cereal detected " #print_name " specialization but no " #print_name " serialize function" ) + + //! Generates a test for specialization for versioned and unversioned functions + /*! This creates checks that can be queried to see if a given type of serialization function + has been specialized for this type */ + #define CEREAL_MAKE_IS_SPECIALIZED(name, versioned_name, spec_name) \ + template \ + struct is_specialized_##name : std::integral_constant::value && detail::is_specialized_##spec_name::value> \ + { CEREAL_MAKE_IS_SPECIALIZED_ASSERT(name, versioned_name, name, spec_name); }; \ + template \ + struct is_specialized_##versioned_name : std::integral_constant::value && detail::is_specialized_##spec_name::value> \ + { CEREAL_MAKE_IS_SPECIALIZED_ASSERT(name, versioned_name, versioned_name, spec_name); } + + CEREAL_MAKE_IS_SPECIALIZED(member_serialize, member_versioned_serialize, member_serialize); + CEREAL_MAKE_IS_SPECIALIZED(non_member_serialize, non_member_versioned_serialize, non_member_serialize); + + CEREAL_MAKE_IS_SPECIALIZED(member_save, member_versioned_save, member_load_save); + CEREAL_MAKE_IS_SPECIALIZED(non_member_save, non_member_versioned_save, non_member_load_save); + CEREAL_MAKE_IS_SPECIALIZED(member_load, member_versioned_load, member_load_save); + CEREAL_MAKE_IS_SPECIALIZED(non_member_load, non_member_versioned_load, non_member_load_save); + + CEREAL_MAKE_IS_SPECIALIZED(member_save_minimal, member_versioned_save_minimal, member_load_save_minimal); + CEREAL_MAKE_IS_SPECIALIZED(non_member_save_minimal, non_member_versioned_save_minimal, non_member_load_save_minimal); + CEREAL_MAKE_IS_SPECIALIZED(member_load_minimal, member_versioned_load_minimal, member_load_save_minimal); + CEREAL_MAKE_IS_SPECIALIZED(non_member_load_minimal, non_member_versioned_load_minimal, non_member_load_save_minimal); + + #undef CEREAL_MAKE_IS_SPECIALIZED_ASSERT + #undef CEREAL_MAKE_IS_SPECIALIZED + + // ###################################################################### + // detects if a type has any active minimal output serialization + template + struct has_minimal_output_serialization : std::integral_constant::value || + ((has_member_save_minimal::value || + has_non_member_save_minimal::value || + has_member_versioned_save_minimal::value || + has_non_member_versioned_save_minimal::value) && + !(is_specialized_member_serialize::value || + is_specialized_member_save::value))> {}; + + // ###################################################################### + // detects if a type has any active minimal input serialization + template + struct has_minimal_input_serialization : std::integral_constant::value || + ((has_member_load_minimal::value || + has_non_member_load_minimal::value || + has_member_versioned_load_minimal::value || + has_non_member_versioned_load_minimal::value) && + !(is_specialized_member_serialize::value || + is_specialized_member_load::value))> {}; + + // ###################################################################### + namespace detail + { + //! The number of output serialization functions available + /*! If specialization is being used, we'll count only those; otherwise we'll count everything */ + template + struct count_output_serializers : std::integral_constant::value ? count_specializations::value : + has_member_save::value + + has_non_member_save::value + + has_member_serialize::value + + has_non_member_serialize::value + + has_member_save_minimal::value + + has_non_member_save_minimal::value + + /*-versioned---------------------------------------------------------*/ + has_member_versioned_save::value + + has_non_member_versioned_save::value + + has_member_versioned_serialize::value + + has_non_member_versioned_serialize::value + + has_member_versioned_save_minimal::value + + has_non_member_versioned_save_minimal::value> {}; + } + + template + struct is_output_serializable : std::integral_constant::value == 1> {}; + + // ###################################################################### + namespace detail + { + //! The number of input serialization functions available + /*! If specialization is being used, we'll count only those; otherwise we'll count everything */ + template + struct count_input_serializers : std::integral_constant::value ? count_specializations::value : + has_member_load::value + + has_non_member_load::value + + has_member_serialize::value + + has_non_member_serialize::value + + has_member_load_minimal::value + + has_non_member_load_minimal::value + + /*-versioned---------------------------------------------------------*/ + has_member_versioned_load::value + + has_non_member_versioned_load::value + + has_member_versioned_serialize::value + + has_non_member_versioned_serialize::value + + has_member_versioned_load_minimal::value + + has_non_member_versioned_load_minimal::value> {}; + } + + template + struct is_input_serializable : std::integral_constant::value == 1> {}; + + // ###################################################################### + // Base Class Support + namespace detail + { + struct base_class_id + { + template + base_class_id(T const * const t) : + type(typeid(T)), + ptr(t), + hash(std::hash()(typeid(T)) ^ (std::hash()(t) << 1)) + { } + + bool operator==(base_class_id const & other) const + { return (type == other.type) && (ptr == other.ptr); } + + std::type_index type; + void const * ptr; + size_t hash; + }; + struct base_class_id_hash { size_t operator()(base_class_id const & id) const { return id.hash; } }; + } // namespace detail + + namespace detail + { + //! Common base type for base class casting + struct BaseCastBase {}; + + template + struct get_base_class; + + template class Cast, class Base> + struct get_base_class> + { + using type = Base; + }; + + //! Base class cast, behave as the test + template class Test, class Archive, + bool IsBaseCast = std::is_base_of::value> + struct has_minimal_base_class_serialization_impl : Test::type, Archive> + { }; + + //! Not a base class cast + template class Test, class Archive> + struct has_minimal_base_class_serialization_impl : std::false_type + { }; + } + + //! Checks to see if the base class used in a cast has a minimal serialization + /*! @tparam Cast Either base_class or virtual_base_class wrapped type + @tparam Test A has_minimal test (for either input or output) + @tparam Archive The archive to use with the test */ + template class Test, class Archive> + struct has_minimal_base_class_serialization : detail::has_minimal_base_class_serialization_impl + { }; + + + // ###################################################################### + namespace detail + { + struct shared_from_this_wrapper + { + template + static auto (check)( U const & t ) -> decltype( ::cereal::access::shared_from_this(t), std::true_type() ); + + static auto (check)( ... ) -> decltype( std::false_type() ); + + template + static auto get( U const & t ) -> decltype( t.shared_from_this() ); + }; + } + + //! Determine if T or any base class of T has inherited from std::enable_shared_from_this + template + struct has_shared_from_this : decltype((detail::shared_from_this_wrapper::check)(std::declval())) + { }; + + //! Get the type of the base class of T which inherited from std::enable_shared_from_this + template + struct get_shared_from_this_base + { + private: + using PtrType = decltype(detail::shared_from_this_wrapper::get(std::declval())); + public: + //! The type of the base of T that inherited from std::enable_shared_from_this + using type = typename std::decay::type; + }; + + // ###################################################################### + //! Extracts the true type from something possibly wrapped in a cereal NoConvert + /*! Internally cereal uses some wrapper classes to test the validity of non-member + minimal load and save functions. This can interfere with user type traits on + templated load and save minimal functions. To get to the correct underlying type, + users should use strip_minimal when performing any enable_if type type trait checks. + + See the enum serialization in types/common.hpp for an example of using this */ + template ::value> + struct strip_minimal + { + using type = T; + }; + + //! Specialization for types wrapped in a NoConvert + template + struct strip_minimal + { + using type = typename T::type; + }; + + // ###################################################################### + //! Determines whether the class T can be default constructed by cereal::access + template + struct is_default_constructible + { + #ifdef CEREAL_OLDER_GCC + template + struct test : no {}; + template + struct test() ) >::type> : yes {}; + static const bool value = test(); + #else // NOT CEREAL_OLDER_GCC ========================================= + template + static auto test(int) -> decltype( cereal::access::construct(), yes()); + template + static no test(...); + static const bool value = std::is_same(0)), yes>::value; + #endif // NOT CEREAL_OLDER_GCC + }; + + // ###################################################################### + namespace detail + { + //! Removes all qualifiers and minimal wrappers from an archive + template + using decay_archive = typename std::decay::type>::type; + } + + //! Checks if the provided archive type is equal to some cereal archive type + /*! This automatically does things such as std::decay and removing any other wrappers that may be + on the Archive template parameter. + + Example use: + @code{cpp} + // example use to disable a serialization function + template ::value> = sfinae> + void save( Archive & ar, MyType const & mt ); + @endcode */ + template + struct is_same_archive : std::integral_constant, CerealArchiveT>::value> + { }; + + // ###################################################################### + //! A macro to use to restrict which types of archives your function will work for. + /*! This requires you to have a template class parameter named Archive and replaces the void return + type for your function. + + INTYPE refers to the input archive type you wish to restrict on. + OUTTYPE refers to the output archive type you wish to restrict on. + + For example, if we want to limit a serialize to only work with binary serialization: + + @code{.cpp} + template + CEREAL_ARCHIVE_RESTRICT(BinaryInputArchive, BinaryOutputArchive) + serialize( Archive & ar, MyCoolType & m ) + { + ar & m; + } + @endcode + + If you need to do more restrictions in your enable_if, you will need to do this by hand. + */ + #define CEREAL_ARCHIVE_RESTRICT(INTYPE, OUTTYPE) \ + typename std::enable_if::value || cereal::traits::is_same_archive::value, void>::type + + //! Type traits only struct used to mark an archive as human readable (text based) + /*! Archives that wish to identify as text based/human readable should inherit from + this struct */ + struct TextArchive {}; + + //! Checks if an archive is a text archive (human readable) + template + struct is_text_archive : std::integral_constant>::value> + { }; + } // namespace traits + + // ###################################################################### + namespace detail + { + template ::value, + bool MemberVersioned = traits::has_member_versioned_load_and_construct::value, + bool NonMember = traits::has_non_member_load_and_construct::value, + bool NonMemberVersioned = traits::has_non_member_versioned_load_and_construct::value> + struct Construct + { + static_assert( cereal::traits::detail::delay_static_assert::value, + "cereal found more than one compatible load_and_construct function for the provided type and archive combination. \n\n " + "Types must either have a member load_and_construct function or a non-member specialization of LoadAndConstruct (you may not mix these). \n " + "In addition, you may not mix versioned with non-versioned load_and_construct functions. \n\n " ); + static T * load_andor_construct( A & /*ar*/, construct & /*construct*/ ) + { return nullptr; } + }; + + // no load and construct case + template + struct Construct + { + static_assert( ::cereal::traits::is_default_constructible::value, + "Trying to serialize a an object with no default constructor. \n\n " + "Types must either be default constructible or define either a member or non member Construct function. \n " + "Construct functions generally have the signature: \n\n " + "template \n " + "static void load_and_construct(Archive & ar, cereal::construct & construct) \n " + "{ \n " + " var a; \n " + " ar( a ) \n " + " construct( a ); \n " + "} \n\n" ); + static T * load_andor_construct() + { return ::cereal::access::construct(); } + }; + + // member non-versioned + template + struct Construct + { + static void load_andor_construct( A & ar, construct & construct ) + { + access::load_and_construct( ar, construct ); + } + }; + + // member versioned + template + struct Construct + { + static void load_andor_construct( A & ar, construct & construct ) + { + const auto version = ar.template loadClassVersion(); + access::load_and_construct( ar, construct, version ); + } + }; + + // non-member non-versioned + template + struct Construct + { + static void load_andor_construct( A & ar, construct & construct ) + { + LoadAndConstruct::load_and_construct( ar, construct ); + } + }; + + // non-member versioned + template + struct Construct + { + static void load_andor_construct( A & ar, construct & construct ) + { + const auto version = ar.template loadClassVersion(); + LoadAndConstruct::load_and_construct( ar, construct, version ); + } + }; + } // namespace detail +} // namespace cereal + +#endif // CEREAL_DETAILS_TRAITS_HPP_ diff --git a/third_party/cereal/details/util.hpp b/third_party/cereal/details/util.hpp new file mode 100755 index 0000000..e4aebe9 --- /dev/null +++ b/third_party/cereal/details/util.hpp @@ -0,0 +1,84 @@ +/*! \file util.hpp + \brief Internal misc utilities + \ingroup Internal */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_DETAILS_UTIL_HPP_ +#define CEREAL_DETAILS_UTIL_HPP_ + +#include +#include + +#ifdef _MSC_VER +namespace cereal +{ + namespace util + { + //! Demangles the type encoded in a string + /*! @internal */ + inline std::string demangle( std::string const & name ) + { return name; } + + //! Gets the demangled name of a type + /*! @internal */ + template inline + std::string demangledName() + { return typeid( T ).name(); } + } // namespace util +} // namespace cereal +#else // clang or gcc +#include +#include +namespace cereal +{ + namespace util + { + //! Demangles the type encoded in a string + /*! @internal */ + inline std::string demangle(std::string mangledName) + { + int status = 0; + char *demangledName = nullptr; + std::size_t len; + + demangledName = abi::__cxa_demangle(mangledName.c_str(), 0, &len, &status); + + std::string retName(demangledName); + free(demangledName); + + return retName; + } + + //! Gets the demangled name of a type + /*! @internal */ + template inline + std::string demangledName() + { return demangle(typeid(T).name()); } + } +} // namespace cereal +#endif // clang or gcc branch of _MSC_VER +#endif // CEREAL_DETAILS_UTIL_HPP_ diff --git a/third_party/cereal/external/base64.hpp b/third_party/cereal/external/base64.hpp new file mode 100755 index 0000000..ce32324 --- /dev/null +++ b/third_party/cereal/external/base64.hpp @@ -0,0 +1,134 @@ +/* + Copyright (C) 2004-2008 René Nyffenegger + + This source code is provided 'as-is', without any express or implied + warranty. In no event will the author be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this source code must not be misrepresented; you must not + claim that you wrote the original source code. If you use this source code + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original source code. + + 3. This notice may not be removed or altered from any source distribution. + + René Nyffenegger rene.nyffenegger@adp-gmbh.ch +*/ + +#ifndef CEREAL_EXTERNAL_BASE64_HPP_ +#define CEREAL_EXTERNAL_BASE64_HPP_ + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + +#include + +namespace cereal +{ + namespace base64 + { + static const std::string chars = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+/"; + + static inline bool is_base64(unsigned char c) { + return (isalnum(c) || (c == '+') || (c == '/')); + } + + inline std::string encode(unsigned char const* bytes_to_encode, size_t in_len) { + std::string ret; + int i = 0; + int j = 0; + unsigned char char_array_3[3]; + unsigned char char_array_4[4]; + + while (in_len--) { + char_array_3[i++] = *(bytes_to_encode++); + if (i == 3) { + char_array_4[0] = static_cast((char_array_3[0] & 0xfc) >> 2); + char_array_4[1] = static_cast( ( ( char_array_3[0] & 0x03 ) << 4 ) + ( ( char_array_3[1] & 0xf0 ) >> 4 ) ); + char_array_4[2] = static_cast( ( ( char_array_3[1] & 0x0f ) << 2 ) + ( ( char_array_3[2] & 0xc0 ) >> 6 ) ); + char_array_4[3] = static_cast( char_array_3[2] & 0x3f ); + + for(i = 0; (i <4) ; i++) + ret += chars[char_array_4[i]]; + i = 0; + } + } + + if (i) + { + for(j = i; j < 3; j++) + char_array_3[j] = '\0'; + + char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; + char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); + char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); + char_array_4[3] = char_array_3[2] & 0x3f; + + for (j = 0; (j < i + 1); j++) + ret += chars[char_array_4[j]]; + + while((i++ < 3)) + ret += '='; + } + + return ret; + } + + inline std::string decode(std::string const& encoded_string) { + size_t in_len = encoded_string.size(); + size_t i = 0; + size_t j = 0; + int in_ = 0; + unsigned char char_array_4[4], char_array_3[3]; + std::string ret; + + while (in_len-- && ( encoded_string[in_] != '=') && is_base64(encoded_string[in_])) { + char_array_4[i++] = encoded_string[in_]; in_++; + if (i ==4) { + for (i = 0; i <4; i++) + char_array_4[i] = static_cast(chars.find( char_array_4[i] )); + + char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); + char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); + char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; + + for (i = 0; (i < 3); i++) + ret += char_array_3[i]; + i = 0; + } + } + + if (i) { + for (j = i; j <4; j++) + char_array_4[j] = 0; + + for (j = 0; j <4; j++) + char_array_4[j] = static_cast(chars.find( char_array_4[j] )); + + char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); + char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); + char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; + + for (j = 0; (j < i - 1); j++) ret += char_array_3[j]; + } + + return ret; + } + } // namespace base64 +} // namespace cereal +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif +#endif // CEREAL_EXTERNAL_BASE64_HPP_ diff --git a/third_party/cereal/external/rapidjson/allocators.h b/third_party/cereal/external/rapidjson/allocators.h new file mode 100755 index 0000000..d375e28 --- /dev/null +++ b/third_party/cereal/external/rapidjson/allocators.h @@ -0,0 +1,284 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_ALLOCATORS_H_ +#define CEREAL_RAPIDJSON_ALLOCATORS_H_ + +#include "rapidjson.h" + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// Allocator + +/*! \class rapidjson::Allocator + \brief Concept for allocating, resizing and freeing memory block. + + Note that Malloc() and Realloc() are non-static but Free() is static. + + So if an allocator need to support Free(), it needs to put its pointer in + the header of memory block. + +\code +concept Allocator { + static const bool kNeedFree; //!< Whether this allocator needs to call Free(). + + // Allocate a memory block. + // \param size of the memory block in bytes. + // \returns pointer to the memory block. + void* Malloc(size_t size); + + // Resize a memory block. + // \param originalPtr The pointer to current memory block. Null pointer is permitted. + // \param originalSize The current size in bytes. (Design issue: since some allocator may not book-keep this, explicitly pass to it can save memory.) + // \param newSize the new size in bytes. + void* Realloc(void* originalPtr, size_t originalSize, size_t newSize); + + // Free a memory block. + // \param pointer to the memory block. Null pointer is permitted. + static void Free(void *ptr); +}; +\endcode +*/ + + +/*! \def CEREAL_RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY + \ingroup CEREAL_RAPIDJSON_CONFIG + \brief User-defined kDefaultChunkCapacity definition. + + User can define this as any \c size that is a power of 2. +*/ + +#ifndef CEREAL_RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY +#define CEREAL_RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY (64 * 1024) +#endif + + +/////////////////////////////////////////////////////////////////////////////// +// CrtAllocator + +//! C-runtime library allocator. +/*! This class is just wrapper for standard C library memory routines. + \note implements Allocator concept +*/ +class CrtAllocator { +public: + static const bool kNeedFree = true; + void* Malloc(size_t size) { + if (size) // behavior of malloc(0) is implementation defined. + return std::malloc(size); + else + return NULL; // standardize to returning NULL. + } + void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { + (void)originalSize; + if (newSize == 0) { + std::free(originalPtr); + return NULL; + } + return std::realloc(originalPtr, newSize); + } + static void Free(void *ptr) { std::free(ptr); } +}; + +/////////////////////////////////////////////////////////////////////////////// +// MemoryPoolAllocator + +//! Default memory allocator used by the parser and DOM. +/*! This allocator allocate memory blocks from pre-allocated memory chunks. + + It does not free memory blocks. And Realloc() only allocate new memory. + + The memory chunks are allocated by BaseAllocator, which is CrtAllocator by default. + + User may also supply a buffer as the first chunk. + + If the user-buffer is full then additional chunks are allocated by BaseAllocator. + + The user-buffer is not deallocated by this allocator. + + \tparam BaseAllocator the allocator type for allocating memory chunks. Default is CrtAllocator. + \note implements Allocator concept +*/ +template +class MemoryPoolAllocator { +public: + static const bool kNeedFree = false; //!< Tell users that no need to call Free() with this allocator. (concept Allocator) + + //! Constructor with chunkSize. + /*! \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. + \param baseAllocator The allocator for allocating memory chunks. + */ + MemoryPoolAllocator(size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : + chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(0), baseAllocator_(baseAllocator), ownBaseAllocator_(0) + { + } + + //! Constructor with user-supplied buffer. + /*! The user buffer will be used firstly. When it is full, memory pool allocates new chunk with chunk size. + + The user buffer will not be deallocated when this allocator is destructed. + + \param buffer User supplied buffer. + \param size Size of the buffer in bytes. It must at least larger than sizeof(ChunkHeader). + \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. + \param baseAllocator The allocator for allocating memory chunks. + */ + MemoryPoolAllocator(void *buffer, size_t size, size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : + chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(buffer), baseAllocator_(baseAllocator), ownBaseAllocator_(0) + { + CEREAL_RAPIDJSON_ASSERT(buffer != 0); + CEREAL_RAPIDJSON_ASSERT(size > sizeof(ChunkHeader)); + chunkHead_ = reinterpret_cast(buffer); + chunkHead_->capacity = size - sizeof(ChunkHeader); + chunkHead_->size = 0; + chunkHead_->next = 0; + } + + //! Destructor. + /*! This deallocates all memory chunks, excluding the user-supplied buffer. + */ + ~MemoryPoolAllocator() { + Clear(); + CEREAL_RAPIDJSON_DELETE(ownBaseAllocator_); + } + + //! Deallocates all memory chunks, excluding the user-supplied buffer. + void Clear() { + while (chunkHead_ && chunkHead_ != userBuffer_) { + ChunkHeader* next = chunkHead_->next; + baseAllocator_->Free(chunkHead_); + chunkHead_ = next; + } + if (chunkHead_ && chunkHead_ == userBuffer_) + chunkHead_->size = 0; // Clear user buffer + } + + //! Computes the total capacity of allocated memory chunks. + /*! \return total capacity in bytes. + */ + size_t Capacity() const { + size_t capacity = 0; + for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) + capacity += c->capacity; + return capacity; + } + + //! Computes the memory blocks allocated. + /*! \return total used bytes. + */ + size_t Size() const { + size_t size = 0; + for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) + size += c->size; + return size; + } + + //! Allocates a memory block. (concept Allocator) + void* Malloc(size_t size) { + if (!size) + return NULL; + + size = CEREAL_RAPIDJSON_ALIGN(size); + if (chunkHead_ == 0 || chunkHead_->size + size > chunkHead_->capacity) + if (!AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size)) + return NULL; + + void *buffer = reinterpret_cast(chunkHead_) + CEREAL_RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size; + chunkHead_->size += size; + return buffer; + } + + //! Resizes a memory block (concept Allocator) + void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { + if (originalPtr == 0) + return Malloc(newSize); + + if (newSize == 0) + return NULL; + + originalSize = CEREAL_RAPIDJSON_ALIGN(originalSize); + newSize = CEREAL_RAPIDJSON_ALIGN(newSize); + + // Do not shrink if new size is smaller than original + if (originalSize >= newSize) + return originalPtr; + + // Simply expand it if it is the last allocation and there is sufficient space + if (originalPtr == reinterpret_cast(chunkHead_) + CEREAL_RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size - originalSize) { + size_t increment = static_cast(newSize - originalSize); + if (chunkHead_->size + increment <= chunkHead_->capacity) { + chunkHead_->size += increment; + return originalPtr; + } + } + + // Realloc process: allocate and copy memory, do not free original buffer. + if (void* newBuffer = Malloc(newSize)) { + if (originalSize) + std::memcpy(newBuffer, originalPtr, originalSize); + return newBuffer; + } + else + return NULL; + } + + //! Frees a memory block (concept Allocator) + static void Free(void *ptr) { (void)ptr; } // Do nothing + +private: + //! Copy constructor is not permitted. + MemoryPoolAllocator(const MemoryPoolAllocator& rhs) /* = delete */; + //! Copy assignment operator is not permitted. + MemoryPoolAllocator& operator=(const MemoryPoolAllocator& rhs) /* = delete */; + + //! Creates a new chunk. + /*! \param capacity Capacity of the chunk in bytes. + \return true if success. + */ + bool AddChunk(size_t capacity) { + if (!baseAllocator_) + ownBaseAllocator_ = baseAllocator_ = CEREAL_RAPIDJSON_NEW(BaseAllocator)(); + if (ChunkHeader* chunk = reinterpret_cast(baseAllocator_->Malloc(CEREAL_RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + capacity))) { + chunk->capacity = capacity; + chunk->size = 0; + chunk->next = chunkHead_; + chunkHead_ = chunk; + return true; + } + else + return false; + } + + static const int kDefaultChunkCapacity = CEREAL_RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY; //!< Default chunk capacity. + + //! Chunk header for perpending to each chunk. + /*! Chunks are stored as a singly linked list. + */ + struct ChunkHeader { + size_t capacity; //!< Capacity of the chunk in bytes (excluding the header itself). + size_t size; //!< Current size of allocated memory in bytes. + ChunkHeader *next; //!< Next chunk in the linked list. + }; + + ChunkHeader *chunkHead_; //!< Head of the chunk linked-list. Only the head chunk serves allocation. + size_t chunk_capacity_; //!< The minimum capacity of chunk when they are allocated. + void *userBuffer_; //!< User supplied buffer. + BaseAllocator* baseAllocator_; //!< base allocator for allocating memory chunks. + BaseAllocator* ownBaseAllocator_; //!< base allocator created by this object. +}; + +CEREAL_RAPIDJSON_NAMESPACE_END + +#endif // CEREAL_RAPIDJSON_ENCODINGS_H_ diff --git a/third_party/cereal/external/rapidjson/cursorstreamwrapper.h b/third_party/cereal/external/rapidjson/cursorstreamwrapper.h new file mode 100755 index 0000000..f3d20f7 --- /dev/null +++ b/third_party/cereal/external/rapidjson/cursorstreamwrapper.h @@ -0,0 +1,78 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_CURSORSTREAMWRAPPER_H_ +#define CEREAL_RAPIDJSON_CURSORSTREAMWRAPPER_H_ + +#include "stream.h" + +#if defined(__GNUC__) +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(effc++) +#endif + +#if defined(_MSC_VER) && _MSC_VER <= 1800 +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(4702) // unreachable code +CEREAL_RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN + + +//! Cursor stream wrapper for counting line and column number if error exists. +/*! + \tparam InputStream Any stream that implements Stream Concept +*/ +template > +class CursorStreamWrapper : public GenericStreamWrapper { +public: + typedef typename Encoding::Ch Ch; + + CursorStreamWrapper(InputStream& is): + GenericStreamWrapper(is), line_(1), col_(0) {} + + // counting line and column number + Ch Take() { + Ch ch = this->is_.Take(); + if(ch == '\n') { + line_ ++; + col_ = 0; + } else { + col_ ++; + } + return ch; + } + + //! Get the error line number, if error exists. + size_t GetLine() const { return line_; } + //! Get the error column number, if error exists. + size_t GetColumn() const { return col_; } + +private: + size_t line_; //!< Current Line + size_t col_; //!< Current Column +}; + +#if defined(_MSC_VER) && _MSC_VER <= 1800 +CEREAL_RAPIDJSON_DIAG_POP +#endif + +#if defined(__GNUC__) +CEREAL_RAPIDJSON_DIAG_POP +#endif + +CEREAL_RAPIDJSON_NAMESPACE_END + +#endif // CEREAL_RAPIDJSON_CURSORSTREAMWRAPPER_H_ diff --git a/third_party/cereal/external/rapidjson/document.h b/third_party/cereal/external/rapidjson/document.h new file mode 100755 index 0000000..91c4be8 --- /dev/null +++ b/third_party/cereal/external/rapidjson/document.h @@ -0,0 +1,2659 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_DOCUMENT_H_ +#define CEREAL_RAPIDJSON_DOCUMENT_H_ + +/*! \file document.h */ + +#include "reader.h" +#include "internal/meta.h" +#include "internal/strfunc.h" +#include "memorystream.h" +#include "encodedstream.h" +#include // placement new +#include +#ifdef __cpp_lib_three_way_comparison +#include +#endif + +CEREAL_RAPIDJSON_DIAG_PUSH +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_OFF(padded) +CEREAL_RAPIDJSON_DIAG_OFF(switch-enum) +CEREAL_RAPIDJSON_DIAG_OFF(c++98-compat) +#elif defined(_MSC_VER) +CEREAL_RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant +CEREAL_RAPIDJSON_DIAG_OFF(4244) // conversion from kXxxFlags to 'uint16_t', possible loss of data +#endif + +#ifdef __GNUC__ +CEREAL_RAPIDJSON_DIAG_OFF(effc++) +#endif // __GNUC__ + +#ifndef CEREAL_RAPIDJSON_NOMEMBERITERATORCLASS +#include // std::random_access_iterator_tag +#endif + +#if CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS +#include // std::move +#endif + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN + +// Forward declaration. +template +class GenericValue; + +template +class GenericDocument; + +//! Name-value pair in a JSON object value. +/*! + This class was internal to GenericValue. It used to be a inner struct. + But a compiler (IBM XL C/C++ for AIX) have reported to have problem with that so it moved as a namespace scope struct. + https://code.google.com/p/rapidjson/issues/detail?id=64 +*/ +template +struct GenericMember { + GenericValue name; //!< name of member (must be a string) + GenericValue value; //!< value of member. + + // swap() for std::sort() and other potential use in STL. + friend inline void swap(GenericMember& a, GenericMember& b) CEREAL_RAPIDJSON_NOEXCEPT { + a.name.Swap(b.name); + a.value.Swap(b.value); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// GenericMemberIterator + +#ifndef CEREAL_RAPIDJSON_NOMEMBERITERATORCLASS + +//! (Constant) member iterator for a JSON object value +/*! + \tparam Const Is this a constant iterator? + \tparam Encoding Encoding of the value. (Even non-string values need to have the same encoding in a document) + \tparam Allocator Allocator type for allocating memory of object, array and string. + + This class implements a Random Access Iterator for GenericMember elements + of a GenericValue, see ISO/IEC 14882:2003(E) C++ standard, 24.1 [lib.iterator.requirements]. + + \note This iterator implementation is mainly intended to avoid implicit + conversions from iterator values to \c NULL, + e.g. from GenericValue::FindMember. + + \note Define \c CEREAL_RAPIDJSON_NOMEMBERITERATORCLASS to fall back to a + pointer-based implementation, if your platform doesn't provide + the C++ header. + + \see GenericMember, GenericValue::MemberIterator, GenericValue::ConstMemberIterator + */ +template +class GenericMemberIterator { + + friend class GenericValue; + template friend class GenericMemberIterator; + + typedef GenericMember PlainType; + typedef typename internal::MaybeAddConst::Type ValueType; + +public: + //! Iterator type itself + typedef GenericMemberIterator Iterator; + //! Constant iterator type + typedef GenericMemberIterator ConstIterator; + //! Non-constant iterator type + typedef GenericMemberIterator NonConstIterator; + + /** \name std::iterator_traits support */ + //@{ + typedef ValueType value_type; + typedef ValueType * pointer; + typedef ValueType & reference; + typedef std::ptrdiff_t difference_type; + typedef std::random_access_iterator_tag iterator_category; + //@} + + //! Pointer to (const) GenericMember + typedef pointer Pointer; + //! Reference to (const) GenericMember + typedef reference Reference; + //! Signed integer type (e.g. \c ptrdiff_t) + typedef difference_type DifferenceType; + + //! Default constructor (singular value) + /*! Creates an iterator pointing to no element. + \note All operations, except for comparisons, are undefined on such values. + */ + GenericMemberIterator() : ptr_() {} + + //! Iterator conversions to more const + /*! + \param it (Non-const) iterator to copy from + + Allows the creation of an iterator from another GenericMemberIterator + that is "less const". Especially, creating a non-constant iterator + from a constant iterator are disabled: + \li const -> non-const (not ok) + \li const -> const (ok) + \li non-const -> const (ok) + \li non-const -> non-const (ok) + + \note If the \c Const template parameter is already \c false, this + constructor effectively defines a regular copy-constructor. + Otherwise, the copy constructor is implicitly defined. + */ + GenericMemberIterator(const NonConstIterator & it) : ptr_(it.ptr_) {} + Iterator& operator=(const NonConstIterator & it) { ptr_ = it.ptr_; return *this; } + + //! @name stepping + //@{ + Iterator& operator++(){ ++ptr_; return *this; } + Iterator& operator--(){ --ptr_; return *this; } + Iterator operator++(int){ Iterator old(*this); ++ptr_; return old; } + Iterator operator--(int){ Iterator old(*this); --ptr_; return old; } + //@} + + //! @name increment/decrement + //@{ + Iterator operator+(DifferenceType n) const { return Iterator(ptr_+n); } + Iterator operator-(DifferenceType n) const { return Iterator(ptr_-n); } + + Iterator& operator+=(DifferenceType n) { ptr_+=n; return *this; } + Iterator& operator-=(DifferenceType n) { ptr_-=n; return *this; } + //@} + + //! @name relations + //@{ + template bool operator==(const GenericMemberIterator& that) const { return ptr_ == that.ptr_; } + template bool operator!=(const GenericMemberIterator& that) const { return ptr_ != that.ptr_; } + template bool operator<=(const GenericMemberIterator& that) const { return ptr_ <= that.ptr_; } + template bool operator>=(const GenericMemberIterator& that) const { return ptr_ >= that.ptr_; } + template bool operator< (const GenericMemberIterator& that) const { return ptr_ < that.ptr_; } + template bool operator> (const GenericMemberIterator& that) const { return ptr_ > that.ptr_; } + +#ifdef __cpp_lib_three_way_comparison + template std::strong_ordering operator<=>(const GenericMemberIterator& that) const { return ptr_ <=> that.ptr_; } +#endif + //@} + + //! @name dereference + //@{ + Reference operator*() const { return *ptr_; } + Pointer operator->() const { return ptr_; } + Reference operator[](DifferenceType n) const { return ptr_[n]; } + //@} + + //! Distance + DifferenceType operator-(ConstIterator that) const { return ptr_-that.ptr_; } + +private: + //! Internal constructor from plain pointer + explicit GenericMemberIterator(Pointer p) : ptr_(p) {} + + Pointer ptr_; //!< raw pointer +}; + +#else // CEREAL_RAPIDJSON_NOMEMBERITERATORCLASS + +// class-based member iterator implementation disabled, use plain pointers + +template +class GenericMemberIterator; + +//! non-const GenericMemberIterator +template +class GenericMemberIterator { + //! use plain pointer as iterator type + typedef GenericMember* Iterator; +}; +//! const GenericMemberIterator +template +class GenericMemberIterator { + //! use plain const pointer as iterator type + typedef const GenericMember* Iterator; +}; + +#endif // CEREAL_RAPIDJSON_NOMEMBERITERATORCLASS + +/////////////////////////////////////////////////////////////////////////////// +// GenericStringRef + +//! Reference to a constant string (not taking a copy) +/*! + \tparam CharType character type of the string + + This helper class is used to automatically infer constant string + references for string literals, especially from \c const \b (!) + character arrays. + + The main use is for creating JSON string values without copying the + source string via an \ref Allocator. This requires that the referenced + string pointers have a sufficient lifetime, which exceeds the lifetime + of the associated GenericValue. + + \b Example + \code + Value v("foo"); // ok, no need to copy & calculate length + const char foo[] = "foo"; + v.SetString(foo); // ok + + const char* bar = foo; + // Value x(bar); // not ok, can't rely on bar's lifetime + Value x(StringRef(bar)); // lifetime explicitly guaranteed by user + Value y(StringRef(bar, 3)); // ok, explicitly pass length + \endcode + + \see StringRef, GenericValue::SetString +*/ +template +struct GenericStringRef { + typedef CharType Ch; //!< character type of the string + + //! Create string reference from \c const character array +#ifndef __clang__ // -Wdocumentation + /*! + This constructor implicitly creates a constant string reference from + a \c const character array. It has better performance than + \ref StringRef(const CharType*) by inferring the string \ref length + from the array length, and also supports strings containing null + characters. + + \tparam N length of the string, automatically inferred + + \param str Constant character array, lifetime assumed to be longer + than the use of the string in e.g. a GenericValue + + \post \ref s == str + + \note Constant complexity. + \note There is a hidden, private overload to disallow references to + non-const character arrays to be created via this constructor. + By this, e.g. function-scope arrays used to be filled via + \c snprintf are excluded from consideration. + In such cases, the referenced string should be \b copied to the + GenericValue instead. + */ +#endif + template + GenericStringRef(const CharType (&str)[N]) CEREAL_RAPIDJSON_NOEXCEPT + : s(str), length(N-1) {} + + //! Explicitly create string reference from \c const character pointer +#ifndef __clang__ // -Wdocumentation + /*! + This constructor can be used to \b explicitly create a reference to + a constant string pointer. + + \see StringRef(const CharType*) + + \param str Constant character pointer, lifetime assumed to be longer + than the use of the string in e.g. a GenericValue + + \post \ref s == str + + \note There is a hidden, private overload to disallow references to + non-const character arrays to be created via this constructor. + By this, e.g. function-scope arrays used to be filled via + \c snprintf are excluded from consideration. + In such cases, the referenced string should be \b copied to the + GenericValue instead. + */ +#endif + explicit GenericStringRef(const CharType* str) + : s(str), length(NotNullStrLen(str)) {} + + //! Create constant string reference from pointer and length +#ifndef __clang__ // -Wdocumentation + /*! \param str constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue + \param len length of the string, excluding the trailing NULL terminator + + \post \ref s == str && \ref length == len + \note Constant complexity. + */ +#endif + GenericStringRef(const CharType* str, SizeType len) + : s(CEREAL_RAPIDJSON_LIKELY(str) ? str : emptyString), length(len) { CEREAL_RAPIDJSON_ASSERT(str != 0 || len == 0u); } + + GenericStringRef(const GenericStringRef& rhs) : s(rhs.s), length(rhs.length) {} + + //! implicit conversion to plain CharType pointer + operator const Ch *() const { return s; } + + const Ch* const s; //!< plain CharType pointer + const SizeType length; //!< length of the string (excluding the trailing NULL terminator) + +private: + SizeType NotNullStrLen(const CharType* str) { + CEREAL_RAPIDJSON_ASSERT(str != 0); + return internal::StrLen(str); + } + + /// Empty string - used when passing in a NULL pointer + static const Ch emptyString[]; + + //! Disallow construction from non-const array + template + GenericStringRef(CharType (&str)[N]) /* = delete */; + //! Copy assignment operator not permitted - immutable type + GenericStringRef& operator=(const GenericStringRef& rhs) /* = delete */; +}; + +template +const CharType GenericStringRef::emptyString[] = { CharType() }; + +//! Mark a character pointer as constant string +/*! Mark a plain character pointer as a "string literal". This function + can be used to avoid copying a character string to be referenced as a + value in a JSON GenericValue object, if the string's lifetime is known + to be valid long enough. + \tparam CharType Character type of the string + \param str Constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue + \return GenericStringRef string reference object + \relatesalso GenericStringRef + + \see GenericValue::GenericValue(StringRefType), GenericValue::operator=(StringRefType), GenericValue::SetString(StringRefType), GenericValue::PushBack(StringRefType, Allocator&), GenericValue::AddMember +*/ +template +inline GenericStringRef StringRef(const CharType* str) { + return GenericStringRef(str); +} + +//! Mark a character pointer as constant string +/*! Mark a plain character pointer as a "string literal". This function + can be used to avoid copying a character string to be referenced as a + value in a JSON GenericValue object, if the string's lifetime is known + to be valid long enough. + + This version has better performance with supplied length, and also + supports string containing null characters. + + \tparam CharType character type of the string + \param str Constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue + \param length The length of source string. + \return GenericStringRef string reference object + \relatesalso GenericStringRef +*/ +template +inline GenericStringRef StringRef(const CharType* str, size_t length) { + return GenericStringRef(str, SizeType(length)); +} + +#if CEREAL_RAPIDJSON_HAS_STDSTRING +//! Mark a string object as constant string +/*! Mark a string object (e.g. \c std::string) as a "string literal". + This function can be used to avoid copying a string to be referenced as a + value in a JSON GenericValue object, if the string's lifetime is known + to be valid long enough. + + \tparam CharType character type of the string + \param str Constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue + \return GenericStringRef string reference object + \relatesalso GenericStringRef + \note Requires the definition of the preprocessor symbol \ref CEREAL_RAPIDJSON_HAS_STDSTRING. +*/ +template +inline GenericStringRef StringRef(const std::basic_string& str) { + return GenericStringRef(str.data(), SizeType(str.size())); +} +#endif + +/////////////////////////////////////////////////////////////////////////////// +// GenericValue type traits +namespace internal { + +template +struct IsGenericValueImpl : FalseType {}; + +// select candidates according to nested encoding and allocator types +template struct IsGenericValueImpl::Type, typename Void::Type> + : IsBaseOf, T>::Type {}; + +// helper to match arbitrary GenericValue instantiations, including derived classes +template struct IsGenericValue : IsGenericValueImpl::Type {}; + +} // namespace internal + +/////////////////////////////////////////////////////////////////////////////// +// TypeHelper + +namespace internal { + +template +struct TypeHelper {}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsBool(); } + static bool Get(const ValueType& v) { return v.GetBool(); } + static ValueType& Set(ValueType& v, bool data) { return v.SetBool(data); } + static ValueType& Set(ValueType& v, bool data, typename ValueType::AllocatorType&) { return v.SetBool(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsInt(); } + static int Get(const ValueType& v) { return v.GetInt(); } + static ValueType& Set(ValueType& v, int data) { return v.SetInt(data); } + static ValueType& Set(ValueType& v, int data, typename ValueType::AllocatorType&) { return v.SetInt(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsUint(); } + static unsigned Get(const ValueType& v) { return v.GetUint(); } + static ValueType& Set(ValueType& v, unsigned data) { return v.SetUint(data); } + static ValueType& Set(ValueType& v, unsigned data, typename ValueType::AllocatorType&) { return v.SetUint(data); } +}; + +#ifdef _MSC_VER +CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(long) == sizeof(int)); +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsInt(); } + static long Get(const ValueType& v) { return v.GetInt(); } + static ValueType& Set(ValueType& v, long data) { return v.SetInt(data); } + static ValueType& Set(ValueType& v, long data, typename ValueType::AllocatorType&) { return v.SetInt(data); } +}; + +CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(unsigned long) == sizeof(unsigned)); +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsUint(); } + static unsigned long Get(const ValueType& v) { return v.GetUint(); } + static ValueType& Set(ValueType& v, unsigned long data) { return v.SetUint(data); } + static ValueType& Set(ValueType& v, unsigned long data, typename ValueType::AllocatorType&) { return v.SetUint(data); } +}; +#endif + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsInt64(); } + static int64_t Get(const ValueType& v) { return v.GetInt64(); } + static ValueType& Set(ValueType& v, int64_t data) { return v.SetInt64(data); } + static ValueType& Set(ValueType& v, int64_t data, typename ValueType::AllocatorType&) { return v.SetInt64(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsUint64(); } + static uint64_t Get(const ValueType& v) { return v.GetUint64(); } + static ValueType& Set(ValueType& v, uint64_t data) { return v.SetUint64(data); } + static ValueType& Set(ValueType& v, uint64_t data, typename ValueType::AllocatorType&) { return v.SetUint64(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsDouble(); } + static double Get(const ValueType& v) { return v.GetDouble(); } + static ValueType& Set(ValueType& v, double data) { return v.SetDouble(data); } + static ValueType& Set(ValueType& v, double data, typename ValueType::AllocatorType&) { return v.SetDouble(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsFloat(); } + static float Get(const ValueType& v) { return v.GetFloat(); } + static ValueType& Set(ValueType& v, float data) { return v.SetFloat(data); } + static ValueType& Set(ValueType& v, float data, typename ValueType::AllocatorType&) { return v.SetFloat(data); } +}; + +template +struct TypeHelper { + typedef const typename ValueType::Ch* StringType; + static bool Is(const ValueType& v) { return v.IsString(); } + static StringType Get(const ValueType& v) { return v.GetString(); } + static ValueType& Set(ValueType& v, const StringType data) { return v.SetString(typename ValueType::StringRefType(data)); } + static ValueType& Set(ValueType& v, const StringType data, typename ValueType::AllocatorType& a) { return v.SetString(data, a); } +}; + +#if CEREAL_RAPIDJSON_HAS_STDSTRING +template +struct TypeHelper > { + typedef std::basic_string StringType; + static bool Is(const ValueType& v) { return v.IsString(); } + static StringType Get(const ValueType& v) { return StringType(v.GetString(), v.GetStringLength()); } + static ValueType& Set(ValueType& v, const StringType& data, typename ValueType::AllocatorType& a) { return v.SetString(data, a); } +}; +#endif + +template +struct TypeHelper { + typedef typename ValueType::Array ArrayType; + static bool Is(const ValueType& v) { return v.IsArray(); } + static ArrayType Get(ValueType& v) { return v.GetArray(); } + static ValueType& Set(ValueType& v, ArrayType data) { return v = data; } + static ValueType& Set(ValueType& v, ArrayType data, typename ValueType::AllocatorType&) { return v = data; } +}; + +template +struct TypeHelper { + typedef typename ValueType::ConstArray ArrayType; + static bool Is(const ValueType& v) { return v.IsArray(); } + static ArrayType Get(const ValueType& v) { return v.GetArray(); } +}; + +template +struct TypeHelper { + typedef typename ValueType::Object ObjectType; + static bool Is(const ValueType& v) { return v.IsObject(); } + static ObjectType Get(ValueType& v) { return v.GetObject(); } + static ValueType& Set(ValueType& v, ObjectType data) { return v = data; } + static ValueType& Set(ValueType& v, ObjectType data, typename ValueType::AllocatorType&) { return v = data; } +}; + +template +struct TypeHelper { + typedef typename ValueType::ConstObject ObjectType; + static bool Is(const ValueType& v) { return v.IsObject(); } + static ObjectType Get(const ValueType& v) { return v.GetObject(); } +}; + +} // namespace internal + +// Forward declarations +template class GenericArray; +template class GenericObject; + +/////////////////////////////////////////////////////////////////////////////// +// GenericValue + +//! Represents a JSON value. Use Value for UTF8 encoding and default allocator. +/*! + A JSON value can be one of 7 types. This class is a variant type supporting + these types. + + Use the Value if UTF8 and default allocator + + \tparam Encoding Encoding of the value. (Even non-string values need to have the same encoding in a document) + \tparam Allocator Allocator type for allocating memory of object, array and string. +*/ +template > +class GenericValue { +public: + //! Name-value pair in an object. + typedef GenericMember Member; + typedef Encoding EncodingType; //!< Encoding type from template parameter. + typedef Allocator AllocatorType; //!< Allocator type from template parameter. + typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. + typedef GenericStringRef StringRefType; //!< Reference to a constant string + typedef typename GenericMemberIterator::Iterator MemberIterator; //!< Member iterator for iterating in object. + typedef typename GenericMemberIterator::Iterator ConstMemberIterator; //!< Constant member iterator for iterating in object. + typedef GenericValue* ValueIterator; //!< Value iterator for iterating in array. + typedef const GenericValue* ConstValueIterator; //!< Constant value iterator for iterating in array. + typedef GenericValue ValueType; //!< Value type of itself. + typedef GenericArray Array; + typedef GenericArray ConstArray; + typedef GenericObject Object; + typedef GenericObject ConstObject; + + //!@name Constructors and destructor. + //@{ + + //! Default constructor creates a null value. + GenericValue() CEREAL_RAPIDJSON_NOEXCEPT : data_() { data_.f.flags = kNullFlag; } + +#if CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move constructor in C++11 + GenericValue(GenericValue&& rhs) CEREAL_RAPIDJSON_NOEXCEPT : data_(rhs.data_) { + rhs.data_.f.flags = kNullFlag; // give up contents + } +#endif + +private: + //! Copy constructor is not permitted. + GenericValue(const GenericValue& rhs); + +#if CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Moving from a GenericDocument is not permitted. + template + GenericValue(GenericDocument&& rhs); + + //! Move assignment from a GenericDocument is not permitted. + template + GenericValue& operator=(GenericDocument&& rhs); +#endif + +public: + + //! Constructor with JSON value type. + /*! This creates a Value of specified type with default content. + \param type Type of the value. + \note Default content for number is zero. + */ + explicit GenericValue(Type type) CEREAL_RAPIDJSON_NOEXCEPT : data_() { + static const uint16_t defaultFlags[] = { + kNullFlag, kFalseFlag, kTrueFlag, kObjectFlag, kArrayFlag, kShortStringFlag, + kNumberAnyFlag + }; + CEREAL_RAPIDJSON_NOEXCEPT_ASSERT(type >= kNullType && type <= kNumberType); + data_.f.flags = defaultFlags[type]; + + // Use ShortString to store empty string. + if (type == kStringType) + data_.ss.SetLength(0); + } + + //! Explicit copy constructor (with allocator) + /*! Creates a copy of a Value by using the given Allocator + \tparam SourceAllocator allocator of \c rhs + \param rhs Value to copy from (read-only) + \param allocator Allocator for allocating copied elements and buffers. Commonly use GenericDocument::GetAllocator(). + \param copyConstStrings Force copying of constant strings (e.g. referencing an in-situ buffer) + \see CopyFrom() + */ + template + GenericValue(const GenericValue& rhs, Allocator& allocator, bool copyConstStrings = false) { + switch (rhs.GetType()) { + case kObjectType: { + SizeType count = rhs.data_.o.size; + Member* lm = reinterpret_cast(allocator.Malloc(count * sizeof(Member))); + const typename GenericValue::Member* rm = rhs.GetMembersPointer(); + for (SizeType i = 0; i < count; i++) { + new (&lm[i].name) GenericValue(rm[i].name, allocator, copyConstStrings); + new (&lm[i].value) GenericValue(rm[i].value, allocator, copyConstStrings); + } + data_.f.flags = kObjectFlag; + data_.o.size = data_.o.capacity = count; + SetMembersPointer(lm); + } + break; + case kArrayType: { + SizeType count = rhs.data_.a.size; + GenericValue* le = reinterpret_cast(allocator.Malloc(count * sizeof(GenericValue))); + const GenericValue* re = rhs.GetElementsPointer(); + for (SizeType i = 0; i < count; i++) + new (&le[i]) GenericValue(re[i], allocator, copyConstStrings); + data_.f.flags = kArrayFlag; + data_.a.size = data_.a.capacity = count; + SetElementsPointer(le); + } + break; + case kStringType: + if (rhs.data_.f.flags == kConstStringFlag && !copyConstStrings) { + data_.f.flags = rhs.data_.f.flags; + data_ = *reinterpret_cast(&rhs.data_); + } + else + SetStringRaw(StringRef(rhs.GetString(), rhs.GetStringLength()), allocator); + break; + default: + data_.f.flags = rhs.data_.f.flags; + data_ = *reinterpret_cast(&rhs.data_); + break; + } + } + + //! Constructor for boolean value. + /*! \param b Boolean value + \note This constructor is limited to \em real boolean values and rejects + implicitly converted types like arbitrary pointers. Use an explicit cast + to \c bool, if you want to construct a boolean JSON value in such cases. + */ +#ifndef CEREAL_RAPIDJSON_DOXYGEN_RUNNING // hide SFINAE from Doxygen + template + explicit GenericValue(T b, CEREAL_RAPIDJSON_ENABLEIF((internal::IsSame))) CEREAL_RAPIDJSON_NOEXCEPT // See #472 +#else + explicit GenericValue(bool b) CEREAL_RAPIDJSON_NOEXCEPT +#endif + : data_() { + // safe-guard against failing SFINAE + CEREAL_RAPIDJSON_STATIC_ASSERT((internal::IsSame::Value)); + data_.f.flags = b ? kTrueFlag : kFalseFlag; + } + + //! Constructor for int value. + explicit GenericValue(int i) CEREAL_RAPIDJSON_NOEXCEPT : data_() { + data_.n.i64 = i; + data_.f.flags = (i >= 0) ? (kNumberIntFlag | kUintFlag | kUint64Flag) : kNumberIntFlag; + } + + //! Constructor for unsigned value. + explicit GenericValue(unsigned u) CEREAL_RAPIDJSON_NOEXCEPT : data_() { + data_.n.u64 = u; + data_.f.flags = (u & 0x80000000) ? kNumberUintFlag : (kNumberUintFlag | kIntFlag | kInt64Flag); + } + + //! Constructor for int64_t value. + explicit GenericValue(int64_t i64) CEREAL_RAPIDJSON_NOEXCEPT : data_() { + data_.n.i64 = i64; + data_.f.flags = kNumberInt64Flag; + if (i64 >= 0) { + data_.f.flags |= kNumberUint64Flag; + if (!(static_cast(i64) & CEREAL_RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) + data_.f.flags |= kUintFlag; + if (!(static_cast(i64) & CEREAL_RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) + data_.f.flags |= kIntFlag; + } + else if (i64 >= static_cast(CEREAL_RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) + data_.f.flags |= kIntFlag; + } + + //! Constructor for uint64_t value. + explicit GenericValue(uint64_t u64) CEREAL_RAPIDJSON_NOEXCEPT : data_() { + data_.n.u64 = u64; + data_.f.flags = kNumberUint64Flag; + if (!(u64 & CEREAL_RAPIDJSON_UINT64_C2(0x80000000, 0x00000000))) + data_.f.flags |= kInt64Flag; + if (!(u64 & CEREAL_RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) + data_.f.flags |= kUintFlag; + if (!(u64 & CEREAL_RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) + data_.f.flags |= kIntFlag; + } + + //! Constructor for double value. + explicit GenericValue(double d) CEREAL_RAPIDJSON_NOEXCEPT : data_() { data_.n.d = d; data_.f.flags = kNumberDoubleFlag; } + + //! Constructor for float value. + explicit GenericValue(float f) CEREAL_RAPIDJSON_NOEXCEPT : data_() { data_.n.d = static_cast(f); data_.f.flags = kNumberDoubleFlag; } + + //! Constructor for constant string (i.e. do not make a copy of string) + GenericValue(const Ch* s, SizeType length) CEREAL_RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(StringRef(s, length)); } + + //! Constructor for constant string (i.e. do not make a copy of string) + explicit GenericValue(StringRefType s) CEREAL_RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(s); } + + //! Constructor for copy-string (i.e. do make a copy of string) + GenericValue(const Ch* s, SizeType length, Allocator& allocator) : data_() { SetStringRaw(StringRef(s, length), allocator); } + + //! Constructor for copy-string (i.e. do make a copy of string) + GenericValue(const Ch*s, Allocator& allocator) : data_() { SetStringRaw(StringRef(s), allocator); } + +#if CEREAL_RAPIDJSON_HAS_STDSTRING + //! Constructor for copy-string from a string object (i.e. do make a copy of string) + /*! \note Requires the definition of the preprocessor symbol \ref CEREAL_RAPIDJSON_HAS_STDSTRING. + */ + GenericValue(const std::basic_string& s, Allocator& allocator) : data_() { SetStringRaw(StringRef(s), allocator); } +#endif + + //! Constructor for Array. + /*! + \param a An array obtained by \c GetArray(). + \note \c Array is always pass-by-value. + \note the source array is moved into this value and the sourec array becomes empty. + */ + GenericValue(Array a) CEREAL_RAPIDJSON_NOEXCEPT : data_(a.value_.data_) { + a.value_.data_ = Data(); + a.value_.data_.f.flags = kArrayFlag; + } + + //! Constructor for Object. + /*! + \param o An object obtained by \c GetObject(). + \note \c Object is always pass-by-value. + \note the source object is moved into this value and the sourec object becomes empty. + */ + GenericValue(Object o) CEREAL_RAPIDJSON_NOEXCEPT : data_(o.value_.data_) { + o.value_.data_ = Data(); + o.value_.data_.f.flags = kObjectFlag; + } + + //! Destructor. + /*! Need to destruct elements of array, members of object, or copy-string. + */ + ~GenericValue() { + if (Allocator::kNeedFree) { // Shortcut by Allocator's trait + switch(data_.f.flags) { + case kArrayFlag: + { + GenericValue* e = GetElementsPointer(); + for (GenericValue* v = e; v != e + data_.a.size; ++v) + v->~GenericValue(); + Allocator::Free(e); + } + break; + + case kObjectFlag: + for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) + m->~Member(); + Allocator::Free(GetMembersPointer()); + break; + + case kCopyStringFlag: + Allocator::Free(const_cast(GetStringPointer())); + break; + + default: + break; // Do nothing for other types. + } + } + } + + //@} + + //!@name Assignment operators + //@{ + + //! Assignment with move semantics. + /*! \param rhs Source of the assignment. It will become a null value after assignment. + */ + GenericValue& operator=(GenericValue& rhs) CEREAL_RAPIDJSON_NOEXCEPT { + if (CEREAL_RAPIDJSON_LIKELY(this != &rhs)) { + this->~GenericValue(); + RawAssign(rhs); + } + return *this; + } + +#if CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move assignment in C++11 + GenericValue& operator=(GenericValue&& rhs) CEREAL_RAPIDJSON_NOEXCEPT { + return *this = rhs.Move(); + } +#endif + + //! Assignment of constant string reference (no copy) + /*! \param str Constant string reference to be assigned + \note This overload is needed to avoid clashes with the generic primitive type assignment overload below. + \see GenericStringRef, operator=(T) + */ + GenericValue& operator=(StringRefType str) CEREAL_RAPIDJSON_NOEXCEPT { + GenericValue s(str); + return *this = s; + } + + //! Assignment with primitive types. + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t + \param value The value to be assigned. + + \note The source type \c T explicitly disallows all pointer types, + especially (\c const) \ref Ch*. This helps avoiding implicitly + referencing character strings with insufficient lifetime, use + \ref SetString(const Ch*, Allocator&) (for copying) or + \ref StringRef() (to explicitly mark the pointer as constant) instead. + All other pointer types would implicitly convert to \c bool, + use \ref SetBool() instead. + */ + template + CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::IsPointer), (GenericValue&)) + operator=(T value) { + GenericValue v(value); + return *this = v; + } + + //! Deep-copy assignment from Value + /*! Assigns a \b copy of the Value to the current Value object + \tparam SourceAllocator Allocator type of \c rhs + \param rhs Value to copy from (read-only) + \param allocator Allocator to use for copying + \param copyConstStrings Force copying of constant strings (e.g. referencing an in-situ buffer) + */ + template + GenericValue& CopyFrom(const GenericValue& rhs, Allocator& allocator, bool copyConstStrings = false) { + CEREAL_RAPIDJSON_ASSERT(static_cast(this) != static_cast(&rhs)); + this->~GenericValue(); + new (this) GenericValue(rhs, allocator, copyConstStrings); + return *this; + } + + //! Exchange the contents of this value with those of other. + /*! + \param other Another value. + \note Constant complexity. + */ + GenericValue& Swap(GenericValue& other) CEREAL_RAPIDJSON_NOEXCEPT { + GenericValue temp; + temp.RawAssign(*this); + RawAssign(other); + other.RawAssign(temp); + return *this; + } + + //! free-standing swap function helper + /*! + Helper function to enable support for common swap implementation pattern based on \c std::swap: + \code + void swap(MyClass& a, MyClass& b) { + using std::swap; + swap(a.value, b.value); + // ... + } + \endcode + \see Swap() + */ + friend inline void swap(GenericValue& a, GenericValue& b) CEREAL_RAPIDJSON_NOEXCEPT { a.Swap(b); } + + //! Prepare Value for move semantics + /*! \return *this */ + GenericValue& Move() CEREAL_RAPIDJSON_NOEXCEPT { return *this; } + //@} + + //!@name Equal-to and not-equal-to operators + //@{ + //! Equal-to operator + /*! + \note If an object contains duplicated named member, comparing equality with any object is always \c false. + \note Complexity is quadratic in Object's member number and linear for the rest (number of all values in the subtree and total lengths of all strings). + */ + template + bool operator==(const GenericValue& rhs) const { + typedef GenericValue RhsType; + if (GetType() != rhs.GetType()) + return false; + + switch (GetType()) { + case kObjectType: // Warning: O(n^2) inner-loop + if (data_.o.size != rhs.data_.o.size) + return false; + for (ConstMemberIterator lhsMemberItr = MemberBegin(); lhsMemberItr != MemberEnd(); ++lhsMemberItr) { + typename RhsType::ConstMemberIterator rhsMemberItr = rhs.FindMember(lhsMemberItr->name); + if (rhsMemberItr == rhs.MemberEnd() || lhsMemberItr->value != rhsMemberItr->value) + return false; + } + return true; + + case kArrayType: + if (data_.a.size != rhs.data_.a.size) + return false; + for (SizeType i = 0; i < data_.a.size; i++) + if ((*this)[i] != rhs[i]) + return false; + return true; + + case kStringType: + return StringEqual(rhs); + + case kNumberType: + if (IsDouble() || rhs.IsDouble()) { + double a = GetDouble(); // May convert from integer to double. + double b = rhs.GetDouble(); // Ditto + return a >= b && a <= b; // Prevent -Wfloat-equal + } + else + return data_.n.u64 == rhs.data_.n.u64; + + default: + return true; + } + } + + //! Equal-to operator with const C-string pointer + bool operator==(const Ch* rhs) const { return *this == GenericValue(StringRef(rhs)); } + +#if CEREAL_RAPIDJSON_HAS_STDSTRING + //! Equal-to operator with string object + /*! \note Requires the definition of the preprocessor symbol \ref CEREAL_RAPIDJSON_HAS_STDSTRING. + */ + bool operator==(const std::basic_string& rhs) const { return *this == GenericValue(StringRef(rhs)); } +#endif + + //! Equal-to operator with primitive types + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c double, \c true, \c false + */ + template CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr,internal::IsGenericValue >), (bool)) operator==(const T& rhs) const { return *this == GenericValue(rhs); } + + //! Not-equal-to operator + /*! \return !(*this == rhs) + */ + template + bool operator!=(const GenericValue& rhs) const { return !(*this == rhs); } + + //! Not-equal-to operator with const C-string pointer + bool operator!=(const Ch* rhs) const { return !(*this == rhs); } + + //! Not-equal-to operator with arbitrary types + /*! \return !(*this == rhs) + */ + template CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& rhs) const { return !(*this == rhs); } + + //! Equal-to operator with arbitrary types (symmetric version) + /*! \return (rhs == lhs) + */ + template friend CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator==(const T& lhs, const GenericValue& rhs) { return rhs == lhs; } + + //! Not-Equal-to operator with arbitrary types (symmetric version) + /*! \return !(rhs == lhs) + */ + template friend CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& lhs, const GenericValue& rhs) { return !(rhs == lhs); } + //@} + + //!@name Type + //@{ + + Type GetType() const { return static_cast(data_.f.flags & kTypeMask); } + bool IsNull() const { return data_.f.flags == kNullFlag; } + bool IsFalse() const { return data_.f.flags == kFalseFlag; } + bool IsTrue() const { return data_.f.flags == kTrueFlag; } + bool IsBool() const { return (data_.f.flags & kBoolFlag) != 0; } + bool IsObject() const { return data_.f.flags == kObjectFlag; } + bool IsArray() const { return data_.f.flags == kArrayFlag; } + bool IsNumber() const { return (data_.f.flags & kNumberFlag) != 0; } + bool IsInt() const { return (data_.f.flags & kIntFlag) != 0; } + bool IsUint() const { return (data_.f.flags & kUintFlag) != 0; } + bool IsInt64() const { return (data_.f.flags & kInt64Flag) != 0; } + bool IsUint64() const { return (data_.f.flags & kUint64Flag) != 0; } + bool IsDouble() const { return (data_.f.flags & kDoubleFlag) != 0; } + bool IsString() const { return (data_.f.flags & kStringFlag) != 0; } + + // Checks whether a number can be losslessly converted to a double. + bool IsLosslessDouble() const { + if (!IsNumber()) return false; + if (IsUint64()) { + uint64_t u = GetUint64(); + volatile double d = static_cast(u); + return (d >= 0.0) + && (d < static_cast((std::numeric_limits::max)())) + && (u == static_cast(d)); + } + if (IsInt64()) { + int64_t i = GetInt64(); + volatile double d = static_cast(i); + return (d >= static_cast((std::numeric_limits::min)())) + && (d < static_cast((std::numeric_limits::max)())) + && (i == static_cast(d)); + } + return true; // double, int, uint are always lossless + } + + // Checks whether a number is a float (possible lossy). + bool IsFloat() const { + if ((data_.f.flags & kDoubleFlag) == 0) + return false; + double d = GetDouble(); + return d >= -3.4028234e38 && d <= 3.4028234e38; + } + // Checks whether a number can be losslessly converted to a float. + bool IsLosslessFloat() const { + if (!IsNumber()) return false; + double a = GetDouble(); + if (a < static_cast(-(std::numeric_limits::max)()) + || a > static_cast((std::numeric_limits::max)())) + return false; + double b = static_cast(static_cast(a)); + return a >= b && a <= b; // Prevent -Wfloat-equal + } + + //@} + + //!@name Null + //@{ + + GenericValue& SetNull() { this->~GenericValue(); new (this) GenericValue(); return *this; } + + //@} + + //!@name Bool + //@{ + + bool GetBool() const { CEREAL_RAPIDJSON_ASSERT(IsBool()); return data_.f.flags == kTrueFlag; } + //!< Set boolean value + /*! \post IsBool() == true */ + GenericValue& SetBool(bool b) { this->~GenericValue(); new (this) GenericValue(b); return *this; } + + //@} + + //!@name Object + //@{ + + //! Set this value as an empty object. + /*! \post IsObject() == true */ + GenericValue& SetObject() { this->~GenericValue(); new (this) GenericValue(kObjectType); return *this; } + + //! Get the number of members in the object. + SizeType MemberCount() const { CEREAL_RAPIDJSON_ASSERT(IsObject()); return data_.o.size; } + + //! Get the capacity of object. + SizeType MemberCapacity() const { CEREAL_RAPIDJSON_ASSERT(IsObject()); return data_.o.capacity; } + + //! Check whether the object is empty. + bool ObjectEmpty() const { CEREAL_RAPIDJSON_ASSERT(IsObject()); return data_.o.size == 0; } + + //! Get a value from an object associated with the name. + /*! \pre IsObject() == true + \tparam T Either \c Ch or \c const \c Ch (template used for disambiguation with \ref operator[](SizeType)) + \note In version 0.1x, if the member is not found, this function returns a null value. This makes issue 7. + Since 0.2, if the name is not correct, it will assert. + If user is unsure whether a member exists, user should use HasMember() first. + A better approach is to use FindMember(). + \note Linear time complexity. + */ + template + CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >),(GenericValue&)) operator[](T* name) { + GenericValue n(StringRef(name)); + return (*this)[n]; + } + template + CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >),(const GenericValue&)) operator[](T* name) const { return const_cast(*this)[name]; } + + //! Get a value from an object associated with the name. + /*! \pre IsObject() == true + \tparam SourceAllocator Allocator of the \c name value + + \note Compared to \ref operator[](T*), this version is faster because it does not need a StrLen(). + And it can also handle strings with embedded null characters. + + \note Linear time complexity. + */ + template + GenericValue& operator[](const GenericValue& name) { + MemberIterator member = FindMember(name); + if (member != MemberEnd()) + return member->value; + else { + CEREAL_RAPIDJSON_ASSERT(false); // see above note + + // This will generate -Wexit-time-destructors in clang + // static GenericValue NullValue; + // return NullValue; + + // Use static buffer and placement-new to prevent destruction + static char buffer[sizeof(GenericValue)]; + return *new (buffer) GenericValue(); + } + } + template + const GenericValue& operator[](const GenericValue& name) const { return const_cast(*this)[name]; } + +#if CEREAL_RAPIDJSON_HAS_STDSTRING + //! Get a value from an object associated with name (string object). + GenericValue& operator[](const std::basic_string& name) { return (*this)[GenericValue(StringRef(name))]; } + const GenericValue& operator[](const std::basic_string& name) const { return (*this)[GenericValue(StringRef(name))]; } +#endif + + //! Const member iterator + /*! \pre IsObject() == true */ + ConstMemberIterator MemberBegin() const { CEREAL_RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(GetMembersPointer()); } + //! Const \em past-the-end member iterator + /*! \pre IsObject() == true */ + ConstMemberIterator MemberEnd() const { CEREAL_RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(GetMembersPointer() + data_.o.size); } + //! Member iterator + /*! \pre IsObject() == true */ + MemberIterator MemberBegin() { CEREAL_RAPIDJSON_ASSERT(IsObject()); return MemberIterator(GetMembersPointer()); } + //! \em Past-the-end member iterator + /*! \pre IsObject() == true */ + MemberIterator MemberEnd() { CEREAL_RAPIDJSON_ASSERT(IsObject()); return MemberIterator(GetMembersPointer() + data_.o.size); } + + //! Request the object to have enough capacity to store members. + /*! \param newCapacity The capacity that the object at least need to have. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \note Linear time complexity. + */ + GenericValue& MemberReserve(SizeType newCapacity, Allocator &allocator) { + CEREAL_RAPIDJSON_ASSERT(IsObject()); + if (newCapacity > data_.o.capacity) { + SetMembersPointer(reinterpret_cast(allocator.Realloc(GetMembersPointer(), data_.o.capacity * sizeof(Member), newCapacity * sizeof(Member)))); + data_.o.capacity = newCapacity; + } + return *this; + } + + //! Check whether a member exists in the object. + /*! + \param name Member name to be searched. + \pre IsObject() == true + \return Whether a member with that name exists. + \note It is better to use FindMember() directly if you need the obtain the value as well. + \note Linear time complexity. + */ + bool HasMember(const Ch* name) const { return FindMember(name) != MemberEnd(); } + +#if CEREAL_RAPIDJSON_HAS_STDSTRING + //! Check whether a member exists in the object with string object. + /*! + \param name Member name to be searched. + \pre IsObject() == true + \return Whether a member with that name exists. + \note It is better to use FindMember() directly if you need the obtain the value as well. + \note Linear time complexity. + */ + bool HasMember(const std::basic_string& name) const { return FindMember(name) != MemberEnd(); } +#endif + + //! Check whether a member exists in the object with GenericValue name. + /*! + This version is faster because it does not need a StrLen(). It can also handle string with null character. + \param name Member name to be searched. + \pre IsObject() == true + \return Whether a member with that name exists. + \note It is better to use FindMember() directly if you need the obtain the value as well. + \note Linear time complexity. + */ + template + bool HasMember(const GenericValue& name) const { return FindMember(name) != MemberEnd(); } + + //! Find member by name. + /*! + \param name Member name to be searched. + \pre IsObject() == true + \return Iterator to member, if it exists. + Otherwise returns \ref MemberEnd(). + + \note Earlier versions of Rapidjson returned a \c NULL pointer, in case + the requested member doesn't exist. For consistency with e.g. + \c std::map, this has been changed to MemberEnd() now. + \note Linear time complexity. + */ + MemberIterator FindMember(const Ch* name) { + GenericValue n(StringRef(name)); + return FindMember(n); + } + + ConstMemberIterator FindMember(const Ch* name) const { return const_cast(*this).FindMember(name); } + + //! Find member by name. + /*! + This version is faster because it does not need a StrLen(). It can also handle string with null character. + \param name Member name to be searched. + \pre IsObject() == true + \return Iterator to member, if it exists. + Otherwise returns \ref MemberEnd(). + + \note Earlier versions of Rapidjson returned a \c NULL pointer, in case + the requested member doesn't exist. For consistency with e.g. + \c std::map, this has been changed to MemberEnd() now. + \note Linear time complexity. + */ + template + MemberIterator FindMember(const GenericValue& name) { + CEREAL_RAPIDJSON_ASSERT(IsObject()); + CEREAL_RAPIDJSON_ASSERT(name.IsString()); + MemberIterator member = MemberBegin(); + for ( ; member != MemberEnd(); ++member) + if (name.StringEqual(member->name)) + break; + return member; + } + template ConstMemberIterator FindMember(const GenericValue& name) const { return const_cast(*this).FindMember(name); } + +#if CEREAL_RAPIDJSON_HAS_STDSTRING + //! Find member by string object name. + /*! + \param name Member name to be searched. + \pre IsObject() == true + \return Iterator to member, if it exists. + Otherwise returns \ref MemberEnd(). + */ + MemberIterator FindMember(const std::basic_string& name) { return FindMember(GenericValue(StringRef(name))); } + ConstMemberIterator FindMember(const std::basic_string& name) const { return FindMember(GenericValue(StringRef(name))); } +#endif + + //! Add a member (name-value pair) to the object. + /*! \param name A string value as name of member. + \param value Value of any type. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \note The ownership of \c name and \c value will be transferred to this object on success. + \pre IsObject() && name.IsString() + \post name.IsNull() && value.IsNull() + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(GenericValue& name, GenericValue& value, Allocator& allocator) { + CEREAL_RAPIDJSON_ASSERT(IsObject()); + CEREAL_RAPIDJSON_ASSERT(name.IsString()); + + ObjectData& o = data_.o; + if (o.size >= o.capacity) + MemberReserve(o.capacity == 0 ? kDefaultObjectCapacity : (o.capacity + (o.capacity + 1) / 2), allocator); + Member* members = GetMembersPointer(); + members[o.size].name.RawAssign(name); + members[o.size].value.RawAssign(value); + o.size++; + return *this; + } + + //! Add a constant string value as member (name-value pair) to the object. + /*! \param name A string value as name of member. + \param value constant string reference as value of member. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \pre IsObject() + \note This overload is needed to avoid clashes with the generic primitive type AddMember(GenericValue&,T,Allocator&) overload below. + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(GenericValue& name, StringRefType value, Allocator& allocator) { + GenericValue v(value); + return AddMember(name, v, allocator); + } + +#if CEREAL_RAPIDJSON_HAS_STDSTRING + //! Add a string object as member (name-value pair) to the object. + /*! \param name A string value as name of member. + \param value constant string reference as value of member. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \pre IsObject() + \note This overload is needed to avoid clashes with the generic primitive type AddMember(GenericValue&,T,Allocator&) overload below. + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(GenericValue& name, std::basic_string& value, Allocator& allocator) { + GenericValue v(value, allocator); + return AddMember(name, v, allocator); + } +#endif + + //! Add any primitive value as member (name-value pair) to the object. + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t + \param name A string value as name of member. + \param value Value of primitive type \c T as value of member + \param allocator Allocator for reallocating memory. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \pre IsObject() + + \note The source type \c T explicitly disallows all pointer types, + especially (\c const) \ref Ch*. This helps avoiding implicitly + referencing character strings with insufficient lifetime, use + \ref AddMember(StringRefType, GenericValue&, Allocator&) or \ref + AddMember(StringRefType, StringRefType, Allocator&). + All other pointer types would implicitly convert to \c bool, + use an explicit cast instead, if needed. + \note Amortized Constant time complexity. + */ + template + CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericValue&)) + AddMember(GenericValue& name, T value, Allocator& allocator) { + GenericValue v(value); + return AddMember(name, v, allocator); + } + +#if CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericValue& AddMember(GenericValue&& name, GenericValue&& value, Allocator& allocator) { + return AddMember(name, value, allocator); + } + GenericValue& AddMember(GenericValue&& name, GenericValue& value, Allocator& allocator) { + return AddMember(name, value, allocator); + } + GenericValue& AddMember(GenericValue& name, GenericValue&& value, Allocator& allocator) { + return AddMember(name, value, allocator); + } + GenericValue& AddMember(StringRefType name, GenericValue&& value, Allocator& allocator) { + GenericValue n(name); + return AddMember(n, value, allocator); + } +#endif // CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS + + + //! Add a member (name-value pair) to the object. + /*! \param name A constant string reference as name of member. + \param value Value of any type. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \note The ownership of \c value will be transferred to this object on success. + \pre IsObject() + \post value.IsNull() + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(StringRefType name, GenericValue& value, Allocator& allocator) { + GenericValue n(name); + return AddMember(n, value, allocator); + } + + //! Add a constant string value as member (name-value pair) to the object. + /*! \param name A constant string reference as name of member. + \param value constant string reference as value of member. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \pre IsObject() + \note This overload is needed to avoid clashes with the generic primitive type AddMember(StringRefType,T,Allocator&) overload below. + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(StringRefType name, StringRefType value, Allocator& allocator) { + GenericValue v(value); + return AddMember(name, v, allocator); + } + + //! Add any primitive value as member (name-value pair) to the object. + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t + \param name A constant string reference as name of member. + \param value Value of primitive type \c T as value of member + \param allocator Allocator for reallocating memory. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \pre IsObject() + + \note The source type \c T explicitly disallows all pointer types, + especially (\c const) \ref Ch*. This helps avoiding implicitly + referencing character strings with insufficient lifetime, use + \ref AddMember(StringRefType, GenericValue&, Allocator&) or \ref + AddMember(StringRefType, StringRefType, Allocator&). + All other pointer types would implicitly convert to \c bool, + use an explicit cast instead, if needed. + \note Amortized Constant time complexity. + */ + template + CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericValue&)) + AddMember(StringRefType name, T value, Allocator& allocator) { + GenericValue n(name); + return AddMember(n, value, allocator); + } + + //! Remove all members in the object. + /*! This function do not deallocate memory in the object, i.e. the capacity is unchanged. + \note Linear time complexity. + */ + void RemoveAllMembers() { + CEREAL_RAPIDJSON_ASSERT(IsObject()); + for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) + m->~Member(); + data_.o.size = 0; + } + + //! Remove a member in object by its name. + /*! \param name Name of member to be removed. + \return Whether the member existed. + \note This function may reorder the object members. Use \ref + EraseMember(ConstMemberIterator) if you need to preserve the + relative order of the remaining members. + \note Linear time complexity. + */ + bool RemoveMember(const Ch* name) { + GenericValue n(StringRef(name)); + return RemoveMember(n); + } + +#if CEREAL_RAPIDJSON_HAS_STDSTRING + bool RemoveMember(const std::basic_string& name) { return RemoveMember(GenericValue(StringRef(name))); } +#endif + + template + bool RemoveMember(const GenericValue& name) { + MemberIterator m = FindMember(name); + if (m != MemberEnd()) { + RemoveMember(m); + return true; + } + else + return false; + } + + //! Remove a member in object by iterator. + /*! \param m member iterator (obtained by FindMember() or MemberBegin()). + \return the new iterator after removal. + \note This function may reorder the object members. Use \ref + EraseMember(ConstMemberIterator) if you need to preserve the + relative order of the remaining members. + \note Constant time complexity. + */ + MemberIterator RemoveMember(MemberIterator m) { + CEREAL_RAPIDJSON_ASSERT(IsObject()); + CEREAL_RAPIDJSON_ASSERT(data_.o.size > 0); + CEREAL_RAPIDJSON_ASSERT(GetMembersPointer() != 0); + CEREAL_RAPIDJSON_ASSERT(m >= MemberBegin() && m < MemberEnd()); + + MemberIterator last(GetMembersPointer() + (data_.o.size - 1)); + if (data_.o.size > 1 && m != last) + *m = *last; // Move the last one to this place + else + m->~Member(); // Only one left, just destroy + --data_.o.size; + return m; + } + + //! Remove a member from an object by iterator. + /*! \param pos iterator to the member to remove + \pre IsObject() == true && \ref MemberBegin() <= \c pos < \ref MemberEnd() + \return Iterator following the removed element. + If the iterator \c pos refers to the last element, the \ref MemberEnd() iterator is returned. + \note This function preserves the relative order of the remaining object + members. If you do not need this, use the more efficient \ref RemoveMember(MemberIterator). + \note Linear time complexity. + */ + MemberIterator EraseMember(ConstMemberIterator pos) { + return EraseMember(pos, pos +1); + } + + //! Remove members in the range [first, last) from an object. + /*! \param first iterator to the first member to remove + \param last iterator following the last member to remove + \pre IsObject() == true && \ref MemberBegin() <= \c first <= \c last <= \ref MemberEnd() + \return Iterator following the last removed element. + \note This function preserves the relative order of the remaining object + members. + \note Linear time complexity. + */ + MemberIterator EraseMember(ConstMemberIterator first, ConstMemberIterator last) { + CEREAL_RAPIDJSON_ASSERT(IsObject()); + CEREAL_RAPIDJSON_ASSERT(data_.o.size > 0); + CEREAL_RAPIDJSON_ASSERT(GetMembersPointer() != 0); + CEREAL_RAPIDJSON_ASSERT(first >= MemberBegin()); + CEREAL_RAPIDJSON_ASSERT(first <= last); + CEREAL_RAPIDJSON_ASSERT(last <= MemberEnd()); + + MemberIterator pos = MemberBegin() + (first - MemberBegin()); + for (MemberIterator itr = pos; itr != last; ++itr) + itr->~Member(); + std::memmove(static_cast(&*pos), &*last, static_cast(MemberEnd() - last) * sizeof(Member)); + data_.o.size -= static_cast(last - first); + return pos; + } + + //! Erase a member in object by its name. + /*! \param name Name of member to be removed. + \return Whether the member existed. + \note Linear time complexity. + */ + bool EraseMember(const Ch* name) { + GenericValue n(StringRef(name)); + return EraseMember(n); + } + +#if CEREAL_RAPIDJSON_HAS_STDSTRING + bool EraseMember(const std::basic_string& name) { return EraseMember(GenericValue(StringRef(name))); } +#endif + + template + bool EraseMember(const GenericValue& name) { + MemberIterator m = FindMember(name); + if (m != MemberEnd()) { + EraseMember(m); + return true; + } + else + return false; + } + + Object GetObject() { CEREAL_RAPIDJSON_ASSERT(IsObject()); return Object(*this); } + ConstObject GetObject() const { CEREAL_RAPIDJSON_ASSERT(IsObject()); return ConstObject(*this); } + + //@} + + //!@name Array + //@{ + + //! Set this value as an empty array. + /*! \post IsArray == true */ + GenericValue& SetArray() { this->~GenericValue(); new (this) GenericValue(kArrayType); return *this; } + + //! Get the number of elements in array. + SizeType Size() const { CEREAL_RAPIDJSON_ASSERT(IsArray()); return data_.a.size; } + + //! Get the capacity of array. + SizeType Capacity() const { CEREAL_RAPIDJSON_ASSERT(IsArray()); return data_.a.capacity; } + + //! Check whether the array is empty. + bool Empty() const { CEREAL_RAPIDJSON_ASSERT(IsArray()); return data_.a.size == 0; } + + //! Remove all elements in the array. + /*! This function do not deallocate memory in the array, i.e. the capacity is unchanged. + \note Linear time complexity. + */ + void Clear() { + CEREAL_RAPIDJSON_ASSERT(IsArray()); + GenericValue* e = GetElementsPointer(); + for (GenericValue* v = e; v != e + data_.a.size; ++v) + v->~GenericValue(); + data_.a.size = 0; + } + + //! Get an element from array by index. + /*! \pre IsArray() == true + \param index Zero-based index of element. + \see operator[](T*) + */ + GenericValue& operator[](SizeType index) { + CEREAL_RAPIDJSON_ASSERT(IsArray()); + CEREAL_RAPIDJSON_ASSERT(index < data_.a.size); + return GetElementsPointer()[index]; + } + const GenericValue& operator[](SizeType index) const { return const_cast(*this)[index]; } + + //! Element iterator + /*! \pre IsArray() == true */ + ValueIterator Begin() { CEREAL_RAPIDJSON_ASSERT(IsArray()); return GetElementsPointer(); } + //! \em Past-the-end element iterator + /*! \pre IsArray() == true */ + ValueIterator End() { CEREAL_RAPIDJSON_ASSERT(IsArray()); return GetElementsPointer() + data_.a.size; } + //! Constant element iterator + /*! \pre IsArray() == true */ + ConstValueIterator Begin() const { return const_cast(*this).Begin(); } + //! Constant \em past-the-end element iterator + /*! \pre IsArray() == true */ + ConstValueIterator End() const { return const_cast(*this).End(); } + + //! Request the array to have enough capacity to store elements. + /*! \param newCapacity The capacity that the array at least need to have. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \note Linear time complexity. + */ + GenericValue& Reserve(SizeType newCapacity, Allocator &allocator) { + CEREAL_RAPIDJSON_ASSERT(IsArray()); + if (newCapacity > data_.a.capacity) { + SetElementsPointer(reinterpret_cast(allocator.Realloc(GetElementsPointer(), data_.a.capacity * sizeof(GenericValue), newCapacity * sizeof(GenericValue)))); + data_.a.capacity = newCapacity; + } + return *this; + } + + //! Append a GenericValue at the end of the array. + /*! \param value Value to be appended. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \pre IsArray() == true + \post value.IsNull() == true + \return The value itself for fluent API. + \note The ownership of \c value will be transferred to this array on success. + \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. + \note Amortized constant time complexity. + */ + GenericValue& PushBack(GenericValue& value, Allocator& allocator) { + CEREAL_RAPIDJSON_ASSERT(IsArray()); + if (data_.a.size >= data_.a.capacity) + Reserve(data_.a.capacity == 0 ? kDefaultArrayCapacity : (data_.a.capacity + (data_.a.capacity + 1) / 2), allocator); + GetElementsPointer()[data_.a.size++].RawAssign(value); + return *this; + } + +#if CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericValue& PushBack(GenericValue&& value, Allocator& allocator) { + return PushBack(value, allocator); + } +#endif // CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS + + //! Append a constant string reference at the end of the array. + /*! \param value Constant string reference to be appended. + \param allocator Allocator for reallocating memory. It must be the same one used previously. Commonly use GenericDocument::GetAllocator(). + \pre IsArray() == true + \return The value itself for fluent API. + \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. + \note Amortized constant time complexity. + \see GenericStringRef + */ + GenericValue& PushBack(StringRefType value, Allocator& allocator) { + return (*this).template PushBack(value, allocator); + } + + //! Append a primitive value at the end of the array. + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t + \param value Value of primitive type T to be appended. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \pre IsArray() == true + \return The value itself for fluent API. + \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. + + \note The source type \c T explicitly disallows all pointer types, + especially (\c const) \ref Ch*. This helps avoiding implicitly + referencing character strings with insufficient lifetime, use + \ref PushBack(GenericValue&, Allocator&) or \ref + PushBack(StringRefType, Allocator&). + All other pointer types would implicitly convert to \c bool, + use an explicit cast instead, if needed. + \note Amortized constant time complexity. + */ + template + CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericValue&)) + PushBack(T value, Allocator& allocator) { + GenericValue v(value); + return PushBack(v, allocator); + } + + //! Remove the last element in the array. + /*! + \note Constant time complexity. + */ + GenericValue& PopBack() { + CEREAL_RAPIDJSON_ASSERT(IsArray()); + CEREAL_RAPIDJSON_ASSERT(!Empty()); + GetElementsPointer()[--data_.a.size].~GenericValue(); + return *this; + } + + //! Remove an element of array by iterator. + /*! + \param pos iterator to the element to remove + \pre IsArray() == true && \ref Begin() <= \c pos < \ref End() + \return Iterator following the removed element. If the iterator pos refers to the last element, the End() iterator is returned. + \note Linear time complexity. + */ + ValueIterator Erase(ConstValueIterator pos) { + return Erase(pos, pos + 1); + } + + //! Remove elements in the range [first, last) of the array. + /*! + \param first iterator to the first element to remove + \param last iterator following the last element to remove + \pre IsArray() == true && \ref Begin() <= \c first <= \c last <= \ref End() + \return Iterator following the last removed element. + \note Linear time complexity. + */ + ValueIterator Erase(ConstValueIterator first, ConstValueIterator last) { + CEREAL_RAPIDJSON_ASSERT(IsArray()); + CEREAL_RAPIDJSON_ASSERT(data_.a.size > 0); + CEREAL_RAPIDJSON_ASSERT(GetElementsPointer() != 0); + CEREAL_RAPIDJSON_ASSERT(first >= Begin()); + CEREAL_RAPIDJSON_ASSERT(first <= last); + CEREAL_RAPIDJSON_ASSERT(last <= End()); + ValueIterator pos = Begin() + (first - Begin()); + for (ValueIterator itr = pos; itr != last; ++itr) + itr->~GenericValue(); + std::memmove(static_cast(pos), last, static_cast(End() - last) * sizeof(GenericValue)); + data_.a.size -= static_cast(last - first); + return pos; + } + + Array GetArray() { CEREAL_RAPIDJSON_ASSERT(IsArray()); return Array(*this); } + ConstArray GetArray() const { CEREAL_RAPIDJSON_ASSERT(IsArray()); return ConstArray(*this); } + + //@} + + //!@name Number + //@{ + + int GetInt() const { CEREAL_RAPIDJSON_ASSERT(data_.f.flags & kIntFlag); return data_.n.i.i; } + unsigned GetUint() const { CEREAL_RAPIDJSON_ASSERT(data_.f.flags & kUintFlag); return data_.n.u.u; } + int64_t GetInt64() const { CEREAL_RAPIDJSON_ASSERT(data_.f.flags & kInt64Flag); return data_.n.i64; } + uint64_t GetUint64() const { CEREAL_RAPIDJSON_ASSERT(data_.f.flags & kUint64Flag); return data_.n.u64; } + + //! Get the value as double type. + /*! \note If the value is 64-bit integer type, it may lose precision. Use \c IsLosslessDouble() to check whether the converison is lossless. + */ + double GetDouble() const { + CEREAL_RAPIDJSON_ASSERT(IsNumber()); + if ((data_.f.flags & kDoubleFlag) != 0) return data_.n.d; // exact type, no conversion. + if ((data_.f.flags & kIntFlag) != 0) return data_.n.i.i; // int -> double + if ((data_.f.flags & kUintFlag) != 0) return data_.n.u.u; // unsigned -> double + if ((data_.f.flags & kInt64Flag) != 0) return static_cast(data_.n.i64); // int64_t -> double (may lose precision) + CEREAL_RAPIDJSON_ASSERT((data_.f.flags & kUint64Flag) != 0); return static_cast(data_.n.u64); // uint64_t -> double (may lose precision) + } + + //! Get the value as float type. + /*! \note If the value is 64-bit integer type, it may lose precision. Use \c IsLosslessFloat() to check whether the converison is lossless. + */ + float GetFloat() const { + return static_cast(GetDouble()); + } + + GenericValue& SetInt(int i) { this->~GenericValue(); new (this) GenericValue(i); return *this; } + GenericValue& SetUint(unsigned u) { this->~GenericValue(); new (this) GenericValue(u); return *this; } + GenericValue& SetInt64(int64_t i64) { this->~GenericValue(); new (this) GenericValue(i64); return *this; } + GenericValue& SetUint64(uint64_t u64) { this->~GenericValue(); new (this) GenericValue(u64); return *this; } + GenericValue& SetDouble(double d) { this->~GenericValue(); new (this) GenericValue(d); return *this; } + GenericValue& SetFloat(float f) { this->~GenericValue(); new (this) GenericValue(static_cast(f)); return *this; } + + //@} + + //!@name String + //@{ + + const Ch* GetString() const { CEREAL_RAPIDJSON_ASSERT(IsString()); return (data_.f.flags & kInlineStrFlag) ? data_.ss.str : GetStringPointer(); } + + //! Get the length of string. + /*! Since rapidjson permits "\\u0000" in the json string, strlen(v.GetString()) may not equal to v.GetStringLength(). + */ + SizeType GetStringLength() const { CEREAL_RAPIDJSON_ASSERT(IsString()); return ((data_.f.flags & kInlineStrFlag) ? (data_.ss.GetLength()) : data_.s.length); } + + //! Set this value as a string without copying source string. + /*! This version has better performance with supplied length, and also support string containing null character. + \param s source string pointer. + \param length The length of source string, excluding the trailing null terminator. + \return The value itself for fluent API. + \post IsString() == true && GetString() == s && GetStringLength() == length + \see SetString(StringRefType) + */ + GenericValue& SetString(const Ch* s, SizeType length) { return SetString(StringRef(s, length)); } + + //! Set this value as a string without copying source string. + /*! \param s source string reference + \return The value itself for fluent API. + \post IsString() == true && GetString() == s && GetStringLength() == s.length + */ + GenericValue& SetString(StringRefType s) { this->~GenericValue(); SetStringRaw(s); return *this; } + + //! Set this value as a string by copying from source string. + /*! This version has better performance with supplied length, and also support string containing null character. + \param s source string. + \param length The length of source string, excluding the trailing null terminator. + \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length + */ + GenericValue& SetString(const Ch* s, SizeType length, Allocator& allocator) { return SetString(StringRef(s, length), allocator); } + + //! Set this value as a string by copying from source string. + /*! \param s source string. + \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length + */ + GenericValue& SetString(const Ch* s, Allocator& allocator) { return SetString(StringRef(s), allocator); } + + //! Set this value as a string by copying from source string. + /*! \param s source string reference + \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \post IsString() == true && GetString() != s.s && strcmp(GetString(),s) == 0 && GetStringLength() == length + */ + GenericValue& SetString(StringRefType s, Allocator& allocator) { this->~GenericValue(); SetStringRaw(s, allocator); return *this; } + +#if CEREAL_RAPIDJSON_HAS_STDSTRING + //! Set this value as a string by copying from source string. + /*! \param s source string. + \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \post IsString() == true && GetString() != s.data() && strcmp(GetString(),s.data() == 0 && GetStringLength() == s.size() + \note Requires the definition of the preprocessor symbol \ref CEREAL_RAPIDJSON_HAS_STDSTRING. + */ + GenericValue& SetString(const std::basic_string& s, Allocator& allocator) { return SetString(StringRef(s), allocator); } +#endif + + //@} + + //!@name Array + //@{ + + //! Templated version for checking whether this value is type T. + /*! + \tparam T Either \c bool, \c int, \c unsigned, \c int64_t, \c uint64_t, \c double, \c float, \c const \c char*, \c std::basic_string + */ + template + bool Is() const { return internal::TypeHelper::Is(*this); } + + template + T Get() const { return internal::TypeHelper::Get(*this); } + + template + T Get() { return internal::TypeHelper::Get(*this); } + + template + ValueType& Set(const T& data) { return internal::TypeHelper::Set(*this, data); } + + template + ValueType& Set(const T& data, AllocatorType& allocator) { return internal::TypeHelper::Set(*this, data, allocator); } + + //@} + + //! Generate events of this value to a Handler. + /*! This function adopts the GoF visitor pattern. + Typical usage is to output this JSON value as JSON text via Writer, which is a Handler. + It can also be used to deep clone this value via GenericDocument, which is also a Handler. + \tparam Handler type of handler. + \param handler An object implementing concept Handler. + */ + template + bool Accept(Handler& handler) const { + switch(GetType()) { + case kNullType: return handler.Null(); + case kFalseType: return handler.Bool(false); + case kTrueType: return handler.Bool(true); + + case kObjectType: + if (CEREAL_RAPIDJSON_UNLIKELY(!handler.StartObject())) + return false; + for (ConstMemberIterator m = MemberBegin(); m != MemberEnd(); ++m) { + CEREAL_RAPIDJSON_ASSERT(m->name.IsString()); // User may change the type of name by MemberIterator. + if (CEREAL_RAPIDJSON_UNLIKELY(!handler.Key(m->name.GetString(), m->name.GetStringLength(), (m->name.data_.f.flags & kCopyFlag) != 0))) + return false; + if (CEREAL_RAPIDJSON_UNLIKELY(!m->value.Accept(handler))) + return false; + } + return handler.EndObject(data_.o.size); + + case kArrayType: + if (CEREAL_RAPIDJSON_UNLIKELY(!handler.StartArray())) + return false; + for (const GenericValue* v = Begin(); v != End(); ++v) + if (CEREAL_RAPIDJSON_UNLIKELY(!v->Accept(handler))) + return false; + return handler.EndArray(data_.a.size); + + case kStringType: + return handler.String(GetString(), GetStringLength(), (data_.f.flags & kCopyFlag) != 0); + + default: + CEREAL_RAPIDJSON_ASSERT(GetType() == kNumberType); + if (IsDouble()) return handler.Double(data_.n.d); + else if (IsInt()) return handler.Int(data_.n.i.i); + else if (IsUint()) return handler.Uint(data_.n.u.u); + else if (IsInt64()) return handler.Int64(data_.n.i64); + else return handler.Uint64(data_.n.u64); + } + } + +private: + template friend class GenericValue; + template friend class GenericDocument; + + enum { + kBoolFlag = 0x0008, + kNumberFlag = 0x0010, + kIntFlag = 0x0020, + kUintFlag = 0x0040, + kInt64Flag = 0x0080, + kUint64Flag = 0x0100, + kDoubleFlag = 0x0200, + kStringFlag = 0x0400, + kCopyFlag = 0x0800, + kInlineStrFlag = 0x1000, + + // Initial flags of different types. + kNullFlag = kNullType, + kTrueFlag = kTrueType | kBoolFlag, + kFalseFlag = kFalseType | kBoolFlag, + kNumberIntFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag, + kNumberUintFlag = kNumberType | kNumberFlag | kUintFlag | kUint64Flag | kInt64Flag, + kNumberInt64Flag = kNumberType | kNumberFlag | kInt64Flag, + kNumberUint64Flag = kNumberType | kNumberFlag | kUint64Flag, + kNumberDoubleFlag = kNumberType | kNumberFlag | kDoubleFlag, + kNumberAnyFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag | kUintFlag | kUint64Flag | kDoubleFlag, + kConstStringFlag = kStringType | kStringFlag, + kCopyStringFlag = kStringType | kStringFlag | kCopyFlag, + kShortStringFlag = kStringType | kStringFlag | kCopyFlag | kInlineStrFlag, + kObjectFlag = kObjectType, + kArrayFlag = kArrayType, + + kTypeMask = 0x07 + }; + + static const SizeType kDefaultArrayCapacity = 16; + static const SizeType kDefaultObjectCapacity = 16; + + struct Flag { +#if CEREAL_RAPIDJSON_48BITPOINTER_OPTIMIZATION + char payload[sizeof(SizeType) * 2 + 6]; // 2 x SizeType + lower 48-bit pointer +#elif CEREAL_RAPIDJSON_64BIT + char payload[sizeof(SizeType) * 2 + sizeof(void*) + 6]; // 6 padding bytes +#else + char payload[sizeof(SizeType) * 2 + sizeof(void*) + 2]; // 2 padding bytes +#endif + uint16_t flags; + }; + + struct String { + SizeType length; + SizeType hashcode; //!< reserved + const Ch* str; + }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + + // implementation detail: ShortString can represent zero-terminated strings up to MaxSize chars + // (excluding the terminating zero) and store a value to determine the length of the contained + // string in the last character str[LenPos] by storing "MaxSize - length" there. If the string + // to store has the maximal length of MaxSize then str[LenPos] will be 0 and therefore act as + // the string terminator as well. For getting the string length back from that value just use + // "MaxSize - str[LenPos]". + // This allows to store 13-chars strings in 32-bit mode, 21-chars strings in 64-bit mode, + // 13-chars strings for CEREAL_RAPIDJSON_48BITPOINTER_OPTIMIZATION=1 inline (for `UTF8`-encoded strings). + struct ShortString { + enum { MaxChars = sizeof(static_cast(0)->payload) / sizeof(Ch), MaxSize = MaxChars - 1, LenPos = MaxSize }; + Ch str[MaxChars]; + + inline static bool Usable(SizeType len) { return (MaxSize >= len); } + inline void SetLength(SizeType len) { str[LenPos] = static_cast(MaxSize - len); } + inline SizeType GetLength() const { return static_cast(MaxSize - str[LenPos]); } + }; // at most as many bytes as "String" above => 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + + // By using proper binary layout, retrieval of different integer types do not need conversions. + union Number { +#if CEREAL_RAPIDJSON_ENDIAN == CEREAL_RAPIDJSON_LITTLEENDIAN + struct I { + int i; + char padding[4]; + }i; + struct U { + unsigned u; + char padding2[4]; + }u; +#else + struct I { + char padding[4]; + int i; + }i; + struct U { + char padding2[4]; + unsigned u; + }u; +#endif + int64_t i64; + uint64_t u64; + double d; + }; // 8 bytes + + struct ObjectData { + SizeType size; + SizeType capacity; + Member* members; + }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + + struct ArrayData { + SizeType size; + SizeType capacity; + GenericValue* elements; + }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + + union Data { + String s; + ShortString ss; + Number n; + ObjectData o; + ArrayData a; + Flag f; + }; // 16 bytes in 32-bit mode, 24 bytes in 64-bit mode, 16 bytes in 64-bit with CEREAL_RAPIDJSON_48BITPOINTER_OPTIMIZATION + + CEREAL_RAPIDJSON_FORCEINLINE const Ch* GetStringPointer() const { return CEREAL_RAPIDJSON_GETPOINTER(Ch, data_.s.str); } + CEREAL_RAPIDJSON_FORCEINLINE const Ch* SetStringPointer(const Ch* str) { return CEREAL_RAPIDJSON_SETPOINTER(Ch, data_.s.str, str); } + CEREAL_RAPIDJSON_FORCEINLINE GenericValue* GetElementsPointer() const { return CEREAL_RAPIDJSON_GETPOINTER(GenericValue, data_.a.elements); } + CEREAL_RAPIDJSON_FORCEINLINE GenericValue* SetElementsPointer(GenericValue* elements) { return CEREAL_RAPIDJSON_SETPOINTER(GenericValue, data_.a.elements, elements); } + CEREAL_RAPIDJSON_FORCEINLINE Member* GetMembersPointer() const { return CEREAL_RAPIDJSON_GETPOINTER(Member, data_.o.members); } + CEREAL_RAPIDJSON_FORCEINLINE Member* SetMembersPointer(Member* members) { return CEREAL_RAPIDJSON_SETPOINTER(Member, data_.o.members, members); } + + // Initialize this value as array with initial data, without calling destructor. + void SetArrayRaw(GenericValue* values, SizeType count, Allocator& allocator) { + data_.f.flags = kArrayFlag; + if (count) { + GenericValue* e = static_cast(allocator.Malloc(count * sizeof(GenericValue))); + SetElementsPointer(e); + std::memcpy(static_cast(e), values, count * sizeof(GenericValue)); + } + else + SetElementsPointer(0); + data_.a.size = data_.a.capacity = count; + } + + //! Initialize this value as object with initial data, without calling destructor. + void SetObjectRaw(Member* members, SizeType count, Allocator& allocator) { + data_.f.flags = kObjectFlag; + if (count) { + Member* m = static_cast(allocator.Malloc(count * sizeof(Member))); + SetMembersPointer(m); + std::memcpy(static_cast(m), members, count * sizeof(Member)); + } + else + SetMembersPointer(0); + data_.o.size = data_.o.capacity = count; + } + + //! Initialize this value as constant string, without calling destructor. + void SetStringRaw(StringRefType s) CEREAL_RAPIDJSON_NOEXCEPT { + data_.f.flags = kConstStringFlag; + SetStringPointer(s); + data_.s.length = s.length; + } + + //! Initialize this value as copy string with initial data, without calling destructor. + void SetStringRaw(StringRefType s, Allocator& allocator) { + Ch* str = 0; + if (ShortString::Usable(s.length)) { + data_.f.flags = kShortStringFlag; + data_.ss.SetLength(s.length); + str = data_.ss.str; + } else { + data_.f.flags = kCopyStringFlag; + data_.s.length = s.length; + str = static_cast(allocator.Malloc((s.length + 1) * sizeof(Ch))); + SetStringPointer(str); + } + std::memcpy(str, s, s.length * sizeof(Ch)); + str[s.length] = '\0'; + } + + //! Assignment without calling destructor + void RawAssign(GenericValue& rhs) CEREAL_RAPIDJSON_NOEXCEPT { + data_ = rhs.data_; + // data_.f.flags = rhs.data_.f.flags; + rhs.data_.f.flags = kNullFlag; + } + + template + bool StringEqual(const GenericValue& rhs) const { + CEREAL_RAPIDJSON_ASSERT(IsString()); + CEREAL_RAPIDJSON_ASSERT(rhs.IsString()); + + const SizeType len1 = GetStringLength(); + const SizeType len2 = rhs.GetStringLength(); + if(len1 != len2) { return false; } + + const Ch* const str1 = GetString(); + const Ch* const str2 = rhs.GetString(); + if(str1 == str2) { return true; } // fast path for constant string + + return (std::memcmp(str1, str2, sizeof(Ch) * len1) == 0); + } + + Data data_; +}; + +//! GenericValue with UTF8 encoding +typedef GenericValue > Value; + +/////////////////////////////////////////////////////////////////////////////// +// GenericDocument + +//! A document for parsing JSON text as DOM. +/*! + \note implements Handler concept + \tparam Encoding Encoding for both parsing and string storage. + \tparam Allocator Allocator for allocating memory for the DOM + \tparam StackAllocator Allocator for allocating memory for stack during parsing. + \warning Although GenericDocument inherits from GenericValue, the API does \b not provide any virtual functions, especially no virtual destructor. To avoid memory leaks, do not \c delete a GenericDocument object via a pointer to a GenericValue. +*/ +template , typename StackAllocator = CrtAllocator> +class GenericDocument : public GenericValue { +public: + typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. + typedef GenericValue ValueType; //!< Value type of the document. + typedef Allocator AllocatorType; //!< Allocator type from template parameter. + + //! Constructor + /*! Creates an empty document of specified type. + \param type Mandatory type of object to create. + \param allocator Optional allocator for allocating memory. + \param stackCapacity Optional initial capacity of stack in bytes. + \param stackAllocator Optional allocator for allocating memory for stack. + */ + explicit GenericDocument(Type type, Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity, StackAllocator* stackAllocator = 0) : + GenericValue(type), allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() + { + if (!allocator_) + ownAllocator_ = allocator_ = CEREAL_RAPIDJSON_NEW(Allocator)(); + } + + //! Constructor + /*! Creates an empty document which type is Null. + \param allocator Optional allocator for allocating memory. + \param stackCapacity Optional initial capacity of stack in bytes. + \param stackAllocator Optional allocator for allocating memory for stack. + */ + GenericDocument(Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity, StackAllocator* stackAllocator = 0) : + allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() + { + if (!allocator_) + ownAllocator_ = allocator_ = CEREAL_RAPIDJSON_NEW(Allocator)(); + } + +#if CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move constructor in C++11 + GenericDocument(GenericDocument&& rhs) CEREAL_RAPIDJSON_NOEXCEPT + : ValueType(std::forward(rhs)), // explicit cast to avoid prohibited move from Document + allocator_(rhs.allocator_), + ownAllocator_(rhs.ownAllocator_), + stack_(std::move(rhs.stack_)), + parseResult_(rhs.parseResult_) + { + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + rhs.parseResult_ = ParseResult(); + } +#endif + + ~GenericDocument() { + Destroy(); + } + +#if CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move assignment in C++11 + GenericDocument& operator=(GenericDocument&& rhs) CEREAL_RAPIDJSON_NOEXCEPT + { + // The cast to ValueType is necessary here, because otherwise it would + // attempt to call GenericValue's templated assignment operator. + ValueType::operator=(std::forward(rhs)); + + // Calling the destructor here would prematurely call stack_'s destructor + Destroy(); + + allocator_ = rhs.allocator_; + ownAllocator_ = rhs.ownAllocator_; + stack_ = std::move(rhs.stack_); + parseResult_ = rhs.parseResult_; + + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + rhs.parseResult_ = ParseResult(); + + return *this; + } +#endif + + //! Exchange the contents of this document with those of another. + /*! + \param rhs Another document. + \note Constant complexity. + \see GenericValue::Swap + */ + GenericDocument& Swap(GenericDocument& rhs) CEREAL_RAPIDJSON_NOEXCEPT { + ValueType::Swap(rhs); + stack_.Swap(rhs.stack_); + internal::Swap(allocator_, rhs.allocator_); + internal::Swap(ownAllocator_, rhs.ownAllocator_); + internal::Swap(parseResult_, rhs.parseResult_); + return *this; + } + + // Allow Swap with ValueType. + // Refer to Effective C++ 3rd Edition/Item 33: Avoid hiding inherited names. + using ValueType::Swap; + + //! free-standing swap function helper + /*! + Helper function to enable support for common swap implementation pattern based on \c std::swap: + \code + void swap(MyClass& a, MyClass& b) { + using std::swap; + swap(a.doc, b.doc); + // ... + } + \endcode + \see Swap() + */ + friend inline void swap(GenericDocument& a, GenericDocument& b) CEREAL_RAPIDJSON_NOEXCEPT { a.Swap(b); } + + //! Populate this document by a generator which produces SAX events. + /*! \tparam Generator A functor with bool f(Handler) prototype. + \param g Generator functor which sends SAX events to the parameter. + \return The document itself for fluent API. + */ + template + GenericDocument& Populate(Generator& g) { + ClearStackOnExit scope(*this); + if (g(*this)) { + CEREAL_RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object + ValueType::operator=(*stack_.template Pop(1));// Move value from stack to document + } + return *this; + } + + //!@name Parse from stream + //!@{ + + //! Parse JSON text from an input stream (with Encoding conversion) + /*! \tparam parseFlags Combination of \ref ParseFlag. + \tparam SourceEncoding Encoding of input stream + \tparam InputStream Type of input stream, implementing Stream concept + \param is Input stream to be parsed. + \return The document itself for fluent API. + */ + template + GenericDocument& ParseStream(InputStream& is) { + GenericReader reader( + stack_.HasAllocator() ? &stack_.GetAllocator() : 0); + ClearStackOnExit scope(*this); + parseResult_ = reader.template Parse(is, *this); + if (parseResult_) { + CEREAL_RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object + ValueType::operator=(*stack_.template Pop(1));// Move value from stack to document + } + return *this; + } + + //! Parse JSON text from an input stream + /*! \tparam parseFlags Combination of \ref ParseFlag. + \tparam InputStream Type of input stream, implementing Stream concept + \param is Input stream to be parsed. + \return The document itself for fluent API. + */ + template + GenericDocument& ParseStream(InputStream& is) { + return ParseStream(is); + } + + //! Parse JSON text from an input stream (with \ref kParseDefaultFlags) + /*! \tparam InputStream Type of input stream, implementing Stream concept + \param is Input stream to be parsed. + \return The document itself for fluent API. + */ + template + GenericDocument& ParseStream(InputStream& is) { + return ParseStream(is); + } + //!@} + + //!@name Parse in-place from mutable string + //!@{ + + //! Parse JSON text from a mutable string + /*! \tparam parseFlags Combination of \ref ParseFlag. + \param str Mutable zero-terminated string to be parsed. + \return The document itself for fluent API. + */ + template + GenericDocument& ParseInsitu(Ch* str) { + GenericInsituStringStream s(str); + return ParseStream(s); + } + + //! Parse JSON text from a mutable string (with \ref kParseDefaultFlags) + /*! \param str Mutable zero-terminated string to be parsed. + \return The document itself for fluent API. + */ + GenericDocument& ParseInsitu(Ch* str) { + return ParseInsitu(str); + } + //!@} + + //!@name Parse from read-only string + //!@{ + + //! Parse JSON text from a read-only string (with Encoding conversion) + /*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag). + \tparam SourceEncoding Transcoding from input Encoding + \param str Read-only zero-terminated string to be parsed. + */ + template + GenericDocument& Parse(const typename SourceEncoding::Ch* str) { + CEREAL_RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); + GenericStringStream s(str); + return ParseStream(s); + } + + //! Parse JSON text from a read-only string + /*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag). + \param str Read-only zero-terminated string to be parsed. + */ + template + GenericDocument& Parse(const Ch* str) { + return Parse(str); + } + + //! Parse JSON text from a read-only string (with \ref kParseDefaultFlags) + /*! \param str Read-only zero-terminated string to be parsed. + */ + GenericDocument& Parse(const Ch* str) { + return Parse(str); + } + + template + GenericDocument& Parse(const typename SourceEncoding::Ch* str, size_t length) { + CEREAL_RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); + MemoryStream ms(reinterpret_cast(str), length * sizeof(typename SourceEncoding::Ch)); + EncodedInputStream is(ms); + ParseStream(is); + return *this; + } + + template + GenericDocument& Parse(const Ch* str, size_t length) { + return Parse(str, length); + } + + GenericDocument& Parse(const Ch* str, size_t length) { + return Parse(str, length); + } + +#if CEREAL_RAPIDJSON_HAS_STDSTRING + template + GenericDocument& Parse(const std::basic_string& str) { + // c_str() is constant complexity according to standard. Should be faster than Parse(const char*, size_t) + return Parse(str.c_str()); + } + + template + GenericDocument& Parse(const std::basic_string& str) { + return Parse(str.c_str()); + } + + GenericDocument& Parse(const std::basic_string& str) { + return Parse(str); + } +#endif // CEREAL_RAPIDJSON_HAS_STDSTRING + + //!@} + + //!@name Handling parse errors + //!@{ + + //! Whether a parse error has occurred in the last parsing. + bool HasParseError() const { return parseResult_.IsError(); } + + //! Get the \ref ParseErrorCode of last parsing. + ParseErrorCode GetParseError() const { return parseResult_.Code(); } + + //! Get the position of last parsing error in input, 0 otherwise. + size_t GetErrorOffset() const { return parseResult_.Offset(); } + + //! Implicit conversion to get the last parse result +#ifndef __clang // -Wdocumentation + /*! \return \ref ParseResult of the last parse operation + + \code + Document doc; + ParseResult ok = doc.Parse(json); + if (!ok) + printf( "JSON parse error: %s (%u)\n", GetParseError_En(ok.Code()), ok.Offset()); + \endcode + */ +#endif + operator ParseResult() const { return parseResult_; } + //!@} + + //! Get the allocator of this document. + Allocator& GetAllocator() { + CEREAL_RAPIDJSON_ASSERT(allocator_); + return *allocator_; + } + + //! Get the capacity of stack in bytes. + size_t GetStackCapacity() const { return stack_.GetCapacity(); } + +private: + // clear stack on any exit from ParseStream, e.g. due to exception + struct ClearStackOnExit { + explicit ClearStackOnExit(GenericDocument& d) : d_(d) {} + ~ClearStackOnExit() { d_.ClearStack(); } + private: + ClearStackOnExit(const ClearStackOnExit&); + ClearStackOnExit& operator=(const ClearStackOnExit&); + GenericDocument& d_; + }; + + // callers of the following private Handler functions + // template friend class GenericReader; // for parsing + template friend class GenericValue; // for deep copying + +public: + // Implementation of Handler + bool Null() { new (stack_.template Push()) ValueType(); return true; } + bool Bool(bool b) { new (stack_.template Push()) ValueType(b); return true; } + bool Int(int i) { new (stack_.template Push()) ValueType(i); return true; } + bool Uint(unsigned i) { new (stack_.template Push()) ValueType(i); return true; } + bool Int64(int64_t i) { new (stack_.template Push()) ValueType(i); return true; } + bool Uint64(uint64_t i) { new (stack_.template Push()) ValueType(i); return true; } + bool Double(double d) { new (stack_.template Push()) ValueType(d); return true; } + + bool RawNumber(const Ch* str, SizeType length, bool copy) { + if (copy) + new (stack_.template Push()) ValueType(str, length, GetAllocator()); + else + new (stack_.template Push()) ValueType(str, length); + return true; + } + + bool String(const Ch* str, SizeType length, bool copy) { + if (copy) + new (stack_.template Push()) ValueType(str, length, GetAllocator()); + else + new (stack_.template Push()) ValueType(str, length); + return true; + } + + bool StartObject() { new (stack_.template Push()) ValueType(kObjectType); return true; } + + bool Key(const Ch* str, SizeType length, bool copy) { return String(str, length, copy); } + + bool EndObject(SizeType memberCount) { + typename ValueType::Member* members = stack_.template Pop(memberCount); + stack_.template Top()->SetObjectRaw(members, memberCount, GetAllocator()); + return true; + } + + bool StartArray() { new (stack_.template Push()) ValueType(kArrayType); return true; } + + bool EndArray(SizeType elementCount) { + ValueType* elements = stack_.template Pop(elementCount); + stack_.template Top()->SetArrayRaw(elements, elementCount, GetAllocator()); + return true; + } + +private: + //! Prohibit copying + GenericDocument(const GenericDocument&); + //! Prohibit assignment + GenericDocument& operator=(const GenericDocument&); + + void ClearStack() { + if (Allocator::kNeedFree) + while (stack_.GetSize() > 0) // Here assumes all elements in stack array are GenericValue (Member is actually 2 GenericValue objects) + (stack_.template Pop(1))->~ValueType(); + else + stack_.Clear(); + stack_.ShrinkToFit(); + } + + void Destroy() { + CEREAL_RAPIDJSON_DELETE(ownAllocator_); + } + + static const size_t kDefaultStackCapacity = 1024; + Allocator* allocator_; + Allocator* ownAllocator_; + internal::Stack stack_; + ParseResult parseResult_; +}; + +//! GenericDocument with UTF8 encoding +typedef GenericDocument > Document; + +//! Helper class for accessing Value of array type. +/*! + Instance of this helper class is obtained by \c GenericValue::GetArray(). + In addition to all APIs for array type, it provides range-based for loop if \c CEREAL_RAPIDJSON_HAS_CXX11_RANGE_FOR=1. +*/ +template +class GenericArray { +public: + typedef GenericArray ConstArray; + typedef GenericArray Array; + typedef ValueT PlainType; + typedef typename internal::MaybeAddConst::Type ValueType; + typedef ValueType* ValueIterator; // This may be const or non-const iterator + typedef const ValueT* ConstValueIterator; + typedef typename ValueType::AllocatorType AllocatorType; + typedef typename ValueType::StringRefType StringRefType; + + template + friend class GenericValue; + + GenericArray(const GenericArray& rhs) : value_(rhs.value_) {} + GenericArray& operator=(const GenericArray& rhs) { value_ = rhs.value_; return *this; } + ~GenericArray() {} + + SizeType Size() const { return value_.Size(); } + SizeType Capacity() const { return value_.Capacity(); } + bool Empty() const { return value_.Empty(); } + void Clear() const { value_.Clear(); } + ValueType& operator[](SizeType index) const { return value_[index]; } + ValueIterator Begin() const { return value_.Begin(); } + ValueIterator End() const { return value_.End(); } + GenericArray Reserve(SizeType newCapacity, AllocatorType &allocator) const { value_.Reserve(newCapacity, allocator); return *this; } + GenericArray PushBack(ValueType& value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } +#if CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericArray PushBack(ValueType&& value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } +#endif // CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericArray PushBack(StringRefType value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } + template CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (const GenericArray&)) PushBack(T value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } + GenericArray PopBack() const { value_.PopBack(); return *this; } + ValueIterator Erase(ConstValueIterator pos) const { return value_.Erase(pos); } + ValueIterator Erase(ConstValueIterator first, ConstValueIterator last) const { return value_.Erase(first, last); } + +#if CEREAL_RAPIDJSON_HAS_CXX11_RANGE_FOR + ValueIterator begin() const { return value_.Begin(); } + ValueIterator end() const { return value_.End(); } +#endif + +private: + GenericArray(); + GenericArray(ValueType& value) : value_(value) {} + ValueType& value_; +}; + +//! Helper class for accessing Value of object type. +/*! + Instance of this helper class is obtained by \c GenericValue::GetObject(). + In addition to all APIs for array type, it provides range-based for loop if \c CEREAL_RAPIDJSON_HAS_CXX11_RANGE_FOR=1. +*/ +template +class GenericObject { +public: + typedef GenericObject ConstObject; + typedef GenericObject Object; + typedef ValueT PlainType; + typedef typename internal::MaybeAddConst::Type ValueType; + typedef GenericMemberIterator MemberIterator; // This may be const or non-const iterator + typedef GenericMemberIterator ConstMemberIterator; + typedef typename ValueType::AllocatorType AllocatorType; + typedef typename ValueType::StringRefType StringRefType; + typedef typename ValueType::EncodingType EncodingType; + typedef typename ValueType::Ch Ch; + + template + friend class GenericValue; + + GenericObject(const GenericObject& rhs) : value_(rhs.value_) {} + GenericObject& operator=(const GenericObject& rhs) { value_ = rhs.value_; return *this; } + ~GenericObject() {} + + SizeType MemberCount() const { return value_.MemberCount(); } + SizeType MemberCapacity() const { return value_.MemberCapacity(); } + bool ObjectEmpty() const { return value_.ObjectEmpty(); } + template ValueType& operator[](T* name) const { return value_[name]; } + template ValueType& operator[](const GenericValue& name) const { return value_[name]; } +#if CEREAL_RAPIDJSON_HAS_STDSTRING + ValueType& operator[](const std::basic_string& name) const { return value_[name]; } +#endif + MemberIterator MemberBegin() const { return value_.MemberBegin(); } + MemberIterator MemberEnd() const { return value_.MemberEnd(); } + GenericObject MemberReserve(SizeType newCapacity, AllocatorType &allocator) const { value_.MemberReserve(newCapacity, allocator); return *this; } + bool HasMember(const Ch* name) const { return value_.HasMember(name); } +#if CEREAL_RAPIDJSON_HAS_STDSTRING + bool HasMember(const std::basic_string& name) const { return value_.HasMember(name); } +#endif + template bool HasMember(const GenericValue& name) const { return value_.HasMember(name); } + MemberIterator FindMember(const Ch* name) const { return value_.FindMember(name); } + template MemberIterator FindMember(const GenericValue& name) const { return value_.FindMember(name); } +#if CEREAL_RAPIDJSON_HAS_STDSTRING + MemberIterator FindMember(const std::basic_string& name) const { return value_.FindMember(name); } +#endif + GenericObject AddMember(ValueType& name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(ValueType& name, StringRefType value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } +#if CEREAL_RAPIDJSON_HAS_STDSTRING + GenericObject AddMember(ValueType& name, std::basic_string& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } +#endif + template CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) AddMember(ValueType& name, T value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } +#if CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericObject AddMember(ValueType&& name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(ValueType&& name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(ValueType& name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(StringRefType name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } +#endif // CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericObject AddMember(StringRefType name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(StringRefType name, StringRefType value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + template CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericObject)) AddMember(StringRefType name, T value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + void RemoveAllMembers() { value_.RemoveAllMembers(); } + bool RemoveMember(const Ch* name) const { return value_.RemoveMember(name); } +#if CEREAL_RAPIDJSON_HAS_STDSTRING + bool RemoveMember(const std::basic_string& name) const { return value_.RemoveMember(name); } +#endif + template bool RemoveMember(const GenericValue& name) const { return value_.RemoveMember(name); } + MemberIterator RemoveMember(MemberIterator m) const { return value_.RemoveMember(m); } + MemberIterator EraseMember(ConstMemberIterator pos) const { return value_.EraseMember(pos); } + MemberIterator EraseMember(ConstMemberIterator first, ConstMemberIterator last) const { return value_.EraseMember(first, last); } + bool EraseMember(const Ch* name) const { return value_.EraseMember(name); } +#if CEREAL_RAPIDJSON_HAS_STDSTRING + bool EraseMember(const std::basic_string& name) const { return EraseMember(ValueType(StringRef(name))); } +#endif + template bool EraseMember(const GenericValue& name) const { return value_.EraseMember(name); } + +#if CEREAL_RAPIDJSON_HAS_CXX11_RANGE_FOR + MemberIterator begin() const { return value_.MemberBegin(); } + MemberIterator end() const { return value_.MemberEnd(); } +#endif + +private: + GenericObject(); + GenericObject(ValueType& value) : value_(value) {} + ValueType& value_; +}; + +CEREAL_RAPIDJSON_NAMESPACE_END +CEREAL_RAPIDJSON_DIAG_POP + +#endif // CEREAL_RAPIDJSON_DOCUMENT_H_ diff --git a/third_party/cereal/external/rapidjson/encodedstream.h b/third_party/cereal/external/rapidjson/encodedstream.h new file mode 100755 index 0000000..b04774b --- /dev/null +++ b/third_party/cereal/external/rapidjson/encodedstream.h @@ -0,0 +1,299 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_ENCODEDSTREAM_H_ +#define CEREAL_RAPIDJSON_ENCODEDSTREAM_H_ + +#include "stream.h" +#include "memorystream.h" + +#ifdef __GNUC__ +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(effc++) +#endif + +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(padded) +#endif + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN + +//! Input byte stream wrapper with a statically bound encoding. +/*! + \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE. + \tparam InputByteStream Type of input byte stream. For example, FileReadStream. +*/ +template +class EncodedInputStream { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); +public: + typedef typename Encoding::Ch Ch; + + EncodedInputStream(InputByteStream& is) : is_(is) { + current_ = Encoding::TakeBOM(is_); + } + + Ch Peek() const { return current_; } + Ch Take() { Ch c = current_; current_ = Encoding::Take(is_); return c; } + size_t Tell() const { return is_.Tell(); } + + // Not implemented + void Put(Ch) { CEREAL_RAPIDJSON_ASSERT(false); } + void Flush() { CEREAL_RAPIDJSON_ASSERT(false); } + Ch* PutBegin() { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + +private: + EncodedInputStream(const EncodedInputStream&); + EncodedInputStream& operator=(const EncodedInputStream&); + + InputByteStream& is_; + Ch current_; +}; + +//! Specialized for UTF8 MemoryStream. +template <> +class EncodedInputStream, MemoryStream> { +public: + typedef UTF8<>::Ch Ch; + + EncodedInputStream(MemoryStream& is) : is_(is) { + if (static_cast(is_.Peek()) == 0xEFu) is_.Take(); + if (static_cast(is_.Peek()) == 0xBBu) is_.Take(); + if (static_cast(is_.Peek()) == 0xBFu) is_.Take(); + } + Ch Peek() const { return is_.Peek(); } + Ch Take() { return is_.Take(); } + size_t Tell() const { return is_.Tell(); } + + // Not implemented + void Put(Ch) {} + void Flush() {} + Ch* PutBegin() { return 0; } + size_t PutEnd(Ch*) { return 0; } + + MemoryStream& is_; + +private: + EncodedInputStream(const EncodedInputStream&); + EncodedInputStream& operator=(const EncodedInputStream&); +}; + +//! Output byte stream wrapper with statically bound encoding. +/*! + \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE. + \tparam OutputByteStream Type of input byte stream. For example, FileWriteStream. +*/ +template +class EncodedOutputStream { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); +public: + typedef typename Encoding::Ch Ch; + + EncodedOutputStream(OutputByteStream& os, bool putBOM = true) : os_(os) { + if (putBOM) + Encoding::PutBOM(os_); + } + + void Put(Ch c) { Encoding::Put(os_, c); } + void Flush() { os_.Flush(); } + + // Not implemented + Ch Peek() const { CEREAL_RAPIDJSON_ASSERT(false); return 0;} + Ch Take() { CEREAL_RAPIDJSON_ASSERT(false); return 0;} + size_t Tell() const { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + Ch* PutBegin() { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + +private: + EncodedOutputStream(const EncodedOutputStream&); + EncodedOutputStream& operator=(const EncodedOutputStream&); + + OutputByteStream& os_; +}; + +#define CEREAL_RAPIDJSON_ENCODINGS_FUNC(x) UTF8::x, UTF16LE::x, UTF16BE::x, UTF32LE::x, UTF32BE::x + +//! Input stream wrapper with dynamically bound encoding and automatic encoding detection. +/*! + \tparam CharType Type of character for reading. + \tparam InputByteStream type of input byte stream to be wrapped. +*/ +template +class AutoUTFInputStream { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); +public: + typedef CharType Ch; + + //! Constructor. + /*! + \param is input stream to be wrapped. + \param type UTF encoding type if it is not detected from the stream. + */ + AutoUTFInputStream(InputByteStream& is, UTFType type = kUTF8) : is_(&is), type_(type), hasBOM_(false) { + CEREAL_RAPIDJSON_ASSERT(type >= kUTF8 && type <= kUTF32BE); + DetectType(); + static const TakeFunc f[] = { CEREAL_RAPIDJSON_ENCODINGS_FUNC(Take) }; + takeFunc_ = f[type_]; + current_ = takeFunc_(*is_); + } + + UTFType GetType() const { return type_; } + bool HasBOM() const { return hasBOM_; } + + Ch Peek() const { return current_; } + Ch Take() { Ch c = current_; current_ = takeFunc_(*is_); return c; } + size_t Tell() const { return is_->Tell(); } + + // Not implemented + void Put(Ch) { CEREAL_RAPIDJSON_ASSERT(false); } + void Flush() { CEREAL_RAPIDJSON_ASSERT(false); } + Ch* PutBegin() { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + +private: + AutoUTFInputStream(const AutoUTFInputStream&); + AutoUTFInputStream& operator=(const AutoUTFInputStream&); + + // Detect encoding type with BOM or RFC 4627 + void DetectType() { + // BOM (Byte Order Mark): + // 00 00 FE FF UTF-32BE + // FF FE 00 00 UTF-32LE + // FE FF UTF-16BE + // FF FE UTF-16LE + // EF BB BF UTF-8 + + const unsigned char* c = reinterpret_cast(is_->Peek4()); + if (!c) + return; + + unsigned bom = static_cast(c[0] | (c[1] << 8) | (c[2] << 16) | (c[3] << 24)); + hasBOM_ = false; + if (bom == 0xFFFE0000) { type_ = kUTF32BE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } + else if (bom == 0x0000FEFF) { type_ = kUTF32LE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } + else if ((bom & 0xFFFF) == 0xFFFE) { type_ = kUTF16BE; hasBOM_ = true; is_->Take(); is_->Take(); } + else if ((bom & 0xFFFF) == 0xFEFF) { type_ = kUTF16LE; hasBOM_ = true; is_->Take(); is_->Take(); } + else if ((bom & 0xFFFFFF) == 0xBFBBEF) { type_ = kUTF8; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); } + + // RFC 4627: Section 3 + // "Since the first two characters of a JSON text will always be ASCII + // characters [RFC0020], it is possible to determine whether an octet + // stream is UTF-8, UTF-16 (BE or LE), or UTF-32 (BE or LE) by looking + // at the pattern of nulls in the first four octets." + // 00 00 00 xx UTF-32BE + // 00 xx 00 xx UTF-16BE + // xx 00 00 00 UTF-32LE + // xx 00 xx 00 UTF-16LE + // xx xx xx xx UTF-8 + + if (!hasBOM_) { + int pattern = (c[0] ? 1 : 0) | (c[1] ? 2 : 0) | (c[2] ? 4 : 0) | (c[3] ? 8 : 0); + switch (pattern) { + case 0x08: type_ = kUTF32BE; break; + case 0x0A: type_ = kUTF16BE; break; + case 0x01: type_ = kUTF32LE; break; + case 0x05: type_ = kUTF16LE; break; + case 0x0F: type_ = kUTF8; break; + default: break; // Use type defined by user. + } + } + + // Runtime check whether the size of character type is sufficient. It only perform checks with assertion. + if (type_ == kUTF16LE || type_ == kUTF16BE) CEREAL_RAPIDJSON_ASSERT(sizeof(Ch) >= 2); + if (type_ == kUTF32LE || type_ == kUTF32BE) CEREAL_RAPIDJSON_ASSERT(sizeof(Ch) >= 4); + } + + typedef Ch (*TakeFunc)(InputByteStream& is); + InputByteStream* is_; + UTFType type_; + Ch current_; + TakeFunc takeFunc_; + bool hasBOM_; +}; + +//! Output stream wrapper with dynamically bound encoding and automatic encoding detection. +/*! + \tparam CharType Type of character for writing. + \tparam OutputByteStream type of output byte stream to be wrapped. +*/ +template +class AutoUTFOutputStream { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); +public: + typedef CharType Ch; + + //! Constructor. + /*! + \param os output stream to be wrapped. + \param type UTF encoding type. + \param putBOM Whether to write BOM at the beginning of the stream. + */ + AutoUTFOutputStream(OutputByteStream& os, UTFType type, bool putBOM) : os_(&os), type_(type) { + CEREAL_RAPIDJSON_ASSERT(type >= kUTF8 && type <= kUTF32BE); + + // Runtime check whether the size of character type is sufficient. It only perform checks with assertion. + if (type_ == kUTF16LE || type_ == kUTF16BE) CEREAL_RAPIDJSON_ASSERT(sizeof(Ch) >= 2); + if (type_ == kUTF32LE || type_ == kUTF32BE) CEREAL_RAPIDJSON_ASSERT(sizeof(Ch) >= 4); + + static const PutFunc f[] = { CEREAL_RAPIDJSON_ENCODINGS_FUNC(Put) }; + putFunc_ = f[type_]; + + if (putBOM) + PutBOM(); + } + + UTFType GetType() const { return type_; } + + void Put(Ch c) { putFunc_(*os_, c); } + void Flush() { os_->Flush(); } + + // Not implemented + Ch Peek() const { CEREAL_RAPIDJSON_ASSERT(false); return 0;} + Ch Take() { CEREAL_RAPIDJSON_ASSERT(false); return 0;} + size_t Tell() const { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + Ch* PutBegin() { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + +private: + AutoUTFOutputStream(const AutoUTFOutputStream&); + AutoUTFOutputStream& operator=(const AutoUTFOutputStream&); + + void PutBOM() { + typedef void (*PutBOMFunc)(OutputByteStream&); + static const PutBOMFunc f[] = { CEREAL_RAPIDJSON_ENCODINGS_FUNC(PutBOM) }; + f[type_](*os_); + } + + typedef void (*PutFunc)(OutputByteStream&, Ch); + + OutputByteStream* os_; + UTFType type_; + PutFunc putFunc_; +}; + +#undef CEREAL_RAPIDJSON_ENCODINGS_FUNC + +CEREAL_RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_POP +#endif + +#ifdef __GNUC__ +CEREAL_RAPIDJSON_DIAG_POP +#endif + +#endif // CEREAL_RAPIDJSON_FILESTREAM_H_ diff --git a/third_party/cereal/external/rapidjson/encodings.h b/third_party/cereal/external/rapidjson/encodings.h new file mode 100755 index 0000000..a1c27ec --- /dev/null +++ b/third_party/cereal/external/rapidjson/encodings.h @@ -0,0 +1,716 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_ENCODINGS_H_ +#define CEREAL_RAPIDJSON_ENCODINGS_H_ + +#include "rapidjson.h" + +#if defined(_MSC_VER) && !defined(__clang__) +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(4244) // conversion from 'type1' to 'type2', possible loss of data +CEREAL_RAPIDJSON_DIAG_OFF(4702) // unreachable code +#elif defined(__GNUC__) +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(effc++) +CEREAL_RAPIDJSON_DIAG_OFF(overflow) +#endif + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// Encoding + +/*! \class rapidjson::Encoding + \brief Concept for encoding of Unicode characters. + +\code +concept Encoding { + typename Ch; //! Type of character. A "character" is actually a code unit in unicode's definition. + + enum { supportUnicode = 1 }; // or 0 if not supporting unicode + + //! \brief Encode a Unicode codepoint to an output stream. + //! \param os Output stream. + //! \param codepoint An unicode codepoint, ranging from 0x0 to 0x10FFFF inclusively. + template + static void Encode(OutputStream& os, unsigned codepoint); + + //! \brief Decode a Unicode codepoint from an input stream. + //! \param is Input stream. + //! \param codepoint Output of the unicode codepoint. + //! \return true if a valid codepoint can be decoded from the stream. + template + static bool Decode(InputStream& is, unsigned* codepoint); + + //! \brief Validate one Unicode codepoint from an encoded stream. + //! \param is Input stream to obtain codepoint. + //! \param os Output for copying one codepoint. + //! \return true if it is valid. + //! \note This function just validating and copying the codepoint without actually decode it. + template + static bool Validate(InputStream& is, OutputStream& os); + + // The following functions are deal with byte streams. + + //! Take a character from input byte stream, skip BOM if exist. + template + static CharType TakeBOM(InputByteStream& is); + + //! Take a character from input byte stream. + template + static Ch Take(InputByteStream& is); + + //! Put BOM to output byte stream. + template + static void PutBOM(OutputByteStream& os); + + //! Put a character to output byte stream. + template + static void Put(OutputByteStream& os, Ch c); +}; +\endcode +*/ + +/////////////////////////////////////////////////////////////////////////////// +// UTF8 + +//! UTF-8 encoding. +/*! http://en.wikipedia.org/wiki/UTF-8 + http://tools.ietf.org/html/rfc3629 + \tparam CharType Code unit for storing 8-bit UTF-8 data. Default is char. + \note implements Encoding concept +*/ +template +struct UTF8 { + typedef CharType Ch; + + enum { supportUnicode = 1 }; + + template + static void Encode(OutputStream& os, unsigned codepoint) { + if (codepoint <= 0x7F) + os.Put(static_cast(codepoint & 0xFF)); + else if (codepoint <= 0x7FF) { + os.Put(static_cast(0xC0 | ((codepoint >> 6) & 0xFF))); + os.Put(static_cast(0x80 | ((codepoint & 0x3F)))); + } + else if (codepoint <= 0xFFFF) { + os.Put(static_cast(0xE0 | ((codepoint >> 12) & 0xFF))); + os.Put(static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + os.Put(static_cast(0x80 | (codepoint & 0x3F))); + } + else { + CEREAL_RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + os.Put(static_cast(0xF0 | ((codepoint >> 18) & 0xFF))); + os.Put(static_cast(0x80 | ((codepoint >> 12) & 0x3F))); + os.Put(static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + os.Put(static_cast(0x80 | (codepoint & 0x3F))); + } + } + + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + if (codepoint <= 0x7F) + PutUnsafe(os, static_cast(codepoint & 0xFF)); + else if (codepoint <= 0x7FF) { + PutUnsafe(os, static_cast(0xC0 | ((codepoint >> 6) & 0xFF))); + PutUnsafe(os, static_cast(0x80 | ((codepoint & 0x3F)))); + } + else if (codepoint <= 0xFFFF) { + PutUnsafe(os, static_cast(0xE0 | ((codepoint >> 12) & 0xFF))); + PutUnsafe(os, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + PutUnsafe(os, static_cast(0x80 | (codepoint & 0x3F))); + } + else { + CEREAL_RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + PutUnsafe(os, static_cast(0xF0 | ((codepoint >> 18) & 0xFF))); + PutUnsafe(os, static_cast(0x80 | ((codepoint >> 12) & 0x3F))); + PutUnsafe(os, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + PutUnsafe(os, static_cast(0x80 | (codepoint & 0x3F))); + } + } + + template + static bool Decode(InputStream& is, unsigned* codepoint) { +#define CEREAL_RAPIDJSON_COPY() c = is.Take(); *codepoint = (*codepoint << 6) | (static_cast(c) & 0x3Fu) +#define CEREAL_RAPIDJSON_TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) +#define CEREAL_RAPIDJSON_TAIL() CEREAL_RAPIDJSON_COPY(); CEREAL_RAPIDJSON_TRANS(0x70) + typename InputStream::Ch c = is.Take(); + if (!(c & 0x80)) { + *codepoint = static_cast(c); + return true; + } + + unsigned char type = GetRange(static_cast(c)); + if (type >= 32) { + *codepoint = 0; + } else { + *codepoint = (0xFFu >> type) & static_cast(c); + } + bool result = true; + switch (type) { + case 2: CEREAL_RAPIDJSON_TAIL(); return result; + case 3: CEREAL_RAPIDJSON_TAIL(); CEREAL_RAPIDJSON_TAIL(); return result; + case 4: CEREAL_RAPIDJSON_COPY(); CEREAL_RAPIDJSON_TRANS(0x50); CEREAL_RAPIDJSON_TAIL(); return result; + case 5: CEREAL_RAPIDJSON_COPY(); CEREAL_RAPIDJSON_TRANS(0x10); CEREAL_RAPIDJSON_TAIL(); CEREAL_RAPIDJSON_TAIL(); return result; + case 6: CEREAL_RAPIDJSON_TAIL(); CEREAL_RAPIDJSON_TAIL(); CEREAL_RAPIDJSON_TAIL(); return result; + case 10: CEREAL_RAPIDJSON_COPY(); CEREAL_RAPIDJSON_TRANS(0x20); CEREAL_RAPIDJSON_TAIL(); return result; + case 11: CEREAL_RAPIDJSON_COPY(); CEREAL_RAPIDJSON_TRANS(0x60); CEREAL_RAPIDJSON_TAIL(); CEREAL_RAPIDJSON_TAIL(); return result; + default: return false; + } +#undef CEREAL_RAPIDJSON_COPY +#undef CEREAL_RAPIDJSON_TRANS +#undef CEREAL_RAPIDJSON_TAIL + } + + template + static bool Validate(InputStream& is, OutputStream& os) { +#define CEREAL_RAPIDJSON_COPY() os.Put(c = is.Take()) +#define CEREAL_RAPIDJSON_TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) +#define CEREAL_RAPIDJSON_TAIL() CEREAL_RAPIDJSON_COPY(); CEREAL_RAPIDJSON_TRANS(0x70) + Ch c; + CEREAL_RAPIDJSON_COPY(); + if (!(c & 0x80)) + return true; + + bool result = true; + switch (GetRange(static_cast(c))) { + case 2: CEREAL_RAPIDJSON_TAIL(); return result; + case 3: CEREAL_RAPIDJSON_TAIL(); CEREAL_RAPIDJSON_TAIL(); return result; + case 4: CEREAL_RAPIDJSON_COPY(); CEREAL_RAPIDJSON_TRANS(0x50); CEREAL_RAPIDJSON_TAIL(); return result; + case 5: CEREAL_RAPIDJSON_COPY(); CEREAL_RAPIDJSON_TRANS(0x10); CEREAL_RAPIDJSON_TAIL(); CEREAL_RAPIDJSON_TAIL(); return result; + case 6: CEREAL_RAPIDJSON_TAIL(); CEREAL_RAPIDJSON_TAIL(); CEREAL_RAPIDJSON_TAIL(); return result; + case 10: CEREAL_RAPIDJSON_COPY(); CEREAL_RAPIDJSON_TRANS(0x20); CEREAL_RAPIDJSON_TAIL(); return result; + case 11: CEREAL_RAPIDJSON_COPY(); CEREAL_RAPIDJSON_TRANS(0x60); CEREAL_RAPIDJSON_TAIL(); CEREAL_RAPIDJSON_TAIL(); return result; + default: return false; + } +#undef CEREAL_RAPIDJSON_COPY +#undef CEREAL_RAPIDJSON_TRANS +#undef CEREAL_RAPIDJSON_TAIL + } + + static unsigned char GetRange(unsigned char c) { + // Referring to DFA of http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ + // With new mapping 1 -> 0x10, 7 -> 0x20, 9 -> 0x40, such that AND operation can test multiple types. + static const unsigned char type[] = { + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10, + 0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40, + 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, + 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, + 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, + 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, + }; + return type[c]; + } + + template + static CharType TakeBOM(InputByteStream& is) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + typename InputByteStream::Ch c = Take(is); + if (static_cast(c) != 0xEFu) return c; + c = is.Take(); + if (static_cast(c) != 0xBBu) return c; + c = is.Take(); + if (static_cast(c) != 0xBFu) return c; + c = is.Take(); + return c; + } + + template + static Ch Take(InputByteStream& is) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + return static_cast(is.Take()); + } + + template + static void PutBOM(OutputByteStream& os) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(0xEFu)); + os.Put(static_cast(0xBBu)); + os.Put(static_cast(0xBFu)); + } + + template + static void Put(OutputByteStream& os, Ch c) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(c)); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// UTF16 + +//! UTF-16 encoding. +/*! http://en.wikipedia.org/wiki/UTF-16 + http://tools.ietf.org/html/rfc2781 + \tparam CharType Type for storing 16-bit UTF-16 data. Default is wchar_t. C++11 may use char16_t instead. + \note implements Encoding concept + + \note For in-memory access, no need to concern endianness. The code units and code points are represented by CPU's endianness. + For streaming, use UTF16LE and UTF16BE, which handle endianness. +*/ +template +struct UTF16 { + typedef CharType Ch; + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(Ch) >= 2); + + enum { supportUnicode = 1 }; + + template + static void Encode(OutputStream& os, unsigned codepoint) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); + if (codepoint <= 0xFFFF) { + CEREAL_RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair + os.Put(static_cast(codepoint)); + } + else { + CEREAL_RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + unsigned v = codepoint - 0x10000; + os.Put(static_cast((v >> 10) | 0xD800)); + os.Put(static_cast((v & 0x3FF) | 0xDC00)); + } + } + + + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); + if (codepoint <= 0xFFFF) { + CEREAL_RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair + PutUnsafe(os, static_cast(codepoint)); + } + else { + CEREAL_RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + unsigned v = codepoint - 0x10000; + PutUnsafe(os, static_cast((v >> 10) | 0xD800)); + PutUnsafe(os, static_cast((v & 0x3FF) | 0xDC00)); + } + } + + template + static bool Decode(InputStream& is, unsigned* codepoint) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); + typename InputStream::Ch c = is.Take(); + if (c < 0xD800 || c > 0xDFFF) { + *codepoint = static_cast(c); + return true; + } + else if (c <= 0xDBFF) { + *codepoint = (static_cast(c) & 0x3FF) << 10; + c = is.Take(); + *codepoint |= (static_cast(c) & 0x3FF); + *codepoint += 0x10000; + return c >= 0xDC00 && c <= 0xDFFF; + } + return false; + } + + template + static bool Validate(InputStream& is, OutputStream& os) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); + typename InputStream::Ch c; + os.Put(static_cast(c = is.Take())); + if (c < 0xD800 || c > 0xDFFF) + return true; + else if (c <= 0xDBFF) { + os.Put(c = is.Take()); + return c >= 0xDC00 && c <= 0xDFFF; + } + return false; + } +}; + +//! UTF-16 little endian encoding. +template +struct UTF16LE : UTF16 { + template + static CharType TakeBOM(InputByteStream& is) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = Take(is); + return static_cast(c) == 0xFEFFu ? Take(is) : c; + } + + template + static CharType Take(InputByteStream& is) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + unsigned c = static_cast(is.Take()); + c |= static_cast(static_cast(is.Take())) << 8; + return static_cast(c); + } + + template + static void PutBOM(OutputByteStream& os) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(0xFFu)); + os.Put(static_cast(0xFEu)); + } + + template + static void Put(OutputByteStream& os, CharType c) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(static_cast(c) & 0xFFu)); + os.Put(static_cast((static_cast(c) >> 8) & 0xFFu)); + } +}; + +//! UTF-16 big endian encoding. +template +struct UTF16BE : UTF16 { + template + static CharType TakeBOM(InputByteStream& is) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = Take(is); + return static_cast(c) == 0xFEFFu ? Take(is) : c; + } + + template + static CharType Take(InputByteStream& is) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + unsigned c = static_cast(static_cast(is.Take())) << 8; + c |= static_cast(static_cast(is.Take())); + return static_cast(c); + } + + template + static void PutBOM(OutputByteStream& os) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(0xFEu)); + os.Put(static_cast(0xFFu)); + } + + template + static void Put(OutputByteStream& os, CharType c) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast((static_cast(c) >> 8) & 0xFFu)); + os.Put(static_cast(static_cast(c) & 0xFFu)); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// UTF32 + +//! UTF-32 encoding. +/*! http://en.wikipedia.org/wiki/UTF-32 + \tparam CharType Type for storing 32-bit UTF-32 data. Default is unsigned. C++11 may use char32_t instead. + \note implements Encoding concept + + \note For in-memory access, no need to concern endianness. The code units and code points are represented by CPU's endianness. + For streaming, use UTF32LE and UTF32BE, which handle endianness. +*/ +template +struct UTF32 { + typedef CharType Ch; + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(Ch) >= 4); + + enum { supportUnicode = 1 }; + + template + static void Encode(OutputStream& os, unsigned codepoint) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4); + CEREAL_RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + os.Put(codepoint); + } + + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4); + CEREAL_RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + PutUnsafe(os, codepoint); + } + + template + static bool Decode(InputStream& is, unsigned* codepoint) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); + Ch c = is.Take(); + *codepoint = c; + return c <= 0x10FFFF; + } + + template + static bool Validate(InputStream& is, OutputStream& os) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); + Ch c; + os.Put(c = is.Take()); + return c <= 0x10FFFF; + } +}; + +//! UTF-32 little endian enocoding. +template +struct UTF32LE : UTF32 { + template + static CharType TakeBOM(InputByteStream& is) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = Take(is); + return static_cast(c) == 0x0000FEFFu ? Take(is) : c; + } + + template + static CharType Take(InputByteStream& is) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + unsigned c = static_cast(is.Take()); + c |= static_cast(static_cast(is.Take())) << 8; + c |= static_cast(static_cast(is.Take())) << 16; + c |= static_cast(static_cast(is.Take())) << 24; + return static_cast(c); + } + + template + static void PutBOM(OutputByteStream& os) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(0xFFu)); + os.Put(static_cast(0xFEu)); + os.Put(static_cast(0x00u)); + os.Put(static_cast(0x00u)); + } + + template + static void Put(OutputByteStream& os, CharType c) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(c & 0xFFu)); + os.Put(static_cast((c >> 8) & 0xFFu)); + os.Put(static_cast((c >> 16) & 0xFFu)); + os.Put(static_cast((c >> 24) & 0xFFu)); + } +}; + +//! UTF-32 big endian encoding. +template +struct UTF32BE : UTF32 { + template + static CharType TakeBOM(InputByteStream& is) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = Take(is); + return static_cast(c) == 0x0000FEFFu ? Take(is) : c; + } + + template + static CharType Take(InputByteStream& is) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + unsigned c = static_cast(static_cast(is.Take())) << 24; + c |= static_cast(static_cast(is.Take())) << 16; + c |= static_cast(static_cast(is.Take())) << 8; + c |= static_cast(static_cast(is.Take())); + return static_cast(c); + } + + template + static void PutBOM(OutputByteStream& os) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(0x00u)); + os.Put(static_cast(0x00u)); + os.Put(static_cast(0xFEu)); + os.Put(static_cast(0xFFu)); + } + + template + static void Put(OutputByteStream& os, CharType c) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast((c >> 24) & 0xFFu)); + os.Put(static_cast((c >> 16) & 0xFFu)); + os.Put(static_cast((c >> 8) & 0xFFu)); + os.Put(static_cast(c & 0xFFu)); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// ASCII + +//! ASCII encoding. +/*! http://en.wikipedia.org/wiki/ASCII + \tparam CharType Code unit for storing 7-bit ASCII data. Default is char. + \note implements Encoding concept +*/ +template +struct ASCII { + typedef CharType Ch; + + enum { supportUnicode = 0 }; + + template + static void Encode(OutputStream& os, unsigned codepoint) { + CEREAL_RAPIDJSON_ASSERT(codepoint <= 0x7F); + os.Put(static_cast(codepoint & 0xFF)); + } + + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + CEREAL_RAPIDJSON_ASSERT(codepoint <= 0x7F); + PutUnsafe(os, static_cast(codepoint & 0xFF)); + } + + template + static bool Decode(InputStream& is, unsigned* codepoint) { + uint8_t c = static_cast(is.Take()); + *codepoint = c; + return c <= 0X7F; + } + + template + static bool Validate(InputStream& is, OutputStream& os) { + uint8_t c = static_cast(is.Take()); + os.Put(static_cast(c)); + return c <= 0x7F; + } + + template + static CharType TakeBOM(InputByteStream& is) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + uint8_t c = static_cast(Take(is)); + return static_cast(c); + } + + template + static Ch Take(InputByteStream& is) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + return static_cast(is.Take()); + } + + template + static void PutBOM(OutputByteStream& os) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + (void)os; + } + + template + static void Put(OutputByteStream& os, Ch c) { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(c)); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// AutoUTF + +//! Runtime-specified UTF encoding type of a stream. +enum UTFType { + kUTF8 = 0, //!< UTF-8. + kUTF16LE = 1, //!< UTF-16 little endian. + kUTF16BE = 2, //!< UTF-16 big endian. + kUTF32LE = 3, //!< UTF-32 little endian. + kUTF32BE = 4 //!< UTF-32 big endian. +}; + +//! Dynamically select encoding according to stream's runtime-specified UTF encoding type. +/*! \note This class can be used with AutoUTFInputtStream and AutoUTFOutputStream, which provides GetType(). +*/ +template +struct AutoUTF { + typedef CharType Ch; + + enum { supportUnicode = 1 }; + +#define CEREAL_RAPIDJSON_ENCODINGS_FUNC(x) UTF8::x, UTF16LE::x, UTF16BE::x, UTF32LE::x, UTF32BE::x + + template + static CEREAL_RAPIDJSON_FORCEINLINE void Encode(OutputStream& os, unsigned codepoint) { + typedef void (*EncodeFunc)(OutputStream&, unsigned); + static const EncodeFunc f[] = { CEREAL_RAPIDJSON_ENCODINGS_FUNC(Encode) }; + (*f[os.GetType()])(os, codepoint); + } + + template + static CEREAL_RAPIDJSON_FORCEINLINE void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + typedef void (*EncodeFunc)(OutputStream&, unsigned); + static const EncodeFunc f[] = { CEREAL_RAPIDJSON_ENCODINGS_FUNC(EncodeUnsafe) }; + (*f[os.GetType()])(os, codepoint); + } + + template + static CEREAL_RAPIDJSON_FORCEINLINE bool Decode(InputStream& is, unsigned* codepoint) { + typedef bool (*DecodeFunc)(InputStream&, unsigned*); + static const DecodeFunc f[] = { CEREAL_RAPIDJSON_ENCODINGS_FUNC(Decode) }; + return (*f[is.GetType()])(is, codepoint); + } + + template + static CEREAL_RAPIDJSON_FORCEINLINE bool Validate(InputStream& is, OutputStream& os) { + typedef bool (*ValidateFunc)(InputStream&, OutputStream&); + static const ValidateFunc f[] = { CEREAL_RAPIDJSON_ENCODINGS_FUNC(Validate) }; + return (*f[is.GetType()])(is, os); + } + +#undef CEREAL_RAPIDJSON_ENCODINGS_FUNC +}; + +/////////////////////////////////////////////////////////////////////////////// +// Transcoder + +//! Encoding conversion. +template +struct Transcoder { + //! Take one Unicode codepoint from source encoding, convert it to target encoding and put it to the output stream. + template + static CEREAL_RAPIDJSON_FORCEINLINE bool Transcode(InputStream& is, OutputStream& os) { + unsigned codepoint; + if (!SourceEncoding::Decode(is, &codepoint)) + return false; + TargetEncoding::Encode(os, codepoint); + return true; + } + + template + static CEREAL_RAPIDJSON_FORCEINLINE bool TranscodeUnsafe(InputStream& is, OutputStream& os) { + unsigned codepoint; + if (!SourceEncoding::Decode(is, &codepoint)) + return false; + TargetEncoding::EncodeUnsafe(os, codepoint); + return true; + } + + //! Validate one Unicode codepoint from an encoded stream. + template + static CEREAL_RAPIDJSON_FORCEINLINE bool Validate(InputStream& is, OutputStream& os) { + return Transcode(is, os); // Since source/target encoding is different, must transcode. + } +}; + +// Forward declaration. +template +inline void PutUnsafe(Stream& stream, typename Stream::Ch c); + +//! Specialization of Transcoder with same source and target encoding. +template +struct Transcoder { + template + static CEREAL_RAPIDJSON_FORCEINLINE bool Transcode(InputStream& is, OutputStream& os) { + os.Put(is.Take()); // Just copy one code unit. This semantic is different from primary template class. + return true; + } + + template + static CEREAL_RAPIDJSON_FORCEINLINE bool TranscodeUnsafe(InputStream& is, OutputStream& os) { + PutUnsafe(os, is.Take()); // Just copy one code unit. This semantic is different from primary template class. + return true; + } + + template + static CEREAL_RAPIDJSON_FORCEINLINE bool Validate(InputStream& is, OutputStream& os) { + return Encoding::Validate(is, os); // source/target encoding are the same + } +}; + +CEREAL_RAPIDJSON_NAMESPACE_END + +#if defined(__GNUC__) || (defined(_MSC_VER) && !defined(__clang__)) +CEREAL_RAPIDJSON_DIAG_POP +#endif + +#endif // CEREAL_RAPIDJSON_ENCODINGS_H_ diff --git a/third_party/cereal/external/rapidjson/error/en.h b/third_party/cereal/external/rapidjson/error/en.h new file mode 100755 index 0000000..ea93bf8 --- /dev/null +++ b/third_party/cereal/external/rapidjson/error/en.h @@ -0,0 +1,74 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_ERROR_EN_H_ +#define CEREAL_RAPIDJSON_ERROR_EN_H_ + +#include "error.h" + +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(switch-enum) +CEREAL_RAPIDJSON_DIAG_OFF(covered-switch-default) +#endif + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN + +//! Maps error code of parsing into error message. +/*! + \ingroup CEREAL_RAPIDJSON_ERRORS + \param parseErrorCode Error code obtained in parsing. + \return the error message. + \note User can make a copy of this function for localization. + Using switch-case is safer for future modification of error codes. +*/ +inline const CEREAL_RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErrorCode) { + switch (parseErrorCode) { + case kParseErrorNone: return CEREAL_RAPIDJSON_ERROR_STRING("No error."); + + case kParseErrorDocumentEmpty: return CEREAL_RAPIDJSON_ERROR_STRING("The document is empty."); + case kParseErrorDocumentRootNotSingular: return CEREAL_RAPIDJSON_ERROR_STRING("The document root must not be followed by other values."); + + case kParseErrorValueInvalid: return CEREAL_RAPIDJSON_ERROR_STRING("Invalid value."); + + case kParseErrorObjectMissName: return CEREAL_RAPIDJSON_ERROR_STRING("Missing a name for object member."); + case kParseErrorObjectMissColon: return CEREAL_RAPIDJSON_ERROR_STRING("Missing a colon after a name of object member."); + case kParseErrorObjectMissCommaOrCurlyBracket: return CEREAL_RAPIDJSON_ERROR_STRING("Missing a comma or '}' after an object member."); + + case kParseErrorArrayMissCommaOrSquareBracket: return CEREAL_RAPIDJSON_ERROR_STRING("Missing a comma or ']' after an array element."); + + case kParseErrorStringUnicodeEscapeInvalidHex: return CEREAL_RAPIDJSON_ERROR_STRING("Incorrect hex digit after \\u escape in string."); + case kParseErrorStringUnicodeSurrogateInvalid: return CEREAL_RAPIDJSON_ERROR_STRING("The surrogate pair in string is invalid."); + case kParseErrorStringEscapeInvalid: return CEREAL_RAPIDJSON_ERROR_STRING("Invalid escape character in string."); + case kParseErrorStringMissQuotationMark: return CEREAL_RAPIDJSON_ERROR_STRING("Missing a closing quotation mark in string."); + case kParseErrorStringInvalidEncoding: return CEREAL_RAPIDJSON_ERROR_STRING("Invalid encoding in string."); + + case kParseErrorNumberTooBig: return CEREAL_RAPIDJSON_ERROR_STRING("Number too big to be stored in double."); + case kParseErrorNumberMissFraction: return CEREAL_RAPIDJSON_ERROR_STRING("Miss fraction part in number."); + case kParseErrorNumberMissExponent: return CEREAL_RAPIDJSON_ERROR_STRING("Miss exponent in number."); + + case kParseErrorTermination: return CEREAL_RAPIDJSON_ERROR_STRING("Terminate parsing due to Handler error."); + case kParseErrorUnspecificSyntaxError: return CEREAL_RAPIDJSON_ERROR_STRING("Unspecific syntax error."); + + default: return CEREAL_RAPIDJSON_ERROR_STRING("Unknown error."); + } +} + +CEREAL_RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_POP +#endif + +#endif // CEREAL_RAPIDJSON_ERROR_EN_H_ diff --git a/third_party/cereal/external/rapidjson/error/error.h b/third_party/cereal/external/rapidjson/error/error.h new file mode 100755 index 0000000..7e7cc24 --- /dev/null +++ b/third_party/cereal/external/rapidjson/error/error.h @@ -0,0 +1,161 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_ERROR_ERROR_H_ +#define CEREAL_RAPIDJSON_ERROR_ERROR_H_ + +#include "../rapidjson.h" + +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(padded) +#endif + +/*! \file error.h */ + +/*! \defgroup CEREAL_RAPIDJSON_ERRORS RapidJSON error handling */ + +/////////////////////////////////////////////////////////////////////////////// +// CEREAL_RAPIDJSON_ERROR_CHARTYPE + +//! Character type of error messages. +/*! \ingroup CEREAL_RAPIDJSON_ERRORS + The default character type is \c char. + On Windows, user can define this macro as \c TCHAR for supporting both + unicode/non-unicode settings. +*/ +#ifndef CEREAL_RAPIDJSON_ERROR_CHARTYPE +#define CEREAL_RAPIDJSON_ERROR_CHARTYPE char +#endif + +/////////////////////////////////////////////////////////////////////////////// +// CEREAL_RAPIDJSON_ERROR_STRING + +//! Macro for converting string literial to \ref CEREAL_RAPIDJSON_ERROR_CHARTYPE[]. +/*! \ingroup CEREAL_RAPIDJSON_ERRORS + By default this conversion macro does nothing. + On Windows, user can define this macro as \c _T(x) for supporting both + unicode/non-unicode settings. +*/ +#ifndef CEREAL_RAPIDJSON_ERROR_STRING +#define CEREAL_RAPIDJSON_ERROR_STRING(x) x +#endif + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// ParseErrorCode + +//! Error code of parsing. +/*! \ingroup CEREAL_RAPIDJSON_ERRORS + \see GenericReader::Parse, GenericReader::GetParseErrorCode +*/ +enum ParseErrorCode { + kParseErrorNone = 0, //!< No error. + + kParseErrorDocumentEmpty, //!< The document is empty. + kParseErrorDocumentRootNotSingular, //!< The document root must not follow by other values. + + kParseErrorValueInvalid, //!< Invalid value. + + kParseErrorObjectMissName, //!< Missing a name for object member. + kParseErrorObjectMissColon, //!< Missing a colon after a name of object member. + kParseErrorObjectMissCommaOrCurlyBracket, //!< Missing a comma or '}' after an object member. + + kParseErrorArrayMissCommaOrSquareBracket, //!< Missing a comma or ']' after an array element. + + kParseErrorStringUnicodeEscapeInvalidHex, //!< Incorrect hex digit after \\u escape in string. + kParseErrorStringUnicodeSurrogateInvalid, //!< The surrogate pair in string is invalid. + kParseErrorStringEscapeInvalid, //!< Invalid escape character in string. + kParseErrorStringMissQuotationMark, //!< Missing a closing quotation mark in string. + kParseErrorStringInvalidEncoding, //!< Invalid encoding in string. + + kParseErrorNumberTooBig, //!< Number too big to be stored in double. + kParseErrorNumberMissFraction, //!< Miss fraction part in number. + kParseErrorNumberMissExponent, //!< Miss exponent in number. + + kParseErrorTermination, //!< Parsing was terminated. + kParseErrorUnspecificSyntaxError //!< Unspecific syntax error. +}; + +//! Result of parsing (wraps ParseErrorCode) +/*! + \ingroup CEREAL_RAPIDJSON_ERRORS + \code + Document doc; + ParseResult ok = doc.Parse("[42]"); + if (!ok) { + fprintf(stderr, "JSON parse error: %s (%u)", + GetParseError_En(ok.Code()), ok.Offset()); + exit(EXIT_FAILURE); + } + \endcode + \see GenericReader::Parse, GenericDocument::Parse +*/ +struct ParseResult { + //!! Unspecified boolean type + typedef bool (ParseResult::*BooleanType)() const; +public: + //! Default constructor, no error. + ParseResult() : code_(kParseErrorNone), offset_(0) {} + //! Constructor to set an error. + ParseResult(ParseErrorCode code, size_t offset) : code_(code), offset_(offset) {} + + //! Get the error code. + ParseErrorCode Code() const { return code_; } + //! Get the error offset, if \ref IsError(), 0 otherwise. + size_t Offset() const { return offset_; } + + //! Explicit conversion to \c bool, returns \c true, iff !\ref IsError(). + operator BooleanType() const { return !IsError() ? &ParseResult::IsError : NULL; } + //! Whether the result is an error. + bool IsError() const { return code_ != kParseErrorNone; } + + bool operator==(const ParseResult& that) const { return code_ == that.code_; } + bool operator==(ParseErrorCode code) const { return code_ == code; } + friend bool operator==(ParseErrorCode code, const ParseResult & err) { return code == err.code_; } + + bool operator!=(const ParseResult& that) const { return !(*this == that); } + bool operator!=(ParseErrorCode code) const { return !(*this == code); } + friend bool operator!=(ParseErrorCode code, const ParseResult & err) { return err != code; } + + //! Reset error code. + void Clear() { Set(kParseErrorNone); } + //! Update error code and offset. + void Set(ParseErrorCode code, size_t offset = 0) { code_ = code; offset_ = offset; } + +private: + ParseErrorCode code_; + size_t offset_; +}; + +//! Function pointer type of GetParseError(). +/*! \ingroup CEREAL_RAPIDJSON_ERRORS + + This is the prototype for \c GetParseError_X(), where \c X is a locale. + User can dynamically change locale in runtime, e.g.: +\code + GetParseErrorFunc GetParseError = GetParseError_En; // or whatever + const CEREAL_RAPIDJSON_ERROR_CHARTYPE* s = GetParseError(document.GetParseErrorCode()); +\endcode +*/ +typedef const CEREAL_RAPIDJSON_ERROR_CHARTYPE* (*GetParseErrorFunc)(ParseErrorCode); + +CEREAL_RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_POP +#endif + +#endif // CEREAL_RAPIDJSON_ERROR_ERROR_H_ diff --git a/third_party/cereal/external/rapidjson/filereadstream.h b/third_party/cereal/external/rapidjson/filereadstream.h new file mode 100755 index 0000000..253a7f9 --- /dev/null +++ b/third_party/cereal/external/rapidjson/filereadstream.h @@ -0,0 +1,99 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_FILEREADSTREAM_H_ +#define CEREAL_RAPIDJSON_FILEREADSTREAM_H_ + +#include "stream.h" +#include + +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(padded) +CEREAL_RAPIDJSON_DIAG_OFF(unreachable-code) +CEREAL_RAPIDJSON_DIAG_OFF(missing-noreturn) +#endif + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN + +//! File byte stream for input using fread(). +/*! + \note implements Stream concept +*/ +class FileReadStream { +public: + typedef char Ch; //!< Character type (byte). + + //! Constructor. + /*! + \param fp File pointer opened for read. + \param buffer user-supplied buffer. + \param bufferSize size of buffer in bytes. Must >=4 bytes. + */ + FileReadStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferSize_(bufferSize), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { + CEREAL_RAPIDJSON_ASSERT(fp_ != 0); + CEREAL_RAPIDJSON_ASSERT(bufferSize >= 4); + Read(); + } + + Ch Peek() const { return *current_; } + Ch Take() { Ch c = *current_; Read(); return c; } + size_t Tell() const { return count_ + static_cast(current_ - buffer_); } + + // Not implemented + void Put(Ch) { CEREAL_RAPIDJSON_ASSERT(false); } + void Flush() { CEREAL_RAPIDJSON_ASSERT(false); } + Ch* PutBegin() { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + + // For encoding detection only. + const Ch* Peek4() const { + return (current_ + 4 - !eof_ <= bufferLast_) ? current_ : 0; + } + +private: + void Read() { + if (current_ < bufferLast_) + ++current_; + else if (!eof_) { + count_ += readCount_; + readCount_ = std::fread(buffer_, 1, bufferSize_, fp_); + bufferLast_ = buffer_ + readCount_ - 1; + current_ = buffer_; + + if (readCount_ < bufferSize_) { + buffer_[readCount_] = '\0'; + ++bufferLast_; + eof_ = true; + } + } + } + + std::FILE* fp_; + Ch *buffer_; + size_t bufferSize_; + Ch *bufferLast_; + Ch *current_; + size_t readCount_; + size_t count_; //!< Number of characters read + bool eof_; +}; + +CEREAL_RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_POP +#endif + +#endif // CEREAL_RAPIDJSON_FILESTREAM_H_ diff --git a/third_party/cereal/external/rapidjson/filewritestream.h b/third_party/cereal/external/rapidjson/filewritestream.h new file mode 100755 index 0000000..44ae127 --- /dev/null +++ b/third_party/cereal/external/rapidjson/filewritestream.h @@ -0,0 +1,104 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_FILEWRITESTREAM_H_ +#define CEREAL_RAPIDJSON_FILEWRITESTREAM_H_ + +#include "stream.h" +#include + +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(unreachable-code) +#endif + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN + +//! Wrapper of C file stream for output using fwrite(). +/*! + \note implements Stream concept +*/ +class FileWriteStream { +public: + typedef char Ch; //!< Character type. Only support char. + + FileWriteStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferEnd_(buffer + bufferSize), current_(buffer_) { + CEREAL_RAPIDJSON_ASSERT(fp_ != 0); + } + + void Put(char c) { + if (current_ >= bufferEnd_) + Flush(); + + *current_++ = c; + } + + void PutN(char c, size_t n) { + size_t avail = static_cast(bufferEnd_ - current_); + while (n > avail) { + std::memset(current_, c, avail); + current_ += avail; + Flush(); + n -= avail; + avail = static_cast(bufferEnd_ - current_); + } + + if (n > 0) { + std::memset(current_, c, n); + current_ += n; + } + } + + void Flush() { + if (current_ != buffer_) { + size_t result = std::fwrite(buffer_, 1, static_cast(current_ - buffer_), fp_); + if (result < static_cast(current_ - buffer_)) { + // failure deliberately ignored at this time + // added to avoid warn_unused_result build errors + } + current_ = buffer_; + } + } + + // Not implemented + char Peek() const { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + char Take() { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + size_t Tell() const { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + char* PutBegin() { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(char*) { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + +private: + // Prohibit copy constructor & assignment operator. + FileWriteStream(const FileWriteStream&); + FileWriteStream& operator=(const FileWriteStream&); + + std::FILE* fp_; + char *buffer_; + char *bufferEnd_; + char *current_; +}; + +//! Implement specialized version of PutN() with memset() for better performance. +template<> +inline void PutN(FileWriteStream& stream, char c, size_t n) { + stream.PutN(c, n); +} + +CEREAL_RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_POP +#endif + +#endif // CEREAL_RAPIDJSON_FILESTREAM_H_ diff --git a/third_party/cereal/external/rapidjson/fwd.h b/third_party/cereal/external/rapidjson/fwd.h new file mode 100755 index 0000000..1b15c66 --- /dev/null +++ b/third_party/cereal/external/rapidjson/fwd.h @@ -0,0 +1,151 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_FWD_H_ +#define CEREAL_RAPIDJSON_FWD_H_ + +#include "rapidjson.h" + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN + +// encodings.h + +template struct UTF8; +template struct UTF16; +template struct UTF16BE; +template struct UTF16LE; +template struct UTF32; +template struct UTF32BE; +template struct UTF32LE; +template struct ASCII; +template struct AutoUTF; + +template +struct Transcoder; + +// allocators.h + +class CrtAllocator; + +template +class MemoryPoolAllocator; + +// stream.h + +template +struct GenericStringStream; + +typedef GenericStringStream > StringStream; + +template +struct GenericInsituStringStream; + +typedef GenericInsituStringStream > InsituStringStream; + +// stringbuffer.h + +template +class GenericStringBuffer; + +typedef GenericStringBuffer, CrtAllocator> StringBuffer; + +// filereadstream.h + +class FileReadStream; + +// filewritestream.h + +class FileWriteStream; + +// memorybuffer.h + +template +struct GenericMemoryBuffer; + +typedef GenericMemoryBuffer MemoryBuffer; + +// memorystream.h + +struct MemoryStream; + +// reader.h + +template +struct BaseReaderHandler; + +template +class GenericReader; + +typedef GenericReader, UTF8, CrtAllocator> Reader; + +// writer.h + +template +class Writer; + +// prettywriter.h + +template +class PrettyWriter; + +// document.h + +template +struct GenericMember; + +template +class GenericMemberIterator; + +template +struct GenericStringRef; + +template +class GenericValue; + +typedef GenericValue, MemoryPoolAllocator > Value; + +template +class GenericDocument; + +typedef GenericDocument, MemoryPoolAllocator, CrtAllocator> Document; + +// pointer.h + +template +class GenericPointer; + +typedef GenericPointer Pointer; + +// schema.h + +template +class IGenericRemoteSchemaDocumentProvider; + +template +class GenericSchemaDocument; + +typedef GenericSchemaDocument SchemaDocument; +typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProvider; + +template < + typename SchemaDocumentType, + typename OutputHandler, + typename StateAllocator> +class GenericSchemaValidator; + +typedef GenericSchemaValidator, void>, CrtAllocator> SchemaValidator; + +CEREAL_RAPIDJSON_NAMESPACE_END + +#endif // CEREAL_RAPIDJSON_RAPIDJSONFWD_H_ diff --git a/third_party/cereal/external/rapidjson/internal/biginteger.h b/third_party/cereal/external/rapidjson/internal/biginteger.h new file mode 100755 index 0000000..77a6519 --- /dev/null +++ b/third_party/cereal/external/rapidjson/internal/biginteger.h @@ -0,0 +1,290 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_BIGINTEGER_H_ +#define CEREAL_RAPIDJSON_BIGINTEGER_H_ + +#include "../rapidjson.h" + +#if defined(_MSC_VER) && !__INTEL_COMPILER && defined(_M_AMD64) +#include // for _umul128 +#pragma intrinsic(_umul128) +#endif + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +class BigInteger { +public: + typedef uint64_t Type; + + BigInteger(const BigInteger& rhs) : count_(rhs.count_) { + std::memcpy(digits_, rhs.digits_, count_ * sizeof(Type)); + } + + explicit BigInteger(uint64_t u) : count_(1) { + digits_[0] = u; + } + + BigInteger(const char* decimals, size_t length) : count_(1) { + CEREAL_RAPIDJSON_ASSERT(length > 0); + digits_[0] = 0; + size_t i = 0; + const size_t kMaxDigitPerIteration = 19; // 2^64 = 18446744073709551616 > 10^19 + while (length >= kMaxDigitPerIteration) { + AppendDecimal64(decimals + i, decimals + i + kMaxDigitPerIteration); + length -= kMaxDigitPerIteration; + i += kMaxDigitPerIteration; + } + + if (length > 0) + AppendDecimal64(decimals + i, decimals + i + length); + } + + BigInteger& operator=(const BigInteger &rhs) + { + if (this != &rhs) { + count_ = rhs.count_; + std::memcpy(digits_, rhs.digits_, count_ * sizeof(Type)); + } + return *this; + } + + BigInteger& operator=(uint64_t u) { + digits_[0] = u; + count_ = 1; + return *this; + } + + BigInteger& operator+=(uint64_t u) { + Type backup = digits_[0]; + digits_[0] += u; + for (size_t i = 0; i < count_ - 1; i++) { + if (digits_[i] >= backup) + return *this; // no carry + backup = digits_[i + 1]; + digits_[i + 1] += 1; + } + + // Last carry + if (digits_[count_ - 1] < backup) + PushBack(1); + + return *this; + } + + BigInteger& operator*=(uint64_t u) { + if (u == 0) return *this = 0; + if (u == 1) return *this; + if (*this == 1) return *this = u; + + uint64_t k = 0; + for (size_t i = 0; i < count_; i++) { + uint64_t hi; + digits_[i] = MulAdd64(digits_[i], u, k, &hi); + k = hi; + } + + if (k > 0) + PushBack(k); + + return *this; + } + + BigInteger& operator*=(uint32_t u) { + if (u == 0) return *this = 0; + if (u == 1) return *this; + if (*this == 1) return *this = u; + + uint64_t k = 0; + for (size_t i = 0; i < count_; i++) { + const uint64_t c = digits_[i] >> 32; + const uint64_t d = digits_[i] & 0xFFFFFFFF; + const uint64_t uc = u * c; + const uint64_t ud = u * d; + const uint64_t p0 = ud + k; + const uint64_t p1 = uc + (p0 >> 32); + digits_[i] = (p0 & 0xFFFFFFFF) | (p1 << 32); + k = p1 >> 32; + } + + if (k > 0) + PushBack(k); + + return *this; + } + + BigInteger& operator<<=(size_t shift) { + if (IsZero() || shift == 0) return *this; + + size_t offset = shift / kTypeBit; + size_t interShift = shift % kTypeBit; + CEREAL_RAPIDJSON_ASSERT(count_ + offset <= kCapacity); + + if (interShift == 0) { + std::memmove(digits_ + offset, digits_, count_ * sizeof(Type)); + count_ += offset; + } + else { + digits_[count_] = 0; + for (size_t i = count_; i > 0; i--) + digits_[i + offset] = (digits_[i] << interShift) | (digits_[i - 1] >> (kTypeBit - interShift)); + digits_[offset] = digits_[0] << interShift; + count_ += offset; + if (digits_[count_]) + count_++; + } + + std::memset(digits_, 0, offset * sizeof(Type)); + + return *this; + } + + bool operator==(const BigInteger& rhs) const { + return count_ == rhs.count_ && std::memcmp(digits_, rhs.digits_, count_ * sizeof(Type)) == 0; + } + + bool operator==(const Type rhs) const { + return count_ == 1 && digits_[0] == rhs; + } + + BigInteger& MultiplyPow5(unsigned exp) { + static const uint32_t kPow5[12] = { + 5, + 5 * 5, + 5 * 5 * 5, + 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 + }; + if (exp == 0) return *this; + for (; exp >= 27; exp -= 27) *this *= CEREAL_RAPIDJSON_UINT64_C2(0X6765C793, 0XFA10079D); // 5^27 + for (; exp >= 13; exp -= 13) *this *= static_cast(1220703125u); // 5^13 + if (exp > 0) *this *= kPow5[exp - 1]; + return *this; + } + + // Compute absolute difference of this and rhs. + // Assume this != rhs + bool Difference(const BigInteger& rhs, BigInteger* out) const { + int cmp = Compare(rhs); + CEREAL_RAPIDJSON_ASSERT(cmp != 0); + const BigInteger *a, *b; // Makes a > b + bool ret; + if (cmp < 0) { a = &rhs; b = this; ret = true; } + else { a = this; b = &rhs; ret = false; } + + Type borrow = 0; + for (size_t i = 0; i < a->count_; i++) { + Type d = a->digits_[i] - borrow; + if (i < b->count_) + d -= b->digits_[i]; + borrow = (d > a->digits_[i]) ? 1 : 0; + out->digits_[i] = d; + if (d != 0) + out->count_ = i + 1; + } + + return ret; + } + + int Compare(const BigInteger& rhs) const { + if (count_ != rhs.count_) + return count_ < rhs.count_ ? -1 : 1; + + for (size_t i = count_; i-- > 0;) + if (digits_[i] != rhs.digits_[i]) + return digits_[i] < rhs.digits_[i] ? -1 : 1; + + return 0; + } + + size_t GetCount() const { return count_; } + Type GetDigit(size_t index) const { CEREAL_RAPIDJSON_ASSERT(index < count_); return digits_[index]; } + bool IsZero() const { return count_ == 1 && digits_[0] == 0; } + +private: + void AppendDecimal64(const char* begin, const char* end) { + uint64_t u = ParseUint64(begin, end); + if (IsZero()) + *this = u; + else { + unsigned exp = static_cast(end - begin); + (MultiplyPow5(exp) <<= exp) += u; // *this = *this * 10^exp + u + } + } + + void PushBack(Type digit) { + CEREAL_RAPIDJSON_ASSERT(count_ < kCapacity); + digits_[count_++] = digit; + } + + static uint64_t ParseUint64(const char* begin, const char* end) { + uint64_t r = 0; + for (const char* p = begin; p != end; ++p) { + CEREAL_RAPIDJSON_ASSERT(*p >= '0' && *p <= '9'); + r = r * 10u + static_cast(*p - '0'); + } + return r; + } + + // Assume a * b + k < 2^128 + static uint64_t MulAdd64(uint64_t a, uint64_t b, uint64_t k, uint64_t* outHigh) { +#if defined(_MSC_VER) && defined(_M_AMD64) + uint64_t low = _umul128(a, b, outHigh) + k; + if (low < k) + (*outHigh)++; + return low; +#elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) + __extension__ typedef unsigned __int128 uint128; + uint128 p = static_cast(a) * static_cast(b); + p += k; + *outHigh = static_cast(p >> 64); + return static_cast(p); +#else + const uint64_t a0 = a & 0xFFFFFFFF, a1 = a >> 32, b0 = b & 0xFFFFFFFF, b1 = b >> 32; + uint64_t x0 = a0 * b0, x1 = a0 * b1, x2 = a1 * b0, x3 = a1 * b1; + x1 += (x0 >> 32); // can't give carry + x1 += x2; + if (x1 < x2) + x3 += (static_cast(1) << 32); + uint64_t lo = (x1 << 32) + (x0 & 0xFFFFFFFF); + uint64_t hi = x3 + (x1 >> 32); + + lo += k; + if (lo < k) + hi++; + *outHigh = hi; + return lo; +#endif + } + + static const size_t kBitCount = 3328; // 64bit * 54 > 10^1000 + static const size_t kCapacity = kBitCount / sizeof(Type); + static const size_t kTypeBit = sizeof(Type) * 8; + + Type digits_[kCapacity]; + size_t count_; +}; + +} // namespace internal +CEREAL_RAPIDJSON_NAMESPACE_END + +#endif // CEREAL_RAPIDJSON_BIGINTEGER_H_ diff --git a/third_party/cereal/external/rapidjson/internal/diyfp.h b/third_party/cereal/external/rapidjson/internal/diyfp.h new file mode 100755 index 0000000..3f92a07 --- /dev/null +++ b/third_party/cereal/external/rapidjson/internal/diyfp.h @@ -0,0 +1,271 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +// This is a C++ header-only implementation of Grisu2 algorithm from the publication: +// Loitsch, Florian. "Printing floating-point numbers quickly and accurately with +// integers." ACM Sigplan Notices 45.6 (2010): 233-243. + +#ifndef CEREAL_RAPIDJSON_DIYFP_H_ +#define CEREAL_RAPIDJSON_DIYFP_H_ + +#include "../rapidjson.h" +#include + +#if defined(_MSC_VER) && defined(_M_AMD64) && !defined(__INTEL_COMPILER) +#include +#pragma intrinsic(_BitScanReverse64) +#pragma intrinsic(_umul128) +#endif + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +#ifdef __GNUC__ +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(effc++) +#endif + +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(padded) +#endif + +struct DiyFp { + DiyFp() : f(), e() {} + + DiyFp(uint64_t fp, int exp) : f(fp), e(exp) {} + + explicit DiyFp(double d) { + union { + double d; + uint64_t u64; + } u = { d }; + + int biased_e = static_cast((u.u64 & kDpExponentMask) >> kDpSignificandSize); + uint64_t significand = (u.u64 & kDpSignificandMask); + if (biased_e != 0) { + f = significand + kDpHiddenBit; + e = biased_e - kDpExponentBias; + } + else { + f = significand; + e = kDpMinExponent + 1; + } + } + + DiyFp operator-(const DiyFp& rhs) const { + return DiyFp(f - rhs.f, e); + } + + DiyFp operator*(const DiyFp& rhs) const { +#if defined(_MSC_VER) && defined(_M_AMD64) + uint64_t h; + uint64_t l = _umul128(f, rhs.f, &h); + if (l & (uint64_t(1) << 63)) // rounding + h++; + return DiyFp(h, e + rhs.e + 64); +#elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) + __extension__ typedef unsigned __int128 uint128; + uint128 p = static_cast(f) * static_cast(rhs.f); + uint64_t h = static_cast(p >> 64); + uint64_t l = static_cast(p); + if (l & (uint64_t(1) << 63)) // rounding + h++; + return DiyFp(h, e + rhs.e + 64); +#else + const uint64_t M32 = 0xFFFFFFFF; + const uint64_t a = f >> 32; + const uint64_t b = f & M32; + const uint64_t c = rhs.f >> 32; + const uint64_t d = rhs.f & M32; + const uint64_t ac = a * c; + const uint64_t bc = b * c; + const uint64_t ad = a * d; + const uint64_t bd = b * d; + uint64_t tmp = (bd >> 32) + (ad & M32) + (bc & M32); + tmp += 1U << 31; /// mult_round + return DiyFp(ac + (ad >> 32) + (bc >> 32) + (tmp >> 32), e + rhs.e + 64); +#endif + } + + DiyFp Normalize() const { + CEREAL_RAPIDJSON_ASSERT(f != 0); // https://stackoverflow.com/a/26809183/291737 +#if defined(_MSC_VER) && defined(_M_AMD64) + unsigned long index; + _BitScanReverse64(&index, f); + return DiyFp(f << (63 - index), e - (63 - index)); +#elif defined(__GNUC__) && __GNUC__ >= 4 + int s = __builtin_clzll(f); + return DiyFp(f << s, e - s); +#else + DiyFp res = *this; + while (!(res.f & (static_cast(1) << 63))) { + res.f <<= 1; + res.e--; + } + return res; +#endif + } + + DiyFp NormalizeBoundary() const { + DiyFp res = *this; + while (!(res.f & (kDpHiddenBit << 1))) { + res.f <<= 1; + res.e--; + } + res.f <<= (kDiySignificandSize - kDpSignificandSize - 2); + res.e = res.e - (kDiySignificandSize - kDpSignificandSize - 2); + return res; + } + + void NormalizedBoundaries(DiyFp* minus, DiyFp* plus) const { + DiyFp pl = DiyFp((f << 1) + 1, e - 1).NormalizeBoundary(); + DiyFp mi = (f == kDpHiddenBit) ? DiyFp((f << 2) - 1, e - 2) : DiyFp((f << 1) - 1, e - 1); + mi.f <<= mi.e - pl.e; + mi.e = pl.e; + *plus = pl; + *minus = mi; + } + + double ToDouble() const { + union { + double d; + uint64_t u64; + }u; + CEREAL_RAPIDJSON_ASSERT(f <= kDpHiddenBit + kDpSignificandMask); + if (e < kDpDenormalExponent) { + // Underflow. + return 0.0; + } + if (e >= kDpMaxExponent) { + // Overflow. + return std::numeric_limits::infinity(); + } + const uint64_t be = (e == kDpDenormalExponent && (f & kDpHiddenBit) == 0) ? 0 : + static_cast(e + kDpExponentBias); + u.u64 = (f & kDpSignificandMask) | (be << kDpSignificandSize); + return u.d; + } + + static const int kDiySignificandSize = 64; + static const int kDpSignificandSize = 52; + static const int kDpExponentBias = 0x3FF + kDpSignificandSize; + static const int kDpMaxExponent = 0x7FF - kDpExponentBias; + static const int kDpMinExponent = -kDpExponentBias; + static const int kDpDenormalExponent = -kDpExponentBias + 1; + static const uint64_t kDpExponentMask = CEREAL_RAPIDJSON_UINT64_C2(0x7FF00000, 0x00000000); + static const uint64_t kDpSignificandMask = CEREAL_RAPIDJSON_UINT64_C2(0x000FFFFF, 0xFFFFFFFF); + static const uint64_t kDpHiddenBit = CEREAL_RAPIDJSON_UINT64_C2(0x00100000, 0x00000000); + + uint64_t f; + int e; +}; + +inline DiyFp GetCachedPowerByIndex(size_t index) { + // 10^-348, 10^-340, ..., 10^340 + static const uint64_t kCachedPowers_F[] = { + CEREAL_RAPIDJSON_UINT64_C2(0xfa8fd5a0, 0x081c0288), CEREAL_RAPIDJSON_UINT64_C2(0xbaaee17f, 0xa23ebf76), + CEREAL_RAPIDJSON_UINT64_C2(0x8b16fb20, 0x3055ac76), CEREAL_RAPIDJSON_UINT64_C2(0xcf42894a, 0x5dce35ea), + CEREAL_RAPIDJSON_UINT64_C2(0x9a6bb0aa, 0x55653b2d), CEREAL_RAPIDJSON_UINT64_C2(0xe61acf03, 0x3d1a45df), + CEREAL_RAPIDJSON_UINT64_C2(0xab70fe17, 0xc79ac6ca), CEREAL_RAPIDJSON_UINT64_C2(0xff77b1fc, 0xbebcdc4f), + CEREAL_RAPIDJSON_UINT64_C2(0xbe5691ef, 0x416bd60c), CEREAL_RAPIDJSON_UINT64_C2(0x8dd01fad, 0x907ffc3c), + CEREAL_RAPIDJSON_UINT64_C2(0xd3515c28, 0x31559a83), CEREAL_RAPIDJSON_UINT64_C2(0x9d71ac8f, 0xada6c9b5), + CEREAL_RAPIDJSON_UINT64_C2(0xea9c2277, 0x23ee8bcb), CEREAL_RAPIDJSON_UINT64_C2(0xaecc4991, 0x4078536d), + CEREAL_RAPIDJSON_UINT64_C2(0x823c1279, 0x5db6ce57), CEREAL_RAPIDJSON_UINT64_C2(0xc2109436, 0x4dfb5637), + CEREAL_RAPIDJSON_UINT64_C2(0x9096ea6f, 0x3848984f), CEREAL_RAPIDJSON_UINT64_C2(0xd77485cb, 0x25823ac7), + CEREAL_RAPIDJSON_UINT64_C2(0xa086cfcd, 0x97bf97f4), CEREAL_RAPIDJSON_UINT64_C2(0xef340a98, 0x172aace5), + CEREAL_RAPIDJSON_UINT64_C2(0xb23867fb, 0x2a35b28e), CEREAL_RAPIDJSON_UINT64_C2(0x84c8d4df, 0xd2c63f3b), + CEREAL_RAPIDJSON_UINT64_C2(0xc5dd4427, 0x1ad3cdba), CEREAL_RAPIDJSON_UINT64_C2(0x936b9fce, 0xbb25c996), + CEREAL_RAPIDJSON_UINT64_C2(0xdbac6c24, 0x7d62a584), CEREAL_RAPIDJSON_UINT64_C2(0xa3ab6658, 0x0d5fdaf6), + CEREAL_RAPIDJSON_UINT64_C2(0xf3e2f893, 0xdec3f126), CEREAL_RAPIDJSON_UINT64_C2(0xb5b5ada8, 0xaaff80b8), + CEREAL_RAPIDJSON_UINT64_C2(0x87625f05, 0x6c7c4a8b), CEREAL_RAPIDJSON_UINT64_C2(0xc9bcff60, 0x34c13053), + CEREAL_RAPIDJSON_UINT64_C2(0x964e858c, 0x91ba2655), CEREAL_RAPIDJSON_UINT64_C2(0xdff97724, 0x70297ebd), + CEREAL_RAPIDJSON_UINT64_C2(0xa6dfbd9f, 0xb8e5b88f), CEREAL_RAPIDJSON_UINT64_C2(0xf8a95fcf, 0x88747d94), + CEREAL_RAPIDJSON_UINT64_C2(0xb9447093, 0x8fa89bcf), CEREAL_RAPIDJSON_UINT64_C2(0x8a08f0f8, 0xbf0f156b), + CEREAL_RAPIDJSON_UINT64_C2(0xcdb02555, 0x653131b6), CEREAL_RAPIDJSON_UINT64_C2(0x993fe2c6, 0xd07b7fac), + CEREAL_RAPIDJSON_UINT64_C2(0xe45c10c4, 0x2a2b3b06), CEREAL_RAPIDJSON_UINT64_C2(0xaa242499, 0x697392d3), + CEREAL_RAPIDJSON_UINT64_C2(0xfd87b5f2, 0x8300ca0e), CEREAL_RAPIDJSON_UINT64_C2(0xbce50864, 0x92111aeb), + CEREAL_RAPIDJSON_UINT64_C2(0x8cbccc09, 0x6f5088cc), CEREAL_RAPIDJSON_UINT64_C2(0xd1b71758, 0xe219652c), + CEREAL_RAPIDJSON_UINT64_C2(0x9c400000, 0x00000000), CEREAL_RAPIDJSON_UINT64_C2(0xe8d4a510, 0x00000000), + CEREAL_RAPIDJSON_UINT64_C2(0xad78ebc5, 0xac620000), CEREAL_RAPIDJSON_UINT64_C2(0x813f3978, 0xf8940984), + CEREAL_RAPIDJSON_UINT64_C2(0xc097ce7b, 0xc90715b3), CEREAL_RAPIDJSON_UINT64_C2(0x8f7e32ce, 0x7bea5c70), + CEREAL_RAPIDJSON_UINT64_C2(0xd5d238a4, 0xabe98068), CEREAL_RAPIDJSON_UINT64_C2(0x9f4f2726, 0x179a2245), + CEREAL_RAPIDJSON_UINT64_C2(0xed63a231, 0xd4c4fb27), CEREAL_RAPIDJSON_UINT64_C2(0xb0de6538, 0x8cc8ada8), + CEREAL_RAPIDJSON_UINT64_C2(0x83c7088e, 0x1aab65db), CEREAL_RAPIDJSON_UINT64_C2(0xc45d1df9, 0x42711d9a), + CEREAL_RAPIDJSON_UINT64_C2(0x924d692c, 0xa61be758), CEREAL_RAPIDJSON_UINT64_C2(0xda01ee64, 0x1a708dea), + CEREAL_RAPIDJSON_UINT64_C2(0xa26da399, 0x9aef774a), CEREAL_RAPIDJSON_UINT64_C2(0xf209787b, 0xb47d6b85), + CEREAL_RAPIDJSON_UINT64_C2(0xb454e4a1, 0x79dd1877), CEREAL_RAPIDJSON_UINT64_C2(0x865b8692, 0x5b9bc5c2), + CEREAL_RAPIDJSON_UINT64_C2(0xc83553c5, 0xc8965d3d), CEREAL_RAPIDJSON_UINT64_C2(0x952ab45c, 0xfa97a0b3), + CEREAL_RAPIDJSON_UINT64_C2(0xde469fbd, 0x99a05fe3), CEREAL_RAPIDJSON_UINT64_C2(0xa59bc234, 0xdb398c25), + CEREAL_RAPIDJSON_UINT64_C2(0xf6c69a72, 0xa3989f5c), CEREAL_RAPIDJSON_UINT64_C2(0xb7dcbf53, 0x54e9bece), + CEREAL_RAPIDJSON_UINT64_C2(0x88fcf317, 0xf22241e2), CEREAL_RAPIDJSON_UINT64_C2(0xcc20ce9b, 0xd35c78a5), + CEREAL_RAPIDJSON_UINT64_C2(0x98165af3, 0x7b2153df), CEREAL_RAPIDJSON_UINT64_C2(0xe2a0b5dc, 0x971f303a), + CEREAL_RAPIDJSON_UINT64_C2(0xa8d9d153, 0x5ce3b396), CEREAL_RAPIDJSON_UINT64_C2(0xfb9b7cd9, 0xa4a7443c), + CEREAL_RAPIDJSON_UINT64_C2(0xbb764c4c, 0xa7a44410), CEREAL_RAPIDJSON_UINT64_C2(0x8bab8eef, 0xb6409c1a), + CEREAL_RAPIDJSON_UINT64_C2(0xd01fef10, 0xa657842c), CEREAL_RAPIDJSON_UINT64_C2(0x9b10a4e5, 0xe9913129), + CEREAL_RAPIDJSON_UINT64_C2(0xe7109bfb, 0xa19c0c9d), CEREAL_RAPIDJSON_UINT64_C2(0xac2820d9, 0x623bf429), + CEREAL_RAPIDJSON_UINT64_C2(0x80444b5e, 0x7aa7cf85), CEREAL_RAPIDJSON_UINT64_C2(0xbf21e440, 0x03acdd2d), + CEREAL_RAPIDJSON_UINT64_C2(0x8e679c2f, 0x5e44ff8f), CEREAL_RAPIDJSON_UINT64_C2(0xd433179d, 0x9c8cb841), + CEREAL_RAPIDJSON_UINT64_C2(0x9e19db92, 0xb4e31ba9), CEREAL_RAPIDJSON_UINT64_C2(0xeb96bf6e, 0xbadf77d9), + CEREAL_RAPIDJSON_UINT64_C2(0xaf87023b, 0x9bf0ee6b) + }; + static const int16_t kCachedPowers_E[] = { + -1220, -1193, -1166, -1140, -1113, -1087, -1060, -1034, -1007, -980, + -954, -927, -901, -874, -847, -821, -794, -768, -741, -715, + -688, -661, -635, -608, -582, -555, -529, -502, -475, -449, + -422, -396, -369, -343, -316, -289, -263, -236, -210, -183, + -157, -130, -103, -77, -50, -24, 3, 30, 56, 83, + 109, 136, 162, 189, 216, 242, 269, 295, 322, 348, + 375, 402, 428, 455, 481, 508, 534, 561, 588, 614, + 641, 667, 694, 720, 747, 774, 800, 827, 853, 880, + 907, 933, 960, 986, 1013, 1039, 1066 + }; + CEREAL_RAPIDJSON_ASSERT(index < 87); + return DiyFp(kCachedPowers_F[index], kCachedPowers_E[index]); +} + +inline DiyFp GetCachedPower(int e, int* K) { + + //int k = static_cast(ceil((-61 - e) * 0.30102999566398114)) + 374; + double dk = (-61 - e) * 0.30102999566398114 + 347; // dk must be positive, so can do ceiling in positive + int k = static_cast(dk); + if (dk - k > 0.0) + k++; + + unsigned index = static_cast((k >> 3) + 1); + *K = -(-348 + static_cast(index << 3)); // decimal exponent no need lookup table + + return GetCachedPowerByIndex(index); +} + +inline DiyFp GetCachedPower10(int exp, int *outExp) { + CEREAL_RAPIDJSON_ASSERT(exp >= -348); + unsigned index = static_cast(exp + 348) / 8u; + *outExp = -348 + static_cast(index) * 8; + return GetCachedPowerByIndex(index); +} + +#ifdef __GNUC__ +CEREAL_RAPIDJSON_DIAG_POP +#endif + +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_POP +CEREAL_RAPIDJSON_DIAG_OFF(padded) +#endif + +} // namespace internal +CEREAL_RAPIDJSON_NAMESPACE_END + +#endif // CEREAL_RAPIDJSON_DIYFP_H_ diff --git a/third_party/cereal/external/rapidjson/internal/dtoa.h b/third_party/cereal/external/rapidjson/internal/dtoa.h new file mode 100755 index 0000000..ea62b34 --- /dev/null +++ b/third_party/cereal/external/rapidjson/internal/dtoa.h @@ -0,0 +1,245 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +// This is a C++ header-only implementation of Grisu2 algorithm from the publication: +// Loitsch, Florian. "Printing floating-point numbers quickly and accurately with +// integers." ACM Sigplan Notices 45.6 (2010): 233-243. + +#ifndef CEREAL_RAPIDJSON_DTOA_ +#define CEREAL_RAPIDJSON_DTOA_ + +#include "itoa.h" // GetDigitsLut() +#include "diyfp.h" +#include "ieee754.h" + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +#ifdef __GNUC__ +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(effc++) +CEREAL_RAPIDJSON_DIAG_OFF(array-bounds) // some gcc versions generate wrong warnings https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59124 +#endif + +inline void GrisuRound(char* buffer, int len, uint64_t delta, uint64_t rest, uint64_t ten_kappa, uint64_t wp_w) { + while (rest < wp_w && delta - rest >= ten_kappa && + (rest + ten_kappa < wp_w || /// closer + wp_w - rest > rest + ten_kappa - wp_w)) { + buffer[len - 1]--; + rest += ten_kappa; + } +} + +inline int CountDecimalDigit32(uint32_t n) { + // Simple pure C++ implementation was faster than __builtin_clz version in this situation. + if (n < 10) return 1; + if (n < 100) return 2; + if (n < 1000) return 3; + if (n < 10000) return 4; + if (n < 100000) return 5; + if (n < 1000000) return 6; + if (n < 10000000) return 7; + if (n < 100000000) return 8; + // Will not reach 10 digits in DigitGen() + //if (n < 1000000000) return 9; + //return 10; + return 9; +} + +inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buffer, int* len, int* K) { + static const uint32_t kPow10[] = { 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000 }; + const DiyFp one(uint64_t(1) << -Mp.e, Mp.e); + const DiyFp wp_w = Mp - W; + uint32_t p1 = static_cast(Mp.f >> -one.e); + uint64_t p2 = Mp.f & (one.f - 1); + int kappa = CountDecimalDigit32(p1); // kappa in [0, 9] + *len = 0; + + while (kappa > 0) { + uint32_t d = 0; + switch (kappa) { + case 9: d = p1 / 100000000; p1 %= 100000000; break; + case 8: d = p1 / 10000000; p1 %= 10000000; break; + case 7: d = p1 / 1000000; p1 %= 1000000; break; + case 6: d = p1 / 100000; p1 %= 100000; break; + case 5: d = p1 / 10000; p1 %= 10000; break; + case 4: d = p1 / 1000; p1 %= 1000; break; + case 3: d = p1 / 100; p1 %= 100; break; + case 2: d = p1 / 10; p1 %= 10; break; + case 1: d = p1; p1 = 0; break; + default:; + } + if (d || *len) + buffer[(*len)++] = static_cast('0' + static_cast(d)); + kappa--; + uint64_t tmp = (static_cast(p1) << -one.e) + p2; + if (tmp <= delta) { + *K += kappa; + GrisuRound(buffer, *len, delta, tmp, static_cast(kPow10[kappa]) << -one.e, wp_w.f); + return; + } + } + + // kappa = 0 + for (;;) { + p2 *= 10; + delta *= 10; + char d = static_cast(p2 >> -one.e); + if (d || *len) + buffer[(*len)++] = static_cast('0' + d); + p2 &= one.f - 1; + kappa--; + if (p2 < delta) { + *K += kappa; + int index = -kappa; + GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * (index < 9 ? kPow10[index] : 0)); + return; + } + } +} + +inline void Grisu2(double value, char* buffer, int* length, int* K) { + const DiyFp v(value); + DiyFp w_m, w_p; + v.NormalizedBoundaries(&w_m, &w_p); + + const DiyFp c_mk = GetCachedPower(w_p.e, K); + const DiyFp W = v.Normalize() * c_mk; + DiyFp Wp = w_p * c_mk; + DiyFp Wm = w_m * c_mk; + Wm.f++; + Wp.f--; + DigitGen(W, Wp, Wp.f - Wm.f, buffer, length, K); +} + +inline char* WriteExponent(int K, char* buffer) { + if (K < 0) { + *buffer++ = '-'; + K = -K; + } + + if (K >= 100) { + *buffer++ = static_cast('0' + static_cast(K / 100)); + K %= 100; + const char* d = GetDigitsLut() + K * 2; + *buffer++ = d[0]; + *buffer++ = d[1]; + } + else if (K >= 10) { + const char* d = GetDigitsLut() + K * 2; + *buffer++ = d[0]; + *buffer++ = d[1]; + } + else + *buffer++ = static_cast('0' + static_cast(K)); + + return buffer; +} + +inline char* Prettify(char* buffer, int length, int k, int maxDecimalPlaces) { + const int kk = length + k; // 10^(kk-1) <= v < 10^kk + + if (0 <= k && kk <= 21) { + // 1234e7 -> 12340000000 + for (int i = length; i < kk; i++) + buffer[i] = '0'; + buffer[kk] = '.'; + buffer[kk + 1] = '0'; + return &buffer[kk + 2]; + } + else if (0 < kk && kk <= 21) { + // 1234e-2 -> 12.34 + std::memmove(&buffer[kk + 1], &buffer[kk], static_cast(length - kk)); + buffer[kk] = '.'; + if (0 > k + maxDecimalPlaces) { + // When maxDecimalPlaces = 2, 1.2345 -> 1.23, 1.102 -> 1.1 + // Remove extra trailing zeros (at least one) after truncation. + for (int i = kk + maxDecimalPlaces; i > kk + 1; i--) + if (buffer[i] != '0') + return &buffer[i + 1]; + return &buffer[kk + 2]; // Reserve one zero + } + else + return &buffer[length + 1]; + } + else if (-6 < kk && kk <= 0) { + // 1234e-6 -> 0.001234 + const int offset = 2 - kk; + std::memmove(&buffer[offset], &buffer[0], static_cast(length)); + buffer[0] = '0'; + buffer[1] = '.'; + for (int i = 2; i < offset; i++) + buffer[i] = '0'; + if (length - kk > maxDecimalPlaces) { + // When maxDecimalPlaces = 2, 0.123 -> 0.12, 0.102 -> 0.1 + // Remove extra trailing zeros (at least one) after truncation. + for (int i = maxDecimalPlaces + 1; i > 2; i--) + if (buffer[i] != '0') + return &buffer[i + 1]; + return &buffer[3]; // Reserve one zero + } + else + return &buffer[length + offset]; + } + else if (kk < -maxDecimalPlaces) { + // Truncate to zero + buffer[0] = '0'; + buffer[1] = '.'; + buffer[2] = '0'; + return &buffer[3]; + } + else if (length == 1) { + // 1e30 + buffer[1] = 'e'; + return WriteExponent(kk - 1, &buffer[2]); + } + else { + // 1234e30 -> 1.234e33 + std::memmove(&buffer[2], &buffer[1], static_cast(length - 1)); + buffer[1] = '.'; + buffer[length + 1] = 'e'; + return WriteExponent(kk - 1, &buffer[0 + length + 2]); + } +} + +inline char* dtoa(double value, char* buffer, int maxDecimalPlaces = 324) { + CEREAL_RAPIDJSON_ASSERT(maxDecimalPlaces >= 1); + Double d(value); + if (d.IsZero()) { + if (d.Sign()) + *buffer++ = '-'; // -0.0, Issue #289 + buffer[0] = '0'; + buffer[1] = '.'; + buffer[2] = '0'; + return &buffer[3]; + } + else { + if (value < 0) { + *buffer++ = '-'; + value = -value; + } + int length, K; + Grisu2(value, buffer, &length, &K); + return Prettify(buffer, length, K, maxDecimalPlaces); + } +} + +#ifdef __GNUC__ +CEREAL_RAPIDJSON_DIAG_POP +#endif + +} // namespace internal +CEREAL_RAPIDJSON_NAMESPACE_END + +#endif // CEREAL_RAPIDJSON_DTOA_ diff --git a/third_party/cereal/external/rapidjson/internal/ieee754.h b/third_party/cereal/external/rapidjson/internal/ieee754.h new file mode 100755 index 0000000..6730dfd --- /dev/null +++ b/third_party/cereal/external/rapidjson/internal/ieee754.h @@ -0,0 +1,78 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_IEEE754_ +#define CEREAL_RAPIDJSON_IEEE754_ + +#include "../rapidjson.h" + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +class Double { +public: + Double() {} + Double(double d) : d_(d) {} + Double(uint64_t u) : u_(u) {} + + double Value() const { return d_; } + uint64_t Uint64Value() const { return u_; } + + double NextPositiveDouble() const { + CEREAL_RAPIDJSON_ASSERT(!Sign()); + return Double(u_ + 1).Value(); + } + + bool Sign() const { return (u_ & kSignMask) != 0; } + uint64_t Significand() const { return u_ & kSignificandMask; } + int Exponent() const { return static_cast(((u_ & kExponentMask) >> kSignificandSize) - kExponentBias); } + + bool IsNan() const { return (u_ & kExponentMask) == kExponentMask && Significand() != 0; } + bool IsInf() const { return (u_ & kExponentMask) == kExponentMask && Significand() == 0; } + bool IsNanOrInf() const { return (u_ & kExponentMask) == kExponentMask; } + bool IsNormal() const { return (u_ & kExponentMask) != 0 || Significand() == 0; } + bool IsZero() const { return (u_ & (kExponentMask | kSignificandMask)) == 0; } + + uint64_t IntegerSignificand() const { return IsNormal() ? Significand() | kHiddenBit : Significand(); } + int IntegerExponent() const { return (IsNormal() ? Exponent() : kDenormalExponent) - kSignificandSize; } + uint64_t ToBias() const { return (u_ & kSignMask) ? ~u_ + 1 : u_ | kSignMask; } + + static int EffectiveSignificandSize(int order) { + if (order >= -1021) + return 53; + else if (order <= -1074) + return 0; + else + return order + 1074; + } + +private: + static const int kSignificandSize = 52; + static const int kExponentBias = 0x3FF; + static const int kDenormalExponent = 1 - kExponentBias; + static const uint64_t kSignMask = CEREAL_RAPIDJSON_UINT64_C2(0x80000000, 0x00000000); + static const uint64_t kExponentMask = CEREAL_RAPIDJSON_UINT64_C2(0x7FF00000, 0x00000000); + static const uint64_t kSignificandMask = CEREAL_RAPIDJSON_UINT64_C2(0x000FFFFF, 0xFFFFFFFF); + static const uint64_t kHiddenBit = CEREAL_RAPIDJSON_UINT64_C2(0x00100000, 0x00000000); + + union { + double d_; + uint64_t u_; + }; +}; + +} // namespace internal +CEREAL_RAPIDJSON_NAMESPACE_END + +#endif // CEREAL_RAPIDJSON_IEEE754_ diff --git a/third_party/cereal/external/rapidjson/internal/itoa.h b/third_party/cereal/external/rapidjson/internal/itoa.h new file mode 100755 index 0000000..f04e3fa --- /dev/null +++ b/third_party/cereal/external/rapidjson/internal/itoa.h @@ -0,0 +1,308 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_ITOA_ +#define CEREAL_RAPIDJSON_ITOA_ + +#include "../rapidjson.h" + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +inline const char* GetDigitsLut() { + static const char cDigitsLut[200] = { + '0','0','0','1','0','2','0','3','0','4','0','5','0','6','0','7','0','8','0','9', + '1','0','1','1','1','2','1','3','1','4','1','5','1','6','1','7','1','8','1','9', + '2','0','2','1','2','2','2','3','2','4','2','5','2','6','2','7','2','8','2','9', + '3','0','3','1','3','2','3','3','3','4','3','5','3','6','3','7','3','8','3','9', + '4','0','4','1','4','2','4','3','4','4','4','5','4','6','4','7','4','8','4','9', + '5','0','5','1','5','2','5','3','5','4','5','5','5','6','5','7','5','8','5','9', + '6','0','6','1','6','2','6','3','6','4','6','5','6','6','6','7','6','8','6','9', + '7','0','7','1','7','2','7','3','7','4','7','5','7','6','7','7','7','8','7','9', + '8','0','8','1','8','2','8','3','8','4','8','5','8','6','8','7','8','8','8','9', + '9','0','9','1','9','2','9','3','9','4','9','5','9','6','9','7','9','8','9','9' + }; + return cDigitsLut; +} + +inline char* u32toa(uint32_t value, char* buffer) { + CEREAL_RAPIDJSON_ASSERT(buffer != 0); + + const char* cDigitsLut = GetDigitsLut(); + + if (value < 10000) { + const uint32_t d1 = (value / 100) << 1; + const uint32_t d2 = (value % 100) << 1; + + if (value >= 1000) + *buffer++ = cDigitsLut[d1]; + if (value >= 100) + *buffer++ = cDigitsLut[d1 + 1]; + if (value >= 10) + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + } + else if (value < 100000000) { + // value = bbbbcccc + const uint32_t b = value / 10000; + const uint32_t c = value % 10000; + + const uint32_t d1 = (b / 100) << 1; + const uint32_t d2 = (b % 100) << 1; + + const uint32_t d3 = (c / 100) << 1; + const uint32_t d4 = (c % 100) << 1; + + if (value >= 10000000) + *buffer++ = cDigitsLut[d1]; + if (value >= 1000000) + *buffer++ = cDigitsLut[d1 + 1]; + if (value >= 100000) + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + + *buffer++ = cDigitsLut[d3]; + *buffer++ = cDigitsLut[d3 + 1]; + *buffer++ = cDigitsLut[d4]; + *buffer++ = cDigitsLut[d4 + 1]; + } + else { + // value = aabbbbcccc in decimal + + const uint32_t a = value / 100000000; // 1 to 42 + value %= 100000000; + + if (a >= 10) { + const unsigned i = a << 1; + *buffer++ = cDigitsLut[i]; + *buffer++ = cDigitsLut[i + 1]; + } + else + *buffer++ = static_cast('0' + static_cast(a)); + + const uint32_t b = value / 10000; // 0 to 9999 + const uint32_t c = value % 10000; // 0 to 9999 + + const uint32_t d1 = (b / 100) << 1; + const uint32_t d2 = (b % 100) << 1; + + const uint32_t d3 = (c / 100) << 1; + const uint32_t d4 = (c % 100) << 1; + + *buffer++ = cDigitsLut[d1]; + *buffer++ = cDigitsLut[d1 + 1]; + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + *buffer++ = cDigitsLut[d3]; + *buffer++ = cDigitsLut[d3 + 1]; + *buffer++ = cDigitsLut[d4]; + *buffer++ = cDigitsLut[d4 + 1]; + } + return buffer; +} + +inline char* i32toa(int32_t value, char* buffer) { + CEREAL_RAPIDJSON_ASSERT(buffer != 0); + uint32_t u = static_cast(value); + if (value < 0) { + *buffer++ = '-'; + u = ~u + 1; + } + + return u32toa(u, buffer); +} + +inline char* u64toa(uint64_t value, char* buffer) { + CEREAL_RAPIDJSON_ASSERT(buffer != 0); + const char* cDigitsLut = GetDigitsLut(); + const uint64_t kTen8 = 100000000; + const uint64_t kTen9 = kTen8 * 10; + const uint64_t kTen10 = kTen8 * 100; + const uint64_t kTen11 = kTen8 * 1000; + const uint64_t kTen12 = kTen8 * 10000; + const uint64_t kTen13 = kTen8 * 100000; + const uint64_t kTen14 = kTen8 * 1000000; + const uint64_t kTen15 = kTen8 * 10000000; + const uint64_t kTen16 = kTen8 * kTen8; + + if (value < kTen8) { + uint32_t v = static_cast(value); + if (v < 10000) { + const uint32_t d1 = (v / 100) << 1; + const uint32_t d2 = (v % 100) << 1; + + if (v >= 1000) + *buffer++ = cDigitsLut[d1]; + if (v >= 100) + *buffer++ = cDigitsLut[d1 + 1]; + if (v >= 10) + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + } + else { + // value = bbbbcccc + const uint32_t b = v / 10000; + const uint32_t c = v % 10000; + + const uint32_t d1 = (b / 100) << 1; + const uint32_t d2 = (b % 100) << 1; + + const uint32_t d3 = (c / 100) << 1; + const uint32_t d4 = (c % 100) << 1; + + if (value >= 10000000) + *buffer++ = cDigitsLut[d1]; + if (value >= 1000000) + *buffer++ = cDigitsLut[d1 + 1]; + if (value >= 100000) + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + + *buffer++ = cDigitsLut[d3]; + *buffer++ = cDigitsLut[d3 + 1]; + *buffer++ = cDigitsLut[d4]; + *buffer++ = cDigitsLut[d4 + 1]; + } + } + else if (value < kTen16) { + const uint32_t v0 = static_cast(value / kTen8); + const uint32_t v1 = static_cast(value % kTen8); + + const uint32_t b0 = v0 / 10000; + const uint32_t c0 = v0 % 10000; + + const uint32_t d1 = (b0 / 100) << 1; + const uint32_t d2 = (b0 % 100) << 1; + + const uint32_t d3 = (c0 / 100) << 1; + const uint32_t d4 = (c0 % 100) << 1; + + const uint32_t b1 = v1 / 10000; + const uint32_t c1 = v1 % 10000; + + const uint32_t d5 = (b1 / 100) << 1; + const uint32_t d6 = (b1 % 100) << 1; + + const uint32_t d7 = (c1 / 100) << 1; + const uint32_t d8 = (c1 % 100) << 1; + + if (value >= kTen15) + *buffer++ = cDigitsLut[d1]; + if (value >= kTen14) + *buffer++ = cDigitsLut[d1 + 1]; + if (value >= kTen13) + *buffer++ = cDigitsLut[d2]; + if (value >= kTen12) + *buffer++ = cDigitsLut[d2 + 1]; + if (value >= kTen11) + *buffer++ = cDigitsLut[d3]; + if (value >= kTen10) + *buffer++ = cDigitsLut[d3 + 1]; + if (value >= kTen9) + *buffer++ = cDigitsLut[d4]; + + *buffer++ = cDigitsLut[d4 + 1]; + *buffer++ = cDigitsLut[d5]; + *buffer++ = cDigitsLut[d5 + 1]; + *buffer++ = cDigitsLut[d6]; + *buffer++ = cDigitsLut[d6 + 1]; + *buffer++ = cDigitsLut[d7]; + *buffer++ = cDigitsLut[d7 + 1]; + *buffer++ = cDigitsLut[d8]; + *buffer++ = cDigitsLut[d8 + 1]; + } + else { + const uint32_t a = static_cast(value / kTen16); // 1 to 1844 + value %= kTen16; + + if (a < 10) + *buffer++ = static_cast('0' + static_cast(a)); + else if (a < 100) { + const uint32_t i = a << 1; + *buffer++ = cDigitsLut[i]; + *buffer++ = cDigitsLut[i + 1]; + } + else if (a < 1000) { + *buffer++ = static_cast('0' + static_cast(a / 100)); + + const uint32_t i = (a % 100) << 1; + *buffer++ = cDigitsLut[i]; + *buffer++ = cDigitsLut[i + 1]; + } + else { + const uint32_t i = (a / 100) << 1; + const uint32_t j = (a % 100) << 1; + *buffer++ = cDigitsLut[i]; + *buffer++ = cDigitsLut[i + 1]; + *buffer++ = cDigitsLut[j]; + *buffer++ = cDigitsLut[j + 1]; + } + + const uint32_t v0 = static_cast(value / kTen8); + const uint32_t v1 = static_cast(value % kTen8); + + const uint32_t b0 = v0 / 10000; + const uint32_t c0 = v0 % 10000; + + const uint32_t d1 = (b0 / 100) << 1; + const uint32_t d2 = (b0 % 100) << 1; + + const uint32_t d3 = (c0 / 100) << 1; + const uint32_t d4 = (c0 % 100) << 1; + + const uint32_t b1 = v1 / 10000; + const uint32_t c1 = v1 % 10000; + + const uint32_t d5 = (b1 / 100) << 1; + const uint32_t d6 = (b1 % 100) << 1; + + const uint32_t d7 = (c1 / 100) << 1; + const uint32_t d8 = (c1 % 100) << 1; + + *buffer++ = cDigitsLut[d1]; + *buffer++ = cDigitsLut[d1 + 1]; + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + *buffer++ = cDigitsLut[d3]; + *buffer++ = cDigitsLut[d3 + 1]; + *buffer++ = cDigitsLut[d4]; + *buffer++ = cDigitsLut[d4 + 1]; + *buffer++ = cDigitsLut[d5]; + *buffer++ = cDigitsLut[d5 + 1]; + *buffer++ = cDigitsLut[d6]; + *buffer++ = cDigitsLut[d6 + 1]; + *buffer++ = cDigitsLut[d7]; + *buffer++ = cDigitsLut[d7 + 1]; + *buffer++ = cDigitsLut[d8]; + *buffer++ = cDigitsLut[d8 + 1]; + } + + return buffer; +} + +inline char* i64toa(int64_t value, char* buffer) { + CEREAL_RAPIDJSON_ASSERT(buffer != 0); + uint64_t u = static_cast(value); + if (value < 0) { + *buffer++ = '-'; + u = ~u + 1; + } + + return u64toa(u, buffer); +} + +} // namespace internal +CEREAL_RAPIDJSON_NAMESPACE_END + +#endif // CEREAL_RAPIDJSON_ITOA_ diff --git a/third_party/cereal/external/rapidjson/internal/meta.h b/third_party/cereal/external/rapidjson/internal/meta.h new file mode 100755 index 0000000..f511ff8 --- /dev/null +++ b/third_party/cereal/external/rapidjson/internal/meta.h @@ -0,0 +1,186 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_INTERNAL_META_H_ +#define CEREAL_RAPIDJSON_INTERNAL_META_H_ + +#include "../rapidjson.h" + +#ifdef __GNUC__ +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(effc++) +#endif + +#if defined(_MSC_VER) && !defined(__clang__) +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(6334) +#endif + +#if CEREAL_RAPIDJSON_HAS_CXX11_TYPETRAITS +#include +#endif + +//@cond CEREAL_RAPIDJSON_INTERNAL +CEREAL_RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +// Helper to wrap/convert arbitrary types to void, useful for arbitrary type matching +template struct Void { typedef void Type; }; + +/////////////////////////////////////////////////////////////////////////////// +// BoolType, TrueType, FalseType +// +template struct BoolType { + static const bool Value = Cond; + typedef BoolType Type; +}; +typedef BoolType TrueType; +typedef BoolType FalseType; + + +/////////////////////////////////////////////////////////////////////////////// +// SelectIf, BoolExpr, NotExpr, AndExpr, OrExpr +// + +template struct SelectIfImpl { template struct Apply { typedef T1 Type; }; }; +template <> struct SelectIfImpl { template struct Apply { typedef T2 Type; }; }; +template struct SelectIfCond : SelectIfImpl::template Apply {}; +template struct SelectIf : SelectIfCond {}; + +template struct AndExprCond : FalseType {}; +template <> struct AndExprCond : TrueType {}; +template struct OrExprCond : TrueType {}; +template <> struct OrExprCond : FalseType {}; + +template struct BoolExpr : SelectIf::Type {}; +template struct NotExpr : SelectIf::Type {}; +template struct AndExpr : AndExprCond::Type {}; +template struct OrExpr : OrExprCond::Type {}; + + +/////////////////////////////////////////////////////////////////////////////// +// AddConst, MaybeAddConst, RemoveConst +template struct AddConst { typedef const T Type; }; +template struct MaybeAddConst : SelectIfCond {}; +template struct RemoveConst { typedef T Type; }; +template struct RemoveConst { typedef T Type; }; + + +/////////////////////////////////////////////////////////////////////////////// +// IsSame, IsConst, IsMoreConst, IsPointer +// +template struct IsSame : FalseType {}; +template struct IsSame : TrueType {}; + +template struct IsConst : FalseType {}; +template struct IsConst : TrueType {}; + +template +struct IsMoreConst + : AndExpr::Type, typename RemoveConst::Type>, + BoolType::Value >= IsConst::Value> >::Type {}; + +template struct IsPointer : FalseType {}; +template struct IsPointer : TrueType {}; + +/////////////////////////////////////////////////////////////////////////////// +// IsBaseOf +// +#if CEREAL_RAPIDJSON_HAS_CXX11_TYPETRAITS + +template struct IsBaseOf + : BoolType< ::std::is_base_of::value> {}; + +#else // simplified version adopted from Boost + +template struct IsBaseOfImpl { + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(B) != 0); + CEREAL_RAPIDJSON_STATIC_ASSERT(sizeof(D) != 0); + + typedef char (&Yes)[1]; + typedef char (&No) [2]; + + template + static Yes Check(const D*, T); + static No Check(const B*, int); + + struct Host { + operator const B*() const; + operator const D*(); + }; + + enum { Value = (sizeof(Check(Host(), 0)) == sizeof(Yes)) }; +}; + +template struct IsBaseOf + : OrExpr, BoolExpr > >::Type {}; + +#endif // CEREAL_RAPIDJSON_HAS_CXX11_TYPETRAITS + + +////////////////////////////////////////////////////////////////////////// +// EnableIf / DisableIf +// +template struct EnableIfCond { typedef T Type; }; +template struct EnableIfCond { /* empty */ }; + +template struct DisableIfCond { typedef T Type; }; +template struct DisableIfCond { /* empty */ }; + +template +struct EnableIf : EnableIfCond {}; + +template +struct DisableIf : DisableIfCond {}; + +// SFINAE helpers +struct SfinaeTag {}; +template struct RemoveSfinaeTag; +template struct RemoveSfinaeTag { typedef T Type; }; + +#define CEREAL_RAPIDJSON_REMOVEFPTR_(type) \ + typename ::CEREAL_RAPIDJSON_NAMESPACE::internal::RemoveSfinaeTag \ + < ::CEREAL_RAPIDJSON_NAMESPACE::internal::SfinaeTag&(*) type>::Type + +#define CEREAL_RAPIDJSON_ENABLEIF(cond) \ + typename ::CEREAL_RAPIDJSON_NAMESPACE::internal::EnableIf \ + ::Type * = NULL + +#define CEREAL_RAPIDJSON_DISABLEIF(cond) \ + typename ::CEREAL_RAPIDJSON_NAMESPACE::internal::DisableIf \ + ::Type * = NULL + +#define CEREAL_RAPIDJSON_ENABLEIF_RETURN(cond,returntype) \ + typename ::CEREAL_RAPIDJSON_NAMESPACE::internal::EnableIf \ + ::Type + +#define CEREAL_RAPIDJSON_DISABLEIF_RETURN(cond,returntype) \ + typename ::CEREAL_RAPIDJSON_NAMESPACE::internal::DisableIf \ + ::Type + +} // namespace internal +CEREAL_RAPIDJSON_NAMESPACE_END +//@endcond + +#if defined(_MSC_VER) && !defined(__clang__) +CEREAL_RAPIDJSON_DIAG_POP +#endif + +#ifdef __GNUC__ +CEREAL_RAPIDJSON_DIAG_POP +#endif + +#endif // CEREAL_RAPIDJSON_INTERNAL_META_H_ diff --git a/third_party/cereal/external/rapidjson/internal/pow10.h b/third_party/cereal/external/rapidjson/internal/pow10.h new file mode 100755 index 0000000..7f796a1 --- /dev/null +++ b/third_party/cereal/external/rapidjson/internal/pow10.h @@ -0,0 +1,55 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_POW10_ +#define CEREAL_RAPIDJSON_POW10_ + +#include "../rapidjson.h" + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +//! Computes integer powers of 10 in double (10.0^n). +/*! This function uses lookup table for fast and accurate results. + \param n non-negative exponent. Must <= 308. + \return 10.0^n +*/ +inline double Pow10(int n) { + static const double e[] = { // 1e-0...1e308: 309 * 8 bytes = 2472 bytes + 1e+0, + 1e+1, 1e+2, 1e+3, 1e+4, 1e+5, 1e+6, 1e+7, 1e+8, 1e+9, 1e+10, 1e+11, 1e+12, 1e+13, 1e+14, 1e+15, 1e+16, 1e+17, 1e+18, 1e+19, 1e+20, + 1e+21, 1e+22, 1e+23, 1e+24, 1e+25, 1e+26, 1e+27, 1e+28, 1e+29, 1e+30, 1e+31, 1e+32, 1e+33, 1e+34, 1e+35, 1e+36, 1e+37, 1e+38, 1e+39, 1e+40, + 1e+41, 1e+42, 1e+43, 1e+44, 1e+45, 1e+46, 1e+47, 1e+48, 1e+49, 1e+50, 1e+51, 1e+52, 1e+53, 1e+54, 1e+55, 1e+56, 1e+57, 1e+58, 1e+59, 1e+60, + 1e+61, 1e+62, 1e+63, 1e+64, 1e+65, 1e+66, 1e+67, 1e+68, 1e+69, 1e+70, 1e+71, 1e+72, 1e+73, 1e+74, 1e+75, 1e+76, 1e+77, 1e+78, 1e+79, 1e+80, + 1e+81, 1e+82, 1e+83, 1e+84, 1e+85, 1e+86, 1e+87, 1e+88, 1e+89, 1e+90, 1e+91, 1e+92, 1e+93, 1e+94, 1e+95, 1e+96, 1e+97, 1e+98, 1e+99, 1e+100, + 1e+101,1e+102,1e+103,1e+104,1e+105,1e+106,1e+107,1e+108,1e+109,1e+110,1e+111,1e+112,1e+113,1e+114,1e+115,1e+116,1e+117,1e+118,1e+119,1e+120, + 1e+121,1e+122,1e+123,1e+124,1e+125,1e+126,1e+127,1e+128,1e+129,1e+130,1e+131,1e+132,1e+133,1e+134,1e+135,1e+136,1e+137,1e+138,1e+139,1e+140, + 1e+141,1e+142,1e+143,1e+144,1e+145,1e+146,1e+147,1e+148,1e+149,1e+150,1e+151,1e+152,1e+153,1e+154,1e+155,1e+156,1e+157,1e+158,1e+159,1e+160, + 1e+161,1e+162,1e+163,1e+164,1e+165,1e+166,1e+167,1e+168,1e+169,1e+170,1e+171,1e+172,1e+173,1e+174,1e+175,1e+176,1e+177,1e+178,1e+179,1e+180, + 1e+181,1e+182,1e+183,1e+184,1e+185,1e+186,1e+187,1e+188,1e+189,1e+190,1e+191,1e+192,1e+193,1e+194,1e+195,1e+196,1e+197,1e+198,1e+199,1e+200, + 1e+201,1e+202,1e+203,1e+204,1e+205,1e+206,1e+207,1e+208,1e+209,1e+210,1e+211,1e+212,1e+213,1e+214,1e+215,1e+216,1e+217,1e+218,1e+219,1e+220, + 1e+221,1e+222,1e+223,1e+224,1e+225,1e+226,1e+227,1e+228,1e+229,1e+230,1e+231,1e+232,1e+233,1e+234,1e+235,1e+236,1e+237,1e+238,1e+239,1e+240, + 1e+241,1e+242,1e+243,1e+244,1e+245,1e+246,1e+247,1e+248,1e+249,1e+250,1e+251,1e+252,1e+253,1e+254,1e+255,1e+256,1e+257,1e+258,1e+259,1e+260, + 1e+261,1e+262,1e+263,1e+264,1e+265,1e+266,1e+267,1e+268,1e+269,1e+270,1e+271,1e+272,1e+273,1e+274,1e+275,1e+276,1e+277,1e+278,1e+279,1e+280, + 1e+281,1e+282,1e+283,1e+284,1e+285,1e+286,1e+287,1e+288,1e+289,1e+290,1e+291,1e+292,1e+293,1e+294,1e+295,1e+296,1e+297,1e+298,1e+299,1e+300, + 1e+301,1e+302,1e+303,1e+304,1e+305,1e+306,1e+307,1e+308 + }; + CEREAL_RAPIDJSON_ASSERT(n >= 0 && n <= 308); + return e[n]; +} + +} // namespace internal +CEREAL_RAPIDJSON_NAMESPACE_END + +#endif // CEREAL_RAPIDJSON_POW10_ diff --git a/third_party/cereal/external/rapidjson/internal/regex.h b/third_party/cereal/external/rapidjson/internal/regex.h new file mode 100755 index 0000000..6c2ed6e --- /dev/null +++ b/third_party/cereal/external/rapidjson/internal/regex.h @@ -0,0 +1,740 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_INTERNAL_REGEX_H_ +#define CEREAL_RAPIDJSON_INTERNAL_REGEX_H_ + +#include "../allocators.h" +#include "../stream.h" +#include "stack.h" + +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(padded) +CEREAL_RAPIDJSON_DIAG_OFF(switch-enum) +CEREAL_RAPIDJSON_DIAG_OFF(implicit-fallthrough) +#elif defined(_MSC_VER) +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + +#ifdef __GNUC__ +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(effc++) +#if __GNUC__ >= 7 +CEREAL_RAPIDJSON_DIAG_OFF(implicit-fallthrough) +#endif +#endif + +#ifndef CEREAL_RAPIDJSON_REGEX_VERBOSE +#define CEREAL_RAPIDJSON_REGEX_VERBOSE 0 +#endif + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +/////////////////////////////////////////////////////////////////////////////// +// DecodedStream + +template +class DecodedStream { +public: + DecodedStream(SourceStream& ss) : ss_(ss), codepoint_() { Decode(); } + unsigned Peek() { return codepoint_; } + unsigned Take() { + unsigned c = codepoint_; + if (c) // No further decoding when '\0' + Decode(); + return c; + } + +private: + void Decode() { + if (!Encoding::Decode(ss_, &codepoint_)) + codepoint_ = 0; + } + + SourceStream& ss_; + unsigned codepoint_; +}; + +/////////////////////////////////////////////////////////////////////////////// +// GenericRegex + +static const SizeType kRegexInvalidState = ~SizeType(0); //!< Represents an invalid index in GenericRegex::State::out, out1 +static const SizeType kRegexInvalidRange = ~SizeType(0); + +template +class GenericRegexSearch; + +//! Regular expression engine with subset of ECMAscript grammar. +/*! + Supported regular expression syntax: + - \c ab Concatenation + - \c a|b Alternation + - \c a? Zero or one + - \c a* Zero or more + - \c a+ One or more + - \c a{3} Exactly 3 times + - \c a{3,} At least 3 times + - \c a{3,5} 3 to 5 times + - \c (ab) Grouping + - \c ^a At the beginning + - \c a$ At the end + - \c . Any character + - \c [abc] Character classes + - \c [a-c] Character class range + - \c [a-z0-9_] Character class combination + - \c [^abc] Negated character classes + - \c [^a-c] Negated character class range + - \c [\b] Backspace (U+0008) + - \c \\| \\\\ ... Escape characters + - \c \\f Form feed (U+000C) + - \c \\n Line feed (U+000A) + - \c \\r Carriage return (U+000D) + - \c \\t Tab (U+0009) + - \c \\v Vertical tab (U+000B) + + \note This is a Thompson NFA engine, implemented with reference to + Cox, Russ. "Regular Expression Matching Can Be Simple And Fast (but is slow in Java, Perl, PHP, Python, Ruby,...).", + https://swtch.com/~rsc/regexp/regexp1.html +*/ +template +class GenericRegex { +public: + typedef Encoding EncodingType; + typedef typename Encoding::Ch Ch; + template friend class GenericRegexSearch; + + GenericRegex(const Ch* source, Allocator* allocator = 0) : + ownAllocator_(allocator ? 0 : CEREAL_RAPIDJSON_NEW(Allocator)()), allocator_(allocator ? allocator : ownAllocator_), + states_(allocator_, 256), ranges_(allocator_, 256), root_(kRegexInvalidState), stateCount_(), rangeCount_(), + anchorBegin_(), anchorEnd_() + { + GenericStringStream ss(source); + DecodedStream, Encoding> ds(ss); + Parse(ds); + } + + ~GenericRegex() + { + CEREAL_RAPIDJSON_DELETE(ownAllocator_); + } + + bool IsValid() const { + return root_ != kRegexInvalidState; + } + +private: + enum Operator { + kZeroOrOne, + kZeroOrMore, + kOneOrMore, + kConcatenation, + kAlternation, + kLeftParenthesis + }; + + static const unsigned kAnyCharacterClass = 0xFFFFFFFF; //!< For '.' + static const unsigned kRangeCharacterClass = 0xFFFFFFFE; + static const unsigned kRangeNegationFlag = 0x80000000; + + struct Range { + unsigned start; // + unsigned end; + SizeType next; + }; + + struct State { + SizeType out; //!< Equals to kInvalid for matching state + SizeType out1; //!< Equals to non-kInvalid for split + SizeType rangeStart; + unsigned codepoint; + }; + + struct Frag { + Frag(SizeType s, SizeType o, SizeType m) : start(s), out(o), minIndex(m) {} + SizeType start; + SizeType out; //!< link-list of all output states + SizeType minIndex; + }; + + State& GetState(SizeType index) { + CEREAL_RAPIDJSON_ASSERT(index < stateCount_); + return states_.template Bottom()[index]; + } + + const State& GetState(SizeType index) const { + CEREAL_RAPIDJSON_ASSERT(index < stateCount_); + return states_.template Bottom()[index]; + } + + Range& GetRange(SizeType index) { + CEREAL_RAPIDJSON_ASSERT(index < rangeCount_); + return ranges_.template Bottom()[index]; + } + + const Range& GetRange(SizeType index) const { + CEREAL_RAPIDJSON_ASSERT(index < rangeCount_); + return ranges_.template Bottom()[index]; + } + + template + void Parse(DecodedStream& ds) { + Stack operandStack(allocator_, 256); // Frag + Stack operatorStack(allocator_, 256); // Operator + Stack atomCountStack(allocator_, 256); // unsigned (Atom per parenthesis) + + *atomCountStack.template Push() = 0; + + unsigned codepoint; + while (ds.Peek() != 0) { + switch (codepoint = ds.Take()) { + case '^': + anchorBegin_ = true; + break; + + case '$': + anchorEnd_ = true; + break; + + case '|': + while (!operatorStack.Empty() && *operatorStack.template Top() < kAlternation) + if (!Eval(operandStack, *operatorStack.template Pop(1))) + return; + *operatorStack.template Push() = kAlternation; + *atomCountStack.template Top() = 0; + break; + + case '(': + *operatorStack.template Push() = kLeftParenthesis; + *atomCountStack.template Push() = 0; + break; + + case ')': + while (!operatorStack.Empty() && *operatorStack.template Top() != kLeftParenthesis) + if (!Eval(operandStack, *operatorStack.template Pop(1))) + return; + if (operatorStack.Empty()) + return; + operatorStack.template Pop(1); + atomCountStack.template Pop(1); + ImplicitConcatenation(atomCountStack, operatorStack); + break; + + case '?': + if (!Eval(operandStack, kZeroOrOne)) + return; + break; + + case '*': + if (!Eval(operandStack, kZeroOrMore)) + return; + break; + + case '+': + if (!Eval(operandStack, kOneOrMore)) + return; + break; + + case '{': + { + unsigned n, m; + if (!ParseUnsigned(ds, &n)) + return; + + if (ds.Peek() == ',') { + ds.Take(); + if (ds.Peek() == '}') + m = kInfinityQuantifier; + else if (!ParseUnsigned(ds, &m) || m < n) + return; + } + else + m = n; + + if (!EvalQuantifier(operandStack, n, m) || ds.Peek() != '}') + return; + ds.Take(); + } + break; + + case '.': + PushOperand(operandStack, kAnyCharacterClass); + ImplicitConcatenation(atomCountStack, operatorStack); + break; + + case '[': + { + SizeType range; + if (!ParseRange(ds, &range)) + return; + SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, kRangeCharacterClass); + GetState(s).rangeStart = range; + *operandStack.template Push() = Frag(s, s, s); + } + ImplicitConcatenation(atomCountStack, operatorStack); + break; + + case '\\': // Escape character + if (!CharacterEscape(ds, &codepoint)) + return; // Unsupported escape character + // fall through to default + + default: // Pattern character + PushOperand(operandStack, codepoint); + ImplicitConcatenation(atomCountStack, operatorStack); + } + } + + while (!operatorStack.Empty()) + if (!Eval(operandStack, *operatorStack.template Pop(1))) + return; + + // Link the operand to matching state. + if (operandStack.GetSize() == sizeof(Frag)) { + Frag* e = operandStack.template Pop(1); + Patch(e->out, NewState(kRegexInvalidState, kRegexInvalidState, 0)); + root_ = e->start; + +#if CEREAL_RAPIDJSON_REGEX_VERBOSE + printf("root: %d\n", root_); + for (SizeType i = 0; i < stateCount_ ; i++) { + State& s = GetState(i); + printf("[%2d] out: %2d out1: %2d c: '%c'\n", i, s.out, s.out1, (char)s.codepoint); + } + printf("\n"); +#endif + } + } + + SizeType NewState(SizeType out, SizeType out1, unsigned codepoint) { + State* s = states_.template Push(); + s->out = out; + s->out1 = out1; + s->codepoint = codepoint; + s->rangeStart = kRegexInvalidRange; + return stateCount_++; + } + + void PushOperand(Stack& operandStack, unsigned codepoint) { + SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, codepoint); + *operandStack.template Push() = Frag(s, s, s); + } + + void ImplicitConcatenation(Stack& atomCountStack, Stack& operatorStack) { + if (*atomCountStack.template Top()) + *operatorStack.template Push() = kConcatenation; + (*atomCountStack.template Top())++; + } + + SizeType Append(SizeType l1, SizeType l2) { + SizeType old = l1; + while (GetState(l1).out != kRegexInvalidState) + l1 = GetState(l1).out; + GetState(l1).out = l2; + return old; + } + + void Patch(SizeType l, SizeType s) { + for (SizeType next; l != kRegexInvalidState; l = next) { + next = GetState(l).out; + GetState(l).out = s; + } + } + + bool Eval(Stack& operandStack, Operator op) { + switch (op) { + case kConcatenation: + CEREAL_RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag) * 2); + { + Frag e2 = *operandStack.template Pop(1); + Frag e1 = *operandStack.template Pop(1); + Patch(e1.out, e2.start); + *operandStack.template Push() = Frag(e1.start, e2.out, Min(e1.minIndex, e2.minIndex)); + } + return true; + + case kAlternation: + if (operandStack.GetSize() >= sizeof(Frag) * 2) { + Frag e2 = *operandStack.template Pop(1); + Frag e1 = *operandStack.template Pop(1); + SizeType s = NewState(e1.start, e2.start, 0); + *operandStack.template Push() = Frag(s, Append(e1.out, e2.out), Min(e1.minIndex, e2.minIndex)); + return true; + } + return false; + + case kZeroOrOne: + if (operandStack.GetSize() >= sizeof(Frag)) { + Frag e = *operandStack.template Pop(1); + SizeType s = NewState(kRegexInvalidState, e.start, 0); + *operandStack.template Push() = Frag(s, Append(e.out, s), e.minIndex); + return true; + } + return false; + + case kZeroOrMore: + if (operandStack.GetSize() >= sizeof(Frag)) { + Frag e = *operandStack.template Pop(1); + SizeType s = NewState(kRegexInvalidState, e.start, 0); + Patch(e.out, s); + *operandStack.template Push() = Frag(s, s, e.minIndex); + return true; + } + return false; + + case kOneOrMore: + if (operandStack.GetSize() >= sizeof(Frag)) { + Frag e = *operandStack.template Pop(1); + SizeType s = NewState(kRegexInvalidState, e.start, 0); + Patch(e.out, s); + *operandStack.template Push() = Frag(e.start, s, e.minIndex); + return true; + } + return false; + + default: + // syntax error (e.g. unclosed kLeftParenthesis) + return false; + } + } + + bool EvalQuantifier(Stack& operandStack, unsigned n, unsigned m) { + CEREAL_RAPIDJSON_ASSERT(n <= m); + CEREAL_RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag)); + + if (n == 0) { + if (m == 0) // a{0} not support + return false; + else if (m == kInfinityQuantifier) + Eval(operandStack, kZeroOrMore); // a{0,} -> a* + else { + Eval(operandStack, kZeroOrOne); // a{0,5} -> a? + for (unsigned i = 0; i < m - 1; i++) + CloneTopOperand(operandStack); // a{0,5} -> a? a? a? a? a? + for (unsigned i = 0; i < m - 1; i++) + Eval(operandStack, kConcatenation); // a{0,5} -> a?a?a?a?a? + } + return true; + } + + for (unsigned i = 0; i < n - 1; i++) // a{3} -> a a a + CloneTopOperand(operandStack); + + if (m == kInfinityQuantifier) + Eval(operandStack, kOneOrMore); // a{3,} -> a a a+ + else if (m > n) { + CloneTopOperand(operandStack); // a{3,5} -> a a a a + Eval(operandStack, kZeroOrOne); // a{3,5} -> a a a a? + for (unsigned i = n; i < m - 1; i++) + CloneTopOperand(operandStack); // a{3,5} -> a a a a? a? + for (unsigned i = n; i < m; i++) + Eval(operandStack, kConcatenation); // a{3,5} -> a a aa?a? + } + + for (unsigned i = 0; i < n - 1; i++) + Eval(operandStack, kConcatenation); // a{3} -> aaa, a{3,} -> aaa+, a{3.5} -> aaaa?a? + + return true; + } + + static SizeType Min(SizeType a, SizeType b) { return a < b ? a : b; } + + void CloneTopOperand(Stack& operandStack) { + const Frag src = *operandStack.template Top(); // Copy constructor to prevent invalidation + SizeType count = stateCount_ - src.minIndex; // Assumes top operand contains states in [src->minIndex, stateCount_) + State* s = states_.template Push(count); + memcpy(s, &GetState(src.minIndex), count * sizeof(State)); + for (SizeType j = 0; j < count; j++) { + if (s[j].out != kRegexInvalidState) + s[j].out += count; + if (s[j].out1 != kRegexInvalidState) + s[j].out1 += count; + } + *operandStack.template Push() = Frag(src.start + count, src.out + count, src.minIndex + count); + stateCount_ += count; + } + + template + bool ParseUnsigned(DecodedStream& ds, unsigned* u) { + unsigned r = 0; + if (ds.Peek() < '0' || ds.Peek() > '9') + return false; + while (ds.Peek() >= '0' && ds.Peek() <= '9') { + if (r >= 429496729 && ds.Peek() > '5') // 2^32 - 1 = 4294967295 + return false; // overflow + r = r * 10 + (ds.Take() - '0'); + } + *u = r; + return true; + } + + template + bool ParseRange(DecodedStream& ds, SizeType* range) { + bool isBegin = true; + bool negate = false; + int step = 0; + SizeType start = kRegexInvalidRange; + SizeType current = kRegexInvalidRange; + unsigned codepoint; + while ((codepoint = ds.Take()) != 0) { + if (isBegin) { + isBegin = false; + if (codepoint == '^') { + negate = true; + continue; + } + } + + switch (codepoint) { + case ']': + if (start == kRegexInvalidRange) + return false; // Error: nothing inside [] + if (step == 2) { // Add trailing '-' + SizeType r = NewRange('-'); + CEREAL_RAPIDJSON_ASSERT(current != kRegexInvalidRange); + GetRange(current).next = r; + } + if (negate) + GetRange(start).start |= kRangeNegationFlag; + *range = start; + return true; + + case '\\': + if (ds.Peek() == 'b') { + ds.Take(); + codepoint = 0x0008; // Escape backspace character + } + else if (!CharacterEscape(ds, &codepoint)) + return false; + // fall through to default + + default: + switch (step) { + case 1: + if (codepoint == '-') { + step++; + break; + } + // fall through to step 0 for other characters + + case 0: + { + SizeType r = NewRange(codepoint); + if (current != kRegexInvalidRange) + GetRange(current).next = r; + if (start == kRegexInvalidRange) + start = r; + current = r; + } + step = 1; + break; + + default: + CEREAL_RAPIDJSON_ASSERT(step == 2); + GetRange(current).end = codepoint; + step = 0; + } + } + } + return false; + } + + SizeType NewRange(unsigned codepoint) { + Range* r = ranges_.template Push(); + r->start = r->end = codepoint; + r->next = kRegexInvalidRange; + return rangeCount_++; + } + + template + bool CharacterEscape(DecodedStream& ds, unsigned* escapedCodepoint) { + unsigned codepoint; + switch (codepoint = ds.Take()) { + case '^': + case '$': + case '|': + case '(': + case ')': + case '?': + case '*': + case '+': + case '.': + case '[': + case ']': + case '{': + case '}': + case '\\': + *escapedCodepoint = codepoint; return true; + case 'f': *escapedCodepoint = 0x000C; return true; + case 'n': *escapedCodepoint = 0x000A; return true; + case 'r': *escapedCodepoint = 0x000D; return true; + case 't': *escapedCodepoint = 0x0009; return true; + case 'v': *escapedCodepoint = 0x000B; return true; + default: + return false; // Unsupported escape character + } + } + + Allocator* ownAllocator_; + Allocator* allocator_; + Stack states_; + Stack ranges_; + SizeType root_; + SizeType stateCount_; + SizeType rangeCount_; + + static const unsigned kInfinityQuantifier = ~0u; + + // For SearchWithAnchoring() + bool anchorBegin_; + bool anchorEnd_; +}; + +template +class GenericRegexSearch { +public: + typedef typename RegexType::EncodingType Encoding; + typedef typename Encoding::Ch Ch; + + GenericRegexSearch(const RegexType& regex, Allocator* allocator = 0) : + regex_(regex), allocator_(allocator), ownAllocator_(0), + state0_(allocator, 0), state1_(allocator, 0), stateSet_() + { + CEREAL_RAPIDJSON_ASSERT(regex_.IsValid()); + if (!allocator_) + ownAllocator_ = allocator_ = CEREAL_RAPIDJSON_NEW(Allocator)(); + stateSet_ = static_cast(allocator_->Malloc(GetStateSetSize())); + state0_.template Reserve(regex_.stateCount_); + state1_.template Reserve(regex_.stateCount_); + } + + ~GenericRegexSearch() { + Allocator::Free(stateSet_); + CEREAL_RAPIDJSON_DELETE(ownAllocator_); + } + + template + bool Match(InputStream& is) { + return SearchWithAnchoring(is, true, true); + } + + bool Match(const Ch* s) { + GenericStringStream is(s); + return Match(is); + } + + template + bool Search(InputStream& is) { + return SearchWithAnchoring(is, regex_.anchorBegin_, regex_.anchorEnd_); + } + + bool Search(const Ch* s) { + GenericStringStream is(s); + return Search(is); + } + +private: + typedef typename RegexType::State State; + typedef typename RegexType::Range Range; + + template + bool SearchWithAnchoring(InputStream& is, bool anchorBegin, bool anchorEnd) { + DecodedStream ds(is); + + state0_.Clear(); + Stack *current = &state0_, *next = &state1_; + const size_t stateSetSize = GetStateSetSize(); + std::memset(stateSet_, 0, stateSetSize); + + bool matched = AddState(*current, regex_.root_); + unsigned codepoint; + while (!current->Empty() && (codepoint = ds.Take()) != 0) { + std::memset(stateSet_, 0, stateSetSize); + next->Clear(); + matched = false; + for (const SizeType* s = current->template Bottom(); s != current->template End(); ++s) { + const State& sr = regex_.GetState(*s); + if (sr.codepoint == codepoint || + sr.codepoint == RegexType::kAnyCharacterClass || + (sr.codepoint == RegexType::kRangeCharacterClass && MatchRange(sr.rangeStart, codepoint))) + { + matched = AddState(*next, sr.out) || matched; + if (!anchorEnd && matched) + return true; + } + if (!anchorBegin) + AddState(*next, regex_.root_); + } + internal::Swap(current, next); + } + + return matched; + } + + size_t GetStateSetSize() const { + return (regex_.stateCount_ + 31) / 32 * 4; + } + + // Return whether the added states is a match state + bool AddState(Stack& l, SizeType index) { + CEREAL_RAPIDJSON_ASSERT(index != kRegexInvalidState); + + const State& s = regex_.GetState(index); + if (s.out1 != kRegexInvalidState) { // Split + bool matched = AddState(l, s.out); + return AddState(l, s.out1) || matched; + } + else if (!(stateSet_[index >> 5] & (1u << (index & 31)))) { + stateSet_[index >> 5] |= (1u << (index & 31)); + *l.template PushUnsafe() = index; + } + return s.out == kRegexInvalidState; // by using PushUnsafe() above, we can ensure s is not validated due to reallocation. + } + + bool MatchRange(SizeType rangeIndex, unsigned codepoint) const { + bool yes = (regex_.GetRange(rangeIndex).start & RegexType::kRangeNegationFlag) == 0; + while (rangeIndex != kRegexInvalidRange) { + const Range& r = regex_.GetRange(rangeIndex); + if (codepoint >= (r.start & ~RegexType::kRangeNegationFlag) && codepoint <= r.end) + return yes; + rangeIndex = r.next; + } + return !yes; + } + + const RegexType& regex_; + Allocator* allocator_; + Allocator* ownAllocator_; + Stack state0_; + Stack state1_; + uint32_t* stateSet_; +}; + +typedef GenericRegex > Regex; +typedef GenericRegexSearch RegexSearch; + +} // namespace internal +CEREAL_RAPIDJSON_NAMESPACE_END + +#ifdef __GNUC__ +CEREAL_RAPIDJSON_DIAG_POP +#endif + +#if defined(__clang__) || defined(_MSC_VER) +CEREAL_RAPIDJSON_DIAG_POP +#endif + +#endif // CEREAL_RAPIDJSON_INTERNAL_REGEX_H_ diff --git a/third_party/cereal/external/rapidjson/internal/stack.h b/third_party/cereal/external/rapidjson/internal/stack.h new file mode 100755 index 0000000..be77088 --- /dev/null +++ b/third_party/cereal/external/rapidjson/internal/stack.h @@ -0,0 +1,232 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_INTERNAL_STACK_H_ +#define CEREAL_RAPIDJSON_INTERNAL_STACK_H_ + +#include "../allocators.h" +#include "swap.h" +#include + +#if defined(__clang__) +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +/////////////////////////////////////////////////////////////////////////////// +// Stack + +//! A type-unsafe stack for storing different types of data. +/*! \tparam Allocator Allocator for allocating stack memory. +*/ +template +class Stack { +public: + // Optimization note: Do not allocate memory for stack_ in constructor. + // Do it lazily when first Push() -> Expand() -> Resize(). + Stack(Allocator* allocator, size_t stackCapacity) : allocator_(allocator), ownAllocator_(0), stack_(0), stackTop_(0), stackEnd_(0), initialCapacity_(stackCapacity) { + } + +#if CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS + Stack(Stack&& rhs) + : allocator_(rhs.allocator_), + ownAllocator_(rhs.ownAllocator_), + stack_(rhs.stack_), + stackTop_(rhs.stackTop_), + stackEnd_(rhs.stackEnd_), + initialCapacity_(rhs.initialCapacity_) + { + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + rhs.stack_ = 0; + rhs.stackTop_ = 0; + rhs.stackEnd_ = 0; + rhs.initialCapacity_ = 0; + } +#endif + + ~Stack() { + Destroy(); + } + +#if CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS + Stack& operator=(Stack&& rhs) { + if (&rhs != this) + { + Destroy(); + + allocator_ = rhs.allocator_; + ownAllocator_ = rhs.ownAllocator_; + stack_ = rhs.stack_; + stackTop_ = rhs.stackTop_; + stackEnd_ = rhs.stackEnd_; + initialCapacity_ = rhs.initialCapacity_; + + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + rhs.stack_ = 0; + rhs.stackTop_ = 0; + rhs.stackEnd_ = 0; + rhs.initialCapacity_ = 0; + } + return *this; + } +#endif + + void Swap(Stack& rhs) CEREAL_RAPIDJSON_NOEXCEPT { + internal::Swap(allocator_, rhs.allocator_); + internal::Swap(ownAllocator_, rhs.ownAllocator_); + internal::Swap(stack_, rhs.stack_); + internal::Swap(stackTop_, rhs.stackTop_); + internal::Swap(stackEnd_, rhs.stackEnd_); + internal::Swap(initialCapacity_, rhs.initialCapacity_); + } + + void Clear() { stackTop_ = stack_; } + + void ShrinkToFit() { + if (Empty()) { + // If the stack is empty, completely deallocate the memory. + Allocator::Free(stack_); // NOLINT (+clang-analyzer-unix.Malloc) + stack_ = 0; + stackTop_ = 0; + stackEnd_ = 0; + } + else + Resize(GetSize()); + } + + // Optimization note: try to minimize the size of this function for force inline. + // Expansion is run very infrequently, so it is moved to another (probably non-inline) function. + template + CEREAL_RAPIDJSON_FORCEINLINE void Reserve(size_t count = 1) { + // Expand the stack if needed + if (CEREAL_RAPIDJSON_UNLIKELY(static_cast(sizeof(T) * count) > (stackEnd_ - stackTop_))) + Expand(count); + } + + template + CEREAL_RAPIDJSON_FORCEINLINE T* Push(size_t count = 1) { + Reserve(count); + return PushUnsafe(count); + } + + template + CEREAL_RAPIDJSON_FORCEINLINE T* PushUnsafe(size_t count = 1) { + CEREAL_RAPIDJSON_ASSERT(stackTop_); + CEREAL_RAPIDJSON_ASSERT(static_cast(sizeof(T) * count) <= (stackEnd_ - stackTop_)); + T* ret = reinterpret_cast(stackTop_); + stackTop_ += sizeof(T) * count; + return ret; + } + + template + T* Pop(size_t count) { + CEREAL_RAPIDJSON_ASSERT(GetSize() >= count * sizeof(T)); + stackTop_ -= count * sizeof(T); + return reinterpret_cast(stackTop_); + } + + template + T* Top() { + CEREAL_RAPIDJSON_ASSERT(GetSize() >= sizeof(T)); + return reinterpret_cast(stackTop_ - sizeof(T)); + } + + template + const T* Top() const { + CEREAL_RAPIDJSON_ASSERT(GetSize() >= sizeof(T)); + return reinterpret_cast(stackTop_ - sizeof(T)); + } + + template + T* End() { return reinterpret_cast(stackTop_); } + + template + const T* End() const { return reinterpret_cast(stackTop_); } + + template + T* Bottom() { return reinterpret_cast(stack_); } + + template + const T* Bottom() const { return reinterpret_cast(stack_); } + + bool HasAllocator() const { + return allocator_ != 0; + } + + Allocator& GetAllocator() { + CEREAL_RAPIDJSON_ASSERT(allocator_); + return *allocator_; + } + + bool Empty() const { return stackTop_ == stack_; } + size_t GetSize() const { return static_cast(stackTop_ - stack_); } + size_t GetCapacity() const { return static_cast(stackEnd_ - stack_); } + +private: + template + void Expand(size_t count) { + // Only expand the capacity if the current stack exists. Otherwise just create a stack with initial capacity. + size_t newCapacity; + if (stack_ == 0) { + if (!allocator_) + ownAllocator_ = allocator_ = CEREAL_RAPIDJSON_NEW(Allocator)(); + newCapacity = initialCapacity_; + } else { + newCapacity = GetCapacity(); + newCapacity += (newCapacity + 1) / 2; + } + size_t newSize = GetSize() + sizeof(T) * count; + if (newCapacity < newSize) + newCapacity = newSize; + + Resize(newCapacity); + } + + void Resize(size_t newCapacity) { + const size_t size = GetSize(); // Backup the current size + stack_ = static_cast(allocator_->Realloc(stack_, GetCapacity(), newCapacity)); + stackTop_ = stack_ + size; + stackEnd_ = stack_ + newCapacity; + } + + void Destroy() { + Allocator::Free(stack_); + CEREAL_RAPIDJSON_DELETE(ownAllocator_); // Only delete if it is owned by the stack + } + + // Prohibit copy constructor & assignment operator. + Stack(const Stack&); + Stack& operator=(const Stack&); + + Allocator* allocator_; + Allocator* ownAllocator_; + char *stack_; + char *stackTop_; + char *stackEnd_; + size_t initialCapacity_; +}; + +} // namespace internal +CEREAL_RAPIDJSON_NAMESPACE_END + +#if defined(__clang__) +CEREAL_RAPIDJSON_DIAG_POP +#endif + +#endif // CEREAL_RAPIDJSON_STACK_H_ diff --git a/third_party/cereal/external/rapidjson/internal/strfunc.h b/third_party/cereal/external/rapidjson/internal/strfunc.h new file mode 100755 index 0000000..44af229 --- /dev/null +++ b/third_party/cereal/external/rapidjson/internal/strfunc.h @@ -0,0 +1,69 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_INTERNAL_STRFUNC_H_ +#define CEREAL_RAPIDJSON_INTERNAL_STRFUNC_H_ + +#include "../stream.h" +#include + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +//! Custom strlen() which works on different character types. +/*! \tparam Ch Character type (e.g. char, wchar_t, short) + \param s Null-terminated input string. + \return Number of characters in the string. + \note This has the same semantics as strlen(), the return value is not number of Unicode codepoints. +*/ +template +inline SizeType StrLen(const Ch* s) { + CEREAL_RAPIDJSON_ASSERT(s != 0); + const Ch* p = s; + while (*p) ++p; + return SizeType(p - s); +} + +template <> +inline SizeType StrLen(const char* s) { + return SizeType(std::strlen(s)); +} + +template <> +inline SizeType StrLen(const wchar_t* s) { + return SizeType(std::wcslen(s)); +} + +//! Returns number of code points in a encoded string. +template +bool CountStringCodePoint(const typename Encoding::Ch* s, SizeType length, SizeType* outCount) { + CEREAL_RAPIDJSON_ASSERT(s != 0); + CEREAL_RAPIDJSON_ASSERT(outCount != 0); + GenericStringStream is(s); + const typename Encoding::Ch* end = s + length; + SizeType count = 0; + while (is.src_ < end) { + unsigned codepoint; + if (!Encoding::Decode(is, &codepoint)) + return false; + count++; + } + *outCount = count; + return true; +} + +} // namespace internal +CEREAL_RAPIDJSON_NAMESPACE_END + +#endif // CEREAL_RAPIDJSON_INTERNAL_STRFUNC_H_ diff --git a/third_party/cereal/external/rapidjson/internal/strtod.h b/third_party/cereal/external/rapidjson/internal/strtod.h new file mode 100755 index 0000000..d60c740 --- /dev/null +++ b/third_party/cereal/external/rapidjson/internal/strtod.h @@ -0,0 +1,290 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_STRTOD_ +#define CEREAL_RAPIDJSON_STRTOD_ + +#include "ieee754.h" +#include "biginteger.h" +#include "diyfp.h" +#include "pow10.h" +#include +#include + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +inline double FastPath(double significand, int exp) { + if (exp < -308) + return 0.0; + else if (exp >= 0) + return significand * internal::Pow10(exp); + else + return significand / internal::Pow10(-exp); +} + +inline double StrtodNormalPrecision(double d, int p) { + if (p < -308) { + // Prevent expSum < -308, making Pow10(p) = 0 + d = FastPath(d, -308); + d = FastPath(d, p + 308); + } + else + d = FastPath(d, p); + return d; +} + +template +inline T Min3(T a, T b, T c) { + T m = a; + if (m > b) m = b; + if (m > c) m = c; + return m; +} + +inline int CheckWithinHalfULP(double b, const BigInteger& d, int dExp) { + const Double db(b); + const uint64_t bInt = db.IntegerSignificand(); + const int bExp = db.IntegerExponent(); + const int hExp = bExp - 1; + + int dS_Exp2 = 0, dS_Exp5 = 0, bS_Exp2 = 0, bS_Exp5 = 0, hS_Exp2 = 0, hS_Exp5 = 0; + + // Adjust for decimal exponent + if (dExp >= 0) { + dS_Exp2 += dExp; + dS_Exp5 += dExp; + } + else { + bS_Exp2 -= dExp; + bS_Exp5 -= dExp; + hS_Exp2 -= dExp; + hS_Exp5 -= dExp; + } + + // Adjust for binary exponent + if (bExp >= 0) + bS_Exp2 += bExp; + else { + dS_Exp2 -= bExp; + hS_Exp2 -= bExp; + } + + // Adjust for half ulp exponent + if (hExp >= 0) + hS_Exp2 += hExp; + else { + dS_Exp2 -= hExp; + bS_Exp2 -= hExp; + } + + // Remove common power of two factor from all three scaled values + int common_Exp2 = Min3(dS_Exp2, bS_Exp2, hS_Exp2); + dS_Exp2 -= common_Exp2; + bS_Exp2 -= common_Exp2; + hS_Exp2 -= common_Exp2; + + BigInteger dS = d; + dS.MultiplyPow5(static_cast(dS_Exp5)) <<= static_cast(dS_Exp2); + + BigInteger bS(bInt); + bS.MultiplyPow5(static_cast(bS_Exp5)) <<= static_cast(bS_Exp2); + + BigInteger hS(1); + hS.MultiplyPow5(static_cast(hS_Exp5)) <<= static_cast(hS_Exp2); + + BigInteger delta(0); + dS.Difference(bS, &delta); + + return delta.Compare(hS); +} + +inline bool StrtodFast(double d, int p, double* result) { + // Use fast path for string-to-double conversion if possible + // see http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ + if (p > 22 && p < 22 + 16) { + // Fast Path Cases In Disguise + d *= internal::Pow10(p - 22); + p = 22; + } + + if (p >= -22 && p <= 22 && d <= 9007199254740991.0) { // 2^53 - 1 + *result = FastPath(d, p); + return true; + } + else + return false; +} + +// Compute an approximation and see if it is within 1/2 ULP +inline bool StrtodDiyFp(const char* decimals, int dLen, int dExp, double* result) { + uint64_t significand = 0; + int i = 0; // 2^64 - 1 = 18446744073709551615, 1844674407370955161 = 0x1999999999999999 + for (; i < dLen; i++) { + if (significand > CEREAL_RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || + (significand == CEREAL_RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > '5')) + break; + significand = significand * 10u + static_cast(decimals[i] - '0'); + } + + if (i < dLen && decimals[i] >= '5') // Rounding + significand++; + + int remaining = dLen - i; + const int kUlpShift = 3; + const int kUlp = 1 << kUlpShift; + int64_t error = (remaining == 0) ? 0 : kUlp / 2; + + DiyFp v(significand, 0); + v = v.Normalize(); + error <<= -v.e; + + dExp += remaining; + + int actualExp; + DiyFp cachedPower = GetCachedPower10(dExp, &actualExp); + if (actualExp != dExp) { + static const DiyFp kPow10[] = { + DiyFp(CEREAL_RAPIDJSON_UINT64_C2(0xa0000000, 0x00000000), -60), // 10^1 + DiyFp(CEREAL_RAPIDJSON_UINT64_C2(0xc8000000, 0x00000000), -57), // 10^2 + DiyFp(CEREAL_RAPIDJSON_UINT64_C2(0xfa000000, 0x00000000), -54), // 10^3 + DiyFp(CEREAL_RAPIDJSON_UINT64_C2(0x9c400000, 0x00000000), -50), // 10^4 + DiyFp(CEREAL_RAPIDJSON_UINT64_C2(0xc3500000, 0x00000000), -47), // 10^5 + DiyFp(CEREAL_RAPIDJSON_UINT64_C2(0xf4240000, 0x00000000), -44), // 10^6 + DiyFp(CEREAL_RAPIDJSON_UINT64_C2(0x98968000, 0x00000000), -40) // 10^7 + }; + int adjustment = dExp - actualExp; + CEREAL_RAPIDJSON_ASSERT(adjustment >= 1 && adjustment < 8); + v = v * kPow10[adjustment - 1]; + if (dLen + adjustment > 19) // has more digits than decimal digits in 64-bit + error += kUlp / 2; + } + + v = v * cachedPower; + + error += kUlp + (error == 0 ? 0 : 1); + + const int oldExp = v.e; + v = v.Normalize(); + error <<= oldExp - v.e; + + const int effectiveSignificandSize = Double::EffectiveSignificandSize(64 + v.e); + int precisionSize = 64 - effectiveSignificandSize; + if (precisionSize + kUlpShift >= 64) { + int scaleExp = (precisionSize + kUlpShift) - 63; + v.f >>= scaleExp; + v.e += scaleExp; + error = (error >> scaleExp) + 1 + kUlp; + precisionSize -= scaleExp; + } + + DiyFp rounded(v.f >> precisionSize, v.e + precisionSize); + const uint64_t precisionBits = (v.f & ((uint64_t(1) << precisionSize) - 1)) * kUlp; + const uint64_t halfWay = (uint64_t(1) << (precisionSize - 1)) * kUlp; + if (precisionBits >= halfWay + static_cast(error)) { + rounded.f++; + if (rounded.f & (DiyFp::kDpHiddenBit << 1)) { // rounding overflows mantissa (issue #340) + rounded.f >>= 1; + rounded.e++; + } + } + + *result = rounded.ToDouble(); + + return halfWay - static_cast(error) >= precisionBits || precisionBits >= halfWay + static_cast(error); +} + +inline double StrtodBigInteger(double approx, const char* decimals, int dLen, int dExp) { + CEREAL_RAPIDJSON_ASSERT(dLen >= 0); + const BigInteger dInt(decimals, static_cast(dLen)); + Double a(approx); + int cmp = CheckWithinHalfULP(a.Value(), dInt, dExp); + if (cmp < 0) + return a.Value(); // within half ULP + else if (cmp == 0) { + // Round towards even + if (a.Significand() & 1) + return a.NextPositiveDouble(); + else + return a.Value(); + } + else // adjustment + return a.NextPositiveDouble(); +} + +inline double StrtodFullPrecision(double d, int p, const char* decimals, size_t length, size_t decimalPosition, int exp) { + CEREAL_RAPIDJSON_ASSERT(d >= 0.0); + CEREAL_RAPIDJSON_ASSERT(length >= 1); + + double result = 0.0; + if (StrtodFast(d, p, &result)) + return result; + + CEREAL_RAPIDJSON_ASSERT(length <= INT_MAX); + int dLen = static_cast(length); + + CEREAL_RAPIDJSON_ASSERT(length >= decimalPosition); + CEREAL_RAPIDJSON_ASSERT(length - decimalPosition <= INT_MAX); + int dExpAdjust = static_cast(length - decimalPosition); + + CEREAL_RAPIDJSON_ASSERT(exp >= INT_MIN + dExpAdjust); + int dExp = exp - dExpAdjust; + + // Make sure length+dExp does not overflow + CEREAL_RAPIDJSON_ASSERT(dExp <= INT_MAX - dLen); + + // Trim leading zeros + while (dLen > 0 && *decimals == '0') { + dLen--; + decimals++; + } + + // Trim trailing zeros + while (dLen > 0 && decimals[dLen - 1] == '0') { + dLen--; + dExp++; + } + + if (dLen == 0) { // Buffer only contains zeros. + return 0.0; + } + + // Trim right-most digits + const int kMaxDecimalDigit = 767 + 1; + if (dLen > kMaxDecimalDigit) { + dExp += dLen - kMaxDecimalDigit; + dLen = kMaxDecimalDigit; + } + + // If too small, underflow to zero. + // Any x <= 10^-324 is interpreted as zero. + if (dLen + dExp <= -324) + return 0.0; + + // If too large, overflow to infinity. + // Any x >= 10^309 is interpreted as +infinity. + if (dLen + dExp > 309) + return std::numeric_limits::infinity(); + + if (StrtodDiyFp(decimals, dLen, dExp, &result)) + return result; + + // Use approximation from StrtodDiyFp and make adjustment with BigInteger comparison + return StrtodBigInteger(result, decimals, dLen, dExp); +} + +} // namespace internal +CEREAL_RAPIDJSON_NAMESPACE_END + +#endif // CEREAL_RAPIDJSON_STRTOD_ diff --git a/third_party/cereal/external/rapidjson/internal/swap.h b/third_party/cereal/external/rapidjson/internal/swap.h new file mode 100755 index 0000000..5d8910c --- /dev/null +++ b/third_party/cereal/external/rapidjson/internal/swap.h @@ -0,0 +1,46 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_INTERNAL_SWAP_H_ +#define CEREAL_RAPIDJSON_INTERNAL_SWAP_H_ + +#include "../rapidjson.h" + +#if defined(__clang__) +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +//! Custom swap() to avoid dependency on C++ header +/*! \tparam T Type of the arguments to swap, should be instantiated with primitive C++ types only. + \note This has the same semantics as std::swap(). +*/ +template +inline void Swap(T& a, T& b) CEREAL_RAPIDJSON_NOEXCEPT { + T tmp = a; + a = b; + b = tmp; +} + +} // namespace internal +CEREAL_RAPIDJSON_NAMESPACE_END + +#if defined(__clang__) +CEREAL_RAPIDJSON_DIAG_POP +#endif + +#endif // CEREAL_RAPIDJSON_INTERNAL_SWAP_H_ diff --git a/third_party/cereal/external/rapidjson/istreamwrapper.h b/third_party/cereal/external/rapidjson/istreamwrapper.h new file mode 100755 index 0000000..4df7ce3 --- /dev/null +++ b/third_party/cereal/external/rapidjson/istreamwrapper.h @@ -0,0 +1,128 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_ISTREAMWRAPPER_H_ +#define CEREAL_RAPIDJSON_ISTREAMWRAPPER_H_ + +#include "stream.h" +#include +#include + +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(padded) +#elif defined(_MSC_VER) +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(4351) // new behavior: elements of array 'array' will be default initialized +#endif + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN + +//! Wrapper of \c std::basic_istream into RapidJSON's Stream concept. +/*! + The classes can be wrapped including but not limited to: + + - \c std::istringstream + - \c std::stringstream + - \c std::wistringstream + - \c std::wstringstream + - \c std::ifstream + - \c std::fstream + - \c std::wifstream + - \c std::wfstream + + \tparam StreamType Class derived from \c std::basic_istream. +*/ + +template +class BasicIStreamWrapper { +public: + typedef typename StreamType::char_type Ch; + + //! Constructor. + /*! + \param stream stream opened for read. + */ + BasicIStreamWrapper(StreamType &stream) : stream_(stream), buffer_(peekBuffer_), bufferSize_(4), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { + Read(); + } + + //! Constructor. + /*! + \param stream stream opened for read. + \param buffer user-supplied buffer. + \param bufferSize size of buffer in bytes. Must >=4 bytes. + */ + BasicIStreamWrapper(StreamType &stream, char* buffer, size_t bufferSize) : stream_(stream), buffer_(buffer), bufferSize_(bufferSize), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { + CEREAL_RAPIDJSON_ASSERT(bufferSize >= 4); + Read(); + } + + Ch Peek() const { return *current_; } + Ch Take() { Ch c = *current_; Read(); return c; } + size_t Tell() const { return count_ + static_cast(current_ - buffer_); } + + // Not implemented + void Put(Ch) { CEREAL_RAPIDJSON_ASSERT(false); } + void Flush() { CEREAL_RAPIDJSON_ASSERT(false); } + Ch* PutBegin() { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + + // For encoding detection only. + const Ch* Peek4() const { + return (current_ + 4 - !eof_ <= bufferLast_) ? current_ : 0; + } + +private: + BasicIStreamWrapper(); + BasicIStreamWrapper(const BasicIStreamWrapper&); + BasicIStreamWrapper& operator=(const BasicIStreamWrapper&); + + void Read() { + if (current_ < bufferLast_) + ++current_; + else if (!eof_) { + count_ += readCount_; + readCount_ = bufferSize_; + bufferLast_ = buffer_ + readCount_ - 1; + current_ = buffer_; + + if (!stream_.read(buffer_, static_cast(bufferSize_))) { + readCount_ = static_cast(stream_.gcount()); + *(bufferLast_ = buffer_ + readCount_) = '\0'; + eof_ = true; + } + } + } + + StreamType &stream_; + Ch peekBuffer_[4], *buffer_; + size_t bufferSize_; + Ch *bufferLast_; + Ch *current_; + size_t readCount_; + size_t count_; //!< Number of characters read + bool eof_; +}; + +typedef BasicIStreamWrapper IStreamWrapper; +typedef BasicIStreamWrapper WIStreamWrapper; + +#if defined(__clang__) || defined(_MSC_VER) +CEREAL_RAPIDJSON_DIAG_POP +#endif + +CEREAL_RAPIDJSON_NAMESPACE_END + +#endif // CEREAL_RAPIDJSON_ISTREAMWRAPPER_H_ diff --git a/third_party/cereal/external/rapidjson/memorybuffer.h b/third_party/cereal/external/rapidjson/memorybuffer.h new file mode 100755 index 0000000..0cd75cf --- /dev/null +++ b/third_party/cereal/external/rapidjson/memorybuffer.h @@ -0,0 +1,70 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_MEMORYBUFFER_H_ +#define CEREAL_RAPIDJSON_MEMORYBUFFER_H_ + +#include "stream.h" +#include "internal/stack.h" + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN + +//! Represents an in-memory output byte stream. +/*! + This class is mainly for being wrapped by EncodedOutputStream or AutoUTFOutputStream. + + It is similar to FileWriteBuffer but the destination is an in-memory buffer instead of a file. + + Differences between MemoryBuffer and StringBuffer: + 1. StringBuffer has Encoding but MemoryBuffer is only a byte buffer. + 2. StringBuffer::GetString() returns a null-terminated string. MemoryBuffer::GetBuffer() returns a buffer without terminator. + + \tparam Allocator type for allocating memory buffer. + \note implements Stream concept +*/ +template +struct GenericMemoryBuffer { + typedef char Ch; // byte + + GenericMemoryBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} + + void Put(Ch c) { *stack_.template Push() = c; } + void Flush() {} + + void Clear() { stack_.Clear(); } + void ShrinkToFit() { stack_.ShrinkToFit(); } + Ch* Push(size_t count) { return stack_.template Push(count); } + void Pop(size_t count) { stack_.template Pop(count); } + + const Ch* GetBuffer() const { + return stack_.template Bottom(); + } + + size_t GetSize() const { return stack_.GetSize(); } + + static const size_t kDefaultCapacity = 256; + mutable internal::Stack stack_; +}; + +typedef GenericMemoryBuffer<> MemoryBuffer; + +//! Implement specialized version of PutN() with memset() for better performance. +template<> +inline void PutN(MemoryBuffer& memoryBuffer, char c, size_t n) { + std::memset(memoryBuffer.stack_.Push(n), c, n * sizeof(c)); +} + +CEREAL_RAPIDJSON_NAMESPACE_END + +#endif // CEREAL_RAPIDJSON_MEMORYBUFFER_H_ diff --git a/third_party/cereal/external/rapidjson/memorystream.h b/third_party/cereal/external/rapidjson/memorystream.h new file mode 100755 index 0000000..326bda5 --- /dev/null +++ b/third_party/cereal/external/rapidjson/memorystream.h @@ -0,0 +1,71 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_MEMORYSTREAM_H_ +#define CEREAL_RAPIDJSON_MEMORYSTREAM_H_ + +#include "stream.h" + +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(unreachable-code) +CEREAL_RAPIDJSON_DIAG_OFF(missing-noreturn) +#endif + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN + +//! Represents an in-memory input byte stream. +/*! + This class is mainly for being wrapped by EncodedInputStream or AutoUTFInputStream. + + It is similar to FileReadBuffer but the source is an in-memory buffer instead of a file. + + Differences between MemoryStream and StringStream: + 1. StringStream has encoding but MemoryStream is a byte stream. + 2. MemoryStream needs size of the source buffer and the buffer don't need to be null terminated. StringStream assume null-terminated string as source. + 3. MemoryStream supports Peek4() for encoding detection. StringStream is specified with an encoding so it should not have Peek4(). + \note implements Stream concept +*/ +struct MemoryStream { + typedef char Ch; // byte + + MemoryStream(const Ch *src, size_t size) : src_(src), begin_(src), end_(src + size), size_(size) {} + + Ch Peek() const { return CEREAL_RAPIDJSON_UNLIKELY(src_ == end_) ? '\0' : *src_; } + Ch Take() { return CEREAL_RAPIDJSON_UNLIKELY(src_ == end_) ? '\0' : *src_++; } + size_t Tell() const { return static_cast(src_ - begin_); } + + Ch* PutBegin() { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + void Put(Ch) { CEREAL_RAPIDJSON_ASSERT(false); } + void Flush() { CEREAL_RAPIDJSON_ASSERT(false); } + size_t PutEnd(Ch*) { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + + // For encoding detection only. + const Ch* Peek4() const { + return Tell() + 4 <= size_ ? src_ : 0; + } + + const Ch* src_; //!< Current read position. + const Ch* begin_; //!< Original head of the string. + const Ch* end_; //!< End of stream. + size_t size_; //!< Size of the stream. +}; + +CEREAL_RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_POP +#endif + +#endif // CEREAL_RAPIDJSON_MEMORYBUFFER_H_ diff --git a/third_party/cereal/external/rapidjson/msinttypes/inttypes.h b/third_party/cereal/external/rapidjson/msinttypes/inttypes.h new file mode 100755 index 0000000..1811128 --- /dev/null +++ b/third_party/cereal/external/rapidjson/msinttypes/inttypes.h @@ -0,0 +1,316 @@ +// ISO C9x compliant inttypes.h for Microsoft Visual Studio +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// +// Copyright (c) 2006-2013 Alexander Chemeris +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the product nor the names of its contributors may +// be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +/////////////////////////////////////////////////////////////////////////////// + +// The above software in this distribution may have been modified by +// THL A29 Limited ("Tencent Modifications"). +// All Tencent Modifications are Copyright (C) 2015 THL A29 Limited. + +#ifndef _MSC_VER // [ +#error "Use this header only with Microsoft Visual C++ compilers!" +#endif // _MSC_VER ] + +#ifndef _MSC_INTTYPES_H_ // [ +#define _MSC_INTTYPES_H_ + +#if _MSC_VER > 1000 +#pragma once +#endif + +#include "stdint.h" + +// miloyip: VC supports inttypes.h since VC2013 +#if _MSC_VER >= 1800 +#include +#else + +// 7.8 Format conversion of integer types + +typedef struct { + intmax_t quot; + intmax_t rem; +} imaxdiv_t; + +// 7.8.1 Macros for format specifiers + +#if !defined(__cplusplus) || defined(__STDC_FORMAT_MACROS) // [ See footnote 185 at page 198 + +// The fprintf macros for signed integers are: +#define PRId8 "d" +#define PRIi8 "i" +#define PRIdLEAST8 "d" +#define PRIiLEAST8 "i" +#define PRIdFAST8 "d" +#define PRIiFAST8 "i" + +#define PRId16 "hd" +#define PRIi16 "hi" +#define PRIdLEAST16 "hd" +#define PRIiLEAST16 "hi" +#define PRIdFAST16 "hd" +#define PRIiFAST16 "hi" + +#define PRId32 "I32d" +#define PRIi32 "I32i" +#define PRIdLEAST32 "I32d" +#define PRIiLEAST32 "I32i" +#define PRIdFAST32 "I32d" +#define PRIiFAST32 "I32i" + +#define PRId64 "I64d" +#define PRIi64 "I64i" +#define PRIdLEAST64 "I64d" +#define PRIiLEAST64 "I64i" +#define PRIdFAST64 "I64d" +#define PRIiFAST64 "I64i" + +#define PRIdMAX "I64d" +#define PRIiMAX "I64i" + +#define PRIdPTR "Id" +#define PRIiPTR "Ii" + +// The fprintf macros for unsigned integers are: +#define PRIo8 "o" +#define PRIu8 "u" +#define PRIx8 "x" +#define PRIX8 "X" +#define PRIoLEAST8 "o" +#define PRIuLEAST8 "u" +#define PRIxLEAST8 "x" +#define PRIXLEAST8 "X" +#define PRIoFAST8 "o" +#define PRIuFAST8 "u" +#define PRIxFAST8 "x" +#define PRIXFAST8 "X" + +#define PRIo16 "ho" +#define PRIu16 "hu" +#define PRIx16 "hx" +#define PRIX16 "hX" +#define PRIoLEAST16 "ho" +#define PRIuLEAST16 "hu" +#define PRIxLEAST16 "hx" +#define PRIXLEAST16 "hX" +#define PRIoFAST16 "ho" +#define PRIuFAST16 "hu" +#define PRIxFAST16 "hx" +#define PRIXFAST16 "hX" + +#define PRIo32 "I32o" +#define PRIu32 "I32u" +#define PRIx32 "I32x" +#define PRIX32 "I32X" +#define PRIoLEAST32 "I32o" +#define PRIuLEAST32 "I32u" +#define PRIxLEAST32 "I32x" +#define PRIXLEAST32 "I32X" +#define PRIoFAST32 "I32o" +#define PRIuFAST32 "I32u" +#define PRIxFAST32 "I32x" +#define PRIXFAST32 "I32X" + +#define PRIo64 "I64o" +#define PRIu64 "I64u" +#define PRIx64 "I64x" +#define PRIX64 "I64X" +#define PRIoLEAST64 "I64o" +#define PRIuLEAST64 "I64u" +#define PRIxLEAST64 "I64x" +#define PRIXLEAST64 "I64X" +#define PRIoFAST64 "I64o" +#define PRIuFAST64 "I64u" +#define PRIxFAST64 "I64x" +#define PRIXFAST64 "I64X" + +#define PRIoMAX "I64o" +#define PRIuMAX "I64u" +#define PRIxMAX "I64x" +#define PRIXMAX "I64X" + +#define PRIoPTR "Io" +#define PRIuPTR "Iu" +#define PRIxPTR "Ix" +#define PRIXPTR "IX" + +// The fscanf macros for signed integers are: +#define SCNd8 "d" +#define SCNi8 "i" +#define SCNdLEAST8 "d" +#define SCNiLEAST8 "i" +#define SCNdFAST8 "d" +#define SCNiFAST8 "i" + +#define SCNd16 "hd" +#define SCNi16 "hi" +#define SCNdLEAST16 "hd" +#define SCNiLEAST16 "hi" +#define SCNdFAST16 "hd" +#define SCNiFAST16 "hi" + +#define SCNd32 "ld" +#define SCNi32 "li" +#define SCNdLEAST32 "ld" +#define SCNiLEAST32 "li" +#define SCNdFAST32 "ld" +#define SCNiFAST32 "li" + +#define SCNd64 "I64d" +#define SCNi64 "I64i" +#define SCNdLEAST64 "I64d" +#define SCNiLEAST64 "I64i" +#define SCNdFAST64 "I64d" +#define SCNiFAST64 "I64i" + +#define SCNdMAX "I64d" +#define SCNiMAX "I64i" + +#ifdef _WIN64 // [ +# define SCNdPTR "I64d" +# define SCNiPTR "I64i" +#else // _WIN64 ][ +# define SCNdPTR "ld" +# define SCNiPTR "li" +#endif // _WIN64 ] + +// The fscanf macros for unsigned integers are: +#define SCNo8 "o" +#define SCNu8 "u" +#define SCNx8 "x" +#define SCNX8 "X" +#define SCNoLEAST8 "o" +#define SCNuLEAST8 "u" +#define SCNxLEAST8 "x" +#define SCNXLEAST8 "X" +#define SCNoFAST8 "o" +#define SCNuFAST8 "u" +#define SCNxFAST8 "x" +#define SCNXFAST8 "X" + +#define SCNo16 "ho" +#define SCNu16 "hu" +#define SCNx16 "hx" +#define SCNX16 "hX" +#define SCNoLEAST16 "ho" +#define SCNuLEAST16 "hu" +#define SCNxLEAST16 "hx" +#define SCNXLEAST16 "hX" +#define SCNoFAST16 "ho" +#define SCNuFAST16 "hu" +#define SCNxFAST16 "hx" +#define SCNXFAST16 "hX" + +#define SCNo32 "lo" +#define SCNu32 "lu" +#define SCNx32 "lx" +#define SCNX32 "lX" +#define SCNoLEAST32 "lo" +#define SCNuLEAST32 "lu" +#define SCNxLEAST32 "lx" +#define SCNXLEAST32 "lX" +#define SCNoFAST32 "lo" +#define SCNuFAST32 "lu" +#define SCNxFAST32 "lx" +#define SCNXFAST32 "lX" + +#define SCNo64 "I64o" +#define SCNu64 "I64u" +#define SCNx64 "I64x" +#define SCNX64 "I64X" +#define SCNoLEAST64 "I64o" +#define SCNuLEAST64 "I64u" +#define SCNxLEAST64 "I64x" +#define SCNXLEAST64 "I64X" +#define SCNoFAST64 "I64o" +#define SCNuFAST64 "I64u" +#define SCNxFAST64 "I64x" +#define SCNXFAST64 "I64X" + +#define SCNoMAX "I64o" +#define SCNuMAX "I64u" +#define SCNxMAX "I64x" +#define SCNXMAX "I64X" + +#ifdef _WIN64 // [ +# define SCNoPTR "I64o" +# define SCNuPTR "I64u" +# define SCNxPTR "I64x" +# define SCNXPTR "I64X" +#else // _WIN64 ][ +# define SCNoPTR "lo" +# define SCNuPTR "lu" +# define SCNxPTR "lx" +# define SCNXPTR "lX" +#endif // _WIN64 ] + +#endif // __STDC_FORMAT_MACROS ] + +// 7.8.2 Functions for greatest-width integer types + +// 7.8.2.1 The imaxabs function +#define imaxabs _abs64 + +// 7.8.2.2 The imaxdiv function + +// This is modified version of div() function from Microsoft's div.c found +// in %MSVC.NET%\crt\src\div.c +#ifdef STATIC_IMAXDIV // [ +static +#else // STATIC_IMAXDIV ][ +_inline +#endif // STATIC_IMAXDIV ] +imaxdiv_t __cdecl imaxdiv(intmax_t numer, intmax_t denom) +{ + imaxdiv_t result; + + result.quot = numer / denom; + result.rem = numer % denom; + + if (numer < 0 && result.rem > 0) { + // did division wrong; must fix up + ++result.quot; + result.rem -= denom; + } + + return result; +} + +// 7.8.2.3 The strtoimax and strtoumax functions +#define strtoimax _strtoi64 +#define strtoumax _strtoui64 + +// 7.8.2.4 The wcstoimax and wcstoumax functions +#define wcstoimax _wcstoi64 +#define wcstoumax _wcstoui64 + +#endif // _MSC_VER >= 1800 + +#endif // _MSC_INTTYPES_H_ ] diff --git a/third_party/cereal/external/rapidjson/msinttypes/stdint.h b/third_party/cereal/external/rapidjson/msinttypes/stdint.h new file mode 100755 index 0000000..3d4477b --- /dev/null +++ b/third_party/cereal/external/rapidjson/msinttypes/stdint.h @@ -0,0 +1,300 @@ +// ISO C9x compliant stdint.h for Microsoft Visual Studio +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// +// Copyright (c) 2006-2013 Alexander Chemeris +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the product nor the names of its contributors may +// be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +/////////////////////////////////////////////////////////////////////////////// + +// The above software in this distribution may have been modified by +// THL A29 Limited ("Tencent Modifications"). +// All Tencent Modifications are Copyright (C) 2015 THL A29 Limited. + +#ifndef _MSC_VER // [ +#error "Use this header only with Microsoft Visual C++ compilers!" +#endif // _MSC_VER ] + +#ifndef _MSC_STDINT_H_ // [ +#define _MSC_STDINT_H_ + +#if _MSC_VER > 1000 +#pragma once +#endif + +// miloyip: Originally Visual Studio 2010 uses its own stdint.h. However it generates warning with INT64_C(), so change to use this file for vs2010. +#if _MSC_VER >= 1600 // [ +#include + +#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 + +#undef INT8_C +#undef INT16_C +#undef INT32_C +#undef INT64_C +#undef UINT8_C +#undef UINT16_C +#undef UINT32_C +#undef UINT64_C + +// 7.18.4.1 Macros for minimum-width integer constants + +#define INT8_C(val) val##i8 +#define INT16_C(val) val##i16 +#define INT32_C(val) val##i32 +#define INT64_C(val) val##i64 + +#define UINT8_C(val) val##ui8 +#define UINT16_C(val) val##ui16 +#define UINT32_C(val) val##ui32 +#define UINT64_C(val) val##ui64 + +// 7.18.4.2 Macros for greatest-width integer constants +// These #ifndef's are needed to prevent collisions with . +// Check out Issue 9 for the details. +#ifndef INTMAX_C // [ +# define INTMAX_C INT64_C +#endif // INTMAX_C ] +#ifndef UINTMAX_C // [ +# define UINTMAX_C UINT64_C +#endif // UINTMAX_C ] + +#endif // __STDC_CONSTANT_MACROS ] + +#else // ] _MSC_VER >= 1700 [ + +#include + +// For Visual Studio 6 in C++ mode and for many Visual Studio versions when +// compiling for ARM we have to wrap include with 'extern "C++" {}' +// or compiler would give many errors like this: +// error C2733: second C linkage of overloaded function 'wmemchr' not allowed +#if defined(__cplusplus) && !defined(_M_ARM) +extern "C" { +#endif +# include +#if defined(__cplusplus) && !defined(_M_ARM) +} +#endif + +// Define _W64 macros to mark types changing their size, like intptr_t. +#ifndef _W64 +# if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 +# define _W64 __w64 +# else +# define _W64 +# endif +#endif + + +// 7.18.1 Integer types + +// 7.18.1.1 Exact-width integer types + +// Visual Studio 6 and Embedded Visual C++ 4 doesn't +// realize that, e.g. char has the same size as __int8 +// so we give up on __intX for them. +#if (_MSC_VER < 1300) + typedef signed char int8_t; + typedef signed short int16_t; + typedef signed int int32_t; + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; +#else + typedef signed __int8 int8_t; + typedef signed __int16 int16_t; + typedef signed __int32 int32_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; +#endif +typedef signed __int64 int64_t; +typedef unsigned __int64 uint64_t; + + +// 7.18.1.2 Minimum-width integer types +typedef int8_t int_least8_t; +typedef int16_t int_least16_t; +typedef int32_t int_least32_t; +typedef int64_t int_least64_t; +typedef uint8_t uint_least8_t; +typedef uint16_t uint_least16_t; +typedef uint32_t uint_least32_t; +typedef uint64_t uint_least64_t; + +// 7.18.1.3 Fastest minimum-width integer types +typedef int8_t int_fast8_t; +typedef int16_t int_fast16_t; +typedef int32_t int_fast32_t; +typedef int64_t int_fast64_t; +typedef uint8_t uint_fast8_t; +typedef uint16_t uint_fast16_t; +typedef uint32_t uint_fast32_t; +typedef uint64_t uint_fast64_t; + +// 7.18.1.4 Integer types capable of holding object pointers +#ifdef _WIN64 // [ + typedef signed __int64 intptr_t; + typedef unsigned __int64 uintptr_t; +#else // _WIN64 ][ + typedef _W64 signed int intptr_t; + typedef _W64 unsigned int uintptr_t; +#endif // _WIN64 ] + +// 7.18.1.5 Greatest-width integer types +typedef int64_t intmax_t; +typedef uint64_t uintmax_t; + + +// 7.18.2 Limits of specified-width integer types + +#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259 + +// 7.18.2.1 Limits of exact-width integer types +#define INT8_MIN ((int8_t)_I8_MIN) +#define INT8_MAX _I8_MAX +#define INT16_MIN ((int16_t)_I16_MIN) +#define INT16_MAX _I16_MAX +#define INT32_MIN ((int32_t)_I32_MIN) +#define INT32_MAX _I32_MAX +#define INT64_MIN ((int64_t)_I64_MIN) +#define INT64_MAX _I64_MAX +#define UINT8_MAX _UI8_MAX +#define UINT16_MAX _UI16_MAX +#define UINT32_MAX _UI32_MAX +#define UINT64_MAX _UI64_MAX + +// 7.18.2.2 Limits of minimum-width integer types +#define INT_LEAST8_MIN INT8_MIN +#define INT_LEAST8_MAX INT8_MAX +#define INT_LEAST16_MIN INT16_MIN +#define INT_LEAST16_MAX INT16_MAX +#define INT_LEAST32_MIN INT32_MIN +#define INT_LEAST32_MAX INT32_MAX +#define INT_LEAST64_MIN INT64_MIN +#define INT_LEAST64_MAX INT64_MAX +#define UINT_LEAST8_MAX UINT8_MAX +#define UINT_LEAST16_MAX UINT16_MAX +#define UINT_LEAST32_MAX UINT32_MAX +#define UINT_LEAST64_MAX UINT64_MAX + +// 7.18.2.3 Limits of fastest minimum-width integer types +#define INT_FAST8_MIN INT8_MIN +#define INT_FAST8_MAX INT8_MAX +#define INT_FAST16_MIN INT16_MIN +#define INT_FAST16_MAX INT16_MAX +#define INT_FAST32_MIN INT32_MIN +#define INT_FAST32_MAX INT32_MAX +#define INT_FAST64_MIN INT64_MIN +#define INT_FAST64_MAX INT64_MAX +#define UINT_FAST8_MAX UINT8_MAX +#define UINT_FAST16_MAX UINT16_MAX +#define UINT_FAST32_MAX UINT32_MAX +#define UINT_FAST64_MAX UINT64_MAX + +// 7.18.2.4 Limits of integer types capable of holding object pointers +#ifdef _WIN64 // [ +# define INTPTR_MIN INT64_MIN +# define INTPTR_MAX INT64_MAX +# define UINTPTR_MAX UINT64_MAX +#else // _WIN64 ][ +# define INTPTR_MIN INT32_MIN +# define INTPTR_MAX INT32_MAX +# define UINTPTR_MAX UINT32_MAX +#endif // _WIN64 ] + +// 7.18.2.5 Limits of greatest-width integer types +#define INTMAX_MIN INT64_MIN +#define INTMAX_MAX INT64_MAX +#define UINTMAX_MAX UINT64_MAX + +// 7.18.3 Limits of other integer types + +#ifdef _WIN64 // [ +# define PTRDIFF_MIN _I64_MIN +# define PTRDIFF_MAX _I64_MAX +#else // _WIN64 ][ +# define PTRDIFF_MIN _I32_MIN +# define PTRDIFF_MAX _I32_MAX +#endif // _WIN64 ] + +#define SIG_ATOMIC_MIN INT_MIN +#define SIG_ATOMIC_MAX INT_MAX + +#ifndef SIZE_MAX // [ +# ifdef _WIN64 // [ +# define SIZE_MAX _UI64_MAX +# else // _WIN64 ][ +# define SIZE_MAX _UI32_MAX +# endif // _WIN64 ] +#endif // SIZE_MAX ] + +// WCHAR_MIN and WCHAR_MAX are also defined in +#ifndef WCHAR_MIN // [ +# define WCHAR_MIN 0 +#endif // WCHAR_MIN ] +#ifndef WCHAR_MAX // [ +# define WCHAR_MAX _UI16_MAX +#endif // WCHAR_MAX ] + +#define WINT_MIN 0 +#define WINT_MAX _UI16_MAX + +#endif // __STDC_LIMIT_MACROS ] + + +// 7.18.4 Limits of other integer types + +#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 + +// 7.18.4.1 Macros for minimum-width integer constants + +#define INT8_C(val) val##i8 +#define INT16_C(val) val##i16 +#define INT32_C(val) val##i32 +#define INT64_C(val) val##i64 + +#define UINT8_C(val) val##ui8 +#define UINT16_C(val) val##ui16 +#define UINT32_C(val) val##ui32 +#define UINT64_C(val) val##ui64 + +// 7.18.4.2 Macros for greatest-width integer constants +// These #ifndef's are needed to prevent collisions with . +// Check out Issue 9 for the details. +#ifndef INTMAX_C // [ +# define INTMAX_C INT64_C +#endif // INTMAX_C ] +#ifndef UINTMAX_C // [ +# define UINTMAX_C UINT64_C +#endif // UINTMAX_C ] + +#endif // __STDC_CONSTANT_MACROS ] + +#endif // _MSC_VER >= 1600 ] + +#endif // _MSC_STDINT_H_ ] diff --git a/third_party/cereal/external/rapidjson/ostreamwrapper.h b/third_party/cereal/external/rapidjson/ostreamwrapper.h new file mode 100755 index 0000000..58c034a --- /dev/null +++ b/third_party/cereal/external/rapidjson/ostreamwrapper.h @@ -0,0 +1,81 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_OSTREAMWRAPPER_H_ +#define CEREAL_RAPIDJSON_OSTREAMWRAPPER_H_ + +#include "stream.h" +#include + +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(padded) +#endif + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN + +//! Wrapper of \c std::basic_ostream into RapidJSON's Stream concept. +/*! + The classes can be wrapped including but not limited to: + + - \c std::ostringstream + - \c std::stringstream + - \c std::wpstringstream + - \c std::wstringstream + - \c std::ifstream + - \c std::fstream + - \c std::wofstream + - \c std::wfstream + + \tparam StreamType Class derived from \c std::basic_ostream. +*/ + +template +class BasicOStreamWrapper { +public: + typedef typename StreamType::char_type Ch; + BasicOStreamWrapper(StreamType& stream) : stream_(stream) {} + + void Put(Ch c) { + stream_.put(c); + } + + void Flush() { + stream_.flush(); + } + + // Not implemented + char Peek() const { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + char Take() { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + size_t Tell() const { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + char* PutBegin() { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(char*) { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + +private: + BasicOStreamWrapper(const BasicOStreamWrapper&); + BasicOStreamWrapper& operator=(const BasicOStreamWrapper&); + + StreamType& stream_; +}; + +typedef BasicOStreamWrapper OStreamWrapper; +typedef BasicOStreamWrapper WOStreamWrapper; + +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_POP +#endif + +CEREAL_RAPIDJSON_NAMESPACE_END + +#endif // CEREAL_RAPIDJSON_OSTREAMWRAPPER_H_ diff --git a/third_party/cereal/external/rapidjson/pointer.h b/third_party/cereal/external/rapidjson/pointer.h new file mode 100755 index 0000000..930970f --- /dev/null +++ b/third_party/cereal/external/rapidjson/pointer.h @@ -0,0 +1,1414 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_POINTER_H_ +#define CEREAL_RAPIDJSON_POINTER_H_ + +#include "document.h" +#include "internal/itoa.h" + +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(switch-enum) +#elif defined(_MSC_VER) +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN + +static const SizeType kPointerInvalidIndex = ~SizeType(0); //!< Represents an invalid index in GenericPointer::Token + +//! Error code of parsing. +/*! \ingroup CEREAL_RAPIDJSON_ERRORS + \see GenericPointer::GenericPointer, GenericPointer::GetParseErrorCode +*/ +enum PointerParseErrorCode { + kPointerParseErrorNone = 0, //!< The parse is successful + + kPointerParseErrorTokenMustBeginWithSolidus, //!< A token must begin with a '/' + kPointerParseErrorInvalidEscape, //!< Invalid escape + kPointerParseErrorInvalidPercentEncoding, //!< Invalid percent encoding in URI fragment + kPointerParseErrorCharacterMustPercentEncode //!< A character must percent encoded in URI fragment +}; + +/////////////////////////////////////////////////////////////////////////////// +// GenericPointer + +//! Represents a JSON Pointer. Use Pointer for UTF8 encoding and default allocator. +/*! + This class implements RFC 6901 "JavaScript Object Notation (JSON) Pointer" + (https://tools.ietf.org/html/rfc6901). + + A JSON pointer is for identifying a specific value in a JSON document + (GenericDocument). It can simplify coding of DOM tree manipulation, because it + can access multiple-level depth of DOM tree with single API call. + + After it parses a string representation (e.g. "/foo/0" or URI fragment + representation (e.g. "#/foo/0") into its internal representation (tokens), + it can be used to resolve a specific value in multiple documents, or sub-tree + of documents. + + Contrary to GenericValue, Pointer can be copy constructed and copy assigned. + Apart from assignment, a Pointer cannot be modified after construction. + + Although Pointer is very convenient, please aware that constructing Pointer + involves parsing and dynamic memory allocation. A special constructor with user- + supplied tokens eliminates these. + + GenericPointer depends on GenericDocument and GenericValue. + + \tparam ValueType The value type of the DOM tree. E.g. GenericValue > + \tparam Allocator The allocator type for allocating memory for internal representation. + + \note GenericPointer uses same encoding of ValueType. + However, Allocator of GenericPointer is independent of Allocator of Value. +*/ +template +class GenericPointer { +public: + typedef typename ValueType::EncodingType EncodingType; //!< Encoding type from Value + typedef typename ValueType::Ch Ch; //!< Character type from Value + + //! A token is the basic units of internal representation. + /*! + A JSON pointer string representation "/foo/123" is parsed to two tokens: + "foo" and 123. 123 will be represented in both numeric form and string form. + They are resolved according to the actual value type (object or array). + + For token that are not numbers, or the numeric value is out of bound + (greater than limits of SizeType), they are only treated as string form + (i.e. the token's index will be equal to kPointerInvalidIndex). + + This struct is public so that user can create a Pointer without parsing and + allocation, using a special constructor. + */ + struct Token { + const Ch* name; //!< Name of the token. It has null character at the end but it can contain null character. + SizeType length; //!< Length of the name. + SizeType index; //!< A valid array index, if it is not equal to kPointerInvalidIndex. + }; + + //!@name Constructors and destructor. + //@{ + + //! Default constructor. + GenericPointer(Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} + + //! Constructor that parses a string or URI fragment representation. + /*! + \param source A null-terminated, string or URI fragment representation of JSON pointer. + \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. + */ + explicit GenericPointer(const Ch* source, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + Parse(source, internal::StrLen(source)); + } + +#if CEREAL_RAPIDJSON_HAS_STDSTRING + //! Constructor that parses a string or URI fragment representation. + /*! + \param source A string or URI fragment representation of JSON pointer. + \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. + \note Requires the definition of the preprocessor symbol \ref CEREAL_RAPIDJSON_HAS_STDSTRING. + */ + explicit GenericPointer(const std::basic_string& source, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + Parse(source.c_str(), source.size()); + } +#endif + + //! Constructor that parses a string or URI fragment representation, with length of the source string. + /*! + \param source A string or URI fragment representation of JSON pointer. + \param length Length of source. + \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. + \note Slightly faster than the overload without length. + */ + GenericPointer(const Ch* source, size_t length, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + Parse(source, length); + } + + //! Constructor with user-supplied tokens. + /*! + This constructor let user supplies const array of tokens. + This prevents the parsing process and eliminates allocation. + This is preferred for memory constrained environments. + + \param tokens An constant array of tokens representing the JSON pointer. + \param tokenCount Number of tokens. + + \b Example + \code + #define NAME(s) { s, sizeof(s) / sizeof(s[0]) - 1, kPointerInvalidIndex } + #define INDEX(i) { #i, sizeof(#i) - 1, i } + + static const Pointer::Token kTokens[] = { NAME("foo"), INDEX(123) }; + static const Pointer p(kTokens, sizeof(kTokens) / sizeof(kTokens[0])); + // Equivalent to static const Pointer p("/foo/123"); + + #undef NAME + #undef INDEX + \endcode + */ + GenericPointer(const Token* tokens, size_t tokenCount) : allocator_(), ownAllocator_(), nameBuffer_(), tokens_(const_cast(tokens)), tokenCount_(tokenCount), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} + + //! Copy constructor. + GenericPointer(const GenericPointer& rhs) : allocator_(rhs.allocator_), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + *this = rhs; + } + + //! Copy constructor. + GenericPointer(const GenericPointer& rhs, Allocator* allocator) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + *this = rhs; + } + + //! Destructor. + ~GenericPointer() { + if (nameBuffer_) // If user-supplied tokens constructor is used, nameBuffer_ is nullptr and tokens_ are not deallocated. + Allocator::Free(tokens_); + CEREAL_RAPIDJSON_DELETE(ownAllocator_); + } + + //! Assignment operator. + GenericPointer& operator=(const GenericPointer& rhs) { + if (this != &rhs) { + // Do not delete ownAllcator + if (nameBuffer_) + Allocator::Free(tokens_); + + tokenCount_ = rhs.tokenCount_; + parseErrorOffset_ = rhs.parseErrorOffset_; + parseErrorCode_ = rhs.parseErrorCode_; + + if (rhs.nameBuffer_) + CopyFromRaw(rhs); // Normally parsed tokens. + else { + tokens_ = rhs.tokens_; // User supplied const tokens. + nameBuffer_ = 0; + } + } + return *this; + } + + //! Swap the content of this pointer with an other. + /*! + \param other The pointer to swap with. + \note Constant complexity. + */ + GenericPointer& Swap(GenericPointer& other) CEREAL_RAPIDJSON_NOEXCEPT { + internal::Swap(allocator_, other.allocator_); + internal::Swap(ownAllocator_, other.ownAllocator_); + internal::Swap(nameBuffer_, other.nameBuffer_); + internal::Swap(tokens_, other.tokens_); + internal::Swap(tokenCount_, other.tokenCount_); + internal::Swap(parseErrorOffset_, other.parseErrorOffset_); + internal::Swap(parseErrorCode_, other.parseErrorCode_); + return *this; + } + + //! free-standing swap function helper + /*! + Helper function to enable support for common swap implementation pattern based on \c std::swap: + \code + void swap(MyClass& a, MyClass& b) { + using std::swap; + swap(a.pointer, b.pointer); + // ... + } + \endcode + \see Swap() + */ + friend inline void swap(GenericPointer& a, GenericPointer& b) CEREAL_RAPIDJSON_NOEXCEPT { a.Swap(b); } + + //@} + + //!@name Append token + //@{ + + //! Append a token and return a new Pointer + /*! + \param token Token to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(const Token& token, Allocator* allocator = 0) const { + GenericPointer r; + r.allocator_ = allocator; + Ch *p = r.CopyFromRaw(*this, 1, token.length + 1); + std::memcpy(p, token.name, (token.length + 1) * sizeof(Ch)); + r.tokens_[tokenCount_].name = p; + r.tokens_[tokenCount_].length = token.length; + r.tokens_[tokenCount_].index = token.index; + return r; + } + + //! Append a name token with length, and return a new Pointer + /*! + \param name Name to be appended. + \param length Length of name. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(const Ch* name, SizeType length, Allocator* allocator = 0) const { + Token token = { name, length, kPointerInvalidIndex }; + return Append(token, allocator); + } + + //! Append a name token without length, and return a new Pointer + /*! + \param name Name (const Ch*) to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + template + CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >), (GenericPointer)) + Append(T* name, Allocator* allocator = 0) const { + return Append(name, internal::StrLen(name), allocator); + } + +#if CEREAL_RAPIDJSON_HAS_STDSTRING + //! Append a name token, and return a new Pointer + /*! + \param name Name to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(const std::basic_string& name, Allocator* allocator = 0) const { + return Append(name.c_str(), static_cast(name.size()), allocator); + } +#endif + + //! Append a index token, and return a new Pointer + /*! + \param index Index to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(SizeType index, Allocator* allocator = 0) const { + char buffer[21]; + char* end = sizeof(SizeType) == 4 ? internal::u32toa(index, buffer) : internal::u64toa(index, buffer); + SizeType length = static_cast(end - buffer); + buffer[length] = '\0'; + + if (sizeof(Ch) == 1) { + Token token = { reinterpret_cast(buffer), length, index }; + return Append(token, allocator); + } + else { + Ch name[21]; + for (size_t i = 0; i <= length; i++) + name[i] = static_cast(buffer[i]); + Token token = { name, length, index }; + return Append(token, allocator); + } + } + + //! Append a token by value, and return a new Pointer + /*! + \param token token to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(const ValueType& token, Allocator* allocator = 0) const { + if (token.IsString()) + return Append(token.GetString(), token.GetStringLength(), allocator); + else { + CEREAL_RAPIDJSON_ASSERT(token.IsUint64()); + CEREAL_RAPIDJSON_ASSERT(token.GetUint64() <= SizeType(~0)); + return Append(static_cast(token.GetUint64()), allocator); + } + } + + //!@name Handling Parse Error + //@{ + + //! Check whether this is a valid pointer. + bool IsValid() const { return parseErrorCode_ == kPointerParseErrorNone; } + + //! Get the parsing error offset in code unit. + size_t GetParseErrorOffset() const { return parseErrorOffset_; } + + //! Get the parsing error code. + PointerParseErrorCode GetParseErrorCode() const { return parseErrorCode_; } + + //@} + + //! Get the allocator of this pointer. + Allocator& GetAllocator() { return *allocator_; } + + //!@name Tokens + //@{ + + //! Get the token array (const version only). + const Token* GetTokens() const { return tokens_; } + + //! Get the number of tokens. + size_t GetTokenCount() const { return tokenCount_; } + + //@} + + //!@name Equality/inequality operators + //@{ + + //! Equality operator. + /*! + \note When any pointers are invalid, always returns false. + */ + bool operator==(const GenericPointer& rhs) const { + if (!IsValid() || !rhs.IsValid() || tokenCount_ != rhs.tokenCount_) + return false; + + for (size_t i = 0; i < tokenCount_; i++) { + if (tokens_[i].index != rhs.tokens_[i].index || + tokens_[i].length != rhs.tokens_[i].length || + (tokens_[i].length != 0 && std::memcmp(tokens_[i].name, rhs.tokens_[i].name, sizeof(Ch)* tokens_[i].length) != 0)) + { + return false; + } + } + + return true; + } + + //! Inequality operator. + /*! + \note When any pointers are invalid, always returns true. + */ + bool operator!=(const GenericPointer& rhs) const { return !(*this == rhs); } + + //! Less than operator. + /*! + \note Invalid pointers are always greater than valid ones. + */ + bool operator<(const GenericPointer& rhs) const { + if (!IsValid()) + return false; + if (!rhs.IsValid()) + return true; + + if (tokenCount_ != rhs.tokenCount_) + return tokenCount_ < rhs.tokenCount_; + + for (size_t i = 0; i < tokenCount_; i++) { + if (tokens_[i].index != rhs.tokens_[i].index) + return tokens_[i].index < rhs.tokens_[i].index; + + if (tokens_[i].length != rhs.tokens_[i].length) + return tokens_[i].length < rhs.tokens_[i].length; + + if (int cmp = std::memcmp(tokens_[i].name, rhs.tokens_[i].name, sizeof(Ch) * tokens_[i].length)) + return cmp < 0; + } + + return false; + } + + //@} + + //!@name Stringify + //@{ + + //! Stringify the pointer into string representation. + /*! + \tparam OutputStream Type of output stream. + \param os The output stream. + */ + template + bool Stringify(OutputStream& os) const { + return Stringify(os); + } + + //! Stringify the pointer into URI fragment representation. + /*! + \tparam OutputStream Type of output stream. + \param os The output stream. + */ + template + bool StringifyUriFragment(OutputStream& os) const { + return Stringify(os); + } + + //@} + + //!@name Create value + //@{ + + //! Create a value in a subtree. + /*! + If the value is not exist, it creates all parent values and a JSON Null value. + So it always succeed and return the newly created or existing value. + + Remind that it may change types of parents according to tokens, so it + potentially removes previously stored values. For example, if a document + was an array, and "/foo" is used to create a value, then the document + will be changed to an object, and all existing array elements are lost. + + \param root Root value of a DOM subtree to be resolved. It can be any value other than document root. + \param allocator Allocator for creating the values if the specified value or its parents are not exist. + \param alreadyExist If non-null, it stores whether the resolved value is already exist. + \return The resolved newly created (a JSON Null value), or already exists value. + */ + ValueType& Create(ValueType& root, typename ValueType::AllocatorType& allocator, bool* alreadyExist = 0) const { + CEREAL_RAPIDJSON_ASSERT(IsValid()); + ValueType* v = &root; + bool exist = true; + for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { + if (v->IsArray() && t->name[0] == '-' && t->length == 1) { + v->PushBack(ValueType().Move(), allocator); + v = &((*v)[v->Size() - 1]); + exist = false; + } + else { + if (t->index == kPointerInvalidIndex) { // must be object name + if (!v->IsObject()) + v->SetObject(); // Change to Object + } + else { // object name or array index + if (!v->IsArray() && !v->IsObject()) + v->SetArray(); // Change to Array + } + + if (v->IsArray()) { + if (t->index >= v->Size()) { + v->Reserve(t->index + 1, allocator); + while (t->index >= v->Size()) + v->PushBack(ValueType().Move(), allocator); + exist = false; + } + v = &((*v)[t->index]); + } + else { + typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); + if (m == v->MemberEnd()) { + v->AddMember(ValueType(t->name, t->length, allocator).Move(), ValueType().Move(), allocator); + v = &(--v->MemberEnd())->value; // Assumes AddMember() appends at the end + exist = false; + } + else + v = &m->value; + } + } + } + + if (alreadyExist) + *alreadyExist = exist; + + return *v; + } + + //! Creates a value in a document. + /*! + \param document A document to be resolved. + \param alreadyExist If non-null, it stores whether the resolved value is already exist. + \return The resolved newly created, or already exists value. + */ + template + ValueType& Create(GenericDocument& document, bool* alreadyExist = 0) const { + return Create(document, document.GetAllocator(), alreadyExist); + } + + //@} + + //!@name Query value + //@{ + + //! Query a value in a subtree. + /*! + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \param unresolvedTokenIndex If the pointer cannot resolve a token in the pointer, this parameter can obtain the index of unresolved token. + \return Pointer to the value if it can be resolved. Otherwise null. + + \note + There are only 3 situations when a value cannot be resolved: + 1. A value in the path is not an array nor object. + 2. An object value does not contain the token. + 3. A token is out of range of an array value. + + Use unresolvedTokenIndex to retrieve the token index. + */ + ValueType* Get(ValueType& root, size_t* unresolvedTokenIndex = 0) const { + CEREAL_RAPIDJSON_ASSERT(IsValid()); + ValueType* v = &root; + for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { + switch (v->GetType()) { + case kObjectType: + { + typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); + if (m == v->MemberEnd()) + break; + v = &m->value; + } + continue; + case kArrayType: + if (t->index == kPointerInvalidIndex || t->index >= v->Size()) + break; + v = &((*v)[t->index]); + continue; + default: + break; + } + + // Error: unresolved token + if (unresolvedTokenIndex) + *unresolvedTokenIndex = static_cast(t - tokens_); + return 0; + } + return v; + } + + //! Query a const value in a const subtree. + /*! + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \return Pointer to the value if it can be resolved. Otherwise null. + */ + const ValueType* Get(const ValueType& root, size_t* unresolvedTokenIndex = 0) const { + return Get(const_cast(root), unresolvedTokenIndex); + } + + //@} + + //!@name Query a value with default + //@{ + + //! Query a value in a subtree with default value. + /*! + Similar to Get(), but if the specified value do not exists, it creates all parents and clone the default value. + So that this function always succeed. + + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \param defaultValue Default value to be cloned if the value was not exists. + \param allocator Allocator for creating the values if the specified value or its parents are not exist. + \see Create() + */ + ValueType& GetWithDefault(ValueType& root, const ValueType& defaultValue, typename ValueType::AllocatorType& allocator) const { + bool alreadyExist; + ValueType& v = Create(root, allocator, &alreadyExist); + return alreadyExist ? v : v.CopyFrom(defaultValue, allocator); + } + + //! Query a value in a subtree with default null-terminated string. + ValueType& GetWithDefault(ValueType& root, const Ch* defaultValue, typename ValueType::AllocatorType& allocator) const { + bool alreadyExist; + ValueType& v = Create(root, allocator, &alreadyExist); + return alreadyExist ? v : v.SetString(defaultValue, allocator); + } + +#if CEREAL_RAPIDJSON_HAS_STDSTRING + //! Query a value in a subtree with default std::basic_string. + ValueType& GetWithDefault(ValueType& root, const std::basic_string& defaultValue, typename ValueType::AllocatorType& allocator) const { + bool alreadyExist; + ValueType& v = Create(root, allocator, &alreadyExist); + return alreadyExist ? v : v.SetString(defaultValue, allocator); + } +#endif + + //! Query a value in a subtree with default primitive value. + /*! + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + */ + template + CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) + GetWithDefault(ValueType& root, T defaultValue, typename ValueType::AllocatorType& allocator) const { + return GetWithDefault(root, ValueType(defaultValue).Move(), allocator); + } + + //! Query a value in a document with default value. + template + ValueType& GetWithDefault(GenericDocument& document, const ValueType& defaultValue) const { + return GetWithDefault(document, defaultValue, document.GetAllocator()); + } + + //! Query a value in a document with default null-terminated string. + template + ValueType& GetWithDefault(GenericDocument& document, const Ch* defaultValue) const { + return GetWithDefault(document, defaultValue, document.GetAllocator()); + } + +#if CEREAL_RAPIDJSON_HAS_STDSTRING + //! Query a value in a document with default std::basic_string. + template + ValueType& GetWithDefault(GenericDocument& document, const std::basic_string& defaultValue) const { + return GetWithDefault(document, defaultValue, document.GetAllocator()); + } +#endif + + //! Query a value in a document with default primitive value. + /*! + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + */ + template + CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) + GetWithDefault(GenericDocument& document, T defaultValue) const { + return GetWithDefault(document, defaultValue, document.GetAllocator()); + } + + //@} + + //!@name Set a value + //@{ + + //! Set a value in a subtree, with move semantics. + /*! + It creates all parents if they are not exist or types are different to the tokens. + So this function always succeeds but potentially remove existing values. + + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \param value Value to be set. + \param allocator Allocator for creating the values if the specified value or its parents are not exist. + \see Create() + */ + ValueType& Set(ValueType& root, ValueType& value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator) = value; + } + + //! Set a value in a subtree, with copy semantics. + ValueType& Set(ValueType& root, const ValueType& value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator).CopyFrom(value, allocator); + } + + //! Set a null-terminated string in a subtree. + ValueType& Set(ValueType& root, const Ch* value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator) = ValueType(value, allocator).Move(); + } + +#if CEREAL_RAPIDJSON_HAS_STDSTRING + //! Set a std::basic_string in a subtree. + ValueType& Set(ValueType& root, const std::basic_string& value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator) = ValueType(value, allocator).Move(); + } +#endif + + //! Set a primitive value in a subtree. + /*! + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + */ + template + CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) + Set(ValueType& root, T value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator) = ValueType(value).Move(); + } + + //! Set a value in a document, with move semantics. + template + ValueType& Set(GenericDocument& document, ValueType& value) const { + return Create(document) = value; + } + + //! Set a value in a document, with copy semantics. + template + ValueType& Set(GenericDocument& document, const ValueType& value) const { + return Create(document).CopyFrom(value, document.GetAllocator()); + } + + //! Set a null-terminated string in a document. + template + ValueType& Set(GenericDocument& document, const Ch* value) const { + return Create(document) = ValueType(value, document.GetAllocator()).Move(); + } + +#if CEREAL_RAPIDJSON_HAS_STDSTRING + //! Sets a std::basic_string in a document. + template + ValueType& Set(GenericDocument& document, const std::basic_string& value) const { + return Create(document) = ValueType(value, document.GetAllocator()).Move(); + } +#endif + + //! Set a primitive value in a document. + /*! + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + */ + template + CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) + Set(GenericDocument& document, T value) const { + return Create(document) = value; + } + + //@} + + //!@name Swap a value + //@{ + + //! Swap a value with a value in a subtree. + /*! + It creates all parents if they are not exist or types are different to the tokens. + So this function always succeeds but potentially remove existing values. + + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \param value Value to be swapped. + \param allocator Allocator for creating the values if the specified value or its parents are not exist. + \see Create() + */ + ValueType& Swap(ValueType& root, ValueType& value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator).Swap(value); + } + + //! Swap a value with a value in a document. + template + ValueType& Swap(GenericDocument& document, ValueType& value) const { + return Create(document).Swap(value); + } + + //@} + + //! Erase a value in a subtree. + /*! + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \return Whether the resolved value is found and erased. + + \note Erasing with an empty pointer \c Pointer(""), i.e. the root, always fail and return false. + */ + bool Erase(ValueType& root) const { + CEREAL_RAPIDJSON_ASSERT(IsValid()); + if (tokenCount_ == 0) // Cannot erase the root + return false; + + ValueType* v = &root; + const Token* last = tokens_ + (tokenCount_ - 1); + for (const Token *t = tokens_; t != last; ++t) { + switch (v->GetType()) { + case kObjectType: + { + typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); + if (m == v->MemberEnd()) + return false; + v = &m->value; + } + break; + case kArrayType: + if (t->index == kPointerInvalidIndex || t->index >= v->Size()) + return false; + v = &((*v)[t->index]); + break; + default: + return false; + } + } + + switch (v->GetType()) { + case kObjectType: + return v->EraseMember(GenericStringRef(last->name, last->length)); + case kArrayType: + if (last->index == kPointerInvalidIndex || last->index >= v->Size()) + return false; + v->Erase(v->Begin() + last->index); + return true; + default: + return false; + } + } + +private: + //! Clone the content from rhs to this. + /*! + \param rhs Source pointer. + \param extraToken Extra tokens to be allocated. + \param extraNameBufferSize Extra name buffer size (in number of Ch) to be allocated. + \return Start of non-occupied name buffer, for storing extra names. + */ + Ch* CopyFromRaw(const GenericPointer& rhs, size_t extraToken = 0, size_t extraNameBufferSize = 0) { + if (!allocator_) // allocator is independently owned. + ownAllocator_ = allocator_ = CEREAL_RAPIDJSON_NEW(Allocator)(); + + size_t nameBufferSize = rhs.tokenCount_; // null terminators for tokens + for (Token *t = rhs.tokens_; t != rhs.tokens_ + rhs.tokenCount_; ++t) + nameBufferSize += t->length; + + tokenCount_ = rhs.tokenCount_ + extraToken; + tokens_ = static_cast(allocator_->Malloc(tokenCount_ * sizeof(Token) + (nameBufferSize + extraNameBufferSize) * sizeof(Ch))); + nameBuffer_ = reinterpret_cast(tokens_ + tokenCount_); + if (rhs.tokenCount_ > 0) { + std::memcpy(tokens_, rhs.tokens_, rhs.tokenCount_ * sizeof(Token)); + } + if (nameBufferSize > 0) { + std::memcpy(nameBuffer_, rhs.nameBuffer_, nameBufferSize * sizeof(Ch)); + } + + // Adjust pointers to name buffer + std::ptrdiff_t diff = nameBuffer_ - rhs.nameBuffer_; + for (Token *t = tokens_; t != tokens_ + rhs.tokenCount_; ++t) + t->name += diff; + + return nameBuffer_ + nameBufferSize; + } + + //! Check whether a character should be percent-encoded. + /*! + According to RFC 3986 2.3 Unreserved Characters. + \param c The character (code unit) to be tested. + */ + bool NeedPercentEncode(Ch c) const { + return !((c >= '0' && c <= '9') || (c >= 'A' && c <='Z') || (c >= 'a' && c <= 'z') || c == '-' || c == '.' || c == '_' || c =='~'); + } + + //! Parse a JSON String or its URI fragment representation into tokens. +#ifndef __clang__ // -Wdocumentation + /*! + \param source Either a JSON Pointer string, or its URI fragment representation. Not need to be null terminated. + \param length Length of the source string. + \note Source cannot be JSON String Representation of JSON Pointer, e.g. In "/\u0000", \u0000 will not be unescaped. + */ +#endif + void Parse(const Ch* source, size_t length) { + CEREAL_RAPIDJSON_ASSERT(source != NULL); + CEREAL_RAPIDJSON_ASSERT(nameBuffer_ == 0); + CEREAL_RAPIDJSON_ASSERT(tokens_ == 0); + + // Create own allocator if user did not supply. + if (!allocator_) + ownAllocator_ = allocator_ = CEREAL_RAPIDJSON_NEW(Allocator)(); + + // Count number of '/' as tokenCount + tokenCount_ = 0; + for (const Ch* s = source; s != source + length; s++) + if (*s == '/') + tokenCount_++; + + Token* token = tokens_ = static_cast(allocator_->Malloc(tokenCount_ * sizeof(Token) + length * sizeof(Ch))); + Ch* name = nameBuffer_ = reinterpret_cast(tokens_ + tokenCount_); + size_t i = 0; + + // Detect if it is a URI fragment + bool uriFragment = false; + if (source[i] == '#') { + uriFragment = true; + i++; + } + + if (i != length && source[i] != '/') { + parseErrorCode_ = kPointerParseErrorTokenMustBeginWithSolidus; + goto error; + } + + while (i < length) { + CEREAL_RAPIDJSON_ASSERT(source[i] == '/'); + i++; // consumes '/' + + token->name = name; + bool isNumber = true; + + while (i < length && source[i] != '/') { + Ch c = source[i]; + if (uriFragment) { + // Decoding percent-encoding for URI fragment + if (c == '%') { + PercentDecodeStream is(&source[i], source + length); + GenericInsituStringStream os(name); + Ch* begin = os.PutBegin(); + if (!Transcoder, EncodingType>().Validate(is, os) || !is.IsValid()) { + parseErrorCode_ = kPointerParseErrorInvalidPercentEncoding; + goto error; + } + size_t len = os.PutEnd(begin); + i += is.Tell() - 1; + if (len == 1) + c = *name; + else { + name += len; + isNumber = false; + i++; + continue; + } + } + else if (NeedPercentEncode(c)) { + parseErrorCode_ = kPointerParseErrorCharacterMustPercentEncode; + goto error; + } + } + + i++; + + // Escaping "~0" -> '~', "~1" -> '/' + if (c == '~') { + if (i < length) { + c = source[i]; + if (c == '0') c = '~'; + else if (c == '1') c = '/'; + else { + parseErrorCode_ = kPointerParseErrorInvalidEscape; + goto error; + } + i++; + } + else { + parseErrorCode_ = kPointerParseErrorInvalidEscape; + goto error; + } + } + + // First check for index: all of characters are digit + if (c < '0' || c > '9') + isNumber = false; + + *name++ = c; + } + token->length = static_cast(name - token->name); + if (token->length == 0) + isNumber = false; + *name++ = '\0'; // Null terminator + + // Second check for index: more than one digit cannot have leading zero + if (isNumber && token->length > 1 && token->name[0] == '0') + isNumber = false; + + // String to SizeType conversion + SizeType n = 0; + if (isNumber) { + for (size_t j = 0; j < token->length; j++) { + SizeType m = n * 10 + static_cast(token->name[j] - '0'); + if (m < n) { // overflow detection + isNumber = false; + break; + } + n = m; + } + } + + token->index = isNumber ? n : kPointerInvalidIndex; + token++; + } + + CEREAL_RAPIDJSON_ASSERT(name <= nameBuffer_ + length); // Should not overflow buffer + parseErrorCode_ = kPointerParseErrorNone; + return; + + error: + Allocator::Free(tokens_); + nameBuffer_ = 0; + tokens_ = 0; + tokenCount_ = 0; + parseErrorOffset_ = i; + return; + } + + //! Stringify to string or URI fragment representation. + /*! + \tparam uriFragment True for stringifying to URI fragment representation. False for string representation. + \tparam OutputStream type of output stream. + \param os The output stream. + */ + template + bool Stringify(OutputStream& os) const { + CEREAL_RAPIDJSON_ASSERT(IsValid()); + + if (uriFragment) + os.Put('#'); + + for (Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { + os.Put('/'); + for (size_t j = 0; j < t->length; j++) { + Ch c = t->name[j]; + if (c == '~') { + os.Put('~'); + os.Put('0'); + } + else if (c == '/') { + os.Put('~'); + os.Put('1'); + } + else if (uriFragment && NeedPercentEncode(c)) { + // Transcode to UTF8 sequence + GenericStringStream source(&t->name[j]); + PercentEncodeStream target(os); + if (!Transcoder >().Validate(source, target)) + return false; + j += source.Tell() - 1; + } + else + os.Put(c); + } + } + return true; + } + + //! A helper stream for decoding a percent-encoded sequence into code unit. + /*! + This stream decodes %XY triplet into code unit (0-255). + If it encounters invalid characters, it sets output code unit as 0 and + mark invalid, and to be checked by IsValid(). + */ + class PercentDecodeStream { + public: + typedef typename ValueType::Ch Ch; + + //! Constructor + /*! + \param source Start of the stream + \param end Past-the-end of the stream. + */ + PercentDecodeStream(const Ch* source, const Ch* end) : src_(source), head_(source), end_(end), valid_(true) {} + + Ch Take() { + if (*src_ != '%' || src_ + 3 > end_) { // %XY triplet + valid_ = false; + return 0; + } + src_++; + Ch c = 0; + for (int j = 0; j < 2; j++) { + c = static_cast(c << 4); + Ch h = *src_; + if (h >= '0' && h <= '9') c = static_cast(c + h - '0'); + else if (h >= 'A' && h <= 'F') c = static_cast(c + h - 'A' + 10); + else if (h >= 'a' && h <= 'f') c = static_cast(c + h - 'a' + 10); + else { + valid_ = false; + return 0; + } + src_++; + } + return c; + } + + size_t Tell() const { return static_cast(src_ - head_); } + bool IsValid() const { return valid_; } + + private: + const Ch* src_; //!< Current read position. + const Ch* head_; //!< Original head of the string. + const Ch* end_; //!< Past-the-end position. + bool valid_; //!< Whether the parsing is valid. + }; + + //! A helper stream to encode character (UTF-8 code unit) into percent-encoded sequence. + template + class PercentEncodeStream { + public: + PercentEncodeStream(OutputStream& os) : os_(os) {} + void Put(char c) { // UTF-8 must be byte + unsigned char u = static_cast(c); + static const char hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; + os_.Put('%'); + os_.Put(static_cast(hexDigits[u >> 4])); + os_.Put(static_cast(hexDigits[u & 15])); + } + private: + OutputStream& os_; + }; + + Allocator* allocator_; //!< The current allocator. It is either user-supplied or equal to ownAllocator_. + Allocator* ownAllocator_; //!< Allocator owned by this Pointer. + Ch* nameBuffer_; //!< A buffer containing all names in tokens. + Token* tokens_; //!< A list of tokens. + size_t tokenCount_; //!< Number of tokens in tokens_. + size_t parseErrorOffset_; //!< Offset in code unit when parsing fail. + PointerParseErrorCode parseErrorCode_; //!< Parsing error code. +}; + +//! GenericPointer for Value (UTF-8, default allocator). +typedef GenericPointer Pointer; + +//!@name Helper functions for GenericPointer +//@{ + +////////////////////////////////////////////////////////////////////////////// + +template +typename T::ValueType& CreateValueByPointer(T& root, const GenericPointer& pointer, typename T::AllocatorType& a) { + return pointer.Create(root, a); +} + +template +typename T::ValueType& CreateValueByPointer(T& root, const CharType(&source)[N], typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Create(root, a); +} + +// No allocator parameter + +template +typename DocumentType::ValueType& CreateValueByPointer(DocumentType& document, const GenericPointer& pointer) { + return pointer.Create(document); +} + +template +typename DocumentType::ValueType& CreateValueByPointer(DocumentType& document, const CharType(&source)[N]) { + return GenericPointer(source, N - 1).Create(document); +} + +////////////////////////////////////////////////////////////////////////////// + +template +typename T::ValueType* GetValueByPointer(T& root, const GenericPointer& pointer, size_t* unresolvedTokenIndex = 0) { + return pointer.Get(root, unresolvedTokenIndex); +} + +template +const typename T::ValueType* GetValueByPointer(const T& root, const GenericPointer& pointer, size_t* unresolvedTokenIndex = 0) { + return pointer.Get(root, unresolvedTokenIndex); +} + +template +typename T::ValueType* GetValueByPointer(T& root, const CharType (&source)[N], size_t* unresolvedTokenIndex = 0) { + return GenericPointer(source, N - 1).Get(root, unresolvedTokenIndex); +} + +template +const typename T::ValueType* GetValueByPointer(const T& root, const CharType(&source)[N], size_t* unresolvedTokenIndex = 0) { + return GenericPointer(source, N - 1).Get(root, unresolvedTokenIndex); +} + +////////////////////////////////////////////////////////////////////////////// + +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const typename T::ValueType& defaultValue, typename T::AllocatorType& a) { + return pointer.GetWithDefault(root, defaultValue, a); +} + +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const typename T::Ch* defaultValue, typename T::AllocatorType& a) { + return pointer.GetWithDefault(root, defaultValue, a); +} + +#if CEREAL_RAPIDJSON_HAS_STDSTRING +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const std::basic_string& defaultValue, typename T::AllocatorType& a) { + return pointer.GetWithDefault(root, defaultValue, a); +} +#endif + +template +CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) +GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, T2 defaultValue, typename T::AllocatorType& a) { + return pointer.GetWithDefault(root, defaultValue, a); +} + +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const typename T::ValueType& defaultValue, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); +} + +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const typename T::Ch* defaultValue, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); +} + +#if CEREAL_RAPIDJSON_HAS_STDSTRING +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const std::basic_string& defaultValue, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); +} +#endif + +template +CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) +GetValueByPointerWithDefault(T& root, const CharType(&source)[N], T2 defaultValue, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); +} + +// No allocator parameter + +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::ValueType& defaultValue) { + return pointer.GetWithDefault(document, defaultValue); +} + +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::Ch* defaultValue) { + return pointer.GetWithDefault(document, defaultValue); +} + +#if CEREAL_RAPIDJSON_HAS_STDSTRING +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const std::basic_string& defaultValue) { + return pointer.GetWithDefault(document, defaultValue); +} +#endif + +template +CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) +GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, T2 defaultValue) { + return pointer.GetWithDefault(document, defaultValue); +} + +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const typename DocumentType::ValueType& defaultValue) { + return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); +} + +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const typename DocumentType::Ch* defaultValue) { + return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); +} + +#if CEREAL_RAPIDJSON_HAS_STDSTRING +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const std::basic_string& defaultValue) { + return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); +} +#endif + +template +CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) +GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], T2 defaultValue) { + return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); +} + +////////////////////////////////////////////////////////////////////////////// + +template +typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, typename T::ValueType& value, typename T::AllocatorType& a) { + return pointer.Set(root, value, a); +} + +template +typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const typename T::ValueType& value, typename T::AllocatorType& a) { + return pointer.Set(root, value, a); +} + +template +typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const typename T::Ch* value, typename T::AllocatorType& a) { + return pointer.Set(root, value, a); +} + +#if CEREAL_RAPIDJSON_HAS_STDSTRING +template +typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const std::basic_string& value, typename T::AllocatorType& a) { + return pointer.Set(root, value, a); +} +#endif + +template +CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) +SetValueByPointer(T& root, const GenericPointer& pointer, T2 value, typename T::AllocatorType& a) { + return pointer.Set(root, value, a); +} + +template +typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], typename T::ValueType& value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Set(root, value, a); +} + +template +typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const typename T::ValueType& value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Set(root, value, a); +} + +template +typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const typename T::Ch* value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Set(root, value, a); +} + +#if CEREAL_RAPIDJSON_HAS_STDSTRING +template +typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const std::basic_string& value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Set(root, value, a); +} +#endif + +template +CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) +SetValueByPointer(T& root, const CharType(&source)[N], T2 value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Set(root, value, a); +} + +// No allocator parameter + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, typename DocumentType::ValueType& value) { + return pointer.Set(document, value); +} + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::ValueType& value) { + return pointer.Set(document, value); +} + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::Ch* value) { + return pointer.Set(document, value); +} + +#if CEREAL_RAPIDJSON_HAS_STDSTRING +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const std::basic_string& value) { + return pointer.Set(document, value); +} +#endif + +template +CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) +SetValueByPointer(DocumentType& document, const GenericPointer& pointer, T2 value) { + return pointer.Set(document, value); +} + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], typename DocumentType::ValueType& value) { + return GenericPointer(source, N - 1).Set(document, value); +} + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const typename DocumentType::ValueType& value) { + return GenericPointer(source, N - 1).Set(document, value); +} + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const typename DocumentType::Ch* value) { + return GenericPointer(source, N - 1).Set(document, value); +} + +#if CEREAL_RAPIDJSON_HAS_STDSTRING +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const std::basic_string& value) { + return GenericPointer(source, N - 1).Set(document, value); +} +#endif + +template +CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) +SetValueByPointer(DocumentType& document, const CharType(&source)[N], T2 value) { + return GenericPointer(source, N - 1).Set(document, value); +} + +////////////////////////////////////////////////////////////////////////////// + +template +typename T::ValueType& SwapValueByPointer(T& root, const GenericPointer& pointer, typename T::ValueType& value, typename T::AllocatorType& a) { + return pointer.Swap(root, value, a); +} + +template +typename T::ValueType& SwapValueByPointer(T& root, const CharType(&source)[N], typename T::ValueType& value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Swap(root, value, a); +} + +template +typename DocumentType::ValueType& SwapValueByPointer(DocumentType& document, const GenericPointer& pointer, typename DocumentType::ValueType& value) { + return pointer.Swap(document, value); +} + +template +typename DocumentType::ValueType& SwapValueByPointer(DocumentType& document, const CharType(&source)[N], typename DocumentType::ValueType& value) { + return GenericPointer(source, N - 1).Swap(document, value); +} + +////////////////////////////////////////////////////////////////////////////// + +template +bool EraseValueByPointer(T& root, const GenericPointer& pointer) { + return pointer.Erase(root); +} + +template +bool EraseValueByPointer(T& root, const CharType(&source)[N]) { + return GenericPointer(source, N - 1).Erase(root); +} + +//@} + +CEREAL_RAPIDJSON_NAMESPACE_END + +#if defined(__clang__) || defined(_MSC_VER) +CEREAL_RAPIDJSON_DIAG_POP +#endif + +#endif // CEREAL_RAPIDJSON_POINTER_H_ diff --git a/third_party/cereal/external/rapidjson/prettywriter.h b/third_party/cereal/external/rapidjson/prettywriter.h new file mode 100755 index 0000000..0c2beb4 --- /dev/null +++ b/third_party/cereal/external/rapidjson/prettywriter.h @@ -0,0 +1,277 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_PRETTYWRITER_H_ +#define CEREAL_RAPIDJSON_PRETTYWRITER_H_ + +#include "writer.h" + +#ifdef __GNUC__ +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(effc++) +#endif + +#if defined(__clang__) +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN + +//! Combination of PrettyWriter format flags. +/*! \see PrettyWriter::SetFormatOptions + */ +enum PrettyFormatOptions { + kFormatDefault = 0, //!< Default pretty formatting. + kFormatSingleLineArray = 1 //!< Format arrays on a single line. +}; + +//! Writer with indentation and spacing. +/*! + \tparam OutputStream Type of output os. + \tparam SourceEncoding Encoding of source string. + \tparam TargetEncoding Encoding of output stream. + \tparam StackAllocator Type of allocator for allocating memory of stack. +*/ +template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator, unsigned writeFlags = kWriteDefaultFlags> +class PrettyWriter : public Writer { +public: + typedef Writer Base; + typedef typename Base::Ch Ch; + + //! Constructor + /*! \param os Output stream. + \param allocator User supplied allocator. If it is null, it will create a private one. + \param levelDepth Initial capacity of stack. + */ + explicit PrettyWriter(OutputStream& os, StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : + Base(os, allocator, levelDepth), indentChar_(' '), indentCharCount_(4), formatOptions_(kFormatDefault) {} + + + explicit PrettyWriter(StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : + Base(allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} + +#if CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS + PrettyWriter(PrettyWriter&& rhs) : + Base(std::forward(rhs)), indentChar_(rhs.indentChar_), indentCharCount_(rhs.indentCharCount_), formatOptions_(rhs.formatOptions_) {} +#endif + + //! Set custom indentation. + /*! \param indentChar Character for indentation. Must be whitespace character (' ', '\\t', '\\n', '\\r'). + \param indentCharCount Number of indent characters for each indentation level. + \note The default indentation is 4 spaces. + */ + PrettyWriter& SetIndent(Ch indentChar, unsigned indentCharCount) { + CEREAL_RAPIDJSON_ASSERT(indentChar == ' ' || indentChar == '\t' || indentChar == '\n' || indentChar == '\r'); + indentChar_ = indentChar; + indentCharCount_ = indentCharCount; + return *this; + } + + //! Set pretty writer formatting options. + /*! \param options Formatting options. + */ + PrettyWriter& SetFormatOptions(PrettyFormatOptions options) { + formatOptions_ = options; + return *this; + } + + /*! @name Implementation of Handler + \see Handler + */ + //@{ + + bool Null() { PrettyPrefix(kNullType); return Base::EndValue(Base::WriteNull()); } + bool Bool(bool b) { PrettyPrefix(b ? kTrueType : kFalseType); return Base::EndValue(Base::WriteBool(b)); } + bool Int(int i) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteInt(i)); } + bool Uint(unsigned u) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteUint(u)); } + bool Int64(int64_t i64) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteInt64(i64)); } + bool Uint64(uint64_t u64) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteUint64(u64)); } + bool Double(double d) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteDouble(d)); } + + bool RawNumber(const Ch* str, SizeType length, bool copy = false) { + CEREAL_RAPIDJSON_ASSERT(str != 0); + (void)copy; + PrettyPrefix(kNumberType); + return Base::EndValue(Base::WriteString(str, length)); + } + + bool String(const Ch* str, SizeType length, bool copy = false) { + CEREAL_RAPIDJSON_ASSERT(str != 0); + (void)copy; + PrettyPrefix(kStringType); + return Base::EndValue(Base::WriteString(str, length)); + } + +#if CEREAL_RAPIDJSON_HAS_STDSTRING + bool String(const std::basic_string& str) { + return String(str.data(), SizeType(str.size())); + } +#endif + + bool StartObject() { + PrettyPrefix(kObjectType); + new (Base::level_stack_.template Push()) typename Base::Level(false); + return Base::WriteStartObject(); + } + + bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } + +#if CEREAL_RAPIDJSON_HAS_STDSTRING + bool Key(const std::basic_string& str) { + return Key(str.data(), SizeType(str.size())); + } +#endif + + bool EndObject(SizeType memberCount = 0) { + (void)memberCount; + CEREAL_RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); // not inside an Object + CEREAL_RAPIDJSON_ASSERT(!Base::level_stack_.template Top()->inArray); // currently inside an Array, not Object + CEREAL_RAPIDJSON_ASSERT(0 == Base::level_stack_.template Top()->valueCount % 2); // Object has a Key without a Value + + bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; + + if (!empty) { + Base::os_->Put('\n'); + WriteIndent(); + } + bool ret = Base::EndValue(Base::WriteEndObject()); + (void)ret; + CEREAL_RAPIDJSON_ASSERT(ret == true); + if (Base::level_stack_.Empty()) // end of json text + Base::Flush(); + return true; + } + + bool StartArray() { + PrettyPrefix(kArrayType); + new (Base::level_stack_.template Push()) typename Base::Level(true); + return Base::WriteStartArray(); + } + + bool EndArray(SizeType memberCount = 0) { + (void)memberCount; + CEREAL_RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); + CEREAL_RAPIDJSON_ASSERT(Base::level_stack_.template Top()->inArray); + bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; + + if (!empty && !(formatOptions_ & kFormatSingleLineArray)) { + Base::os_->Put('\n'); + WriteIndent(); + } + bool ret = Base::EndValue(Base::WriteEndArray()); + (void)ret; + CEREAL_RAPIDJSON_ASSERT(ret == true); + if (Base::level_stack_.Empty()) // end of json text + Base::Flush(); + return true; + } + + //@} + + /*! @name Convenience extensions */ + //@{ + + //! Simpler but slower overload. + bool String(const Ch* str) { return String(str, internal::StrLen(str)); } + bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } + + //@} + + //! Write a raw JSON value. + /*! + For user to write a stringified JSON as a value. + + \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. + \param length Length of the json. + \param type Type of the root of json. + \note When using PrettyWriter::RawValue(), the result json may not be indented correctly. + */ + bool RawValue(const Ch* json, size_t length, Type type) { + CEREAL_RAPIDJSON_ASSERT(json != 0); + PrettyPrefix(type); + return Base::EndValue(Base::WriteRawValue(json, length)); + } + +protected: + void PrettyPrefix(Type type) { + (void)type; + if (Base::level_stack_.GetSize() != 0) { // this value is not at root + typename Base::Level* level = Base::level_stack_.template Top(); + + if (level->inArray) { + if (level->valueCount > 0) { + Base::os_->Put(','); // add comma if it is not the first element in array + if (formatOptions_ & kFormatSingleLineArray) + Base::os_->Put(' '); + } + + if (!(formatOptions_ & kFormatSingleLineArray)) { + Base::os_->Put('\n'); + WriteIndent(); + } + } + else { // in object + if (level->valueCount > 0) { + if (level->valueCount % 2 == 0) { + Base::os_->Put(','); + Base::os_->Put('\n'); + } + else { + Base::os_->Put(':'); + Base::os_->Put(' '); + } + } + else + Base::os_->Put('\n'); + + if (level->valueCount % 2 == 0) + WriteIndent(); + } + if (!level->inArray && level->valueCount % 2 == 0) + CEREAL_RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name + level->valueCount++; + } + else { + CEREAL_RAPIDJSON_ASSERT(!Base::hasRoot_); // Should only has one and only one root. + Base::hasRoot_ = true; + } + } + + void WriteIndent() { + size_t count = (Base::level_stack_.GetSize() / sizeof(typename Base::Level)) * indentCharCount_; + PutN(*Base::os_, static_cast(indentChar_), count); + } + + Ch indentChar_; + unsigned indentCharCount_; + PrettyFormatOptions formatOptions_; + +private: + // Prohibit copy constructor & assignment operator. + PrettyWriter(const PrettyWriter&); + PrettyWriter& operator=(const PrettyWriter&); +}; + +CEREAL_RAPIDJSON_NAMESPACE_END + +#if defined(__clang__) +CEREAL_RAPIDJSON_DIAG_POP +#endif + +#ifdef __GNUC__ +CEREAL_RAPIDJSON_DIAG_POP +#endif + +#endif // CEREAL_RAPIDJSON_CEREAL_RAPIDJSON_H_ diff --git a/third_party/cereal/external/rapidjson/rapidjson.h b/third_party/cereal/external/rapidjson/rapidjson.h new file mode 100755 index 0000000..3eefe60 --- /dev/null +++ b/third_party/cereal/external/rapidjson/rapidjson.h @@ -0,0 +1,656 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_CEREAL_RAPIDJSON_H_ +#define CEREAL_RAPIDJSON_CEREAL_RAPIDJSON_H_ + +/*!\file rapidjson.h + \brief common definitions and configuration + + \see CEREAL_RAPIDJSON_CONFIG + */ + +/*! \defgroup CEREAL_RAPIDJSON_CONFIG RapidJSON configuration + \brief Configuration macros for library features + + Some RapidJSON features are configurable to adapt the library to a wide + variety of platforms, environments and usage scenarios. Most of the + features can be configured in terms of overridden or predefined + preprocessor macros at compile-time. + + Some additional customization is available in the \ref CEREAL_RAPIDJSON_ERRORS APIs. + + \note These macros should be given on the compiler command-line + (where applicable) to avoid inconsistent values when compiling + different translation units of a single application. + */ + +#include // malloc(), realloc(), free(), size_t +#include // memset(), memcpy(), memmove(), memcmp() + +/////////////////////////////////////////////////////////////////////////////// +// CEREAL_RAPIDJSON_VERSION_STRING +// +// ALWAYS synchronize the following 3 macros with corresponding variables in /CMakeLists.txt. +// + +//!@cond CEREAL_RAPIDJSON_HIDDEN_FROM_DOXYGEN +// token stringification +#define CEREAL_RAPIDJSON_STRINGIFY(x) CEREAL_RAPIDJSON_DO_STRINGIFY(x) +#define CEREAL_RAPIDJSON_DO_STRINGIFY(x) #x + +// token concatenation +#define CEREAL_RAPIDJSON_JOIN(X, Y) CEREAL_RAPIDJSON_DO_JOIN(X, Y) +#define CEREAL_RAPIDJSON_DO_JOIN(X, Y) CEREAL_RAPIDJSON_DO_JOIN2(X, Y) +#define CEREAL_RAPIDJSON_DO_JOIN2(X, Y) X##Y +//!@endcond + +/*! \def CEREAL_RAPIDJSON_MAJOR_VERSION + \ingroup CEREAL_RAPIDJSON_CONFIG + \brief Major version of RapidJSON in integer. +*/ +/*! \def CEREAL_RAPIDJSON_MINOR_VERSION + \ingroup CEREAL_RAPIDJSON_CONFIG + \brief Minor version of RapidJSON in integer. +*/ +/*! \def CEREAL_RAPIDJSON_PATCH_VERSION + \ingroup CEREAL_RAPIDJSON_CONFIG + \brief Patch version of RapidJSON in integer. +*/ +/*! \def CEREAL_RAPIDJSON_VERSION_STRING + \ingroup CEREAL_RAPIDJSON_CONFIG + \brief Version of RapidJSON in ".." string format. +*/ +#define CEREAL_RAPIDJSON_MAJOR_VERSION 1 +#define CEREAL_RAPIDJSON_MINOR_VERSION 1 +#define CEREAL_RAPIDJSON_PATCH_VERSION 0 +#define CEREAL_RAPIDJSON_VERSION_STRING \ + CEREAL_RAPIDJSON_STRINGIFY(CEREAL_RAPIDJSON_MAJOR_VERSION.CEREAL_RAPIDJSON_MINOR_VERSION.CEREAL_RAPIDJSON_PATCH_VERSION) + +/////////////////////////////////////////////////////////////////////////////// +// CEREAL_RAPIDJSON_NAMESPACE_(BEGIN|END) +/*! \def CEREAL_RAPIDJSON_NAMESPACE + \ingroup CEREAL_RAPIDJSON_CONFIG + \brief provide custom rapidjson namespace + + In order to avoid symbol clashes and/or "One Definition Rule" errors + between multiple inclusions of (different versions of) RapidJSON in + a single binary, users can customize the name of the main RapidJSON + namespace. + + In case of a single nesting level, defining \c CEREAL_RAPIDJSON_NAMESPACE + to a custom name (e.g. \c MyRapidJSON) is sufficient. If multiple + levels are needed, both \ref CEREAL_RAPIDJSON_NAMESPACE_BEGIN and \ref + CEREAL_RAPIDJSON_NAMESPACE_END need to be defined as well: + + \code + // in some .cpp file + #define CEREAL_RAPIDJSON_NAMESPACE my::rapidjson + #define CEREAL_RAPIDJSON_NAMESPACE_BEGIN namespace my { namespace rapidjson { + #define CEREAL_RAPIDJSON_NAMESPACE_END } } + #include "rapidjson/..." + \endcode + + \see rapidjson + */ +/*! \def CEREAL_RAPIDJSON_NAMESPACE_BEGIN + \ingroup CEREAL_RAPIDJSON_CONFIG + \brief provide custom rapidjson namespace (opening expression) + \see CEREAL_RAPIDJSON_NAMESPACE +*/ +/*! \def CEREAL_RAPIDJSON_NAMESPACE_END + \ingroup CEREAL_RAPIDJSON_CONFIG + \brief provide custom rapidjson namespace (closing expression) + \see CEREAL_RAPIDJSON_NAMESPACE +*/ +#ifndef CEREAL_RAPIDJSON_NAMESPACE +#define CEREAL_RAPIDJSON_NAMESPACE rapidjson +#endif +#ifndef CEREAL_RAPIDJSON_NAMESPACE_BEGIN +#define CEREAL_RAPIDJSON_NAMESPACE_BEGIN namespace CEREAL_RAPIDJSON_NAMESPACE { +#endif +#ifndef CEREAL_RAPIDJSON_NAMESPACE_END +#define CEREAL_RAPIDJSON_NAMESPACE_END } +#endif + +/////////////////////////////////////////////////////////////////////////////// +// CEREAL_RAPIDJSON_HAS_STDSTRING + +#ifndef CEREAL_RAPIDJSON_HAS_STDSTRING +#ifdef CEREAL_RAPIDJSON_DOXYGEN_RUNNING +#define CEREAL_RAPIDJSON_HAS_STDSTRING 1 // force generation of documentation +#else +#define CEREAL_RAPIDJSON_HAS_STDSTRING 0 // no std::string support by default +#endif +/*! \def CEREAL_RAPIDJSON_HAS_STDSTRING + \ingroup CEREAL_RAPIDJSON_CONFIG + \brief Enable RapidJSON support for \c std::string + + By defining this preprocessor symbol to \c 1, several convenience functions for using + \ref rapidjson::GenericValue with \c std::string are enabled, especially + for construction and comparison. + + \hideinitializer +*/ +#endif // !defined(CEREAL_RAPIDJSON_HAS_STDSTRING) + +#if CEREAL_RAPIDJSON_HAS_STDSTRING +#include +#endif // CEREAL_RAPIDJSON_HAS_STDSTRING + +/////////////////////////////////////////////////////////////////////////////// +// CEREAL_RAPIDJSON_NO_INT64DEFINE + +/*! \def CEREAL_RAPIDJSON_NO_INT64DEFINE + \ingroup CEREAL_RAPIDJSON_CONFIG + \brief Use external 64-bit integer types. + + RapidJSON requires the 64-bit integer types \c int64_t and \c uint64_t types + to be available at global scope. + + If users have their own definition, define CEREAL_RAPIDJSON_NO_INT64DEFINE to + prevent RapidJSON from defining its own types. +*/ +#ifndef CEREAL_RAPIDJSON_NO_INT64DEFINE +//!@cond CEREAL_RAPIDJSON_HIDDEN_FROM_DOXYGEN +#if defined(_MSC_VER) && (_MSC_VER < 1800) // Visual Studio 2013 +#include "msinttypes/stdint.h" +#include "msinttypes/inttypes.h" +#else +// Other compilers should have this. +#include +#include +#endif +//!@endcond +#ifdef CEREAL_RAPIDJSON_DOXYGEN_RUNNING +#define CEREAL_RAPIDJSON_NO_INT64DEFINE +#endif +#endif // CEREAL_RAPIDJSON_NO_INT64TYPEDEF + +/////////////////////////////////////////////////////////////////////////////// +// CEREAL_RAPIDJSON_FORCEINLINE + +#ifndef CEREAL_RAPIDJSON_FORCEINLINE +//!@cond CEREAL_RAPIDJSON_HIDDEN_FROM_DOXYGEN +#if defined(_MSC_VER) && defined(NDEBUG) +#define CEREAL_RAPIDJSON_FORCEINLINE __forceinline +#elif defined(__GNUC__) && __GNUC__ >= 4 && defined(NDEBUG) +#define CEREAL_RAPIDJSON_FORCEINLINE __attribute__((always_inline)) +#else +#define CEREAL_RAPIDJSON_FORCEINLINE +#endif +//!@endcond +#endif // CEREAL_RAPIDJSON_FORCEINLINE + +/////////////////////////////////////////////////////////////////////////////// +// CEREAL_RAPIDJSON_ENDIAN +#define CEREAL_RAPIDJSON_LITTLEENDIAN 0 //!< Little endian machine +#define CEREAL_RAPIDJSON_BIGENDIAN 1 //!< Big endian machine + +//! Endianness of the machine. +/*! + \def CEREAL_RAPIDJSON_ENDIAN + \ingroup CEREAL_RAPIDJSON_CONFIG + + GCC 4.6 provided macro for detecting endianness of the target machine. But other + compilers may not have this. User can define CEREAL_RAPIDJSON_ENDIAN to either + \ref CEREAL_RAPIDJSON_LITTLEENDIAN or \ref CEREAL_RAPIDJSON_BIGENDIAN. + + Default detection implemented with reference to + \li https://gcc.gnu.org/onlinedocs/gcc-4.6.0/cpp/Common-Predefined-Macros.html + \li http://www.boost.org/doc/libs/1_42_0/boost/detail/endian.hpp +*/ +#ifndef CEREAL_RAPIDJSON_ENDIAN +// Detect with GCC 4.6's macro +# ifdef __BYTE_ORDER__ +# if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +# define CEREAL_RAPIDJSON_ENDIAN CEREAL_RAPIDJSON_LITTLEENDIAN +# elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +# define CEREAL_RAPIDJSON_ENDIAN CEREAL_RAPIDJSON_BIGENDIAN +# else +# error Unknown machine endianness detected. User needs to define CEREAL_RAPIDJSON_ENDIAN. +# endif // __BYTE_ORDER__ +// Detect with GLIBC's endian.h +# elif defined(__GLIBC__) +# include +# if (__BYTE_ORDER == __LITTLE_ENDIAN) +# define CEREAL_RAPIDJSON_ENDIAN CEREAL_RAPIDJSON_LITTLEENDIAN +# elif (__BYTE_ORDER == __BIG_ENDIAN) +# define CEREAL_RAPIDJSON_ENDIAN CEREAL_RAPIDJSON_BIGENDIAN +# else +# error Unknown machine endianness detected. User needs to define CEREAL_RAPIDJSON_ENDIAN. +# endif // __GLIBC__ +// Detect with _LITTLE_ENDIAN and _BIG_ENDIAN macro +# elif defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN) +# define CEREAL_RAPIDJSON_ENDIAN CEREAL_RAPIDJSON_LITTLEENDIAN +# elif defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN) +# define CEREAL_RAPIDJSON_ENDIAN CEREAL_RAPIDJSON_BIGENDIAN +// Detect with architecture macros +# elif defined(__sparc) || defined(__sparc__) || defined(_POWER) || defined(__powerpc__) || defined(__ppc__) || defined(__hpux) || defined(__hppa) || defined(_MIPSEB) || defined(_POWER) || defined(__s390__) +# define CEREAL_RAPIDJSON_ENDIAN CEREAL_RAPIDJSON_BIGENDIAN +# elif defined(__i386__) || defined(__alpha__) || defined(__ia64) || defined(__ia64__) || defined(_M_IX86) || defined(_M_IA64) || defined(_M_ALPHA) || defined(__amd64) || defined(__amd64__) || defined(_M_AMD64) || defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || defined(__bfin__) +# define CEREAL_RAPIDJSON_ENDIAN CEREAL_RAPIDJSON_LITTLEENDIAN +# elif defined(_MSC_VER) && (defined(_M_ARM) || defined(_M_ARM64)) +# define CEREAL_RAPIDJSON_ENDIAN CEREAL_RAPIDJSON_LITTLEENDIAN +# elif defined(CEREAL_RAPIDJSON_DOXYGEN_RUNNING) +# define CEREAL_RAPIDJSON_ENDIAN +# else +# error Unknown machine endianness detected. User needs to define CEREAL_RAPIDJSON_ENDIAN. +# endif +#endif // CEREAL_RAPIDJSON_ENDIAN + +/////////////////////////////////////////////////////////////////////////////// +// CEREAL_RAPIDJSON_64BIT + +//! Whether using 64-bit architecture +#ifndef CEREAL_RAPIDJSON_64BIT +#if defined(__LP64__) || (defined(__x86_64__) && defined(__ILP32__)) || defined(_WIN64) || defined(__EMSCRIPTEN__) +#define CEREAL_RAPIDJSON_64BIT 1 +#else +#define CEREAL_RAPIDJSON_64BIT 0 +#endif +#endif // CEREAL_RAPIDJSON_64BIT + +/////////////////////////////////////////////////////////////////////////////// +// CEREAL_RAPIDJSON_ALIGN + +//! Data alignment of the machine. +/*! \ingroup CEREAL_RAPIDJSON_CONFIG + \param x pointer to align + + Some machines require strict data alignment. The default is 8 bytes. + User can customize by defining the CEREAL_RAPIDJSON_ALIGN function macro. +*/ +#ifndef CEREAL_RAPIDJSON_ALIGN +#define CEREAL_RAPIDJSON_ALIGN(x) (((x) + static_cast(7u)) & ~static_cast(7u)) +#endif + +/////////////////////////////////////////////////////////////////////////////// +// CEREAL_RAPIDJSON_UINT64_C2 + +//! Construct a 64-bit literal by a pair of 32-bit integer. +/*! + 64-bit literal with or without ULL suffix is prone to compiler warnings. + UINT64_C() is C macro which cause compilation problems. + Use this macro to define 64-bit constants by a pair of 32-bit integer. +*/ +#ifndef CEREAL_RAPIDJSON_UINT64_C2 +#define CEREAL_RAPIDJSON_UINT64_C2(high32, low32) ((static_cast(high32) << 32) | static_cast(low32)) +#endif + +/////////////////////////////////////////////////////////////////////////////// +// CEREAL_RAPIDJSON_48BITPOINTER_OPTIMIZATION + +//! Use only lower 48-bit address for some pointers. +/*! + \ingroup CEREAL_RAPIDJSON_CONFIG + + This optimization uses the fact that current X86-64 architecture only implement lower 48-bit virtual address. + The higher 16-bit can be used for storing other data. + \c GenericValue uses this optimization to reduce its size form 24 bytes to 16 bytes in 64-bit architecture. +*/ +#ifndef CEREAL_RAPIDJSON_48BITPOINTER_OPTIMIZATION +#if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64) +#define CEREAL_RAPIDJSON_48BITPOINTER_OPTIMIZATION 1 +#else +#define CEREAL_RAPIDJSON_48BITPOINTER_OPTIMIZATION 0 +#endif +#endif // CEREAL_RAPIDJSON_48BITPOINTER_OPTIMIZATION + +#if CEREAL_RAPIDJSON_48BITPOINTER_OPTIMIZATION == 1 +#if CEREAL_RAPIDJSON_64BIT != 1 +#error CEREAL_RAPIDJSON_48BITPOINTER_OPTIMIZATION can only be set to 1 when CEREAL_RAPIDJSON_64BIT=1 +#endif +#define CEREAL_RAPIDJSON_SETPOINTER(type, p, x) (p = reinterpret_cast((reinterpret_cast(p) & static_cast(CEREAL_RAPIDJSON_UINT64_C2(0xFFFF0000, 0x00000000))) | reinterpret_cast(reinterpret_cast(x)))) +#define CEREAL_RAPIDJSON_GETPOINTER(type, p) (reinterpret_cast(reinterpret_cast(p) & static_cast(CEREAL_RAPIDJSON_UINT64_C2(0x0000FFFF, 0xFFFFFFFF)))) +#else +#define CEREAL_RAPIDJSON_SETPOINTER(type, p, x) (p = (x)) +#define CEREAL_RAPIDJSON_GETPOINTER(type, p) (p) +#endif + +/////////////////////////////////////////////////////////////////////////////// +// CEREAL_RAPIDJSON_SSE2/CEREAL_RAPIDJSON_SSE42/CEREAL_RAPIDJSON_NEON/CEREAL_RAPIDJSON_SIMD + +/*! \def CEREAL_RAPIDJSON_SIMD + \ingroup CEREAL_RAPIDJSON_CONFIG + \brief Enable SSE2/SSE4.2/Neon optimization. + + RapidJSON supports optimized implementations for some parsing operations + based on the SSE2, SSE4.2 or NEon SIMD extensions on modern Intel + or ARM compatible processors. + + To enable these optimizations, three different symbols can be defined; + \code + // Enable SSE2 optimization. + #define CEREAL_RAPIDJSON_SSE2 + + // Enable SSE4.2 optimization. + #define CEREAL_RAPIDJSON_SSE42 + \endcode + + // Enable ARM Neon optimization. + #define CEREAL_RAPIDJSON_NEON + \endcode + + \c CEREAL_RAPIDJSON_SSE42 takes precedence over SSE2, if both are defined. + + If any of these symbols is defined, RapidJSON defines the macro + \c CEREAL_RAPIDJSON_SIMD to indicate the availability of the optimized code. +*/ +#if defined(CEREAL_RAPIDJSON_SSE2) || defined(CEREAL_RAPIDJSON_SSE42) \ + || defined(CEREAL_RAPIDJSON_NEON) || defined(CEREAL_RAPIDJSON_DOXYGEN_RUNNING) +#define CEREAL_RAPIDJSON_SIMD +#endif + +/////////////////////////////////////////////////////////////////////////////// +// CEREAL_RAPIDJSON_NO_SIZETYPEDEFINE + +#ifndef CEREAL_RAPIDJSON_NO_SIZETYPEDEFINE +/*! \def CEREAL_RAPIDJSON_NO_SIZETYPEDEFINE + \ingroup CEREAL_RAPIDJSON_CONFIG + \brief User-provided \c SizeType definition. + + In order to avoid using 32-bit size types for indexing strings and arrays, + define this preprocessor symbol and provide the type rapidjson::SizeType + before including RapidJSON: + \code + #define CEREAL_RAPIDJSON_NO_SIZETYPEDEFINE + namespace rapidjson { typedef ::std::size_t SizeType; } + #include "rapidjson/..." + \endcode + + \see rapidjson::SizeType +*/ +#ifdef CEREAL_RAPIDJSON_DOXYGEN_RUNNING +#define CEREAL_RAPIDJSON_NO_SIZETYPEDEFINE +#endif +CEREAL_RAPIDJSON_NAMESPACE_BEGIN +//! Size type (for string lengths, array sizes, etc.) +/*! RapidJSON uses 32-bit array/string indices even on 64-bit platforms, + instead of using \c size_t. Users may override the SizeType by defining + \ref CEREAL_RAPIDJSON_NO_SIZETYPEDEFINE. +*/ +typedef unsigned SizeType; +CEREAL_RAPIDJSON_NAMESPACE_END +#endif + +// always import std::size_t to rapidjson namespace +CEREAL_RAPIDJSON_NAMESPACE_BEGIN +using std::size_t; +CEREAL_RAPIDJSON_NAMESPACE_END + +/////////////////////////////////////////////////////////////////////////////// +// CEREAL_RAPIDJSON_ASSERT + +//! Assertion. +/*! \ingroup CEREAL_RAPIDJSON_CONFIG + By default, rapidjson uses C \c assert() for internal assertions. + User can override it by defining CEREAL_RAPIDJSON_ASSERT(x) macro. + + \note Parsing errors are handled and can be customized by the + \ref CEREAL_RAPIDJSON_ERRORS APIs. +*/ +#ifndef CEREAL_RAPIDJSON_ASSERT +#include +#define CEREAL_RAPIDJSON_ASSERT(x) assert(x) +#endif // CEREAL_RAPIDJSON_ASSERT + +/////////////////////////////////////////////////////////////////////////////// +// CEREAL_RAPIDJSON_STATIC_ASSERT + +// Prefer C++11 static_assert, if available +#ifndef CEREAL_RAPIDJSON_STATIC_ASSERT +#if __cplusplus >= 201103L || ( defined(_MSC_VER) && _MSC_VER >= 1800 ) +#define CEREAL_RAPIDJSON_STATIC_ASSERT(x) \ + static_assert(x, CEREAL_RAPIDJSON_STRINGIFY(x)) +#endif // C++11 +#endif // CEREAL_RAPIDJSON_STATIC_ASSERT + +// Adopt C++03 implementation from boost +#ifndef CEREAL_RAPIDJSON_STATIC_ASSERT +#ifndef __clang__ +//!@cond CEREAL_RAPIDJSON_HIDDEN_FROM_DOXYGEN +#endif +CEREAL_RAPIDJSON_NAMESPACE_BEGIN +template struct STATIC_ASSERTION_FAILURE; +template <> struct STATIC_ASSERTION_FAILURE { enum { value = 1 }; }; +template struct StaticAssertTest {}; +CEREAL_RAPIDJSON_NAMESPACE_END + +#if defined(__GNUC__) || defined(__clang__) +#define CEREAL_RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE __attribute__((unused)) +#else +#define CEREAL_RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE +#endif +#ifndef __clang__ +//!@endcond +#endif + +/*! \def CEREAL_RAPIDJSON_STATIC_ASSERT + \brief (Internal) macro to check for conditions at compile-time + \param x compile-time condition + \hideinitializer + */ +#define CEREAL_RAPIDJSON_STATIC_ASSERT(x) \ + typedef ::CEREAL_RAPIDJSON_NAMESPACE::StaticAssertTest< \ + sizeof(::CEREAL_RAPIDJSON_NAMESPACE::STATIC_ASSERTION_FAILURE)> \ + CEREAL_RAPIDJSON_JOIN(StaticAssertTypedef, __LINE__) CEREAL_RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE +#endif // CEREAL_RAPIDJSON_STATIC_ASSERT + +/////////////////////////////////////////////////////////////////////////////// +// CEREAL_RAPIDJSON_LIKELY, CEREAL_RAPIDJSON_UNLIKELY + +//! Compiler branching hint for expression with high probability to be true. +/*! + \ingroup CEREAL_RAPIDJSON_CONFIG + \param x Boolean expression likely to be true. +*/ +#ifndef CEREAL_RAPIDJSON_LIKELY +#if defined(__GNUC__) || defined(__clang__) +#define CEREAL_RAPIDJSON_LIKELY(x) __builtin_expect(!!(x), 1) +#else +#define CEREAL_RAPIDJSON_LIKELY(x) (x) +#endif +#endif + +//! Compiler branching hint for expression with low probability to be true. +/*! + \ingroup CEREAL_RAPIDJSON_CONFIG + \param x Boolean expression unlikely to be true. +*/ +#ifndef CEREAL_RAPIDJSON_UNLIKELY +#if defined(__GNUC__) || defined(__clang__) +#define CEREAL_RAPIDJSON_UNLIKELY(x) __builtin_expect(!!(x), 0) +#else +#define CEREAL_RAPIDJSON_UNLIKELY(x) (x) +#endif +#endif + +/////////////////////////////////////////////////////////////////////////////// +// Helpers + +//!@cond CEREAL_RAPIDJSON_HIDDEN_FROM_DOXYGEN + +#define CEREAL_RAPIDJSON_MULTILINEMACRO_BEGIN do { +#define CEREAL_RAPIDJSON_MULTILINEMACRO_END \ +} while((void)0, 0) + +// adopted from Boost +#define CEREAL_RAPIDJSON_VERSION_CODE(x,y,z) \ + (((x)*100000) + ((y)*100) + (z)) + +/////////////////////////////////////////////////////////////////////////////// +// CEREAL_RAPIDJSON_DIAG_PUSH/POP, CEREAL_RAPIDJSON_DIAG_OFF + +#if defined(__GNUC__) +#define CEREAL_RAPIDJSON_GNUC \ + CEREAL_RAPIDJSON_VERSION_CODE(__GNUC__,__GNUC_MINOR__,__GNUC_PATCHLEVEL__) +#endif + +#if defined(__clang__) || (defined(CEREAL_RAPIDJSON_GNUC) && CEREAL_RAPIDJSON_GNUC >= CEREAL_RAPIDJSON_VERSION_CODE(4,2,0)) + +#define CEREAL_RAPIDJSON_PRAGMA(x) _Pragma(CEREAL_RAPIDJSON_STRINGIFY(x)) +#define CEREAL_RAPIDJSON_DIAG_PRAGMA(x) CEREAL_RAPIDJSON_PRAGMA(GCC diagnostic x) +#define CEREAL_RAPIDJSON_DIAG_OFF(x) \ + CEREAL_RAPIDJSON_DIAG_PRAGMA(ignored CEREAL_RAPIDJSON_STRINGIFY(CEREAL_RAPIDJSON_JOIN(-W,x))) + +// push/pop support in Clang and GCC>=4.6 +#if defined(__clang__) || (defined(CEREAL_RAPIDJSON_GNUC) && CEREAL_RAPIDJSON_GNUC >= CEREAL_RAPIDJSON_VERSION_CODE(4,6,0)) +#define CEREAL_RAPIDJSON_DIAG_PUSH CEREAL_RAPIDJSON_DIAG_PRAGMA(push) +#define CEREAL_RAPIDJSON_DIAG_POP CEREAL_RAPIDJSON_DIAG_PRAGMA(pop) +#else // GCC >= 4.2, < 4.6 +#define CEREAL_RAPIDJSON_DIAG_PUSH /* ignored */ +#define CEREAL_RAPIDJSON_DIAG_POP /* ignored */ +#endif + +#elif defined(_MSC_VER) + +// pragma (MSVC specific) +#define CEREAL_RAPIDJSON_PRAGMA(x) __pragma(x) +#define CEREAL_RAPIDJSON_DIAG_PRAGMA(x) CEREAL_RAPIDJSON_PRAGMA(warning(x)) + +#define CEREAL_RAPIDJSON_DIAG_OFF(x) CEREAL_RAPIDJSON_DIAG_PRAGMA(disable: x) +#define CEREAL_RAPIDJSON_DIAG_PUSH CEREAL_RAPIDJSON_DIAG_PRAGMA(push) +#define CEREAL_RAPIDJSON_DIAG_POP CEREAL_RAPIDJSON_DIAG_PRAGMA(pop) + +#else + +#define CEREAL_RAPIDJSON_DIAG_OFF(x) /* ignored */ +#define CEREAL_RAPIDJSON_DIAG_PUSH /* ignored */ +#define CEREAL_RAPIDJSON_DIAG_POP /* ignored */ + +#endif // CEREAL_RAPIDJSON_DIAG_* + +/////////////////////////////////////////////////////////////////////////////// +// C++11 features + +#ifndef CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS +#if defined(__clang__) +#if __has_feature(cxx_rvalue_references) && \ + (defined(_MSC_VER) || defined(_LIBCPP_VERSION) || defined(__GLIBCXX__) && __GLIBCXX__ >= 20080306) +#define CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 +#else +#define CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS 0 +#endif +#elif (defined(CEREAL_RAPIDJSON_GNUC) && (CEREAL_RAPIDJSON_GNUC >= CEREAL_RAPIDJSON_VERSION_CODE(4,3,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ + (defined(_MSC_VER) && _MSC_VER >= 1600) || \ + (defined(__SUNPRO_CC) && __SUNPRO_CC >= 0x5140 && defined(__GXX_EXPERIMENTAL_CXX0X__)) + +#define CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 +#else +#define CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS 0 +#endif +#endif // CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS + +#ifndef CEREAL_RAPIDJSON_HAS_CXX11_NOEXCEPT +#if defined(__clang__) +#define CEREAL_RAPIDJSON_HAS_CXX11_NOEXCEPT __has_feature(cxx_noexcept) +#elif (defined(CEREAL_RAPIDJSON_GNUC) && (CEREAL_RAPIDJSON_GNUC >= CEREAL_RAPIDJSON_VERSION_CODE(4,6,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ + (defined(_MSC_VER) && _MSC_VER >= 1900) || \ + (defined(__SUNPRO_CC) && __SUNPRO_CC >= 0x5140 && defined(__GXX_EXPERIMENTAL_CXX0X__)) +#define CEREAL_RAPIDJSON_HAS_CXX11_NOEXCEPT 1 +#else +#define CEREAL_RAPIDJSON_HAS_CXX11_NOEXCEPT 0 +#endif +#endif +#if CEREAL_RAPIDJSON_HAS_CXX11_NOEXCEPT +#define CEREAL_RAPIDJSON_NOEXCEPT noexcept +#else +#define CEREAL_RAPIDJSON_NOEXCEPT /* noexcept */ +#endif // CEREAL_RAPIDJSON_HAS_CXX11_NOEXCEPT + +// no automatic detection, yet +#ifndef CEREAL_RAPIDJSON_HAS_CXX11_TYPETRAITS +#if (defined(_MSC_VER) && _MSC_VER >= 1700) +#define CEREAL_RAPIDJSON_HAS_CXX11_TYPETRAITS 1 +#else +#define CEREAL_RAPIDJSON_HAS_CXX11_TYPETRAITS 0 +#endif +#endif + +#ifndef CEREAL_RAPIDJSON_HAS_CXX11_RANGE_FOR +#if defined(__clang__) +#define CEREAL_RAPIDJSON_HAS_CXX11_RANGE_FOR __has_feature(cxx_range_for) +#elif (defined(CEREAL_RAPIDJSON_GNUC) && (CEREAL_RAPIDJSON_GNUC >= CEREAL_RAPIDJSON_VERSION_CODE(4,6,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ + (defined(_MSC_VER) && _MSC_VER >= 1700) || \ + (defined(__SUNPRO_CC) && __SUNPRO_CC >= 0x5140 && defined(__GXX_EXPERIMENTAL_CXX0X__)) +#define CEREAL_RAPIDJSON_HAS_CXX11_RANGE_FOR 1 +#else +#define CEREAL_RAPIDJSON_HAS_CXX11_RANGE_FOR 0 +#endif +#endif // CEREAL_RAPIDJSON_HAS_CXX11_RANGE_FOR + +//!@endcond + +//! Assertion (in non-throwing contexts). + /*! \ingroup CEREAL_RAPIDJSON_CONFIG + Some functions provide a \c noexcept guarantee, if the compiler supports it. + In these cases, the \ref CEREAL_RAPIDJSON_ASSERT macro cannot be overridden to + throw an exception. This macro adds a separate customization point for + such cases. + + Defaults to C \c assert() (as \ref CEREAL_RAPIDJSON_ASSERT), if \c noexcept is + supported, and to \ref CEREAL_RAPIDJSON_ASSERT otherwise. + */ + +/////////////////////////////////////////////////////////////////////////////// +// CEREAL_RAPIDJSON_NOEXCEPT_ASSERT + +#ifndef CEREAL_RAPIDJSON_NOEXCEPT_ASSERT +#ifdef CEREAL_RAPIDJSON_ASSERT_THROWS +#if CEREAL_RAPIDJSON_HAS_CXX11_NOEXCEPT +#define CEREAL_RAPIDJSON_NOEXCEPT_ASSERT(x) +#else +#define CEREAL_RAPIDJSON_NOEXCEPT_ASSERT(x) CEREAL_RAPIDJSON_ASSERT(x) +#endif // CEREAL_RAPIDJSON_HAS_CXX11_NOEXCEPT +#else +#define CEREAL_RAPIDJSON_NOEXCEPT_ASSERT(x) CEREAL_RAPIDJSON_ASSERT(x) +#endif // CEREAL_RAPIDJSON_ASSERT_THROWS +#endif // CEREAL_RAPIDJSON_NOEXCEPT_ASSERT + +/////////////////////////////////////////////////////////////////////////////// +// new/delete + +#ifndef CEREAL_RAPIDJSON_NEW +///! customization point for global \c new +#define CEREAL_RAPIDJSON_NEW(TypeName) new TypeName +#endif +#ifndef CEREAL_RAPIDJSON_DELETE +///! customization point for global \c delete +#define CEREAL_RAPIDJSON_DELETE(x) delete x +#endif + +/////////////////////////////////////////////////////////////////////////////// +// Type + +/*! \namespace rapidjson + \brief main RapidJSON namespace + \see CEREAL_RAPIDJSON_NAMESPACE +*/ +CEREAL_RAPIDJSON_NAMESPACE_BEGIN + +//! Type of JSON value +enum Type { + kNullType = 0, //!< null + kFalseType = 1, //!< false + kTrueType = 2, //!< true + kObjectType = 3, //!< object + kArrayType = 4, //!< array + kStringType = 5, //!< string + kNumberType = 6 //!< number +}; + +CEREAL_RAPIDJSON_NAMESPACE_END + +#endif // CEREAL_RAPIDJSON_CEREAL_RAPIDJSON_H_ diff --git a/third_party/cereal/external/rapidjson/reader.h b/third_party/cereal/external/rapidjson/reader.h new file mode 100755 index 0000000..43eb525 --- /dev/null +++ b/third_party/cereal/external/rapidjson/reader.h @@ -0,0 +1,2230 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_READER_H_ +#define CEREAL_RAPIDJSON_READER_H_ + +/*! \file reader.h */ + +#include "allocators.h" +#include "stream.h" +#include "encodedstream.h" +#include "internal/meta.h" +#include "internal/stack.h" +#include "internal/strtod.h" +#include + +#if defined(CEREAL_RAPIDJSON_SIMD) && defined(_MSC_VER) +#include +#pragma intrinsic(_BitScanForward) +#endif +#ifdef CEREAL_RAPIDJSON_SSE42 +#include +#elif defined(CEREAL_RAPIDJSON_SSE2) +#include +#elif defined(CEREAL_RAPIDJSON_NEON) +#include +#endif + +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(old-style-cast) +CEREAL_RAPIDJSON_DIAG_OFF(padded) +CEREAL_RAPIDJSON_DIAG_OFF(switch-enum) +#elif defined(_MSC_VER) +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant +CEREAL_RAPIDJSON_DIAG_OFF(4702) // unreachable code +#endif + +#ifdef __GNUC__ +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(effc++) +#endif + +//!@cond CEREAL_RAPIDJSON_HIDDEN_FROM_DOXYGEN +#define CEREAL_RAPIDJSON_NOTHING /* deliberately empty */ +#ifndef CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN +#define CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN(value) \ + CEREAL_RAPIDJSON_MULTILINEMACRO_BEGIN \ + if (CEREAL_RAPIDJSON_UNLIKELY(HasParseError())) { return value; } \ + CEREAL_RAPIDJSON_MULTILINEMACRO_END +#endif +#define CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID \ + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN(CEREAL_RAPIDJSON_NOTHING) +//!@endcond + +/*! \def CEREAL_RAPIDJSON_PARSE_ERROR_NORETURN + \ingroup CEREAL_RAPIDJSON_ERRORS + \brief Macro to indicate a parse error. + \param parseErrorCode \ref rapidjson::ParseErrorCode of the error + \param offset position of the error in JSON input (\c size_t) + + This macros can be used as a customization point for the internal + error handling mechanism of RapidJSON. + + A common usage model is to throw an exception instead of requiring the + caller to explicitly check the \ref rapidjson::GenericReader::Parse's + return value: + + \code + #define CEREAL_RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode,offset) \ + throw ParseException(parseErrorCode, #parseErrorCode, offset) + + #include // std::runtime_error + #include "rapidjson/error/error.h" // rapidjson::ParseResult + + struct ParseException : std::runtime_error, rapidjson::ParseResult { + ParseException(rapidjson::ParseErrorCode code, const char* msg, size_t offset) + : std::runtime_error(msg), ParseResult(code, offset) {} + }; + + #include "rapidjson/reader.h" + \endcode + + \see CEREAL_RAPIDJSON_PARSE_ERROR, rapidjson::GenericReader::Parse + */ +#ifndef CEREAL_RAPIDJSON_PARSE_ERROR_NORETURN +#define CEREAL_RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode, offset) \ + CEREAL_RAPIDJSON_MULTILINEMACRO_BEGIN \ + CEREAL_RAPIDJSON_ASSERT(!HasParseError()); /* Error can only be assigned once */ \ + SetParseError(parseErrorCode, offset); \ + CEREAL_RAPIDJSON_MULTILINEMACRO_END +#endif + +/*! \def CEREAL_RAPIDJSON_PARSE_ERROR + \ingroup CEREAL_RAPIDJSON_ERRORS + \brief (Internal) macro to indicate and handle a parse error. + \param parseErrorCode \ref rapidjson::ParseErrorCode of the error + \param offset position of the error in JSON input (\c size_t) + + Invokes CEREAL_RAPIDJSON_PARSE_ERROR_NORETURN and stops the parsing. + + \see CEREAL_RAPIDJSON_PARSE_ERROR_NORETURN + \hideinitializer + */ +#ifndef CEREAL_RAPIDJSON_PARSE_ERROR +#define CEREAL_RAPIDJSON_PARSE_ERROR(parseErrorCode, offset) \ + CEREAL_RAPIDJSON_MULTILINEMACRO_BEGIN \ + CEREAL_RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode, offset); \ + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; \ + CEREAL_RAPIDJSON_MULTILINEMACRO_END +#endif + +#include "error/error.h" // ParseErrorCode, ParseResult + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// ParseFlag + +/*! \def CEREAL_RAPIDJSON_PARSE_DEFAULT_FLAGS + \ingroup CEREAL_RAPIDJSON_CONFIG + \brief User-defined kParseDefaultFlags definition. + + User can define this as any \c ParseFlag combinations. +*/ +#ifndef CEREAL_RAPIDJSON_PARSE_DEFAULT_FLAGS +#define CEREAL_RAPIDJSON_PARSE_DEFAULT_FLAGS kParseNoFlags +#endif + +//! Combination of parseFlags +/*! \see Reader::Parse, Document::Parse, Document::ParseInsitu, Document::ParseStream + */ +enum ParseFlag { + kParseNoFlags = 0, //!< No flags are set. + kParseInsituFlag = 1, //!< In-situ(destructive) parsing. + kParseValidateEncodingFlag = 2, //!< Validate encoding of JSON strings. + kParseIterativeFlag = 4, //!< Iterative(constant complexity in terms of function call stack size) parsing. + kParseStopWhenDoneFlag = 8, //!< After parsing a complete JSON root from stream, stop further processing the rest of stream. When this flag is used, parser will not generate kParseErrorDocumentRootNotSingular error. + kParseFullPrecisionFlag = 16, //!< Parse number in full precision (but slower). + kParseCommentsFlag = 32, //!< Allow one-line (//) and multi-line (/**/) comments. + kParseNumbersAsStringsFlag = 64, //!< Parse all numbers (ints/doubles) as strings. + kParseTrailingCommasFlag = 128, //!< Allow trailing commas at the end of objects and arrays. + kParseNanAndInfFlag = 256, //!< Allow parsing NaN, Inf, Infinity, -Inf and -Infinity as doubles. + kParseDefaultFlags = CEREAL_RAPIDJSON_PARSE_DEFAULT_FLAGS //!< Default parse flags. Can be customized by defining CEREAL_RAPIDJSON_PARSE_DEFAULT_FLAGS +}; + +/////////////////////////////////////////////////////////////////////////////// +// Handler + +/*! \class rapidjson::Handler + \brief Concept for receiving events from GenericReader upon parsing. + The functions return true if no error occurs. If they return false, + the event publisher should terminate the process. +\code +concept Handler { + typename Ch; + + bool Null(); + bool Bool(bool b); + bool Int(int i); + bool Uint(unsigned i); + bool Int64(int64_t i); + bool Uint64(uint64_t i); + bool Double(double d); + /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) + bool RawNumber(const Ch* str, SizeType length, bool copy); + bool String(const Ch* str, SizeType length, bool copy); + bool StartObject(); + bool Key(const Ch* str, SizeType length, bool copy); + bool EndObject(SizeType memberCount); + bool StartArray(); + bool EndArray(SizeType elementCount); +}; +\endcode +*/ +/////////////////////////////////////////////////////////////////////////////// +// BaseReaderHandler + +//! Default implementation of Handler. +/*! This can be used as base class of any reader handler. + \note implements Handler concept +*/ +template, typename Derived = void> +struct BaseReaderHandler { + typedef typename Encoding::Ch Ch; + + typedef typename internal::SelectIf, BaseReaderHandler, Derived>::Type Override; + + bool Default() { return true; } + bool Null() { return static_cast(*this).Default(); } + bool Bool(bool) { return static_cast(*this).Default(); } + bool Int(int) { return static_cast(*this).Default(); } + bool Uint(unsigned) { return static_cast(*this).Default(); } + bool Int64(int64_t) { return static_cast(*this).Default(); } + bool Uint64(uint64_t) { return static_cast(*this).Default(); } + bool Double(double) { return static_cast(*this).Default(); } + /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) + bool RawNumber(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } + bool String(const Ch*, SizeType, bool) { return static_cast(*this).Default(); } + bool StartObject() { return static_cast(*this).Default(); } + bool Key(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } + bool EndObject(SizeType) { return static_cast(*this).Default(); } + bool StartArray() { return static_cast(*this).Default(); } + bool EndArray(SizeType) { return static_cast(*this).Default(); } +}; + +/////////////////////////////////////////////////////////////////////////////// +// StreamLocalCopy + +namespace internal { + +template::copyOptimization> +class StreamLocalCopy; + +//! Do copy optimization. +template +class StreamLocalCopy { +public: + StreamLocalCopy(Stream& original) : s(original), original_(original) {} + ~StreamLocalCopy() { original_ = s; } + + Stream s; + +private: + StreamLocalCopy& operator=(const StreamLocalCopy&) /* = delete */; + + Stream& original_; +}; + +//! Keep reference. +template +class StreamLocalCopy { +public: + StreamLocalCopy(Stream& original) : s(original) {} + + Stream& s; + +private: + StreamLocalCopy& operator=(const StreamLocalCopy&) /* = delete */; +}; + +} // namespace internal + +/////////////////////////////////////////////////////////////////////////////// +// SkipWhitespace + +//! Skip the JSON white spaces in a stream. +/*! \param is A input stream for skipping white spaces. + \note This function has SSE2/SSE4.2 specialization. +*/ +template +void SkipWhitespace(InputStream& is) { + internal::StreamLocalCopy copy(is); + InputStream& s(copy.s); + + typename InputStream::Ch c; + while ((c = s.Peek()) == ' ' || c == '\n' || c == '\r' || c == '\t') + s.Take(); +} + +inline const char* SkipWhitespace(const char* p, const char* end) { + while (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) + ++p; + return p; +} + +#ifdef CEREAL_RAPIDJSON_SSE42 +//! Skip whitespace with SSE 4.2 pcmpistrm instruction, testing 16 8-byte characters at once. +inline const char *SkipWhitespace_SIMD(const char* p) { + // Fast return for single non-whitespace + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + // 16-byte align to the next boundary + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + // The rest of string using SIMD + static const char whitespace[16] = " \n\r\t"; + const __m128i w = _mm_loadu_si128(reinterpret_cast(&whitespace[0])); + + for (;; p += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const int r = _mm_cmpistri(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT | _SIDD_NEGATIVE_POLARITY); + if (r != 16) // some of characters is non-whitespace + return p + r; + } +} + +inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { + // Fast return for single non-whitespace + if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) + ++p; + else + return p; + + // The middle of string using SIMD + static const char whitespace[16] = " \n\r\t"; + const __m128i w = _mm_loadu_si128(reinterpret_cast(&whitespace[0])); + + for (; p <= end - 16; p += 16) { + const __m128i s = _mm_loadu_si128(reinterpret_cast(p)); + const int r = _mm_cmpistri(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT | _SIDD_NEGATIVE_POLARITY); + if (r != 16) // some of characters is non-whitespace + return p + r; + } + + return SkipWhitespace(p, end); +} + +#elif defined(CEREAL_RAPIDJSON_SSE2) + +//! Skip whitespace with SSE2 instructions, testing 16 8-byte characters at once. +inline const char *SkipWhitespace_SIMD(const char* p) { + // Fast return for single non-whitespace + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + // 16-byte align to the next boundary + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + // The rest of string + #define C16(c) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c } + static const char whitespaces[4][16] = { C16(' '), C16('\n'), C16('\r'), C16('\t') }; + #undef C16 + + const __m128i w0 = _mm_loadu_si128(reinterpret_cast(&whitespaces[0][0])); + const __m128i w1 = _mm_loadu_si128(reinterpret_cast(&whitespaces[1][0])); + const __m128i w2 = _mm_loadu_si128(reinterpret_cast(&whitespaces[2][0])); + const __m128i w3 = _mm_loadu_si128(reinterpret_cast(&whitespaces[3][0])); + + for (;; p += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + __m128i x = _mm_cmpeq_epi8(s, w0); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); + unsigned short r = static_cast(~_mm_movemask_epi8(x)); + if (r != 0) { // some of characters may be non-whitespace +#ifdef _MSC_VER // Find the index of first non-whitespace + unsigned long offset; + _BitScanForward(&offset, r); + return p + offset; +#else + return p + __builtin_ffs(r) - 1; +#endif + } + } +} + +inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { + // Fast return for single non-whitespace + if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) + ++p; + else + return p; + + // The rest of string + #define C16(c) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c } + static const char whitespaces[4][16] = { C16(' '), C16('\n'), C16('\r'), C16('\t') }; + #undef C16 + + const __m128i w0 = _mm_loadu_si128(reinterpret_cast(&whitespaces[0][0])); + const __m128i w1 = _mm_loadu_si128(reinterpret_cast(&whitespaces[1][0])); + const __m128i w2 = _mm_loadu_si128(reinterpret_cast(&whitespaces[2][0])); + const __m128i w3 = _mm_loadu_si128(reinterpret_cast(&whitespaces[3][0])); + + for (; p <= end - 16; p += 16) { + const __m128i s = _mm_loadu_si128(reinterpret_cast(p)); + __m128i x = _mm_cmpeq_epi8(s, w0); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); + unsigned short r = static_cast(~_mm_movemask_epi8(x)); + if (r != 0) { // some of characters may be non-whitespace +#ifdef _MSC_VER // Find the index of first non-whitespace + unsigned long offset; + _BitScanForward(&offset, r); + return p + offset; +#else + return p + __builtin_ffs(r) - 1; +#endif + } + } + + return SkipWhitespace(p, end); +} + +#elif defined(CEREAL_RAPIDJSON_NEON) + +//! Skip whitespace with ARM Neon instructions, testing 16 8-byte characters at once. +inline const char *SkipWhitespace_SIMD(const char* p) { + // Fast return for single non-whitespace + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + // 16-byte align to the next boundary + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + const uint8x16_t w0 = vmovq_n_u8(' '); + const uint8x16_t w1 = vmovq_n_u8('\n'); + const uint8x16_t w2 = vmovq_n_u8('\r'); + const uint8x16_t w3 = vmovq_n_u8('\t'); + + for (;; p += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, w0); + x = vorrq_u8(x, vceqq_u8(s, w1)); + x = vorrq_u8(x, vceqq_u8(s, w2)); + x = vorrq_u8(x, vceqq_u8(s, w3)); + + x = vmvnq_u8(x); // Negate + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + if (low == 0) { + if (high != 0) { + int lz =__builtin_clzll(high);; + return p + 8 + (lz >> 3); + } + } else { + int lz = __builtin_clzll(low);; + return p + (lz >> 3); + } + } +} + +inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { + // Fast return for single non-whitespace + if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) + ++p; + else + return p; + + const uint8x16_t w0 = vmovq_n_u8(' '); + const uint8x16_t w1 = vmovq_n_u8('\n'); + const uint8x16_t w2 = vmovq_n_u8('\r'); + const uint8x16_t w3 = vmovq_n_u8('\t'); + + for (; p <= end - 16; p += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, w0); + x = vorrq_u8(x, vceqq_u8(s, w1)); + x = vorrq_u8(x, vceqq_u8(s, w2)); + x = vorrq_u8(x, vceqq_u8(s, w3)); + + x = vmvnq_u8(x); // Negate + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + if (low == 0) { + if (high != 0) { + int lz = __builtin_clzll(high); + return p + 8 + (lz >> 3); + } + } else { + int lz = __builtin_clzll(low); + return p + (lz >> 3); + } + } + + return SkipWhitespace(p, end); +} + +#endif // CEREAL_RAPIDJSON_NEON + +#ifdef CEREAL_RAPIDJSON_SIMD +//! Template function specialization for InsituStringStream +template<> inline void SkipWhitespace(InsituStringStream& is) { + is.src_ = const_cast(SkipWhitespace_SIMD(is.src_)); +} + +//! Template function specialization for StringStream +template<> inline void SkipWhitespace(StringStream& is) { + is.src_ = SkipWhitespace_SIMD(is.src_); +} + +template<> inline void SkipWhitespace(EncodedInputStream, MemoryStream>& is) { + is.is_.src_ = SkipWhitespace_SIMD(is.is_.src_, is.is_.end_); +} +#endif // CEREAL_RAPIDJSON_SIMD + +/////////////////////////////////////////////////////////////////////////////// +// GenericReader + +//! SAX-style JSON parser. Use \ref Reader for UTF8 encoding and default allocator. +/*! GenericReader parses JSON text from a stream, and send events synchronously to an + object implementing Handler concept. + + It needs to allocate a stack for storing a single decoded string during + non-destructive parsing. + + For in-situ parsing, the decoded string is directly written to the source + text string, no temporary buffer is required. + + A GenericReader object can be reused for parsing multiple JSON text. + + \tparam SourceEncoding Encoding of the input stream. + \tparam TargetEncoding Encoding of the parse output. + \tparam StackAllocator Allocator type for stack. +*/ +template +class GenericReader { +public: + typedef typename SourceEncoding::Ch Ch; //!< SourceEncoding character type + + //! Constructor. + /*! \param stackAllocator Optional allocator for allocating stack memory. (Only use for non-destructive parsing) + \param stackCapacity stack capacity in bytes for storing a single decoded string. (Only use for non-destructive parsing) + */ + GenericReader(StackAllocator* stackAllocator = 0, size_t stackCapacity = kDefaultStackCapacity) : + stack_(stackAllocator, stackCapacity), parseResult_(), state_(IterativeParsingStartState) {} + + //! Parse JSON text. + /*! \tparam parseFlags Combination of \ref ParseFlag. + \tparam InputStream Type of input stream, implementing Stream concept. + \tparam Handler Type of handler, implementing Handler concept. + \param is Input stream to be parsed. + \param handler The handler to receive events. + \return Whether the parsing is successful. + */ + template + ParseResult Parse(InputStream& is, Handler& handler) { + if (parseFlags & kParseIterativeFlag) + return IterativeParse(is, handler); + + parseResult_.Clear(); + + ClearStackOnExit scope(*this); + + SkipWhitespaceAndComments(is); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + + if (CEREAL_RAPIDJSON_UNLIKELY(is.Peek() == '\0')) { + CEREAL_RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentEmpty, is.Tell()); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + } + else { + ParseValue(is, handler); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + + if (!(parseFlags & kParseStopWhenDoneFlag)) { + SkipWhitespaceAndComments(is); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + + if (CEREAL_RAPIDJSON_UNLIKELY(is.Peek() != '\0')) { + CEREAL_RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentRootNotSingular, is.Tell()); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + } + } + } + + return parseResult_; + } + + //! Parse JSON text (with \ref kParseDefaultFlags) + /*! \tparam InputStream Type of input stream, implementing Stream concept + \tparam Handler Type of handler, implementing Handler concept. + \param is Input stream to be parsed. + \param handler The handler to receive events. + \return Whether the parsing is successful. + */ + template + ParseResult Parse(InputStream& is, Handler& handler) { + return Parse(is, handler); + } + + //! Initialize JSON text token-by-token parsing + /*! + */ + void IterativeParseInit() { + parseResult_.Clear(); + state_ = IterativeParsingStartState; + } + + //! Parse one token from JSON text + /*! \tparam InputStream Type of input stream, implementing Stream concept + \tparam Handler Type of handler, implementing Handler concept. + \param is Input stream to be parsed. + \param handler The handler to receive events. + \return Whether the parsing is successful. + */ + template + bool IterativeParseNext(InputStream& is, Handler& handler) { + while (CEREAL_RAPIDJSON_LIKELY(is.Peek() != '\0')) { + SkipWhitespaceAndComments(is); + + Token t = Tokenize(is.Peek()); + IterativeParsingState n = Predict(state_, t); + IterativeParsingState d = Transit(state_, t, n, is, handler); + + // If we've finished or hit an error... + if (CEREAL_RAPIDJSON_UNLIKELY(IsIterativeParsingCompleteState(d))) { + // Report errors. + if (d == IterativeParsingErrorState) { + HandleError(state_, is); + return false; + } + + // Transition to the finish state. + CEREAL_RAPIDJSON_ASSERT(d == IterativeParsingFinishState); + state_ = d; + + // If StopWhenDone is not set... + if (!(parseFlags & kParseStopWhenDoneFlag)) { + // ... and extra non-whitespace data is found... + SkipWhitespaceAndComments(is); + if (is.Peek() != '\0') { + // ... this is considered an error. + HandleError(state_, is); + return false; + } + } + + // Success! We are done! + return true; + } + + // Transition to the new state. + state_ = d; + + // If we parsed anything other than a delimiter, we invoked the handler, so we can return true now. + if (!IsIterativeParsingDelimiterState(n)) + return true; + } + + // We reached the end of file. + stack_.Clear(); + + if (state_ != IterativeParsingFinishState) { + HandleError(state_, is); + return false; + } + + return true; + } + + //! Check if token-by-token parsing JSON text is complete + /*! \return Whether the JSON has been fully decoded. + */ + CEREAL_RAPIDJSON_FORCEINLINE bool IterativeParseComplete() const { + return IsIterativeParsingCompleteState(state_); + } + + //! Whether a parse error has occurred in the last parsing. + bool HasParseError() const { return parseResult_.IsError(); } + + //! Get the \ref ParseErrorCode of last parsing. + ParseErrorCode GetParseErrorCode() const { return parseResult_.Code(); } + + //! Get the position of last parsing error in input, 0 otherwise. + size_t GetErrorOffset() const { return parseResult_.Offset(); } + +protected: + void SetParseError(ParseErrorCode code, size_t offset) { parseResult_.Set(code, offset); } + +private: + // Prohibit copy constructor & assignment operator. + GenericReader(const GenericReader&); + GenericReader& operator=(const GenericReader&); + + void ClearStack() { stack_.Clear(); } + + // clear stack on any exit from ParseStream, e.g. due to exception + struct ClearStackOnExit { + explicit ClearStackOnExit(GenericReader& r) : r_(r) {} + ~ClearStackOnExit() { r_.ClearStack(); } + private: + GenericReader& r_; + ClearStackOnExit(const ClearStackOnExit&); + ClearStackOnExit& operator=(const ClearStackOnExit&); + }; + + template + void SkipWhitespaceAndComments(InputStream& is) { + SkipWhitespace(is); + + if (parseFlags & kParseCommentsFlag) { + while (CEREAL_RAPIDJSON_UNLIKELY(Consume(is, '/'))) { + if (Consume(is, '*')) { + while (true) { + if (CEREAL_RAPIDJSON_UNLIKELY(is.Peek() == '\0')) + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); + else if (Consume(is, '*')) { + if (Consume(is, '/')) + break; + } + else + is.Take(); + } + } + else if (CEREAL_RAPIDJSON_LIKELY(Consume(is, '/'))) + while (is.Peek() != '\0' && is.Take() != '\n') {} + else + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); + + SkipWhitespace(is); + } + } + } + + // Parse object: { string : value, ... } + template + void ParseObject(InputStream& is, Handler& handler) { + CEREAL_RAPIDJSON_ASSERT(is.Peek() == '{'); + is.Take(); // Skip '{' + + if (CEREAL_RAPIDJSON_UNLIKELY(!handler.StartObject())) + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + + SkipWhitespaceAndComments(is); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + if (Consume(is, '}')) { + if (CEREAL_RAPIDJSON_UNLIKELY(!handler.EndObject(0))) // empty object + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + return; + } + + for (SizeType memberCount = 0;;) { + if (CEREAL_RAPIDJSON_UNLIKELY(is.Peek() != '"')) + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); + + ParseString(is, handler, true); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + SkipWhitespaceAndComments(is); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + if (CEREAL_RAPIDJSON_UNLIKELY(!Consume(is, ':'))) + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); + + SkipWhitespaceAndComments(is); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + ParseValue(is, handler); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + SkipWhitespaceAndComments(is); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + ++memberCount; + + switch (is.Peek()) { + case ',': + is.Take(); + SkipWhitespaceAndComments(is); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + break; + case '}': + is.Take(); + if (CEREAL_RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount))) + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + return; + default: + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); break; // This useless break is only for making warning and coverage happy + } + + if (parseFlags & kParseTrailingCommasFlag) { + if (is.Peek() == '}') { + if (CEREAL_RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount))) + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + is.Take(); + return; + } + } + } + } + + // Parse array: [ value, ... ] + template + void ParseArray(InputStream& is, Handler& handler) { + CEREAL_RAPIDJSON_ASSERT(is.Peek() == '['); + is.Take(); // Skip '[' + + if (CEREAL_RAPIDJSON_UNLIKELY(!handler.StartArray())) + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + + SkipWhitespaceAndComments(is); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + if (Consume(is, ']')) { + if (CEREAL_RAPIDJSON_UNLIKELY(!handler.EndArray(0))) // empty array + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + return; + } + + for (SizeType elementCount = 0;;) { + ParseValue(is, handler); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + ++elementCount; + SkipWhitespaceAndComments(is); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + if (Consume(is, ',')) { + SkipWhitespaceAndComments(is); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + } + else if (Consume(is, ']')) { + if (CEREAL_RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount))) + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + return; + } + else + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); + + if (parseFlags & kParseTrailingCommasFlag) { + if (is.Peek() == ']') { + if (CEREAL_RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount))) + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + is.Take(); + return; + } + } + } + } + + template + void ParseNull(InputStream& is, Handler& handler) { + CEREAL_RAPIDJSON_ASSERT(is.Peek() == 'n'); + is.Take(); + + if (CEREAL_RAPIDJSON_LIKELY(Consume(is, 'u') && Consume(is, 'l') && Consume(is, 'l'))) { + if (CEREAL_RAPIDJSON_UNLIKELY(!handler.Null())) + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + } + else + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); + } + + template + void ParseTrue(InputStream& is, Handler& handler) { + CEREAL_RAPIDJSON_ASSERT(is.Peek() == 't'); + is.Take(); + + if (CEREAL_RAPIDJSON_LIKELY(Consume(is, 'r') && Consume(is, 'u') && Consume(is, 'e'))) { + if (CEREAL_RAPIDJSON_UNLIKELY(!handler.Bool(true))) + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + } + else + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); + } + + template + void ParseFalse(InputStream& is, Handler& handler) { + CEREAL_RAPIDJSON_ASSERT(is.Peek() == 'f'); + is.Take(); + + if (CEREAL_RAPIDJSON_LIKELY(Consume(is, 'a') && Consume(is, 'l') && Consume(is, 's') && Consume(is, 'e'))) { + if (CEREAL_RAPIDJSON_UNLIKELY(!handler.Bool(false))) + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + } + else + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); + } + + template + CEREAL_RAPIDJSON_FORCEINLINE static bool Consume(InputStream& is, typename InputStream::Ch expect) { + if (CEREAL_RAPIDJSON_LIKELY(is.Peek() == expect)) { + is.Take(); + return true; + } + else + return false; + } + + // Helper function to parse four hexadecimal digits in \uXXXX in ParseString(). + template + unsigned ParseHex4(InputStream& is, size_t escapeOffset) { + unsigned codepoint = 0; + for (int i = 0; i < 4; i++) { + Ch c = is.Peek(); + codepoint <<= 4; + codepoint += static_cast(c); + if (c >= '0' && c <= '9') + codepoint -= '0'; + else if (c >= 'A' && c <= 'F') + codepoint -= 'A' - 10; + else if (c >= 'a' && c <= 'f') + codepoint -= 'a' - 10; + else { + CEREAL_RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorStringUnicodeEscapeInvalidHex, escapeOffset); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN(0); + } + is.Take(); + } + return codepoint; + } + + template + class StackStream { + public: + typedef CharType Ch; + + StackStream(internal::Stack& stack) : stack_(stack), length_(0) {} + CEREAL_RAPIDJSON_FORCEINLINE void Put(Ch c) { + *stack_.template Push() = c; + ++length_; + } + + CEREAL_RAPIDJSON_FORCEINLINE void* Push(SizeType count) { + length_ += count; + return stack_.template Push(count); + } + + size_t Length() const { return length_; } + + Ch* Pop() { + return stack_.template Pop(length_); + } + + private: + StackStream(const StackStream&); + StackStream& operator=(const StackStream&); + + internal::Stack& stack_; + SizeType length_; + }; + + // Parse string and generate String event. Different code paths for kParseInsituFlag. + template + void ParseString(InputStream& is, Handler& handler, bool isKey = false) { + internal::StreamLocalCopy copy(is); + InputStream& s(copy.s); + + CEREAL_RAPIDJSON_ASSERT(s.Peek() == '\"'); + s.Take(); // Skip '\"' + + bool success = false; + if (parseFlags & kParseInsituFlag) { + typename InputStream::Ch *head = s.PutBegin(); + ParseStringToStream(s, s); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + size_t length = s.PutEnd(head) - 1; + CEREAL_RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); + const typename TargetEncoding::Ch* const str = reinterpret_cast(head); + success = (isKey ? handler.Key(str, SizeType(length), false) : handler.String(str, SizeType(length), false)); + } + else { + StackStream stackStream(stack_); + ParseStringToStream(s, stackStream); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + SizeType length = static_cast(stackStream.Length()) - 1; + const typename TargetEncoding::Ch* const str = stackStream.Pop(); + success = (isKey ? handler.Key(str, length, true) : handler.String(str, length, true)); + } + if (CEREAL_RAPIDJSON_UNLIKELY(!success)) + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorTermination, s.Tell()); + } + + // Parse string to an output is + // This function handles the prefix/suffix double quotes, escaping, and optional encoding validation. + template + CEREAL_RAPIDJSON_FORCEINLINE void ParseStringToStream(InputStream& is, OutputStream& os) { +//!@cond CEREAL_RAPIDJSON_HIDDEN_FROM_DOXYGEN +#define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + static const char escape[256] = { + Z16, Z16, 0, 0,'\"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'/', + Z16, Z16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, + 0, 0,'\b', 0, 0, 0,'\f', 0, 0, 0, 0, 0, 0, 0,'\n', 0, + 0, 0,'\r', 0,'\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 + }; +#undef Z16 +//!@endcond + + for (;;) { + // Scan and copy string before "\\\"" or < 0x20. This is an optional optimzation. + if (!(parseFlags & kParseValidateEncodingFlag)) + ScanCopyUnescapedString(is, os); + + Ch c = is.Peek(); + if (CEREAL_RAPIDJSON_UNLIKELY(c == '\\')) { // Escape + size_t escapeOffset = is.Tell(); // For invalid escaping, report the initial '\\' as error offset + is.Take(); + Ch e = is.Peek(); + if ((sizeof(Ch) == 1 || unsigned(e) < 256) && CEREAL_RAPIDJSON_LIKELY(escape[static_cast(e)])) { + is.Take(); + os.Put(static_cast(escape[static_cast(e)])); + } + else if (CEREAL_RAPIDJSON_LIKELY(e == 'u')) { // Unicode + is.Take(); + unsigned codepoint = ParseHex4(is, escapeOffset); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + if (CEREAL_RAPIDJSON_UNLIKELY(codepoint >= 0xD800 && codepoint <= 0xDBFF)) { + // Handle UTF-16 surrogate pair + if (CEREAL_RAPIDJSON_UNLIKELY(!Consume(is, '\\') || !Consume(is, 'u'))) + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); + unsigned codepoint2 = ParseHex4(is, escapeOffset); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + if (CEREAL_RAPIDJSON_UNLIKELY(codepoint2 < 0xDC00 || codepoint2 > 0xDFFF)) + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); + codepoint = (((codepoint - 0xD800) << 10) | (codepoint2 - 0xDC00)) + 0x10000; + } + TEncoding::Encode(os, codepoint); + } + else + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, escapeOffset); + } + else if (CEREAL_RAPIDJSON_UNLIKELY(c == '"')) { // Closing double quote + is.Take(); + os.Put('\0'); // null-terminate the string + return; + } + else if (CEREAL_RAPIDJSON_UNLIKELY(static_cast(c) < 0x20)) { // RFC 4627: unescaped = %x20-21 / %x23-5B / %x5D-10FFFF + if (c == '\0') + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorStringMissQuotationMark, is.Tell()); + else + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorStringInvalidEncoding, is.Tell()); + } + else { + size_t offset = is.Tell(); + if (CEREAL_RAPIDJSON_UNLIKELY((parseFlags & kParseValidateEncodingFlag ? + !Transcoder::Validate(is, os) : + !Transcoder::Transcode(is, os)))) + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorStringInvalidEncoding, offset); + } + } + } + + template + static CEREAL_RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InputStream&, OutputStream&) { + // Do nothing for generic version + } + +#if defined(CEREAL_RAPIDJSON_SSE2) || defined(CEREAL_RAPIDJSON_SSE42) + // StringStream -> StackStream + static CEREAL_RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(StringStream& is, StackStream& os) { + const char* p = is.src_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (CEREAL_RAPIDJSON_UNLIKELY(*p == '\"') || CEREAL_RAPIDJSON_UNLIKELY(*p == '\\') || CEREAL_RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = p; + return; + } + else + os.Put(*p++); + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (;; p += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x1F) == 0x1F + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (CEREAL_RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped + SizeType length; + #ifdef _MSC_VER // Find the index of first escaped + unsigned long offset; + _BitScanForward(&offset, r); + length = offset; + #else + length = static_cast(__builtin_ffs(r) - 1); + #endif + if (length != 0) { + char* q = reinterpret_cast(os.Push(length)); + for (size_t i = 0; i < length; i++) + q[i] = p[i]; + + p += length; + } + break; + } + _mm_storeu_si128(reinterpret_cast<__m128i *>(os.Push(16)), s); + } + + is.src_ = p; + } + + // InsituStringStream -> InsituStringStream + static CEREAL_RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InsituStringStream& is, InsituStringStream& os) { + CEREAL_RAPIDJSON_ASSERT(&is == &os); + (void)os; + + if (is.src_ == is.dst_) { + SkipUnescapedString(is); + return; + } + + char* p = is.src_; + char *q = is.dst_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (CEREAL_RAPIDJSON_UNLIKELY(*p == '\"') || CEREAL_RAPIDJSON_UNLIKELY(*p == '\\') || CEREAL_RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = p; + is.dst_ = q; + return; + } + else + *q++ = *p++; + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (;; p += 16, q += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x1F) == 0x1F + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (CEREAL_RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped + size_t length; +#ifdef _MSC_VER // Find the index of first escaped + unsigned long offset; + _BitScanForward(&offset, r); + length = offset; +#else + length = static_cast(__builtin_ffs(r) - 1); +#endif + for (const char* pend = p + length; p != pend; ) + *q++ = *p++; + break; + } + _mm_storeu_si128(reinterpret_cast<__m128i *>(q), s); + } + + is.src_ = p; + is.dst_ = q; + } + + // When read/write pointers are the same for insitu stream, just skip unescaped characters + static CEREAL_RAPIDJSON_FORCEINLINE void SkipUnescapedString(InsituStringStream& is) { + CEREAL_RAPIDJSON_ASSERT(is.src_ == is.dst_); + char* p = is.src_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + for (; p != nextAligned; p++) + if (CEREAL_RAPIDJSON_UNLIKELY(*p == '\"') || CEREAL_RAPIDJSON_UNLIKELY(*p == '\\') || CEREAL_RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = is.dst_ = p; + return; + } + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (;; p += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x1F) == 0x1F + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (CEREAL_RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped + size_t length; +#ifdef _MSC_VER // Find the index of first escaped + unsigned long offset; + _BitScanForward(&offset, r); + length = offset; +#else + length = static_cast(__builtin_ffs(r) - 1); +#endif + p += length; + break; + } + } + + is.src_ = is.dst_ = p; + } +#elif defined(CEREAL_RAPIDJSON_NEON) + // StringStream -> StackStream + static CEREAL_RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(StringStream& is, StackStream& os) { + const char* p = is.src_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (CEREAL_RAPIDJSON_UNLIKELY(*p == '\"') || CEREAL_RAPIDJSON_UNLIKELY(*p == '\\') || CEREAL_RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = p; + return; + } + else + os.Put(*p++); + + // The rest of string using SIMD + const uint8x16_t s0 = vmovq_n_u8('"'); + const uint8x16_t s1 = vmovq_n_u8('\\'); + const uint8x16_t s2 = vmovq_n_u8('\b'); + const uint8x16_t s3 = vmovq_n_u8(32); + + for (;; p += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, s0); + x = vorrq_u8(x, vceqq_u8(s, s1)); + x = vorrq_u8(x, vceqq_u8(s, s2)); + x = vorrq_u8(x, vcltq_u8(s, s3)); + + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + SizeType length = 0; + bool escaped = false; + if (low == 0) { + if (high != 0) { + unsigned lz = (unsigned)__builtin_clzll(high);; + length = 8 + (lz >> 3); + escaped = true; + } + } else { + unsigned lz = (unsigned)__builtin_clzll(low);; + length = lz >> 3; + escaped = true; + } + if (CEREAL_RAPIDJSON_UNLIKELY(escaped)) { // some of characters is escaped + if (length != 0) { + char* q = reinterpret_cast(os.Push(length)); + for (size_t i = 0; i < length; i++) + q[i] = p[i]; + + p += length; + } + break; + } + vst1q_u8(reinterpret_cast(os.Push(16)), s); + } + + is.src_ = p; + } + + // InsituStringStream -> InsituStringStream + static CEREAL_RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InsituStringStream& is, InsituStringStream& os) { + CEREAL_RAPIDJSON_ASSERT(&is == &os); + (void)os; + + if (is.src_ == is.dst_) { + SkipUnescapedString(is); + return; + } + + char* p = is.src_; + char *q = is.dst_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (CEREAL_RAPIDJSON_UNLIKELY(*p == '\"') || CEREAL_RAPIDJSON_UNLIKELY(*p == '\\') || CEREAL_RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = p; + is.dst_ = q; + return; + } + else + *q++ = *p++; + + // The rest of string using SIMD + const uint8x16_t s0 = vmovq_n_u8('"'); + const uint8x16_t s1 = vmovq_n_u8('\\'); + const uint8x16_t s2 = vmovq_n_u8('\b'); + const uint8x16_t s3 = vmovq_n_u8(32); + + for (;; p += 16, q += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, s0); + x = vorrq_u8(x, vceqq_u8(s, s1)); + x = vorrq_u8(x, vceqq_u8(s, s2)); + x = vorrq_u8(x, vcltq_u8(s, s3)); + + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + SizeType length = 0; + bool escaped = false; + if (low == 0) { + if (high != 0) { + unsigned lz = (unsigned)__builtin_clzll(high); + length = 8 + (lz >> 3); + escaped = true; + } + } else { + unsigned lz = (unsigned)__builtin_clzll(low); + length = lz >> 3; + escaped = true; + } + if (CEREAL_RAPIDJSON_UNLIKELY(escaped)) { // some of characters is escaped + for (const char* pend = p + length; p != pend; ) { + *q++ = *p++; + } + break; + } + vst1q_u8(reinterpret_cast(q), s); + } + + is.src_ = p; + is.dst_ = q; + } + + // When read/write pointers are the same for insitu stream, just skip unescaped characters + static CEREAL_RAPIDJSON_FORCEINLINE void SkipUnescapedString(InsituStringStream& is) { + CEREAL_RAPIDJSON_ASSERT(is.src_ == is.dst_); + char* p = is.src_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + for (; p != nextAligned; p++) + if (CEREAL_RAPIDJSON_UNLIKELY(*p == '\"') || CEREAL_RAPIDJSON_UNLIKELY(*p == '\\') || CEREAL_RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = is.dst_ = p; + return; + } + + // The rest of string using SIMD + const uint8x16_t s0 = vmovq_n_u8('"'); + const uint8x16_t s1 = vmovq_n_u8('\\'); + const uint8x16_t s2 = vmovq_n_u8('\b'); + const uint8x16_t s3 = vmovq_n_u8(32); + + for (;; p += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, s0); + x = vorrq_u8(x, vceqq_u8(s, s1)); + x = vorrq_u8(x, vceqq_u8(s, s2)); + x = vorrq_u8(x, vcltq_u8(s, s3)); + + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + if (low == 0) { + if (high != 0) { + int lz = __builtin_clzll(high); + p += 8 + (lz >> 3); + break; + } + } else { + int lz = __builtin_clzll(low); + p += lz >> 3; + break; + } + } + + is.src_ = is.dst_ = p; + } +#endif // CEREAL_RAPIDJSON_NEON + + template + class NumberStream; + + template + class NumberStream { + public: + typedef typename InputStream::Ch Ch; + + NumberStream(GenericReader& reader, InputStream& s) : is(s) { (void)reader; } + + CEREAL_RAPIDJSON_FORCEINLINE Ch Peek() const { return is.Peek(); } + CEREAL_RAPIDJSON_FORCEINLINE Ch TakePush() { return is.Take(); } + CEREAL_RAPIDJSON_FORCEINLINE Ch Take() { return is.Take(); } + CEREAL_RAPIDJSON_FORCEINLINE void Push(char) {} + + size_t Tell() { return is.Tell(); } + size_t Length() { return 0; } + const char* Pop() { return 0; } + + protected: + NumberStream& operator=(const NumberStream&); + + InputStream& is; + }; + + template + class NumberStream : public NumberStream { + typedef NumberStream Base; + public: + NumberStream(GenericReader& reader, InputStream& s) : Base(reader, s), stackStream(reader.stack_) {} + + CEREAL_RAPIDJSON_FORCEINLINE Ch TakePush() { + stackStream.Put(static_cast(Base::is.Peek())); + return Base::is.Take(); + } + + CEREAL_RAPIDJSON_FORCEINLINE void Push(char c) { + stackStream.Put(c); + } + + size_t Length() { return stackStream.Length(); } + + const char* Pop() { + stackStream.Put('\0'); + return stackStream.Pop(); + } + + private: + StackStream stackStream; + }; + + template + class NumberStream : public NumberStream { + typedef NumberStream Base; + public: + NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is) {} + + CEREAL_RAPIDJSON_FORCEINLINE Ch Take() { return Base::TakePush(); } + }; + + template + void ParseNumber(InputStream& is, Handler& handler) { + internal::StreamLocalCopy copy(is); + NumberStream s(*this, copy.s); + + size_t startOffset = s.Tell(); + double d = 0.0; + bool useNanOrInf = false; + + // Parse minus + bool minus = Consume(s, '-'); + + // Parse int: zero / ( digit1-9 *DIGIT ) + unsigned i = 0; + uint64_t i64 = 0; + bool use64bit = false; + int significandDigit = 0; + if (CEREAL_RAPIDJSON_UNLIKELY(s.Peek() == '0')) { + i = 0; + s.TakePush(); + } + else if (CEREAL_RAPIDJSON_LIKELY(s.Peek() >= '1' && s.Peek() <= '9')) { + i = static_cast(s.TakePush() - '0'); + + if (minus) + while (CEREAL_RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (CEREAL_RAPIDJSON_UNLIKELY(i >= 214748364)) { // 2^31 = 2147483648 + if (CEREAL_RAPIDJSON_LIKELY(i != 214748364 || s.Peek() > '8')) { + i64 = i; + use64bit = true; + break; + } + } + i = i * 10 + static_cast(s.TakePush() - '0'); + significandDigit++; + } + else + while (CEREAL_RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (CEREAL_RAPIDJSON_UNLIKELY(i >= 429496729)) { // 2^32 - 1 = 4294967295 + if (CEREAL_RAPIDJSON_LIKELY(i != 429496729 || s.Peek() > '5')) { + i64 = i; + use64bit = true; + break; + } + } + i = i * 10 + static_cast(s.TakePush() - '0'); + significandDigit++; + } + } + // Parse NaN or Infinity here + else if ((parseFlags & kParseNanAndInfFlag) && CEREAL_RAPIDJSON_LIKELY((s.Peek() == 'I' || s.Peek() == 'N'))) { + if (Consume(s, 'N')) { + if (Consume(s, 'a') && Consume(s, 'N')) { + d = std::numeric_limits::quiet_NaN(); + useNanOrInf = true; + } + } + else if (CEREAL_RAPIDJSON_LIKELY(Consume(s, 'I'))) { + if (Consume(s, 'n') && Consume(s, 'f')) { + d = (minus ? -std::numeric_limits::infinity() : std::numeric_limits::infinity()); + useNanOrInf = true; + + if (CEREAL_RAPIDJSON_UNLIKELY(s.Peek() == 'i' && !(Consume(s, 'i') && Consume(s, 'n') + && Consume(s, 'i') && Consume(s, 't') && Consume(s, 'y')))) { + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); + } + } + } + + if (CEREAL_RAPIDJSON_UNLIKELY(!useNanOrInf)) { + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); + } + } + else + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); + + // Parse 64bit int + bool useDouble = false; + if (use64bit) { + if (minus) + while (CEREAL_RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (CEREAL_RAPIDJSON_UNLIKELY(i64 >= CEREAL_RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC))) // 2^63 = 9223372036854775808 + if (CEREAL_RAPIDJSON_LIKELY(i64 != CEREAL_RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC) || s.Peek() > '8')) { + d = static_cast(i64); + useDouble = true; + break; + } + i64 = i64 * 10 + static_cast(s.TakePush() - '0'); + significandDigit++; + } + else + while (CEREAL_RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (CEREAL_RAPIDJSON_UNLIKELY(i64 >= CEREAL_RAPIDJSON_UINT64_C2(0x19999999, 0x99999999))) // 2^64 - 1 = 18446744073709551615 + if (CEREAL_RAPIDJSON_LIKELY(i64 != CEREAL_RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || s.Peek() > '5')) { + d = static_cast(i64); + useDouble = true; + break; + } + i64 = i64 * 10 + static_cast(s.TakePush() - '0'); + significandDigit++; + } + } + + // Force double for big integer + if (useDouble) { + while (CEREAL_RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + d = d * 10 + (s.TakePush() - '0'); + } + } + + // Parse frac = decimal-point 1*DIGIT + int expFrac = 0; + size_t decimalPosition; + if (Consume(s, '.')) { + decimalPosition = s.Length(); + + if (CEREAL_RAPIDJSON_UNLIKELY(!(s.Peek() >= '0' && s.Peek() <= '9'))) + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorNumberMissFraction, s.Tell()); + + if (!useDouble) { +#if CEREAL_RAPIDJSON_64BIT + // Use i64 to store significand in 64-bit architecture + if (!use64bit) + i64 = i; + + while (CEREAL_RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (i64 > CEREAL_RAPIDJSON_UINT64_C2(0x1FFFFF, 0xFFFFFFFF)) // 2^53 - 1 for fast path + break; + else { + i64 = i64 * 10 + static_cast(s.TakePush() - '0'); + --expFrac; + if (i64 != 0) + significandDigit++; + } + } + + d = static_cast(i64); +#else + // Use double to store significand in 32-bit architecture + d = static_cast(use64bit ? i64 : i); +#endif + useDouble = true; + } + + while (CEREAL_RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (significandDigit < 17) { + d = d * 10.0 + (s.TakePush() - '0'); + --expFrac; + if (CEREAL_RAPIDJSON_LIKELY(d > 0.0)) + significandDigit++; + } + else + s.TakePush(); + } + } + else + decimalPosition = s.Length(); // decimal position at the end of integer. + + // Parse exp = e [ minus / plus ] 1*DIGIT + int exp = 0; + if (Consume(s, 'e') || Consume(s, 'E')) { + if (!useDouble) { + d = static_cast(use64bit ? i64 : i); + useDouble = true; + } + + bool expMinus = false; + if (Consume(s, '+')) + ; + else if (Consume(s, '-')) + expMinus = true; + + if (CEREAL_RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + exp = static_cast(s.Take() - '0'); + if (expMinus) { + // (exp + expFrac) must not underflow int => we're detecting when -exp gets + // dangerously close to INT_MIN (a pessimistic next digit 9 would push it into + // underflow territory): + // + // -(exp * 10 + 9) + expFrac >= INT_MIN + // <=> exp <= (expFrac - INT_MIN - 9) / 10 + CEREAL_RAPIDJSON_ASSERT(expFrac <= 0); + int maxExp = (expFrac + 2147483639) / 10; + + while (CEREAL_RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + exp = exp * 10 + static_cast(s.Take() - '0'); + if (CEREAL_RAPIDJSON_UNLIKELY(exp > maxExp)) { + while (CEREAL_RAPIDJSON_UNLIKELY(s.Peek() >= '0' && s.Peek() <= '9')) // Consume the rest of exponent + s.Take(); + } + } + } + else { // positive exp + int maxExp = 308 - expFrac; + while (CEREAL_RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + exp = exp * 10 + static_cast(s.Take() - '0'); + if (CEREAL_RAPIDJSON_UNLIKELY(exp > maxExp)) + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset); + } + } + } + else + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorNumberMissExponent, s.Tell()); + + if (expMinus) + exp = -exp; + } + + // Finish parsing, call event according to the type of number. + bool cont = true; + + if (parseFlags & kParseNumbersAsStringsFlag) { + if (parseFlags & kParseInsituFlag) { + s.Pop(); // Pop stack no matter if it will be used or not. + typename InputStream::Ch* head = is.PutBegin(); + const size_t length = s.Tell() - startOffset; + CEREAL_RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); + // unable to insert the \0 character here, it will erase the comma after this number + const typename TargetEncoding::Ch* const str = reinterpret_cast(head); + cont = handler.RawNumber(str, SizeType(length), false); + } + else { + SizeType numCharsToCopy = static_cast(s.Length()); + StringStream srcStream(s.Pop()); + StackStream dstStream(stack_); + while (numCharsToCopy--) { + Transcoder, TargetEncoding>::Transcode(srcStream, dstStream); + } + dstStream.Put('\0'); + const typename TargetEncoding::Ch* str = dstStream.Pop(); + const SizeType length = static_cast(dstStream.Length()) - 1; + cont = handler.RawNumber(str, SizeType(length), true); + } + } + else { + size_t length = s.Length(); + const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not. + + if (useDouble) { + int p = exp + expFrac; + if (parseFlags & kParseFullPrecisionFlag) + d = internal::StrtodFullPrecision(d, p, decimal, length, decimalPosition, exp); + else + d = internal::StrtodNormalPrecision(d, p); + + // Use > max, instead of == inf, to fix bogus warning -Wfloat-equal + if (d > (std::numeric_limits::max)()) { + // Overflow + // TODO: internal::StrtodX should report overflow (or underflow) + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset); + } + + cont = handler.Double(minus ? -d : d); + } + else if (useNanOrInf) { + cont = handler.Double(d); + } + else { + if (use64bit) { + if (minus) + cont = handler.Int64(static_cast(~i64 + 1)); + else + cont = handler.Uint64(i64); + } + else { + if (minus) + cont = handler.Int(static_cast(~i + 1)); + else + cont = handler.Uint(i); + } + } + } + if (CEREAL_RAPIDJSON_UNLIKELY(!cont)) + CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorTermination, startOffset); + } + + // Parse any JSON value + template + void ParseValue(InputStream& is, Handler& handler) { + switch (is.Peek()) { + case 'n': ParseNull (is, handler); break; + case 't': ParseTrue (is, handler); break; + case 'f': ParseFalse (is, handler); break; + case '"': ParseString(is, handler); break; + case '{': ParseObject(is, handler); break; + case '[': ParseArray (is, handler); break; + default : + ParseNumber(is, handler); + break; + + } + } + + // Iterative Parsing + + // States + enum IterativeParsingState { + IterativeParsingFinishState = 0, // sink states at top + IterativeParsingErrorState, // sink states at top + IterativeParsingStartState, + + // Object states + IterativeParsingObjectInitialState, + IterativeParsingMemberKeyState, + IterativeParsingMemberValueState, + IterativeParsingObjectFinishState, + + // Array states + IterativeParsingArrayInitialState, + IterativeParsingElementState, + IterativeParsingArrayFinishState, + + // Single value state + IterativeParsingValueState, + + // Delimiter states (at bottom) + IterativeParsingElementDelimiterState, + IterativeParsingMemberDelimiterState, + IterativeParsingKeyValueDelimiterState, + + cIterativeParsingStateCount + }; + + // Tokens + enum Token { + LeftBracketToken = 0, + RightBracketToken, + + LeftCurlyBracketToken, + RightCurlyBracketToken, + + CommaToken, + ColonToken, + + StringToken, + FalseToken, + TrueToken, + NullToken, + NumberToken, + + kTokenCount + }; + + CEREAL_RAPIDJSON_FORCEINLINE Token Tokenize(Ch c) const { + +//!@cond CEREAL_RAPIDJSON_HIDDEN_FROM_DOXYGEN +#define N NumberToken +#define N16 N,N,N,N,N,N,N,N,N,N,N,N,N,N,N,N + // Maps from ASCII to Token + static const unsigned char tokenMap[256] = { + N16, // 00~0F + N16, // 10~1F + N, N, StringToken, N, N, N, N, N, N, N, N, N, CommaToken, N, N, N, // 20~2F + N, N, N, N, N, N, N, N, N, N, ColonToken, N, N, N, N, N, // 30~3F + N16, // 40~4F + N, N, N, N, N, N, N, N, N, N, N, LeftBracketToken, N, RightBracketToken, N, N, // 50~5F + N, N, N, N, N, N, FalseToken, N, N, N, N, N, N, N, NullToken, N, // 60~6F + N, N, N, N, TrueToken, N, N, N, N, N, N, LeftCurlyBracketToken, N, RightCurlyBracketToken, N, N, // 70~7F + N16, N16, N16, N16, N16, N16, N16, N16 // 80~FF + }; +#undef N +#undef N16 +//!@endcond + + if (sizeof(Ch) == 1 || static_cast(c) < 256) + return static_cast(tokenMap[static_cast(c)]); + else + return NumberToken; + } + + CEREAL_RAPIDJSON_FORCEINLINE IterativeParsingState Predict(IterativeParsingState state, Token token) const { + // current state x one lookahead token -> new state + static const char G[cIterativeParsingStateCount][kTokenCount] = { + // Finish(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // Error(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // Start + { + IterativeParsingArrayInitialState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingValueState, // String + IterativeParsingValueState, // False + IterativeParsingValueState, // True + IterativeParsingValueState, // Null + IterativeParsingValueState // Number + }, + // ObjectInitial + { + IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingObjectFinishState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberKeyState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // MemberKey + { + IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingKeyValueDelimiterState, // Colon + IterativeParsingErrorState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // MemberValue + { + IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingObjectFinishState, // Right curly bracket + IterativeParsingMemberDelimiterState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingErrorState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // ObjectFinish(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // ArrayInitial + { + IterativeParsingArrayInitialState, // Left bracket(push Element state) + IterativeParsingArrayFinishState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket(push Element state) + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingElementState, // String + IterativeParsingElementState, // False + IterativeParsingElementState, // True + IterativeParsingElementState, // Null + IterativeParsingElementState // Number + }, + // Element + { + IterativeParsingErrorState, // Left bracket + IterativeParsingArrayFinishState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingErrorState, // Right curly bracket + IterativeParsingElementDelimiterState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingErrorState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // ArrayFinish(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // Single Value (sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // ElementDelimiter + { + IterativeParsingArrayInitialState, // Left bracket(push Element state) + IterativeParsingArrayFinishState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket(push Element state) + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingElementState, // String + IterativeParsingElementState, // False + IterativeParsingElementState, // True + IterativeParsingElementState, // Null + IterativeParsingElementState // Number + }, + // MemberDelimiter + { + IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingObjectFinishState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberKeyState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // KeyValueDelimiter + { + IterativeParsingArrayInitialState, // Left bracket(push MemberValue state) + IterativeParsingErrorState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket(push MemberValue state) + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberValueState, // String + IterativeParsingMemberValueState, // False + IterativeParsingMemberValueState, // True + IterativeParsingMemberValueState, // Null + IterativeParsingMemberValueState // Number + }, + }; // End of G + + return static_cast(G[state][token]); + } + + // Make an advance in the token stream and state based on the candidate destination state which was returned by Transit(). + // May return a new state on state pop. + template + CEREAL_RAPIDJSON_FORCEINLINE IterativeParsingState Transit(IterativeParsingState src, Token token, IterativeParsingState dst, InputStream& is, Handler& handler) { + (void)token; + + switch (dst) { + case IterativeParsingErrorState: + return dst; + + case IterativeParsingObjectInitialState: + case IterativeParsingArrayInitialState: + { + // Push the state(Element or MemeberValue) if we are nested in another array or value of member. + // In this way we can get the correct state on ObjectFinish or ArrayFinish by frame pop. + IterativeParsingState n = src; + if (src == IterativeParsingArrayInitialState || src == IterativeParsingElementDelimiterState) + n = IterativeParsingElementState; + else if (src == IterativeParsingKeyValueDelimiterState) + n = IterativeParsingMemberValueState; + // Push current state. + *stack_.template Push(1) = n; + // Initialize and push the member/element count. + *stack_.template Push(1) = 0; + // Call handler + bool hr = (dst == IterativeParsingObjectInitialState) ? handler.StartObject() : handler.StartArray(); + // On handler short circuits the parsing. + if (!hr) { + CEREAL_RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); + return IterativeParsingErrorState; + } + else { + is.Take(); + return dst; + } + } + + case IterativeParsingMemberKeyState: + ParseString(is, handler, true); + if (HasParseError()) + return IterativeParsingErrorState; + else + return dst; + + case IterativeParsingKeyValueDelimiterState: + CEREAL_RAPIDJSON_ASSERT(token == ColonToken); + is.Take(); + return dst; + + case IterativeParsingMemberValueState: + // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. + ParseValue(is, handler); + if (HasParseError()) { + return IterativeParsingErrorState; + } + return dst; + + case IterativeParsingElementState: + // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. + ParseValue(is, handler); + if (HasParseError()) { + return IterativeParsingErrorState; + } + return dst; + + case IterativeParsingMemberDelimiterState: + case IterativeParsingElementDelimiterState: + is.Take(); + // Update member/element count. + *stack_.template Top() = *stack_.template Top() + 1; + return dst; + + case IterativeParsingObjectFinishState: + { + // Transit from delimiter is only allowed when trailing commas are enabled + if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingMemberDelimiterState) { + CEREAL_RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorObjectMissName, is.Tell()); + return IterativeParsingErrorState; + } + // Get member count. + SizeType c = *stack_.template Pop(1); + // If the object is not empty, count the last member. + if (src == IterativeParsingMemberValueState) + ++c; + // Restore the state. + IterativeParsingState n = static_cast(*stack_.template Pop(1)); + // Transit to Finish state if this is the topmost scope. + if (n == IterativeParsingStartState) + n = IterativeParsingFinishState; + // Call handler + bool hr = handler.EndObject(c); + // On handler short circuits the parsing. + if (!hr) { + CEREAL_RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); + return IterativeParsingErrorState; + } + else { + is.Take(); + return n; + } + } + + case IterativeParsingArrayFinishState: + { + // Transit from delimiter is only allowed when trailing commas are enabled + if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingElementDelimiterState) { + CEREAL_RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorValueInvalid, is.Tell()); + return IterativeParsingErrorState; + } + // Get element count. + SizeType c = *stack_.template Pop(1); + // If the array is not empty, count the last element. + if (src == IterativeParsingElementState) + ++c; + // Restore the state. + IterativeParsingState n = static_cast(*stack_.template Pop(1)); + // Transit to Finish state if this is the topmost scope. + if (n == IterativeParsingStartState) + n = IterativeParsingFinishState; + // Call handler + bool hr = handler.EndArray(c); + // On handler short circuits the parsing. + if (!hr) { + CEREAL_RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); + return IterativeParsingErrorState; + } + else { + is.Take(); + return n; + } + } + + default: + // This branch is for IterativeParsingValueState actually. + // Use `default:` rather than + // `case IterativeParsingValueState:` is for code coverage. + + // The IterativeParsingStartState is not enumerated in this switch-case. + // It is impossible for that case. And it can be caught by following assertion. + + // The IterativeParsingFinishState is not enumerated in this switch-case either. + // It is a "derivative" state which cannot triggered from Predict() directly. + // Therefore it cannot happen here. And it can be caught by following assertion. + CEREAL_RAPIDJSON_ASSERT(dst == IterativeParsingValueState); + + // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. + ParseValue(is, handler); + if (HasParseError()) { + return IterativeParsingErrorState; + } + return IterativeParsingFinishState; + } + } + + template + void HandleError(IterativeParsingState src, InputStream& is) { + if (HasParseError()) { + // Error flag has been set. + return; + } + + switch (src) { + case IterativeParsingStartState: CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorDocumentEmpty, is.Tell()); return; + case IterativeParsingFinishState: CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotSingular, is.Tell()); return; + case IterativeParsingObjectInitialState: + case IterativeParsingMemberDelimiterState: CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); return; + case IterativeParsingMemberKeyState: CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); return; + case IterativeParsingMemberValueState: CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); return; + case IterativeParsingKeyValueDelimiterState: + case IterativeParsingArrayInitialState: + case IterativeParsingElementDelimiterState: CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); return; + default: CEREAL_RAPIDJSON_ASSERT(src == IterativeParsingElementState); CEREAL_RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); return; + } + } + + CEREAL_RAPIDJSON_FORCEINLINE bool IsIterativeParsingDelimiterState(IterativeParsingState s) const { + return s >= IterativeParsingElementDelimiterState; + } + + CEREAL_RAPIDJSON_FORCEINLINE bool IsIterativeParsingCompleteState(IterativeParsingState s) const { + return s <= IterativeParsingErrorState; + } + + template + ParseResult IterativeParse(InputStream& is, Handler& handler) { + parseResult_.Clear(); + ClearStackOnExit scope(*this); + IterativeParsingState state = IterativeParsingStartState; + + SkipWhitespaceAndComments(is); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + while (is.Peek() != '\0') { + Token t = Tokenize(is.Peek()); + IterativeParsingState n = Predict(state, t); + IterativeParsingState d = Transit(state, t, n, is, handler); + + if (d == IterativeParsingErrorState) { + HandleError(state, is); + break; + } + + state = d; + + // Do not further consume streams if a root JSON has been parsed. + if ((parseFlags & kParseStopWhenDoneFlag) && state == IterativeParsingFinishState) + break; + + SkipWhitespaceAndComments(is); + CEREAL_RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + } + + // Handle the end of file. + if (state != IterativeParsingFinishState) + HandleError(state, is); + + return parseResult_; + } + + static const size_t kDefaultStackCapacity = 256; //!< Default stack capacity in bytes for storing a single decoded string. + internal::Stack stack_; //!< A stack for storing decoded string temporarily during non-destructive parsing. + ParseResult parseResult_; + IterativeParsingState state_; +}; // class GenericReader + +//! Reader with UTF8 encoding and default allocator. +typedef GenericReader, UTF8<> > Reader; + +CEREAL_RAPIDJSON_NAMESPACE_END + +#if defined(__clang__) || defined(_MSC_VER) +CEREAL_RAPIDJSON_DIAG_POP +#endif + + +#ifdef __GNUC__ +CEREAL_RAPIDJSON_DIAG_POP +#endif + +#endif // CEREAL_RAPIDJSON_READER_H_ diff --git a/third_party/cereal/external/rapidjson/schema.h b/third_party/cereal/external/rapidjson/schema.h new file mode 100755 index 0000000..17366c6 --- /dev/null +++ b/third_party/cereal/external/rapidjson/schema.h @@ -0,0 +1,2497 @@ +// Tencent is pleased to support the open source community by making RapidJSON available-> +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip-> All rights reserved-> +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License-> You may obtain a copy of the License at +// +// http://opensource->org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied-> See the License for the +// specific language governing permissions and limitations under the License-> + +#ifndef CEREAL_RAPIDJSON_SCHEMA_H_ +#define CEREAL_RAPIDJSON_SCHEMA_H_ + +#include "document.h" +#include "pointer.h" +#include "stringbuffer.h" +#include // abs, floor + +#if !defined(CEREAL_RAPIDJSON_SCHEMA_USE_INTERNALREGEX) +#define CEREAL_RAPIDJSON_SCHEMA_USE_INTERNALREGEX 1 +#else +#define CEREAL_RAPIDJSON_SCHEMA_USE_INTERNALREGEX 0 +#endif + +#if !CEREAL_RAPIDJSON_SCHEMA_USE_INTERNALREGEX && defined(CEREAL_RAPIDJSON_SCHEMA_USE_STDREGEX) && (__cplusplus >=201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)) +#define CEREAL_RAPIDJSON_SCHEMA_USE_STDREGEX 1 +#else +#define CEREAL_RAPIDJSON_SCHEMA_USE_STDREGEX 0 +#endif + +#if CEREAL_RAPIDJSON_SCHEMA_USE_INTERNALREGEX +#include "internal/regex.h" +#elif CEREAL_RAPIDJSON_SCHEMA_USE_STDREGEX +#include +#endif + +#if CEREAL_RAPIDJSON_SCHEMA_USE_INTERNALREGEX || CEREAL_RAPIDJSON_SCHEMA_USE_STDREGEX +#define CEREAL_RAPIDJSON_SCHEMA_HAS_REGEX 1 +#else +#define CEREAL_RAPIDJSON_SCHEMA_HAS_REGEX 0 +#endif + +#ifndef CEREAL_RAPIDJSON_SCHEMA_VERBOSE +#define CEREAL_RAPIDJSON_SCHEMA_VERBOSE 0 +#endif + +#if CEREAL_RAPIDJSON_SCHEMA_VERBOSE +#include "stringbuffer.h" +#endif + +CEREAL_RAPIDJSON_DIAG_PUSH + +#if defined(__GNUC__) +CEREAL_RAPIDJSON_DIAG_OFF(effc++) +#endif + +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_OFF(weak-vtables) +CEREAL_RAPIDJSON_DIAG_OFF(exit-time-destructors) +CEREAL_RAPIDJSON_DIAG_OFF(c++98-compat-pedantic) +CEREAL_RAPIDJSON_DIAG_OFF(variadic-macros) +#elif defined(_MSC_VER) +CEREAL_RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// Verbose Utilities + +#if CEREAL_RAPIDJSON_SCHEMA_VERBOSE + +namespace internal { + +inline void PrintInvalidKeyword(const char* keyword) { + printf("Fail keyword: %s\n", keyword); +} + +inline void PrintInvalidKeyword(const wchar_t* keyword) { + wprintf(L"Fail keyword: %ls\n", keyword); +} + +inline void PrintInvalidDocument(const char* document) { + printf("Fail document: %s\n\n", document); +} + +inline void PrintInvalidDocument(const wchar_t* document) { + wprintf(L"Fail document: %ls\n\n", document); +} + +inline void PrintValidatorPointers(unsigned depth, const char* s, const char* d) { + printf("S: %*s%s\nD: %*s%s\n\n", depth * 4, " ", s, depth * 4, " ", d); +} + +inline void PrintValidatorPointers(unsigned depth, const wchar_t* s, const wchar_t* d) { + wprintf(L"S: %*ls%ls\nD: %*ls%ls\n\n", depth * 4, L" ", s, depth * 4, L" ", d); +} + +} // namespace internal + +#endif // CEREAL_RAPIDJSON_SCHEMA_VERBOSE + +/////////////////////////////////////////////////////////////////////////////// +// CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN + +#if CEREAL_RAPIDJSON_SCHEMA_VERBOSE +#define CEREAL_RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword) internal::PrintInvalidKeyword(keyword) +#else +#define CEREAL_RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword) +#endif + +#define CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(keyword)\ +CEREAL_RAPIDJSON_MULTILINEMACRO_BEGIN\ + context.invalidKeyword = keyword.GetString();\ + CEREAL_RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword.GetString());\ + return false;\ +CEREAL_RAPIDJSON_MULTILINEMACRO_END + +/////////////////////////////////////////////////////////////////////////////// +// Forward declarations + +template +class GenericSchemaDocument; + +namespace internal { + +template +class Schema; + +/////////////////////////////////////////////////////////////////////////////// +// ISchemaValidator + +class ISchemaValidator { +public: + virtual ~ISchemaValidator() {} + virtual bool IsValid() const = 0; +}; + +/////////////////////////////////////////////////////////////////////////////// +// ISchemaStateFactory + +template +class ISchemaStateFactory { +public: + virtual ~ISchemaStateFactory() {} + virtual ISchemaValidator* CreateSchemaValidator(const SchemaType&) = 0; + virtual void DestroySchemaValidator(ISchemaValidator* validator) = 0; + virtual void* CreateHasher() = 0; + virtual uint64_t GetHashCode(void* hasher) = 0; + virtual void DestroryHasher(void* hasher) = 0; + virtual void* MallocState(size_t size) = 0; + virtual void FreeState(void* p) = 0; +}; + +/////////////////////////////////////////////////////////////////////////////// +// IValidationErrorHandler + +template +class IValidationErrorHandler { +public: + typedef typename SchemaType::Ch Ch; + typedef typename SchemaType::SValue SValue; + + virtual ~IValidationErrorHandler() {} + + virtual void NotMultipleOf(int64_t actual, const SValue& expected) = 0; + virtual void NotMultipleOf(uint64_t actual, const SValue& expected) = 0; + virtual void NotMultipleOf(double actual, const SValue& expected) = 0; + virtual void AboveMaximum(int64_t actual, const SValue& expected, bool exclusive) = 0; + virtual void AboveMaximum(uint64_t actual, const SValue& expected, bool exclusive) = 0; + virtual void AboveMaximum(double actual, const SValue& expected, bool exclusive) = 0; + virtual void BelowMinimum(int64_t actual, const SValue& expected, bool exclusive) = 0; + virtual void BelowMinimum(uint64_t actual, const SValue& expected, bool exclusive) = 0; + virtual void BelowMinimum(double actual, const SValue& expected, bool exclusive) = 0; + + virtual void TooLong(const Ch* str, SizeType length, SizeType expected) = 0; + virtual void TooShort(const Ch* str, SizeType length, SizeType expected) = 0; + virtual void DoesNotMatch(const Ch* str, SizeType length) = 0; + + virtual void DisallowedItem(SizeType index) = 0; + virtual void TooFewItems(SizeType actualCount, SizeType expectedCount) = 0; + virtual void TooManyItems(SizeType actualCount, SizeType expectedCount) = 0; + virtual void DuplicateItems(SizeType index1, SizeType index2) = 0; + + virtual void TooManyProperties(SizeType actualCount, SizeType expectedCount) = 0; + virtual void TooFewProperties(SizeType actualCount, SizeType expectedCount) = 0; + virtual void StartMissingProperties() = 0; + virtual void AddMissingProperty(const SValue& name) = 0; + virtual bool EndMissingProperties() = 0; + virtual void PropertyViolations(ISchemaValidator** subvalidators, SizeType count) = 0; + virtual void DisallowedProperty(const Ch* name, SizeType length) = 0; + + virtual void StartDependencyErrors() = 0; + virtual void StartMissingDependentProperties() = 0; + virtual void AddMissingDependentProperty(const SValue& targetName) = 0; + virtual void EndMissingDependentProperties(const SValue& sourceName) = 0; + virtual void AddDependencySchemaError(const SValue& souceName, ISchemaValidator* subvalidator) = 0; + virtual bool EndDependencyErrors() = 0; + + virtual void DisallowedValue() = 0; + virtual void StartDisallowedType() = 0; + virtual void AddExpectedType(const typename SchemaType::ValueType& expectedType) = 0; + virtual void EndDisallowedType(const typename SchemaType::ValueType& actualType) = 0; + virtual void NotAllOf(ISchemaValidator** subvalidators, SizeType count) = 0; + virtual void NoneOf(ISchemaValidator** subvalidators, SizeType count) = 0; + virtual void NotOneOf(ISchemaValidator** subvalidators, SizeType count) = 0; + virtual void Disallowed() = 0; +}; + + +/////////////////////////////////////////////////////////////////////////////// +// Hasher + +// For comparison of compound value +template +class Hasher { +public: + typedef typename Encoding::Ch Ch; + + Hasher(Allocator* allocator = 0, size_t stackCapacity = kDefaultSize) : stack_(allocator, stackCapacity) {} + + bool Null() { return WriteType(kNullType); } + bool Bool(bool b) { return WriteType(b ? kTrueType : kFalseType); } + bool Int(int i) { Number n; n.u.i = i; n.d = static_cast(i); return WriteNumber(n); } + bool Uint(unsigned u) { Number n; n.u.u = u; n.d = static_cast(u); return WriteNumber(n); } + bool Int64(int64_t i) { Number n; n.u.i = i; n.d = static_cast(i); return WriteNumber(n); } + bool Uint64(uint64_t u) { Number n; n.u.u = u; n.d = static_cast(u); return WriteNumber(n); } + bool Double(double d) { + Number n; + if (d < 0) n.u.i = static_cast(d); + else n.u.u = static_cast(d); + n.d = d; + return WriteNumber(n); + } + + bool RawNumber(const Ch* str, SizeType len, bool) { + WriteBuffer(kNumberType, str, len * sizeof(Ch)); + return true; + } + + bool String(const Ch* str, SizeType len, bool) { + WriteBuffer(kStringType, str, len * sizeof(Ch)); + return true; + } + + bool StartObject() { return true; } + bool Key(const Ch* str, SizeType len, bool copy) { return String(str, len, copy); } + bool EndObject(SizeType memberCount) { + uint64_t h = Hash(0, kObjectType); + uint64_t* kv = stack_.template Pop(memberCount * 2); + for (SizeType i = 0; i < memberCount; i++) + h ^= Hash(kv[i * 2], kv[i * 2 + 1]); // Use xor to achieve member order insensitive + *stack_.template Push() = h; + return true; + } + + bool StartArray() { return true; } + bool EndArray(SizeType elementCount) { + uint64_t h = Hash(0, kArrayType); + uint64_t* e = stack_.template Pop(elementCount); + for (SizeType i = 0; i < elementCount; i++) + h = Hash(h, e[i]); // Use hash to achieve element order sensitive + *stack_.template Push() = h; + return true; + } + + bool IsValid() const { return stack_.GetSize() == sizeof(uint64_t); } + + uint64_t GetHashCode() const { + CEREAL_RAPIDJSON_ASSERT(IsValid()); + return *stack_.template Top(); + } + +private: + static const size_t kDefaultSize = 256; + struct Number { + union U { + uint64_t u; + int64_t i; + }u; + double d; + }; + + bool WriteType(Type type) { return WriteBuffer(type, 0, 0); } + + bool WriteNumber(const Number& n) { return WriteBuffer(kNumberType, &n, sizeof(n)); } + + bool WriteBuffer(Type type, const void* data, size_t len) { + // FNV-1a from http://isthe.com/chongo/tech/comp/fnv/ + uint64_t h = Hash(CEREAL_RAPIDJSON_UINT64_C2(0x84222325, 0xcbf29ce4), type); + const unsigned char* d = static_cast(data); + for (size_t i = 0; i < len; i++) + h = Hash(h, d[i]); + *stack_.template Push() = h; + return true; + } + + static uint64_t Hash(uint64_t h, uint64_t d) { + static const uint64_t kPrime = CEREAL_RAPIDJSON_UINT64_C2(0x00000100, 0x000001b3); + h ^= d; + h *= kPrime; + return h; + } + + Stack stack_; +}; + +/////////////////////////////////////////////////////////////////////////////// +// SchemaValidationContext + +template +struct SchemaValidationContext { + typedef Schema SchemaType; + typedef ISchemaStateFactory SchemaValidatorFactoryType; + typedef IValidationErrorHandler ErrorHandlerType; + typedef typename SchemaType::ValueType ValueType; + typedef typename ValueType::Ch Ch; + + enum PatternValidatorType { + kPatternValidatorOnly, + kPatternValidatorWithProperty, + kPatternValidatorWithAdditionalProperty + }; + + SchemaValidationContext(SchemaValidatorFactoryType& f, ErrorHandlerType& eh, const SchemaType* s) : + factory(f), + error_handler(eh), + schema(s), + valueSchema(), + invalidKeyword(), + hasher(), + arrayElementHashCodes(), + validators(), + validatorCount(), + patternPropertiesValidators(), + patternPropertiesValidatorCount(), + patternPropertiesSchemas(), + patternPropertiesSchemaCount(), + valuePatternValidatorType(kPatternValidatorOnly), + propertyExist(), + inArray(false), + valueUniqueness(false), + arrayUniqueness(false) + { + } + + ~SchemaValidationContext() { + if (hasher) + factory.DestroryHasher(hasher); + if (validators) { + for (SizeType i = 0; i < validatorCount; i++) + factory.DestroySchemaValidator(validators[i]); + factory.FreeState(validators); + } + if (patternPropertiesValidators) { + for (SizeType i = 0; i < patternPropertiesValidatorCount; i++) + factory.DestroySchemaValidator(patternPropertiesValidators[i]); + factory.FreeState(patternPropertiesValidators); + } + if (patternPropertiesSchemas) + factory.FreeState(patternPropertiesSchemas); + if (propertyExist) + factory.FreeState(propertyExist); + } + + SchemaValidatorFactoryType& factory; + ErrorHandlerType& error_handler; + const SchemaType* schema; + const SchemaType* valueSchema; + const Ch* invalidKeyword; + void* hasher; // Only validator access + void* arrayElementHashCodes; // Only validator access this + ISchemaValidator** validators; + SizeType validatorCount; + ISchemaValidator** patternPropertiesValidators; + SizeType patternPropertiesValidatorCount; + const SchemaType** patternPropertiesSchemas; + SizeType patternPropertiesSchemaCount; + PatternValidatorType valuePatternValidatorType; + PatternValidatorType objectPatternValidatorType; + SizeType arrayElementIndex; + bool* propertyExist; + bool inArray; + bool valueUniqueness; + bool arrayUniqueness; +}; + +/////////////////////////////////////////////////////////////////////////////// +// Schema + +template +class Schema { +public: + typedef typename SchemaDocumentType::ValueType ValueType; + typedef typename SchemaDocumentType::AllocatorType AllocatorType; + typedef typename SchemaDocumentType::PointerType PointerType; + typedef typename ValueType::EncodingType EncodingType; + typedef typename EncodingType::Ch Ch; + typedef SchemaValidationContext Context; + typedef Schema SchemaType; + typedef GenericValue SValue; + typedef IValidationErrorHandler ErrorHandler; + friend class GenericSchemaDocument; + + Schema(SchemaDocumentType* schemaDocument, const PointerType& p, const ValueType& value, const ValueType& document, AllocatorType* allocator) : + allocator_(allocator), + uri_(schemaDocument->GetURI(), *allocator), + pointer_(p, allocator), + typeless_(schemaDocument->GetTypeless()), + enum_(), + enumCount_(), + not_(), + type_((1 << kTotalSchemaType) - 1), // typeless + validatorCount_(), + notValidatorIndex_(), + properties_(), + additionalPropertiesSchema_(), + patternProperties_(), + patternPropertyCount_(), + propertyCount_(), + minProperties_(), + maxProperties_(SizeType(~0)), + additionalProperties_(true), + hasDependencies_(), + hasRequired_(), + hasSchemaDependencies_(), + additionalItemsSchema_(), + itemsList_(), + itemsTuple_(), + itemsTupleCount_(), + minItems_(), + maxItems_(SizeType(~0)), + additionalItems_(true), + uniqueItems_(false), + pattern_(), + minLength_(0), + maxLength_(~SizeType(0)), + exclusiveMinimum_(false), + exclusiveMaximum_(false), + defaultValueLength_(0) + { + typedef typename SchemaDocumentType::ValueType ValueType; + typedef typename ValueType::ConstValueIterator ConstValueIterator; + typedef typename ValueType::ConstMemberIterator ConstMemberIterator; + + if (!value.IsObject()) + return; + + if (const ValueType* v = GetMember(value, GetTypeString())) { + type_ = 0; + if (v->IsString()) + AddType(*v); + else if (v->IsArray()) + for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) + AddType(*itr); + } + + if (const ValueType* v = GetMember(value, GetEnumString())) + if (v->IsArray() && v->Size() > 0) { + enum_ = static_cast(allocator_->Malloc(sizeof(uint64_t) * v->Size())); + for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) { + typedef Hasher > EnumHasherType; + char buffer[256u + 24]; + MemoryPoolAllocator<> hasherAllocator(buffer, sizeof(buffer)); + EnumHasherType h(&hasherAllocator, 256); + itr->Accept(h); + enum_[enumCount_++] = h.GetHashCode(); + } + } + + if (schemaDocument) { + AssignIfExist(allOf_, *schemaDocument, p, value, GetAllOfString(), document); + AssignIfExist(anyOf_, *schemaDocument, p, value, GetAnyOfString(), document); + AssignIfExist(oneOf_, *schemaDocument, p, value, GetOneOfString(), document); + } + + if (const ValueType* v = GetMember(value, GetNotString())) { + schemaDocument->CreateSchema(¬_, p.Append(GetNotString(), allocator_), *v, document); + notValidatorIndex_ = validatorCount_; + validatorCount_++; + } + + // Object + + const ValueType* properties = GetMember(value, GetPropertiesString()); + const ValueType* required = GetMember(value, GetRequiredString()); + const ValueType* dependencies = GetMember(value, GetDependenciesString()); + { + // Gather properties from properties/required/dependencies + SValue allProperties(kArrayType); + + if (properties && properties->IsObject()) + for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) + AddUniqueElement(allProperties, itr->name); + + if (required && required->IsArray()) + for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) + if (itr->IsString()) + AddUniqueElement(allProperties, *itr); + + if (dependencies && dependencies->IsObject()) + for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { + AddUniqueElement(allProperties, itr->name); + if (itr->value.IsArray()) + for (ConstValueIterator i = itr->value.Begin(); i != itr->value.End(); ++i) + if (i->IsString()) + AddUniqueElement(allProperties, *i); + } + + if (allProperties.Size() > 0) { + propertyCount_ = allProperties.Size(); + properties_ = static_cast(allocator_->Malloc(sizeof(Property) * propertyCount_)); + for (SizeType i = 0; i < propertyCount_; i++) { + new (&properties_[i]) Property(); + properties_[i].name = allProperties[i]; + properties_[i].schema = typeless_; + } + } + } + + if (properties && properties->IsObject()) { + PointerType q = p.Append(GetPropertiesString(), allocator_); + for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) { + SizeType index; + if (FindPropertyIndex(itr->name, &index)) + schemaDocument->CreateSchema(&properties_[index].schema, q.Append(itr->name, allocator_), itr->value, document); + } + } + + if (const ValueType* v = GetMember(value, GetPatternPropertiesString())) { + PointerType q = p.Append(GetPatternPropertiesString(), allocator_); + patternProperties_ = static_cast(allocator_->Malloc(sizeof(PatternProperty) * v->MemberCount())); + patternPropertyCount_ = 0; + + for (ConstMemberIterator itr = v->MemberBegin(); itr != v->MemberEnd(); ++itr) { + new (&patternProperties_[patternPropertyCount_]) PatternProperty(); + patternProperties_[patternPropertyCount_].pattern = CreatePattern(itr->name); + schemaDocument->CreateSchema(&patternProperties_[patternPropertyCount_].schema, q.Append(itr->name, allocator_), itr->value, document); + patternPropertyCount_++; + } + } + + if (required && required->IsArray()) + for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) + if (itr->IsString()) { + SizeType index; + if (FindPropertyIndex(*itr, &index)) { + properties_[index].required = true; + hasRequired_ = true; + } + } + + if (dependencies && dependencies->IsObject()) { + PointerType q = p.Append(GetDependenciesString(), allocator_); + hasDependencies_ = true; + for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { + SizeType sourceIndex; + if (FindPropertyIndex(itr->name, &sourceIndex)) { + if (itr->value.IsArray()) { + properties_[sourceIndex].dependencies = static_cast(allocator_->Malloc(sizeof(bool) * propertyCount_)); + std::memset(properties_[sourceIndex].dependencies, 0, sizeof(bool)* propertyCount_); + for (ConstValueIterator targetItr = itr->value.Begin(); targetItr != itr->value.End(); ++targetItr) { + SizeType targetIndex; + if (FindPropertyIndex(*targetItr, &targetIndex)) + properties_[sourceIndex].dependencies[targetIndex] = true; + } + } + else if (itr->value.IsObject()) { + hasSchemaDependencies_ = true; + schemaDocument->CreateSchema(&properties_[sourceIndex].dependenciesSchema, q.Append(itr->name, allocator_), itr->value, document); + properties_[sourceIndex].dependenciesValidatorIndex = validatorCount_; + validatorCount_++; + } + } + } + } + + if (const ValueType* v = GetMember(value, GetAdditionalPropertiesString())) { + if (v->IsBool()) + additionalProperties_ = v->GetBool(); + else if (v->IsObject()) + schemaDocument->CreateSchema(&additionalPropertiesSchema_, p.Append(GetAdditionalPropertiesString(), allocator_), *v, document); + } + + AssignIfExist(minProperties_, value, GetMinPropertiesString()); + AssignIfExist(maxProperties_, value, GetMaxPropertiesString()); + + // Array + if (const ValueType* v = GetMember(value, GetItemsString())) { + PointerType q = p.Append(GetItemsString(), allocator_); + if (v->IsObject()) // List validation + schemaDocument->CreateSchema(&itemsList_, q, *v, document); + else if (v->IsArray()) { // Tuple validation + itemsTuple_ = static_cast(allocator_->Malloc(sizeof(const Schema*) * v->Size())); + SizeType index = 0; + for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr, index++) + schemaDocument->CreateSchema(&itemsTuple_[itemsTupleCount_++], q.Append(index, allocator_), *itr, document); + } + } + + AssignIfExist(minItems_, value, GetMinItemsString()); + AssignIfExist(maxItems_, value, GetMaxItemsString()); + + if (const ValueType* v = GetMember(value, GetAdditionalItemsString())) { + if (v->IsBool()) + additionalItems_ = v->GetBool(); + else if (v->IsObject()) + schemaDocument->CreateSchema(&additionalItemsSchema_, p.Append(GetAdditionalItemsString(), allocator_), *v, document); + } + + AssignIfExist(uniqueItems_, value, GetUniqueItemsString()); + + // String + AssignIfExist(minLength_, value, GetMinLengthString()); + AssignIfExist(maxLength_, value, GetMaxLengthString()); + + if (const ValueType* v = GetMember(value, GetPatternString())) + pattern_ = CreatePattern(*v); + + // Number + if (const ValueType* v = GetMember(value, GetMinimumString())) + if (v->IsNumber()) + minimum_.CopyFrom(*v, *allocator_); + + if (const ValueType* v = GetMember(value, GetMaximumString())) + if (v->IsNumber()) + maximum_.CopyFrom(*v, *allocator_); + + AssignIfExist(exclusiveMinimum_, value, GetExclusiveMinimumString()); + AssignIfExist(exclusiveMaximum_, value, GetExclusiveMaximumString()); + + if (const ValueType* v = GetMember(value, GetMultipleOfString())) + if (v->IsNumber() && v->GetDouble() > 0.0) + multipleOf_.CopyFrom(*v, *allocator_); + + // Default + if (const ValueType* v = GetMember(value, GetDefaultValueString())) + if (v->IsString()) + defaultValueLength_ = v->GetStringLength(); + + } + + ~Schema() { + AllocatorType::Free(enum_); + if (properties_) { + for (SizeType i = 0; i < propertyCount_; i++) + properties_[i].~Property(); + AllocatorType::Free(properties_); + } + if (patternProperties_) { + for (SizeType i = 0; i < patternPropertyCount_; i++) + patternProperties_[i].~PatternProperty(); + AllocatorType::Free(patternProperties_); + } + AllocatorType::Free(itemsTuple_); +#if CEREAL_RAPIDJSON_SCHEMA_HAS_REGEX + if (pattern_) { + pattern_->~RegexType(); + AllocatorType::Free(pattern_); + } +#endif + } + + const SValue& GetURI() const { + return uri_; + } + + const PointerType& GetPointer() const { + return pointer_; + } + + bool BeginValue(Context& context) const { + if (context.inArray) { + if (uniqueItems_) + context.valueUniqueness = true; + + if (itemsList_) + context.valueSchema = itemsList_; + else if (itemsTuple_) { + if (context.arrayElementIndex < itemsTupleCount_) + context.valueSchema = itemsTuple_[context.arrayElementIndex]; + else if (additionalItemsSchema_) + context.valueSchema = additionalItemsSchema_; + else if (additionalItems_) + context.valueSchema = typeless_; + else { + context.error_handler.DisallowedItem(context.arrayElementIndex); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetItemsString()); + } + } + else + context.valueSchema = typeless_; + + context.arrayElementIndex++; + } + return true; + } + + CEREAL_RAPIDJSON_FORCEINLINE bool EndValue(Context& context) const { + if (context.patternPropertiesValidatorCount > 0) { + bool otherValid = false; + SizeType count = context.patternPropertiesValidatorCount; + if (context.objectPatternValidatorType != Context::kPatternValidatorOnly) + otherValid = context.patternPropertiesValidators[--count]->IsValid(); + + bool patternValid = true; + for (SizeType i = 0; i < count; i++) + if (!context.patternPropertiesValidators[i]->IsValid()) { + patternValid = false; + break; + } + + if (context.objectPatternValidatorType == Context::kPatternValidatorOnly) { + if (!patternValid) { + context.error_handler.PropertyViolations(context.patternPropertiesValidators, count); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } + } + else if (context.objectPatternValidatorType == Context::kPatternValidatorWithProperty) { + if (!patternValid || !otherValid) { + context.error_handler.PropertyViolations(context.patternPropertiesValidators, count + 1); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } + } + else if (!patternValid && !otherValid) { // kPatternValidatorWithAdditionalProperty) + context.error_handler.PropertyViolations(context.patternPropertiesValidators, count + 1); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } + } + + if (enum_) { + const uint64_t h = context.factory.GetHashCode(context.hasher); + for (SizeType i = 0; i < enumCount_; i++) + if (enum_[i] == h) + goto foundEnum; + context.error_handler.DisallowedValue(); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetEnumString()); + foundEnum:; + } + + if (allOf_.schemas) + for (SizeType i = allOf_.begin; i < allOf_.begin + allOf_.count; i++) + if (!context.validators[i]->IsValid()) { + context.error_handler.NotAllOf(&context.validators[allOf_.begin], allOf_.count); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetAllOfString()); + } + + if (anyOf_.schemas) { + for (SizeType i = anyOf_.begin; i < anyOf_.begin + anyOf_.count; i++) + if (context.validators[i]->IsValid()) + goto foundAny; + context.error_handler.NoneOf(&context.validators[anyOf_.begin], anyOf_.count); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetAnyOfString()); + foundAny:; + } + + if (oneOf_.schemas) { + bool oneValid = false; + for (SizeType i = oneOf_.begin; i < oneOf_.begin + oneOf_.count; i++) + if (context.validators[i]->IsValid()) { + if (oneValid) { + context.error_handler.NotOneOf(&context.validators[oneOf_.begin], oneOf_.count); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); + } else + oneValid = true; + } + if (!oneValid) { + context.error_handler.NotOneOf(&context.validators[oneOf_.begin], oneOf_.count); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); + } + } + + if (not_ && context.validators[notValidatorIndex_]->IsValid()) { + context.error_handler.Disallowed(); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetNotString()); + } + + return true; + } + + bool Null(Context& context) const { + if (!(type_ & (1 << kNullSchemaType))) { + DisallowedType(context, GetNullString()); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } + return CreateParallelValidator(context); + } + + bool Bool(Context& context, bool) const { + if (!(type_ & (1 << kBooleanSchemaType))) { + DisallowedType(context, GetBooleanString()); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } + return CreateParallelValidator(context); + } + + bool Int(Context& context, int i) const { + if (!CheckInt(context, i)) + return false; + return CreateParallelValidator(context); + } + + bool Uint(Context& context, unsigned u) const { + if (!CheckUint(context, u)) + return false; + return CreateParallelValidator(context); + } + + bool Int64(Context& context, int64_t i) const { + if (!CheckInt(context, i)) + return false; + return CreateParallelValidator(context); + } + + bool Uint64(Context& context, uint64_t u) const { + if (!CheckUint(context, u)) + return false; + return CreateParallelValidator(context); + } + + bool Double(Context& context, double d) const { + if (!(type_ & (1 << kNumberSchemaType))) { + DisallowedType(context, GetNumberString()); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } + + if (!minimum_.IsNull() && !CheckDoubleMinimum(context, d)) + return false; + + if (!maximum_.IsNull() && !CheckDoubleMaximum(context, d)) + return false; + + if (!multipleOf_.IsNull() && !CheckDoubleMultipleOf(context, d)) + return false; + + return CreateParallelValidator(context); + } + + bool String(Context& context, const Ch* str, SizeType length, bool) const { + if (!(type_ & (1 << kStringSchemaType))) { + DisallowedType(context, GetStringString()); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } + + if (minLength_ != 0 || maxLength_ != SizeType(~0)) { + SizeType count; + if (internal::CountStringCodePoint(str, length, &count)) { + if (count < minLength_) { + context.error_handler.TooShort(str, length, minLength_); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinLengthString()); + } + if (count > maxLength_) { + context.error_handler.TooLong(str, length, maxLength_); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxLengthString()); + } + } + } + + if (pattern_ && !IsPatternMatch(pattern_, str, length)) { + context.error_handler.DoesNotMatch(str, length); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternString()); + } + + return CreateParallelValidator(context); + } + + bool StartObject(Context& context) const { + if (!(type_ & (1 << kObjectSchemaType))) { + DisallowedType(context, GetObjectString()); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } + + if (hasDependencies_ || hasRequired_) { + context.propertyExist = static_cast(context.factory.MallocState(sizeof(bool) * propertyCount_)); + std::memset(context.propertyExist, 0, sizeof(bool) * propertyCount_); + } + + if (patternProperties_) { // pre-allocate schema array + SizeType count = patternPropertyCount_ + 1; // extra for valuePatternValidatorType + context.patternPropertiesSchemas = static_cast(context.factory.MallocState(sizeof(const SchemaType*) * count)); + context.patternPropertiesSchemaCount = 0; + std::memset(context.patternPropertiesSchemas, 0, sizeof(SchemaType*) * count); + } + + return CreateParallelValidator(context); + } + + bool Key(Context& context, const Ch* str, SizeType len, bool) const { + if (patternProperties_) { + context.patternPropertiesSchemaCount = 0; + for (SizeType i = 0; i < patternPropertyCount_; i++) + if (patternProperties_[i].pattern && IsPatternMatch(patternProperties_[i].pattern, str, len)) { + context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = patternProperties_[i].schema; + context.valueSchema = typeless_; + } + } + + SizeType index; + if (FindPropertyIndex(ValueType(str, len).Move(), &index)) { + if (context.patternPropertiesSchemaCount > 0) { + context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = properties_[index].schema; + context.valueSchema = typeless_; + context.valuePatternValidatorType = Context::kPatternValidatorWithProperty; + } + else + context.valueSchema = properties_[index].schema; + + if (context.propertyExist) + context.propertyExist[index] = true; + + return true; + } + + if (additionalPropertiesSchema_) { + if (additionalPropertiesSchema_ && context.patternPropertiesSchemaCount > 0) { + context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = additionalPropertiesSchema_; + context.valueSchema = typeless_; + context.valuePatternValidatorType = Context::kPatternValidatorWithAdditionalProperty; + } + else + context.valueSchema = additionalPropertiesSchema_; + return true; + } + else if (additionalProperties_) { + context.valueSchema = typeless_; + return true; + } + + if (context.patternPropertiesSchemaCount == 0) { // patternProperties are not additional properties + context.error_handler.DisallowedProperty(str, len); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetAdditionalPropertiesString()); + } + + return true; + } + + bool EndObject(Context& context, SizeType memberCount) const { + if (hasRequired_) { + context.error_handler.StartMissingProperties(); + for (SizeType index = 0; index < propertyCount_; index++) + if (properties_[index].required && !context.propertyExist[index]) + if (properties_[index].schema->defaultValueLength_ == 0 ) + context.error_handler.AddMissingProperty(properties_[index].name); + if (context.error_handler.EndMissingProperties()) + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetRequiredString()); + } + + if (memberCount < minProperties_) { + context.error_handler.TooFewProperties(memberCount, minProperties_); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinPropertiesString()); + } + + if (memberCount > maxProperties_) { + context.error_handler.TooManyProperties(memberCount, maxProperties_); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxPropertiesString()); + } + + if (hasDependencies_) { + context.error_handler.StartDependencyErrors(); + for (SizeType sourceIndex = 0; sourceIndex < propertyCount_; sourceIndex++) { + const Property& source = properties_[sourceIndex]; + if (context.propertyExist[sourceIndex]) { + if (source.dependencies) { + context.error_handler.StartMissingDependentProperties(); + for (SizeType targetIndex = 0; targetIndex < propertyCount_; targetIndex++) + if (source.dependencies[targetIndex] && !context.propertyExist[targetIndex]) + context.error_handler.AddMissingDependentProperty(properties_[targetIndex].name); + context.error_handler.EndMissingDependentProperties(source.name); + } + else if (source.dependenciesSchema) { + ISchemaValidator* dependenciesValidator = context.validators[source.dependenciesValidatorIndex]; + if (!dependenciesValidator->IsValid()) + context.error_handler.AddDependencySchemaError(source.name, dependenciesValidator); + } + } + } + if (context.error_handler.EndDependencyErrors()) + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); + } + + return true; + } + + bool StartArray(Context& context) const { + if (!(type_ & (1 << kArraySchemaType))) { + DisallowedType(context, GetArrayString()); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } + + context.arrayElementIndex = 0; + context.inArray = true; + + return CreateParallelValidator(context); + } + + bool EndArray(Context& context, SizeType elementCount) const { + context.inArray = false; + + if (elementCount < minItems_) { + context.error_handler.TooFewItems(elementCount, minItems_); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinItemsString()); + } + + if (elementCount > maxItems_) { + context.error_handler.TooManyItems(elementCount, maxItems_); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxItemsString()); + } + + return true; + } + + // Generate functions for string literal according to Ch +#define CEREAL_RAPIDJSON_STRING_(name, ...) \ + static const ValueType& Get##name##String() {\ + static const Ch s[] = { __VA_ARGS__, '\0' };\ + static const ValueType v(s, static_cast(sizeof(s) / sizeof(Ch) - 1));\ + return v;\ + } + + CEREAL_RAPIDJSON_STRING_(Null, 'n', 'u', 'l', 'l') + CEREAL_RAPIDJSON_STRING_(Boolean, 'b', 'o', 'o', 'l', 'e', 'a', 'n') + CEREAL_RAPIDJSON_STRING_(Object, 'o', 'b', 'j', 'e', 'c', 't') + CEREAL_RAPIDJSON_STRING_(Array, 'a', 'r', 'r', 'a', 'y') + CEREAL_RAPIDJSON_STRING_(String, 's', 't', 'r', 'i', 'n', 'g') + CEREAL_RAPIDJSON_STRING_(Number, 'n', 'u', 'm', 'b', 'e', 'r') + CEREAL_RAPIDJSON_STRING_(Integer, 'i', 'n', 't', 'e', 'g', 'e', 'r') + CEREAL_RAPIDJSON_STRING_(Type, 't', 'y', 'p', 'e') + CEREAL_RAPIDJSON_STRING_(Enum, 'e', 'n', 'u', 'm') + CEREAL_RAPIDJSON_STRING_(AllOf, 'a', 'l', 'l', 'O', 'f') + CEREAL_RAPIDJSON_STRING_(AnyOf, 'a', 'n', 'y', 'O', 'f') + CEREAL_RAPIDJSON_STRING_(OneOf, 'o', 'n', 'e', 'O', 'f') + CEREAL_RAPIDJSON_STRING_(Not, 'n', 'o', 't') + CEREAL_RAPIDJSON_STRING_(Properties, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + CEREAL_RAPIDJSON_STRING_(Required, 'r', 'e', 'q', 'u', 'i', 'r', 'e', 'd') + CEREAL_RAPIDJSON_STRING_(Dependencies, 'd', 'e', 'p', 'e', 'n', 'd', 'e', 'n', 'c', 'i', 'e', 's') + CEREAL_RAPIDJSON_STRING_(PatternProperties, 'p', 'a', 't', 't', 'e', 'r', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + CEREAL_RAPIDJSON_STRING_(AdditionalProperties, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + CEREAL_RAPIDJSON_STRING_(MinProperties, 'm', 'i', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + CEREAL_RAPIDJSON_STRING_(MaxProperties, 'm', 'a', 'x', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + CEREAL_RAPIDJSON_STRING_(Items, 'i', 't', 'e', 'm', 's') + CEREAL_RAPIDJSON_STRING_(MinItems, 'm', 'i', 'n', 'I', 't', 'e', 'm', 's') + CEREAL_RAPIDJSON_STRING_(MaxItems, 'm', 'a', 'x', 'I', 't', 'e', 'm', 's') + CEREAL_RAPIDJSON_STRING_(AdditionalItems, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'I', 't', 'e', 'm', 's') + CEREAL_RAPIDJSON_STRING_(UniqueItems, 'u', 'n', 'i', 'q', 'u', 'e', 'I', 't', 'e', 'm', 's') + CEREAL_RAPIDJSON_STRING_(MinLength, 'm', 'i', 'n', 'L', 'e', 'n', 'g', 't', 'h') + CEREAL_RAPIDJSON_STRING_(MaxLength, 'm', 'a', 'x', 'L', 'e', 'n', 'g', 't', 'h') + CEREAL_RAPIDJSON_STRING_(Pattern, 'p', 'a', 't', 't', 'e', 'r', 'n') + CEREAL_RAPIDJSON_STRING_(Minimum, 'm', 'i', 'n', 'i', 'm', 'u', 'm') + CEREAL_RAPIDJSON_STRING_(Maximum, 'm', 'a', 'x', 'i', 'm', 'u', 'm') + CEREAL_RAPIDJSON_STRING_(ExclusiveMinimum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'i', 'n', 'i', 'm', 'u', 'm') + CEREAL_RAPIDJSON_STRING_(ExclusiveMaximum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'a', 'x', 'i', 'm', 'u', 'm') + CEREAL_RAPIDJSON_STRING_(MultipleOf, 'm', 'u', 'l', 't', 'i', 'p', 'l', 'e', 'O', 'f') + CEREAL_RAPIDJSON_STRING_(DefaultValue, 'd', 'e', 'f', 'a', 'u', 'l', 't') + +#undef CEREAL_RAPIDJSON_STRING_ + +private: + enum SchemaValueType { + kNullSchemaType, + kBooleanSchemaType, + kObjectSchemaType, + kArraySchemaType, + kStringSchemaType, + kNumberSchemaType, + kIntegerSchemaType, + kTotalSchemaType + }; + +#if CEREAL_RAPIDJSON_SCHEMA_USE_INTERNALREGEX + typedef internal::GenericRegex RegexType; +#elif CEREAL_RAPIDJSON_SCHEMA_USE_STDREGEX + typedef std::basic_regex RegexType; +#else + typedef char RegexType; +#endif + + struct SchemaArray { + SchemaArray() : schemas(), count() {} + ~SchemaArray() { AllocatorType::Free(schemas); } + const SchemaType** schemas; + SizeType begin; // begin index of context.validators + SizeType count; + }; + + template + void AddUniqueElement(V1& a, const V2& v) { + for (typename V1::ConstValueIterator itr = a.Begin(); itr != a.End(); ++itr) + if (*itr == v) + return; + V1 c(v, *allocator_); + a.PushBack(c, *allocator_); + } + + static const ValueType* GetMember(const ValueType& value, const ValueType& name) { + typename ValueType::ConstMemberIterator itr = value.FindMember(name); + return itr != value.MemberEnd() ? &(itr->value) : 0; + } + + static void AssignIfExist(bool& out, const ValueType& value, const ValueType& name) { + if (const ValueType* v = GetMember(value, name)) + if (v->IsBool()) + out = v->GetBool(); + } + + static void AssignIfExist(SizeType& out, const ValueType& value, const ValueType& name) { + if (const ValueType* v = GetMember(value, name)) + if (v->IsUint64() && v->GetUint64() <= SizeType(~0)) + out = static_cast(v->GetUint64()); + } + + void AssignIfExist(SchemaArray& out, SchemaDocumentType& schemaDocument, const PointerType& p, const ValueType& value, const ValueType& name, const ValueType& document) { + if (const ValueType* v = GetMember(value, name)) { + if (v->IsArray() && v->Size() > 0) { + PointerType q = p.Append(name, allocator_); + out.count = v->Size(); + out.schemas = static_cast(allocator_->Malloc(out.count * sizeof(const Schema*))); + memset(out.schemas, 0, sizeof(Schema*)* out.count); + for (SizeType i = 0; i < out.count; i++) + schemaDocument.CreateSchema(&out.schemas[i], q.Append(i, allocator_), (*v)[i], document); + out.begin = validatorCount_; + validatorCount_ += out.count; + } + } + } + +#if CEREAL_RAPIDJSON_SCHEMA_USE_INTERNALREGEX + template + RegexType* CreatePattern(const ValueType& value) { + if (value.IsString()) { + RegexType* r = new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString(), allocator_); + if (!r->IsValid()) { + r->~RegexType(); + AllocatorType::Free(r); + r = 0; + } + return r; + } + return 0; + } + + static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType) { + GenericRegexSearch rs(*pattern); + return rs.Search(str); + } +#elif CEREAL_RAPIDJSON_SCHEMA_USE_STDREGEX + template + RegexType* CreatePattern(const ValueType& value) { + if (value.IsString()) { + RegexType *r = static_cast(allocator_->Malloc(sizeof(RegexType))); + try { + return new (r) RegexType(value.GetString(), std::size_t(value.GetStringLength()), std::regex_constants::ECMAScript); + } + catch (const std::regex_error&) { + AllocatorType::Free(r); + } + } + return 0; + } + + static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType length) { + std::match_results r; + return std::regex_search(str, str + length, r, *pattern); + } +#else + template + RegexType* CreatePattern(const ValueType&) { return 0; } + + static bool IsPatternMatch(const RegexType*, const Ch *, SizeType) { return true; } +#endif // CEREAL_RAPIDJSON_SCHEMA_USE_STDREGEX + + void AddType(const ValueType& type) { + if (type == GetNullString() ) type_ |= 1 << kNullSchemaType; + else if (type == GetBooleanString()) type_ |= 1 << kBooleanSchemaType; + else if (type == GetObjectString() ) type_ |= 1 << kObjectSchemaType; + else if (type == GetArrayString() ) type_ |= 1 << kArraySchemaType; + else if (type == GetStringString() ) type_ |= 1 << kStringSchemaType; + else if (type == GetIntegerString()) type_ |= 1 << kIntegerSchemaType; + else if (type == GetNumberString() ) type_ |= (1 << kNumberSchemaType) | (1 << kIntegerSchemaType); + } + + bool CreateParallelValidator(Context& context) const { + if (enum_ || context.arrayUniqueness) + context.hasher = context.factory.CreateHasher(); + + if (validatorCount_) { + CEREAL_RAPIDJSON_ASSERT(context.validators == 0); + context.validators = static_cast(context.factory.MallocState(sizeof(ISchemaValidator*) * validatorCount_)); + context.validatorCount = validatorCount_; + + if (allOf_.schemas) + CreateSchemaValidators(context, allOf_); + + if (anyOf_.schemas) + CreateSchemaValidators(context, anyOf_); + + if (oneOf_.schemas) + CreateSchemaValidators(context, oneOf_); + + if (not_) + context.validators[notValidatorIndex_] = context.factory.CreateSchemaValidator(*not_); + + if (hasSchemaDependencies_) { + for (SizeType i = 0; i < propertyCount_; i++) + if (properties_[i].dependenciesSchema) + context.validators[properties_[i].dependenciesValidatorIndex] = context.factory.CreateSchemaValidator(*properties_[i].dependenciesSchema); + } + } + + return true; + } + + void CreateSchemaValidators(Context& context, const SchemaArray& schemas) const { + for (SizeType i = 0; i < schemas.count; i++) + context.validators[schemas.begin + i] = context.factory.CreateSchemaValidator(*schemas.schemas[i]); + } + + // O(n) + bool FindPropertyIndex(const ValueType& name, SizeType* outIndex) const { + SizeType len = name.GetStringLength(); + const Ch* str = name.GetString(); + for (SizeType index = 0; index < propertyCount_; index++) + if (properties_[index].name.GetStringLength() == len && + (std::memcmp(properties_[index].name.GetString(), str, sizeof(Ch) * len) == 0)) + { + *outIndex = index; + return true; + } + return false; + } + + bool CheckInt(Context& context, int64_t i) const { + if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) { + DisallowedType(context, GetIntegerString()); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } + + if (!minimum_.IsNull()) { + if (minimum_.IsInt64()) { + if (exclusiveMinimum_ ? i <= minimum_.GetInt64() : i < minimum_.GetInt64()) { + context.error_handler.BelowMinimum(i, minimum_, exclusiveMinimum_); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + } + } + else if (minimum_.IsUint64()) { + context.error_handler.BelowMinimum(i, minimum_, exclusiveMinimum_); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); // i <= max(int64_t) < minimum.GetUint64() + } + else if (!CheckDoubleMinimum(context, static_cast(i))) + return false; + } + + if (!maximum_.IsNull()) { + if (maximum_.IsInt64()) { + if (exclusiveMaximum_ ? i >= maximum_.GetInt64() : i > maximum_.GetInt64()) { + context.error_handler.AboveMaximum(i, maximum_, exclusiveMaximum_); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + } + } + else if (maximum_.IsUint64()) { } + /* do nothing */ // i <= max(int64_t) < maximum_.GetUint64() + else if (!CheckDoubleMaximum(context, static_cast(i))) + return false; + } + + if (!multipleOf_.IsNull()) { + if (multipleOf_.IsUint64()) { + if (static_cast(i >= 0 ? i : -i) % multipleOf_.GetUint64() != 0) { + context.error_handler.NotMultipleOf(i, multipleOf_); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + } + } + else if (!CheckDoubleMultipleOf(context, static_cast(i))) + return false; + } + + return true; + } + + bool CheckUint(Context& context, uint64_t i) const { + if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) { + DisallowedType(context, GetIntegerString()); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } + + if (!minimum_.IsNull()) { + if (minimum_.IsUint64()) { + if (exclusiveMinimum_ ? i <= minimum_.GetUint64() : i < minimum_.GetUint64()) { + context.error_handler.BelowMinimum(i, minimum_, exclusiveMinimum_); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + } + } + else if (minimum_.IsInt64()) + /* do nothing */; // i >= 0 > minimum.Getint64() + else if (!CheckDoubleMinimum(context, static_cast(i))) + return false; + } + + if (!maximum_.IsNull()) { + if (maximum_.IsUint64()) { + if (exclusiveMaximum_ ? i >= maximum_.GetUint64() : i > maximum_.GetUint64()) { + context.error_handler.AboveMaximum(i, maximum_, exclusiveMaximum_); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + } + } + else if (maximum_.IsInt64()) { + context.error_handler.AboveMaximum(i, maximum_, exclusiveMaximum_); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); // i >= 0 > maximum_ + } + else if (!CheckDoubleMaximum(context, static_cast(i))) + return false; + } + + if (!multipleOf_.IsNull()) { + if (multipleOf_.IsUint64()) { + if (i % multipleOf_.GetUint64() != 0) { + context.error_handler.NotMultipleOf(i, multipleOf_); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + } + } + else if (!CheckDoubleMultipleOf(context, static_cast(i))) + return false; + } + + return true; + } + + bool CheckDoubleMinimum(Context& context, double d) const { + if (exclusiveMinimum_ ? d <= minimum_.GetDouble() : d < minimum_.GetDouble()) { + context.error_handler.BelowMinimum(d, minimum_, exclusiveMinimum_); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + } + return true; + } + + bool CheckDoubleMaximum(Context& context, double d) const { + if (exclusiveMaximum_ ? d >= maximum_.GetDouble() : d > maximum_.GetDouble()) { + context.error_handler.AboveMaximum(d, maximum_, exclusiveMaximum_); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + } + return true; + } + + bool CheckDoubleMultipleOf(Context& context, double d) const { + double a = std::abs(d), b = std::abs(multipleOf_.GetDouble()); + double q = std::floor(a / b); + double r = a - q * b; + if (r > 0.0) { + context.error_handler.NotMultipleOf(d, multipleOf_); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + } + return true; + } + + void DisallowedType(Context& context, const ValueType& actualType) const { + ErrorHandler& eh = context.error_handler; + eh.StartDisallowedType(); + + if (type_ & (1 << kNullSchemaType)) eh.AddExpectedType(GetNullString()); + if (type_ & (1 << kBooleanSchemaType)) eh.AddExpectedType(GetBooleanString()); + if (type_ & (1 << kObjectSchemaType)) eh.AddExpectedType(GetObjectString()); + if (type_ & (1 << kArraySchemaType)) eh.AddExpectedType(GetArrayString()); + if (type_ & (1 << kStringSchemaType)) eh.AddExpectedType(GetStringString()); + + if (type_ & (1 << kNumberSchemaType)) eh.AddExpectedType(GetNumberString()); + else if (type_ & (1 << kIntegerSchemaType)) eh.AddExpectedType(GetIntegerString()); + + eh.EndDisallowedType(actualType); + } + + struct Property { + Property() : schema(), dependenciesSchema(), dependenciesValidatorIndex(), dependencies(), required(false) {} + ~Property() { AllocatorType::Free(dependencies); } + SValue name; + const SchemaType* schema; + const SchemaType* dependenciesSchema; + SizeType dependenciesValidatorIndex; + bool* dependencies; + bool required; + }; + + struct PatternProperty { + PatternProperty() : schema(), pattern() {} + ~PatternProperty() { + if (pattern) { + pattern->~RegexType(); + AllocatorType::Free(pattern); + } + } + const SchemaType* schema; + RegexType* pattern; + }; + + AllocatorType* allocator_; + SValue uri_; + PointerType pointer_; + const SchemaType* typeless_; + uint64_t* enum_; + SizeType enumCount_; + SchemaArray allOf_; + SchemaArray anyOf_; + SchemaArray oneOf_; + const SchemaType* not_; + unsigned type_; // bitmask of kSchemaType + SizeType validatorCount_; + SizeType notValidatorIndex_; + + Property* properties_; + const SchemaType* additionalPropertiesSchema_; + PatternProperty* patternProperties_; + SizeType patternPropertyCount_; + SizeType propertyCount_; + SizeType minProperties_; + SizeType maxProperties_; + bool additionalProperties_; + bool hasDependencies_; + bool hasRequired_; + bool hasSchemaDependencies_; + + const SchemaType* additionalItemsSchema_; + const SchemaType* itemsList_; + const SchemaType** itemsTuple_; + SizeType itemsTupleCount_; + SizeType minItems_; + SizeType maxItems_; + bool additionalItems_; + bool uniqueItems_; + + RegexType* pattern_; + SizeType minLength_; + SizeType maxLength_; + + SValue minimum_; + SValue maximum_; + SValue multipleOf_; + bool exclusiveMinimum_; + bool exclusiveMaximum_; + + SizeType defaultValueLength_; +}; + +template +struct TokenHelper { + CEREAL_RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { + *documentStack.template Push() = '/'; + char buffer[21]; + size_t length = static_cast((sizeof(SizeType) == 4 ? u32toa(index, buffer) : u64toa(index, buffer)) - buffer); + for (size_t i = 0; i < length; i++) + *documentStack.template Push() = static_cast(buffer[i]); + } +}; + +// Partial specialized version for char to prevent buffer copying. +template +struct TokenHelper { + CEREAL_RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { + if (sizeof(SizeType) == 4) { + char *buffer = documentStack.template Push(1 + 10); // '/' + uint + *buffer++ = '/'; + const char* end = internal::u32toa(index, buffer); + documentStack.template Pop(static_cast(10 - (end - buffer))); + } + else { + char *buffer = documentStack.template Push(1 + 20); // '/' + uint64 + *buffer++ = '/'; + const char* end = internal::u64toa(index, buffer); + documentStack.template Pop(static_cast(20 - (end - buffer))); + } + } +}; + +} // namespace internal + +/////////////////////////////////////////////////////////////////////////////// +// IGenericRemoteSchemaDocumentProvider + +template +class IGenericRemoteSchemaDocumentProvider { +public: + typedef typename SchemaDocumentType::Ch Ch; + + virtual ~IGenericRemoteSchemaDocumentProvider() {} + virtual const SchemaDocumentType* GetRemoteDocument(const Ch* uri, SizeType length) = 0; +}; + +/////////////////////////////////////////////////////////////////////////////// +// GenericSchemaDocument + +//! JSON schema document. +/*! + A JSON schema document is a compiled version of a JSON schema. + It is basically a tree of internal::Schema. + + \note This is an immutable class (i.e. its instance cannot be modified after construction). + \tparam ValueT Type of JSON value (e.g. \c Value ), which also determine the encoding. + \tparam Allocator Allocator type for allocating memory of this document. +*/ +template +class GenericSchemaDocument { +public: + typedef ValueT ValueType; + typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProviderType; + typedef Allocator AllocatorType; + typedef typename ValueType::EncodingType EncodingType; + typedef typename EncodingType::Ch Ch; + typedef internal::Schema SchemaType; + typedef GenericPointer PointerType; + typedef GenericValue URIType; + friend class internal::Schema; + template + friend class GenericSchemaValidator; + + //! Constructor. + /*! + Compile a JSON document into schema document. + + \param document A JSON document as source. + \param uri The base URI of this schema document for purposes of violation reporting. + \param uriLength Length of \c name, in code points. + \param remoteProvider An optional remote schema document provider for resolving remote reference. Can be null. + \param allocator An optional allocator instance for allocating memory. Can be null. + */ + explicit GenericSchemaDocument(const ValueType& document, const Ch* uri = 0, SizeType uriLength = 0, + IRemoteSchemaDocumentProviderType* remoteProvider = 0, Allocator* allocator = 0) : + remoteProvider_(remoteProvider), + allocator_(allocator), + ownAllocator_(), + root_(), + typeless_(), + schemaMap_(allocator, kInitialSchemaMapSize), + schemaRef_(allocator, kInitialSchemaRefSize) + { + if (!allocator_) + ownAllocator_ = allocator_ = CEREAL_RAPIDJSON_NEW(Allocator)(); + + Ch noUri[1] = {0}; + uri_.SetString(uri ? uri : noUri, uriLength, *allocator_); + + typeless_ = static_cast(allocator_->Malloc(sizeof(SchemaType))); + new (typeless_) SchemaType(this, PointerType(), ValueType(kObjectType).Move(), ValueType(kObjectType).Move(), allocator_); + + // Generate root schema, it will call CreateSchema() to create sub-schemas, + // And call AddRefSchema() if there are $ref. + CreateSchemaRecursive(&root_, PointerType(), document, document); + + // Resolve $ref + while (!schemaRef_.Empty()) { + SchemaRefEntry* refEntry = schemaRef_.template Pop(1); + if (const SchemaType* s = GetSchema(refEntry->target)) { + if (refEntry->schema) + *refEntry->schema = s; + + // Create entry in map if not exist + if (!GetSchema(refEntry->source)) { + new (schemaMap_.template Push()) SchemaEntry(refEntry->source, const_cast(s), false, allocator_); + } + } + else if (refEntry->schema) + *refEntry->schema = typeless_; + + refEntry->~SchemaRefEntry(); + } + + CEREAL_RAPIDJSON_ASSERT(root_ != 0); + + schemaRef_.ShrinkToFit(); // Deallocate all memory for ref + } + +#if CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move constructor in C++11 + GenericSchemaDocument(GenericSchemaDocument&& rhs) CEREAL_RAPIDJSON_NOEXCEPT : + remoteProvider_(rhs.remoteProvider_), + allocator_(rhs.allocator_), + ownAllocator_(rhs.ownAllocator_), + root_(rhs.root_), + typeless_(rhs.typeless_), + schemaMap_(std::move(rhs.schemaMap_)), + schemaRef_(std::move(rhs.schemaRef_)), + uri_(std::move(rhs.uri_)) + { + rhs.remoteProvider_ = 0; + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + rhs.typeless_ = 0; + } +#endif + + //! Destructor + ~GenericSchemaDocument() { + while (!schemaMap_.Empty()) + schemaMap_.template Pop(1)->~SchemaEntry(); + + if (typeless_) { + typeless_->~SchemaType(); + Allocator::Free(typeless_); + } + + CEREAL_RAPIDJSON_DELETE(ownAllocator_); + } + + const URIType& GetURI() const { return uri_; } + + //! Get the root schema. + const SchemaType& GetRoot() const { return *root_; } + +private: + //! Prohibit copying + GenericSchemaDocument(const GenericSchemaDocument&); + //! Prohibit assignment + GenericSchemaDocument& operator=(const GenericSchemaDocument&); + + struct SchemaRefEntry { + SchemaRefEntry(const PointerType& s, const PointerType& t, const SchemaType** outSchema, Allocator *allocator) : source(s, allocator), target(t, allocator), schema(outSchema) {} + PointerType source; + PointerType target; + const SchemaType** schema; + }; + + struct SchemaEntry { + SchemaEntry(const PointerType& p, SchemaType* s, bool o, Allocator* allocator) : pointer(p, allocator), schema(s), owned(o) {} + ~SchemaEntry() { + if (owned) { + schema->~SchemaType(); + Allocator::Free(schema); + } + } + PointerType pointer; + SchemaType* schema; + bool owned; + }; + + void CreateSchemaRecursive(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { + if (schema) + *schema = typeless_; + + if (v.GetType() == kObjectType) { + const SchemaType* s = GetSchema(pointer); + if (!s) + CreateSchema(schema, pointer, v, document); + + for (typename ValueType::ConstMemberIterator itr = v.MemberBegin(); itr != v.MemberEnd(); ++itr) + CreateSchemaRecursive(0, pointer.Append(itr->name, allocator_), itr->value, document); + } + else if (v.GetType() == kArrayType) + for (SizeType i = 0; i < v.Size(); i++) + CreateSchemaRecursive(0, pointer.Append(i, allocator_), v[i], document); + } + + void CreateSchema(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { + CEREAL_RAPIDJSON_ASSERT(pointer.IsValid()); + if (v.IsObject()) { + if (!HandleRefSchema(pointer, schema, v, document)) { + SchemaType* s = new (allocator_->Malloc(sizeof(SchemaType))) SchemaType(this, pointer, v, document, allocator_); + new (schemaMap_.template Push()) SchemaEntry(pointer, s, true, allocator_); + if (schema) + *schema = s; + } + } + } + + bool HandleRefSchema(const PointerType& source, const SchemaType** schema, const ValueType& v, const ValueType& document) { + static const Ch kRefString[] = { '$', 'r', 'e', 'f', '\0' }; + static const ValueType kRefValue(kRefString, 4); + + typename ValueType::ConstMemberIterator itr = v.FindMember(kRefValue); + if (itr == v.MemberEnd()) + return false; + + if (itr->value.IsString()) { + SizeType len = itr->value.GetStringLength(); + if (len > 0) { + const Ch* s = itr->value.GetString(); + SizeType i = 0; + while (i < len && s[i] != '#') // Find the first # + i++; + + if (i > 0) { // Remote reference, resolve immediately + if (remoteProvider_) { + if (const GenericSchemaDocument* remoteDocument = remoteProvider_->GetRemoteDocument(s, i)) { + PointerType pointer(&s[i], len - i, allocator_); + if (pointer.IsValid()) { + if (const SchemaType* sc = remoteDocument->GetSchema(pointer)) { + if (schema) + *schema = sc; + new (schemaMap_.template Push()) SchemaEntry(source, const_cast(sc), false, allocator_); + return true; + } + } + } + } + } + else if (s[i] == '#') { // Local reference, defer resolution + PointerType pointer(&s[i], len - i, allocator_); + if (pointer.IsValid()) { + if (const ValueType* nv = pointer.Get(document)) + if (HandleRefSchema(source, schema, *nv, document)) + return true; + + new (schemaRef_.template Push()) SchemaRefEntry(source, pointer, schema, allocator_); + return true; + } + } + } + } + return false; + } + + const SchemaType* GetSchema(const PointerType& pointer) const { + for (const SchemaEntry* target = schemaMap_.template Bottom(); target != schemaMap_.template End(); ++target) + if (pointer == target->pointer) + return target->schema; + return 0; + } + + PointerType GetPointer(const SchemaType* schema) const { + for (const SchemaEntry* target = schemaMap_.template Bottom(); target != schemaMap_.template End(); ++target) + if (schema == target->schema) + return target->pointer; + return PointerType(); + } + + const SchemaType* GetTypeless() const { return typeless_; } + + static const size_t kInitialSchemaMapSize = 64; + static const size_t kInitialSchemaRefSize = 64; + + IRemoteSchemaDocumentProviderType* remoteProvider_; + Allocator *allocator_; + Allocator *ownAllocator_; + const SchemaType* root_; //!< Root schema. + SchemaType* typeless_; + internal::Stack schemaMap_; // Stores created Pointer -> Schemas + internal::Stack schemaRef_; // Stores Pointer from $ref and schema which holds the $ref + URIType uri_; +}; + +//! GenericSchemaDocument using Value type. +typedef GenericSchemaDocument SchemaDocument; +//! IGenericRemoteSchemaDocumentProvider using SchemaDocument. +typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProvider; + +/////////////////////////////////////////////////////////////////////////////// +// GenericSchemaValidator + +//! JSON Schema Validator. +/*! + A SAX style JSON schema validator. + It uses a \c GenericSchemaDocument to validate SAX events. + It delegates the incoming SAX events to an output handler. + The default output handler does nothing. + It can be reused multiple times by calling \c Reset(). + + \tparam SchemaDocumentType Type of schema document. + \tparam OutputHandler Type of output handler. Default handler does nothing. + \tparam StateAllocator Allocator for storing the internal validation states. +*/ +template < + typename SchemaDocumentType, + typename OutputHandler = BaseReaderHandler, + typename StateAllocator = CrtAllocator> +class GenericSchemaValidator : + public internal::ISchemaStateFactory, + public internal::ISchemaValidator, + public internal::IValidationErrorHandler +{ +public: + typedef typename SchemaDocumentType::SchemaType SchemaType; + typedef typename SchemaDocumentType::PointerType PointerType; + typedef typename SchemaType::EncodingType EncodingType; + typedef typename SchemaType::SValue SValue; + typedef typename EncodingType::Ch Ch; + typedef GenericStringRef StringRefType; + typedef GenericValue ValueType; + + //! Constructor without output handler. + /*! + \param schemaDocument The schema document to conform to. + \param allocator Optional allocator for storing internal validation states. + \param schemaStackCapacity Optional initial capacity of schema path stack. + \param documentStackCapacity Optional initial capacity of document path stack. + */ + GenericSchemaValidator( + const SchemaDocumentType& schemaDocument, + StateAllocator* allocator = 0, + size_t schemaStackCapacity = kDefaultSchemaStackCapacity, + size_t documentStackCapacity = kDefaultDocumentStackCapacity) + : + schemaDocument_(&schemaDocument), + root_(schemaDocument.GetRoot()), + stateAllocator_(allocator), + ownStateAllocator_(0), + schemaStack_(allocator, schemaStackCapacity), + documentStack_(allocator, documentStackCapacity), + outputHandler_(0), + error_(kObjectType), + currentError_(), + missingDependents_(), + valid_(true) +#if CEREAL_RAPIDJSON_SCHEMA_VERBOSE + , depth_(0) +#endif + { + } + + //! Constructor with output handler. + /*! + \param schemaDocument The schema document to conform to. + \param allocator Optional allocator for storing internal validation states. + \param schemaStackCapacity Optional initial capacity of schema path stack. + \param documentStackCapacity Optional initial capacity of document path stack. + */ + GenericSchemaValidator( + const SchemaDocumentType& schemaDocument, + OutputHandler& outputHandler, + StateAllocator* allocator = 0, + size_t schemaStackCapacity = kDefaultSchemaStackCapacity, + size_t documentStackCapacity = kDefaultDocumentStackCapacity) + : + schemaDocument_(&schemaDocument), + root_(schemaDocument.GetRoot()), + stateAllocator_(allocator), + ownStateAllocator_(0), + schemaStack_(allocator, schemaStackCapacity), + documentStack_(allocator, documentStackCapacity), + outputHandler_(&outputHandler), + error_(kObjectType), + currentError_(), + missingDependents_(), + valid_(true) +#if CEREAL_RAPIDJSON_SCHEMA_VERBOSE + , depth_(0) +#endif + { + } + + //! Destructor. + ~GenericSchemaValidator() { + Reset(); + CEREAL_RAPIDJSON_DELETE(ownStateAllocator_); + } + + //! Reset the internal states. + void Reset() { + while (!schemaStack_.Empty()) + PopSchema(); + documentStack_.Clear(); + error_.SetObject(); + currentError_.SetNull(); + missingDependents_.SetNull(); + valid_ = true; + } + + //! Checks whether the current state is valid. + // Implementation of ISchemaValidator + virtual bool IsValid() const { return valid_; } + + //! Gets the error object. + ValueType& GetError() { return error_; } + const ValueType& GetError() const { return error_; } + + //! Gets the JSON pointer pointed to the invalid schema. + PointerType GetInvalidSchemaPointer() const { + return schemaStack_.Empty() ? PointerType() : CurrentSchema().GetPointer(); + } + + //! Gets the keyword of invalid schema. + const Ch* GetInvalidSchemaKeyword() const { + return schemaStack_.Empty() ? 0 : CurrentContext().invalidKeyword; + } + + //! Gets the JSON pointer pointed to the invalid value. + PointerType GetInvalidDocumentPointer() const { + if (documentStack_.Empty()) { + return PointerType(); + } + else { + return PointerType(documentStack_.template Bottom(), documentStack_.GetSize() / sizeof(Ch)); + } + } + + void NotMultipleOf(int64_t actual, const SValue& expected) { + AddNumberError(SchemaType::GetMultipleOfString(), ValueType(actual).Move(), expected); + } + void NotMultipleOf(uint64_t actual, const SValue& expected) { + AddNumberError(SchemaType::GetMultipleOfString(), ValueType(actual).Move(), expected); + } + void NotMultipleOf(double actual, const SValue& expected) { + AddNumberError(SchemaType::GetMultipleOfString(), ValueType(actual).Move(), expected); + } + void AboveMaximum(int64_t actual, const SValue& expected, bool exclusive) { + AddNumberError(SchemaType::GetMaximumString(), ValueType(actual).Move(), expected, + exclusive ? &SchemaType::GetExclusiveMaximumString : 0); + } + void AboveMaximum(uint64_t actual, const SValue& expected, bool exclusive) { + AddNumberError(SchemaType::GetMaximumString(), ValueType(actual).Move(), expected, + exclusive ? &SchemaType::GetExclusiveMaximumString : 0); + } + void AboveMaximum(double actual, const SValue& expected, bool exclusive) { + AddNumberError(SchemaType::GetMaximumString(), ValueType(actual).Move(), expected, + exclusive ? &SchemaType::GetExclusiveMaximumString : 0); + } + void BelowMinimum(int64_t actual, const SValue& expected, bool exclusive) { + AddNumberError(SchemaType::GetMinimumString(), ValueType(actual).Move(), expected, + exclusive ? &SchemaType::GetExclusiveMinimumString : 0); + } + void BelowMinimum(uint64_t actual, const SValue& expected, bool exclusive) { + AddNumberError(SchemaType::GetMinimumString(), ValueType(actual).Move(), expected, + exclusive ? &SchemaType::GetExclusiveMinimumString : 0); + } + void BelowMinimum(double actual, const SValue& expected, bool exclusive) { + AddNumberError(SchemaType::GetMinimumString(), ValueType(actual).Move(), expected, + exclusive ? &SchemaType::GetExclusiveMinimumString : 0); + } + + void TooLong(const Ch* str, SizeType length, SizeType expected) { + AddNumberError(SchemaType::GetMaxLengthString(), + ValueType(str, length, GetStateAllocator()).Move(), SValue(expected).Move()); + } + void TooShort(const Ch* str, SizeType length, SizeType expected) { + AddNumberError(SchemaType::GetMinLengthString(), + ValueType(str, length, GetStateAllocator()).Move(), SValue(expected).Move()); + } + void DoesNotMatch(const Ch* str, SizeType length) { + currentError_.SetObject(); + currentError_.AddMember(GetActualString(), ValueType(str, length, GetStateAllocator()).Move(), GetStateAllocator()); + AddCurrentError(SchemaType::GetPatternString()); + } + + void DisallowedItem(SizeType index) { + currentError_.SetObject(); + currentError_.AddMember(GetDisallowedString(), ValueType(index).Move(), GetStateAllocator()); + AddCurrentError(SchemaType::GetAdditionalItemsString(), true); + } + void TooFewItems(SizeType actualCount, SizeType expectedCount) { + AddNumberError(SchemaType::GetMinItemsString(), + ValueType(actualCount).Move(), SValue(expectedCount).Move()); + } + void TooManyItems(SizeType actualCount, SizeType expectedCount) { + AddNumberError(SchemaType::GetMaxItemsString(), + ValueType(actualCount).Move(), SValue(expectedCount).Move()); + } + void DuplicateItems(SizeType index1, SizeType index2) { + ValueType duplicates(kArrayType); + duplicates.PushBack(index1, GetStateAllocator()); + duplicates.PushBack(index2, GetStateAllocator()); + currentError_.SetObject(); + currentError_.AddMember(GetDuplicatesString(), duplicates, GetStateAllocator()); + AddCurrentError(SchemaType::GetUniqueItemsString(), true); + } + + void TooManyProperties(SizeType actualCount, SizeType expectedCount) { + AddNumberError(SchemaType::GetMaxPropertiesString(), + ValueType(actualCount).Move(), SValue(expectedCount).Move()); + } + void TooFewProperties(SizeType actualCount, SizeType expectedCount) { + AddNumberError(SchemaType::GetMinPropertiesString(), + ValueType(actualCount).Move(), SValue(expectedCount).Move()); + } + void StartMissingProperties() { + currentError_.SetArray(); + } + void AddMissingProperty(const SValue& name) { + currentError_.PushBack(ValueType(name, GetStateAllocator()).Move(), GetStateAllocator()); + } + bool EndMissingProperties() { + if (currentError_.Empty()) + return false; + ValueType error(kObjectType); + error.AddMember(GetMissingString(), currentError_, GetStateAllocator()); + currentError_ = error; + AddCurrentError(SchemaType::GetRequiredString()); + return true; + } + void PropertyViolations(ISchemaValidator** subvalidators, SizeType count) { + for (SizeType i = 0; i < count; ++i) + MergeError(static_cast(subvalidators[i])->GetError()); + } + void DisallowedProperty(const Ch* name, SizeType length) { + currentError_.SetObject(); + currentError_.AddMember(GetDisallowedString(), ValueType(name, length, GetStateAllocator()).Move(), GetStateAllocator()); + AddCurrentError(SchemaType::GetAdditionalPropertiesString(), true); + } + + void StartDependencyErrors() { + currentError_.SetObject(); + } + void StartMissingDependentProperties() { + missingDependents_.SetArray(); + } + void AddMissingDependentProperty(const SValue& targetName) { + missingDependents_.PushBack(ValueType(targetName, GetStateAllocator()).Move(), GetStateAllocator()); + } + void EndMissingDependentProperties(const SValue& sourceName) { + if (!missingDependents_.Empty()) + currentError_.AddMember(ValueType(sourceName, GetStateAllocator()).Move(), + missingDependents_, GetStateAllocator()); + } + void AddDependencySchemaError(const SValue& sourceName, ISchemaValidator* subvalidator) { + currentError_.AddMember(ValueType(sourceName, GetStateAllocator()).Move(), + static_cast(subvalidator)->GetError(), GetStateAllocator()); + } + bool EndDependencyErrors() { + if (currentError_.ObjectEmpty()) + return false; + ValueType error(kObjectType); + error.AddMember(GetErrorsString(), currentError_, GetStateAllocator()); + currentError_ = error; + AddCurrentError(SchemaType::GetDependenciesString()); + return true; + } + + void DisallowedValue() { + currentError_.SetObject(); + AddCurrentError(SchemaType::GetEnumString()); + } + void StartDisallowedType() { + currentError_.SetArray(); + } + void AddExpectedType(const typename SchemaType::ValueType& expectedType) { + currentError_.PushBack(ValueType(expectedType, GetStateAllocator()).Move(), GetStateAllocator()); + } + void EndDisallowedType(const typename SchemaType::ValueType& actualType) { + ValueType error(kObjectType); + error.AddMember(GetExpectedString(), currentError_, GetStateAllocator()); + error.AddMember(GetActualString(), ValueType(actualType, GetStateAllocator()).Move(), GetStateAllocator()); + currentError_ = error; + AddCurrentError(SchemaType::GetTypeString()); + } + void NotAllOf(ISchemaValidator** subvalidators, SizeType count) { + for (SizeType i = 0; i < count; ++i) { + MergeError(static_cast(subvalidators[i])->GetError()); + } + } + void NoneOf(ISchemaValidator** subvalidators, SizeType count) { + AddErrorArray(SchemaType::GetAnyOfString(), subvalidators, count); + } + void NotOneOf(ISchemaValidator** subvalidators, SizeType count) { + AddErrorArray(SchemaType::GetOneOfString(), subvalidators, count); + } + void Disallowed() { + currentError_.SetObject(); + AddCurrentError(SchemaType::GetNotString()); + } + +#define CEREAL_RAPIDJSON_STRING_(name, ...) \ + static const StringRefType& Get##name##String() {\ + static const Ch s[] = { __VA_ARGS__, '\0' };\ + static const StringRefType v(s, static_cast(sizeof(s) / sizeof(Ch) - 1)); \ + return v;\ + } + + CEREAL_RAPIDJSON_STRING_(InstanceRef, 'i', 'n', 's', 't', 'a', 'n', 'c', 'e', 'R', 'e', 'f') + CEREAL_RAPIDJSON_STRING_(SchemaRef, 's', 'c', 'h', 'e', 'm', 'a', 'R', 'e', 'f') + CEREAL_RAPIDJSON_STRING_(Expected, 'e', 'x', 'p', 'e', 'c', 't', 'e', 'd') + CEREAL_RAPIDJSON_STRING_(Actual, 'a', 'c', 't', 'u', 'a', 'l') + CEREAL_RAPIDJSON_STRING_(Disallowed, 'd', 'i', 's', 'a', 'l', 'l', 'o', 'w', 'e', 'd') + CEREAL_RAPIDJSON_STRING_(Missing, 'm', 'i', 's', 's', 'i', 'n', 'g') + CEREAL_RAPIDJSON_STRING_(Errors, 'e', 'r', 'r', 'o', 'r', 's') + CEREAL_RAPIDJSON_STRING_(Duplicates, 'd', 'u', 'p', 'l', 'i', 'c', 'a', 't', 'e', 's') + +#undef CEREAL_RAPIDJSON_STRING_ + +#if CEREAL_RAPIDJSON_SCHEMA_VERBOSE +#define CEREAL_RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() \ +CEREAL_RAPIDJSON_MULTILINEMACRO_BEGIN\ + *documentStack_.template Push() = '\0';\ + documentStack_.template Pop(1);\ + internal::PrintInvalidDocument(documentStack_.template Bottom());\ +CEREAL_RAPIDJSON_MULTILINEMACRO_END +#else +#define CEREAL_RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() +#endif + +#define CEREAL_RAPIDJSON_SCHEMA_HANDLE_BEGIN_(method, arg1)\ + if (!valid_) return false; \ + if (!BeginValue() || !CurrentSchema().method arg1) {\ + CEREAL_RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_();\ + return valid_ = false;\ + } + +#define CEREAL_RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2)\ + for (Context* context = schemaStack_.template Bottom(); context != schemaStack_.template End(); context++) {\ + if (context->hasher)\ + static_cast(context->hasher)->method arg2;\ + if (context->validators)\ + for (SizeType i_ = 0; i_ < context->validatorCount; i_++)\ + static_cast(context->validators[i_])->method arg2;\ + if (context->patternPropertiesValidators)\ + for (SizeType i_ = 0; i_ < context->patternPropertiesValidatorCount; i_++)\ + static_cast(context->patternPropertiesValidators[i_])->method arg2;\ + } + +#define CEREAL_RAPIDJSON_SCHEMA_HANDLE_END_(method, arg2)\ + return valid_ = EndValue() && (!outputHandler_ || outputHandler_->method arg2) + +#define CEREAL_RAPIDJSON_SCHEMA_HANDLE_VALUE_(method, arg1, arg2) \ + CEREAL_RAPIDJSON_SCHEMA_HANDLE_BEGIN_ (method, arg1);\ + CEREAL_RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2);\ + CEREAL_RAPIDJSON_SCHEMA_HANDLE_END_ (method, arg2) + + bool Null() { CEREAL_RAPIDJSON_SCHEMA_HANDLE_VALUE_(Null, (CurrentContext()), ( )); } + bool Bool(bool b) { CEREAL_RAPIDJSON_SCHEMA_HANDLE_VALUE_(Bool, (CurrentContext(), b), (b)); } + bool Int(int i) { CEREAL_RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int, (CurrentContext(), i), (i)); } + bool Uint(unsigned u) { CEREAL_RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint, (CurrentContext(), u), (u)); } + bool Int64(int64_t i) { CEREAL_RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int64, (CurrentContext(), i), (i)); } + bool Uint64(uint64_t u) { CEREAL_RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint64, (CurrentContext(), u), (u)); } + bool Double(double d) { CEREAL_RAPIDJSON_SCHEMA_HANDLE_VALUE_(Double, (CurrentContext(), d), (d)); } + bool RawNumber(const Ch* str, SizeType length, bool copy) + { CEREAL_RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } + bool String(const Ch* str, SizeType length, bool copy) + { CEREAL_RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } + + bool StartObject() { + CEREAL_RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartObject, (CurrentContext())); + CEREAL_RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartObject, ()); + return valid_ = !outputHandler_ || outputHandler_->StartObject(); + } + + bool Key(const Ch* str, SizeType len, bool copy) { + if (!valid_) return false; + AppendToken(str, len); + if (!CurrentSchema().Key(CurrentContext(), str, len, copy)) return valid_ = false; + CEREAL_RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(Key, (str, len, copy)); + return valid_ = !outputHandler_ || outputHandler_->Key(str, len, copy); + } + + bool EndObject(SizeType memberCount) { + if (!valid_) return false; + CEREAL_RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndObject, (memberCount)); + if (!CurrentSchema().EndObject(CurrentContext(), memberCount)) return valid_ = false; + CEREAL_RAPIDJSON_SCHEMA_HANDLE_END_(EndObject, (memberCount)); + } + + bool StartArray() { + CEREAL_RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartArray, (CurrentContext())); + CEREAL_RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartArray, ()); + return valid_ = !outputHandler_ || outputHandler_->StartArray(); + } + + bool EndArray(SizeType elementCount) { + if (!valid_) return false; + CEREAL_RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndArray, (elementCount)); + if (!CurrentSchema().EndArray(CurrentContext(), elementCount)) return valid_ = false; + CEREAL_RAPIDJSON_SCHEMA_HANDLE_END_(EndArray, (elementCount)); + } + +#undef CEREAL_RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_ +#undef CEREAL_RAPIDJSON_SCHEMA_HANDLE_BEGIN_ +#undef CEREAL_RAPIDJSON_SCHEMA_HANDLE_PARALLEL_ +#undef CEREAL_RAPIDJSON_SCHEMA_HANDLE_VALUE_ + + // Implementation of ISchemaStateFactory + virtual ISchemaValidator* CreateSchemaValidator(const SchemaType& root) { + return new (GetStateAllocator().Malloc(sizeof(GenericSchemaValidator))) GenericSchemaValidator(*schemaDocument_, root, documentStack_.template Bottom(), documentStack_.GetSize(), +#if CEREAL_RAPIDJSON_SCHEMA_VERBOSE + depth_ + 1, +#endif + &GetStateAllocator()); + } + + virtual void DestroySchemaValidator(ISchemaValidator* validator) { + GenericSchemaValidator* v = static_cast(validator); + v->~GenericSchemaValidator(); + StateAllocator::Free(v); + } + + virtual void* CreateHasher() { + return new (GetStateAllocator().Malloc(sizeof(HasherType))) HasherType(&GetStateAllocator()); + } + + virtual uint64_t GetHashCode(void* hasher) { + return static_cast(hasher)->GetHashCode(); + } + + virtual void DestroryHasher(void* hasher) { + HasherType* h = static_cast(hasher); + h->~HasherType(); + StateAllocator::Free(h); + } + + virtual void* MallocState(size_t size) { + return GetStateAllocator().Malloc(size); + } + + virtual void FreeState(void* p) { + StateAllocator::Free(p); + } + +private: + typedef typename SchemaType::Context Context; + typedef GenericValue, StateAllocator> HashCodeArray; + typedef internal::Hasher HasherType; + + GenericSchemaValidator( + const SchemaDocumentType& schemaDocument, + const SchemaType& root, + const char* basePath, size_t basePathSize, +#if CEREAL_RAPIDJSON_SCHEMA_VERBOSE + unsigned depth, +#endif + StateAllocator* allocator = 0, + size_t schemaStackCapacity = kDefaultSchemaStackCapacity, + size_t documentStackCapacity = kDefaultDocumentStackCapacity) + : + schemaDocument_(&schemaDocument), + root_(root), + stateAllocator_(allocator), + ownStateAllocator_(0), + schemaStack_(allocator, schemaStackCapacity), + documentStack_(allocator, documentStackCapacity), + outputHandler_(0), + error_(kObjectType), + currentError_(), + missingDependents_(), + valid_(true) +#if CEREAL_RAPIDJSON_SCHEMA_VERBOSE + , depth_(depth) +#endif + { + if (basePath && basePathSize) + memcpy(documentStack_.template Push(basePathSize), basePath, basePathSize); + } + + StateAllocator& GetStateAllocator() { + if (!stateAllocator_) + stateAllocator_ = ownStateAllocator_ = CEREAL_RAPIDJSON_NEW(StateAllocator)(); + return *stateAllocator_; + } + + bool BeginValue() { + if (schemaStack_.Empty()) + PushSchema(root_); + else { + if (CurrentContext().inArray) + internal::TokenHelper, Ch>::AppendIndexToken(documentStack_, CurrentContext().arrayElementIndex); + + if (!CurrentSchema().BeginValue(CurrentContext())) + return false; + + SizeType count = CurrentContext().patternPropertiesSchemaCount; + const SchemaType** sa = CurrentContext().patternPropertiesSchemas; + typename Context::PatternValidatorType patternValidatorType = CurrentContext().valuePatternValidatorType; + bool valueUniqueness = CurrentContext().valueUniqueness; + CEREAL_RAPIDJSON_ASSERT(CurrentContext().valueSchema); + PushSchema(*CurrentContext().valueSchema); + + if (count > 0) { + CurrentContext().objectPatternValidatorType = patternValidatorType; + ISchemaValidator**& va = CurrentContext().patternPropertiesValidators; + SizeType& validatorCount = CurrentContext().patternPropertiesValidatorCount; + va = static_cast(MallocState(sizeof(ISchemaValidator*) * count)); + for (SizeType i = 0; i < count; i++) + va[validatorCount++] = CreateSchemaValidator(*sa[i]); + } + + CurrentContext().arrayUniqueness = valueUniqueness; + } + return true; + } + + bool EndValue() { + if (!CurrentSchema().EndValue(CurrentContext())) + return false; + +#if CEREAL_RAPIDJSON_SCHEMA_VERBOSE + GenericStringBuffer sb; + schemaDocument_->GetPointer(&CurrentSchema()).Stringify(sb); + + *documentStack_.template Push() = '\0'; + documentStack_.template Pop(1); + internal::PrintValidatorPointers(depth_, sb.GetString(), documentStack_.template Bottom()); +#endif + + uint64_t h = CurrentContext().arrayUniqueness ? static_cast(CurrentContext().hasher)->GetHashCode() : 0; + + PopSchema(); + + if (!schemaStack_.Empty()) { + Context& context = CurrentContext(); + if (context.valueUniqueness) { + HashCodeArray* a = static_cast(context.arrayElementHashCodes); + if (!a) + CurrentContext().arrayElementHashCodes = a = new (GetStateAllocator().Malloc(sizeof(HashCodeArray))) HashCodeArray(kArrayType); + for (typename HashCodeArray::ConstValueIterator itr = a->Begin(); itr != a->End(); ++itr) + if (itr->GetUint64() == h) { + DuplicateItems(static_cast(itr - a->Begin()), a->Size()); + CEREAL_RAPIDJSON_INVALID_KEYWORD_RETURN(SchemaType::GetUniqueItemsString()); + } + a->PushBack(h, GetStateAllocator()); + } + } + + // Remove the last token of document pointer + while (!documentStack_.Empty() && *documentStack_.template Pop(1) != '/') + ; + + return true; + } + + void AppendToken(const Ch* str, SizeType len) { + documentStack_.template Reserve(1 + len * 2); // worst case all characters are escaped as two characters + *documentStack_.template PushUnsafe() = '/'; + for (SizeType i = 0; i < len; i++) { + if (str[i] == '~') { + *documentStack_.template PushUnsafe() = '~'; + *documentStack_.template PushUnsafe() = '0'; + } + else if (str[i] == '/') { + *documentStack_.template PushUnsafe() = '~'; + *documentStack_.template PushUnsafe() = '1'; + } + else + *documentStack_.template PushUnsafe() = str[i]; + } + } + + CEREAL_RAPIDJSON_FORCEINLINE void PushSchema(const SchemaType& schema) { new (schemaStack_.template Push()) Context(*this, *this, &schema); } + + CEREAL_RAPIDJSON_FORCEINLINE void PopSchema() { + Context* c = schemaStack_.template Pop(1); + if (HashCodeArray* a = static_cast(c->arrayElementHashCodes)) { + a->~HashCodeArray(); + StateAllocator::Free(a); + } + c->~Context(); + } + + void AddErrorLocation(ValueType& result, bool parent) { + GenericStringBuffer sb; + PointerType instancePointer = GetInvalidDocumentPointer(); + ((parent && instancePointer.GetTokenCount() > 0) + ? PointerType(instancePointer.GetTokens(), instancePointer.GetTokenCount() - 1) + : instancePointer).StringifyUriFragment(sb); + ValueType instanceRef(sb.GetString(), static_cast(sb.GetSize() / sizeof(Ch)), + GetStateAllocator()); + result.AddMember(GetInstanceRefString(), instanceRef, GetStateAllocator()); + sb.Clear(); + memcpy(sb.Push(CurrentSchema().GetURI().GetStringLength()), + CurrentSchema().GetURI().GetString(), + CurrentSchema().GetURI().GetStringLength() * sizeof(Ch)); + GetInvalidSchemaPointer().StringifyUriFragment(sb); + ValueType schemaRef(sb.GetString(), static_cast(sb.GetSize() / sizeof(Ch)), + GetStateAllocator()); + result.AddMember(GetSchemaRefString(), schemaRef, GetStateAllocator()); + } + + void AddError(ValueType& keyword, ValueType& error) { + typename ValueType::MemberIterator member = error_.FindMember(keyword); + if (member == error_.MemberEnd()) + error_.AddMember(keyword, error, GetStateAllocator()); + else { + if (member->value.IsObject()) { + ValueType errors(kArrayType); + errors.PushBack(member->value, GetStateAllocator()); + member->value = errors; + } + member->value.PushBack(error, GetStateAllocator()); + } + } + + void AddCurrentError(const typename SchemaType::ValueType& keyword, bool parent = false) { + AddErrorLocation(currentError_, parent); + AddError(ValueType(keyword, GetStateAllocator(), false).Move(), currentError_); + } + + void MergeError(ValueType& other) { + for (typename ValueType::MemberIterator it = other.MemberBegin(), end = other.MemberEnd(); it != end; ++it) { + AddError(it->name, it->value); + } + } + + void AddNumberError(const typename SchemaType::ValueType& keyword, ValueType& actual, const SValue& expected, + const typename SchemaType::ValueType& (*exclusive)() = 0) { + currentError_.SetObject(); + currentError_.AddMember(GetActualString(), actual, GetStateAllocator()); + currentError_.AddMember(GetExpectedString(), ValueType(expected, GetStateAllocator()).Move(), GetStateAllocator()); + if (exclusive) + currentError_.AddMember(ValueType(exclusive(), GetStateAllocator()).Move(), true, GetStateAllocator()); + AddCurrentError(keyword); + } + + void AddErrorArray(const typename SchemaType::ValueType& keyword, + ISchemaValidator** subvalidators, SizeType count) { + ValueType errors(kArrayType); + for (SizeType i = 0; i < count; ++i) + errors.PushBack(static_cast(subvalidators[i])->GetError(), GetStateAllocator()); + currentError_.SetObject(); + currentError_.AddMember(GetErrorsString(), errors, GetStateAllocator()); + AddCurrentError(keyword); + } + + const SchemaType& CurrentSchema() const { return *schemaStack_.template Top()->schema; } + Context& CurrentContext() { return *schemaStack_.template Top(); } + const Context& CurrentContext() const { return *schemaStack_.template Top(); } + + static const size_t kDefaultSchemaStackCapacity = 1024; + static const size_t kDefaultDocumentStackCapacity = 256; + const SchemaDocumentType* schemaDocument_; + const SchemaType& root_; + StateAllocator* stateAllocator_; + StateAllocator* ownStateAllocator_; + internal::Stack schemaStack_; //!< stack to store the current path of schema (BaseSchemaType *) + internal::Stack documentStack_; //!< stack to store the current path of validating document (Ch) + OutputHandler* outputHandler_; + ValueType error_; + ValueType currentError_; + ValueType missingDependents_; + bool valid_; +#if CEREAL_RAPIDJSON_SCHEMA_VERBOSE + unsigned depth_; +#endif +}; + +typedef GenericSchemaValidator SchemaValidator; + +/////////////////////////////////////////////////////////////////////////////// +// SchemaValidatingReader + +//! A helper class for parsing with validation. +/*! + This helper class is a functor, designed as a parameter of \ref GenericDocument::Populate(). + + \tparam parseFlags Combination of \ref ParseFlag. + \tparam InputStream Type of input stream, implementing Stream concept. + \tparam SourceEncoding Encoding of the input stream. + \tparam SchemaDocumentType Type of schema document. + \tparam StackAllocator Allocator type for stack. +*/ +template < + unsigned parseFlags, + typename InputStream, + typename SourceEncoding, + typename SchemaDocumentType = SchemaDocument, + typename StackAllocator = CrtAllocator> +class SchemaValidatingReader { +public: + typedef typename SchemaDocumentType::PointerType PointerType; + typedef typename InputStream::Ch Ch; + typedef GenericValue ValueType; + + //! Constructor + /*! + \param is Input stream. + \param sd Schema document. + */ + SchemaValidatingReader(InputStream& is, const SchemaDocumentType& sd) : is_(is), sd_(sd), invalidSchemaKeyword_(), error_(kObjectType), isValid_(true) {} + + template + bool operator()(Handler& handler) { + GenericReader reader; + GenericSchemaValidator validator(sd_, handler); + parseResult_ = reader.template Parse(is_, validator); + + isValid_ = validator.IsValid(); + if (isValid_) { + invalidSchemaPointer_ = PointerType(); + invalidSchemaKeyword_ = 0; + invalidDocumentPointer_ = PointerType(); + error_.SetObject(); + } + else { + invalidSchemaPointer_ = validator.GetInvalidSchemaPointer(); + invalidSchemaKeyword_ = validator.GetInvalidSchemaKeyword(); + invalidDocumentPointer_ = validator.GetInvalidDocumentPointer(); + error_.CopyFrom(validator.GetError(), allocator_); + } + + return parseResult_; + } + + const ParseResult& GetParseResult() const { return parseResult_; } + bool IsValid() const { return isValid_; } + const PointerType& GetInvalidSchemaPointer() const { return invalidSchemaPointer_; } + const Ch* GetInvalidSchemaKeyword() const { return invalidSchemaKeyword_; } + const PointerType& GetInvalidDocumentPointer() const { return invalidDocumentPointer_; } + const ValueType& GetError() const { return error_; } + +private: + InputStream& is_; + const SchemaDocumentType& sd_; + + ParseResult parseResult_; + PointerType invalidSchemaPointer_; + const Ch* invalidSchemaKeyword_; + PointerType invalidDocumentPointer_; + StackAllocator allocator_; + ValueType error_; + bool isValid_; +}; + +CEREAL_RAPIDJSON_NAMESPACE_END +CEREAL_RAPIDJSON_DIAG_POP + +#endif // CEREAL_RAPIDJSON_SCHEMA_H_ diff --git a/third_party/cereal/external/rapidjson/stream.h b/third_party/cereal/external/rapidjson/stream.h new file mode 100755 index 0000000..abc0153 --- /dev/null +++ b/third_party/cereal/external/rapidjson/stream.h @@ -0,0 +1,223 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#include "rapidjson.h" + +#ifndef CEREAL_RAPIDJSON_STREAM_H_ +#define CEREAL_RAPIDJSON_STREAM_H_ + +#include "encodings.h" + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// Stream + +/*! \class rapidjson::Stream + \brief Concept for reading and writing characters. + + For read-only stream, no need to implement PutBegin(), Put(), Flush() and PutEnd(). + + For write-only stream, only need to implement Put() and Flush(). + +\code +concept Stream { + typename Ch; //!< Character type of the stream. + + //! Read the current character from stream without moving the read cursor. + Ch Peek() const; + + //! Read the current character from stream and moving the read cursor to next character. + Ch Take(); + + //! Get the current read cursor. + //! \return Number of characters read from start. + size_t Tell(); + + //! Begin writing operation at the current read pointer. + //! \return The begin writer pointer. + Ch* PutBegin(); + + //! Write a character. + void Put(Ch c); + + //! Flush the buffer. + void Flush(); + + //! End the writing operation. + //! \param begin The begin write pointer returned by PutBegin(). + //! \return Number of characters written. + size_t PutEnd(Ch* begin); +} +\endcode +*/ + +//! Provides additional information for stream. +/*! + By using traits pattern, this type provides a default configuration for stream. + For custom stream, this type can be specialized for other configuration. + See TEST(Reader, CustomStringStream) in readertest.cpp for example. +*/ +template +struct StreamTraits { + //! Whether to make local copy of stream for optimization during parsing. + /*! + By default, for safety, streams do not use local copy optimization. + Stream that can be copied fast should specialize this, like StreamTraits. + */ + enum { copyOptimization = 0 }; +}; + +//! Reserve n characters for writing to a stream. +template +inline void PutReserve(Stream& stream, size_t count) { + (void)stream; + (void)count; +} + +//! Write character to a stream, presuming buffer is reserved. +template +inline void PutUnsafe(Stream& stream, typename Stream::Ch c) { + stream.Put(c); +} + +//! Put N copies of a character to a stream. +template +inline void PutN(Stream& stream, Ch c, size_t n) { + PutReserve(stream, n); + for (size_t i = 0; i < n; i++) + PutUnsafe(stream, c); +} + +/////////////////////////////////////////////////////////////////////////////// +// GenericStreamWrapper + +//! A Stream Wrapper +/*! \tThis string stream is a wrapper for any stream by just forwarding any + \treceived message to the origin stream. + \note implements Stream concept +*/ + +#if defined(_MSC_VER) && _MSC_VER <= 1800 +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(4702) // unreachable code +CEREAL_RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + +template > +class GenericStreamWrapper { +public: + typedef typename Encoding::Ch Ch; + GenericStreamWrapper(InputStream& is): is_(is) {} + + Ch Peek() const { return is_.Peek(); } + Ch Take() { return is_.Take(); } + size_t Tell() { return is_.Tell(); } + Ch* PutBegin() { return is_.PutBegin(); } + void Put(Ch ch) { is_.Put(ch); } + void Flush() { is_.Flush(); } + size_t PutEnd(Ch* ch) { return is_.PutEnd(ch); } + + // wrapper for MemoryStream + const Ch* Peek4() const { return is_.Peek4(); } + + // wrapper for AutoUTFInputStream + UTFType GetType() const { return is_.GetType(); } + bool HasBOM() const { return is_.HasBOM(); } + +protected: + InputStream& is_; +}; + +#if defined(_MSC_VER) && _MSC_VER <= 1800 +CEREAL_RAPIDJSON_DIAG_POP +#endif + +/////////////////////////////////////////////////////////////////////////////// +// StringStream + +//! Read-only string stream. +/*! \note implements Stream concept +*/ +template +struct GenericStringStream { + typedef typename Encoding::Ch Ch; + + GenericStringStream(const Ch *src) : src_(src), head_(src) {} + + Ch Peek() const { return *src_; } + Ch Take() { return *src_++; } + size_t Tell() const { return static_cast(src_ - head_); } + + Ch* PutBegin() { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + void Put(Ch) { CEREAL_RAPIDJSON_ASSERT(false); } + void Flush() { CEREAL_RAPIDJSON_ASSERT(false); } + size_t PutEnd(Ch*) { CEREAL_RAPIDJSON_ASSERT(false); return 0; } + + const Ch* src_; //!< Current read position. + const Ch* head_; //!< Original head of the string. +}; + +template +struct StreamTraits > { + enum { copyOptimization = 1 }; +}; + +//! String stream with UTF8 encoding. +typedef GenericStringStream > StringStream; + +/////////////////////////////////////////////////////////////////////////////// +// InsituStringStream + +//! A read-write string stream. +/*! This string stream is particularly designed for in-situ parsing. + \note implements Stream concept +*/ +template +struct GenericInsituStringStream { + typedef typename Encoding::Ch Ch; + + GenericInsituStringStream(Ch *src) : src_(src), dst_(0), head_(src) {} + + // Read + Ch Peek() { return *src_; } + Ch Take() { return *src_++; } + size_t Tell() { return static_cast(src_ - head_); } + + // Write + void Put(Ch c) { CEREAL_RAPIDJSON_ASSERT(dst_ != 0); *dst_++ = c; } + + Ch* PutBegin() { return dst_ = src_; } + size_t PutEnd(Ch* begin) { return static_cast(dst_ - begin); } + void Flush() {} + + Ch* Push(size_t count) { Ch* begin = dst_; dst_ += count; return begin; } + void Pop(size_t count) { dst_ -= count; } + + Ch* src_; + Ch* dst_; + Ch* head_; +}; + +template +struct StreamTraits > { + enum { copyOptimization = 1 }; +}; + +//! Insitu string stream with UTF8 encoding. +typedef GenericInsituStringStream > InsituStringStream; + +CEREAL_RAPIDJSON_NAMESPACE_END + +#endif // CEREAL_RAPIDJSON_STREAM_H_ diff --git a/third_party/cereal/external/rapidjson/stringbuffer.h b/third_party/cereal/external/rapidjson/stringbuffer.h new file mode 100755 index 0000000..d010477 --- /dev/null +++ b/third_party/cereal/external/rapidjson/stringbuffer.h @@ -0,0 +1,121 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_STRINGBUFFER_H_ +#define CEREAL_RAPIDJSON_STRINGBUFFER_H_ + +#include "stream.h" +#include "internal/stack.h" + +#if CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS +#include // std::move +#endif + +#include "internal/stack.h" + +#if defined(__clang__) +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN + +//! Represents an in-memory output stream. +/*! + \tparam Encoding Encoding of the stream. + \tparam Allocator type for allocating memory buffer. + \note implements Stream concept +*/ +template +class GenericStringBuffer { +public: + typedef typename Encoding::Ch Ch; + + GenericStringBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} + +#if CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericStringBuffer(GenericStringBuffer&& rhs) : stack_(std::move(rhs.stack_)) {} + GenericStringBuffer& operator=(GenericStringBuffer&& rhs) { + if (&rhs != this) + stack_ = std::move(rhs.stack_); + return *this; + } +#endif + + void Put(Ch c) { *stack_.template Push() = c; } + void PutUnsafe(Ch c) { *stack_.template PushUnsafe() = c; } + void Flush() {} + + void Clear() { stack_.Clear(); } + void ShrinkToFit() { + // Push and pop a null terminator. This is safe. + *stack_.template Push() = '\0'; + stack_.ShrinkToFit(); + stack_.template Pop(1); + } + + void Reserve(size_t count) { stack_.template Reserve(count); } + Ch* Push(size_t count) { return stack_.template Push(count); } + Ch* PushUnsafe(size_t count) { return stack_.template PushUnsafe(count); } + void Pop(size_t count) { stack_.template Pop(count); } + + const Ch* GetString() const { + // Push and pop a null terminator. This is safe. + *stack_.template Push() = '\0'; + stack_.template Pop(1); + + return stack_.template Bottom(); + } + + //! Get the size of string in bytes in the string buffer. + size_t GetSize() const { return stack_.GetSize(); } + + //! Get the length of string in Ch in the string buffer. + size_t GetLength() const { return stack_.GetSize() / sizeof(Ch); } + + static const size_t kDefaultCapacity = 256; + mutable internal::Stack stack_; + +private: + // Prohibit copy constructor & assignment operator. + GenericStringBuffer(const GenericStringBuffer&); + GenericStringBuffer& operator=(const GenericStringBuffer&); +}; + +//! String buffer with UTF8 encoding +typedef GenericStringBuffer > StringBuffer; + +template +inline void PutReserve(GenericStringBuffer& stream, size_t count) { + stream.Reserve(count); +} + +template +inline void PutUnsafe(GenericStringBuffer& stream, typename Encoding::Ch c) { + stream.PutUnsafe(c); +} + +//! Implement specialized version of PutN() with memset() for better performance. +template<> +inline void PutN(GenericStringBuffer >& stream, char c, size_t n) { + std::memset(stream.stack_.Push(n), c, n * sizeof(c)); +} + +CEREAL_RAPIDJSON_NAMESPACE_END + +#if defined(__clang__) +CEREAL_RAPIDJSON_DIAG_POP +#endif + +#endif // CEREAL_RAPIDJSON_STRINGBUFFER_H_ diff --git a/third_party/cereal/external/rapidjson/writer.h b/third_party/cereal/external/rapidjson/writer.h new file mode 100755 index 0000000..b8da773 --- /dev/null +++ b/third_party/cereal/external/rapidjson/writer.h @@ -0,0 +1,709 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef CEREAL_RAPIDJSON_WRITER_H_ +#define CEREAL_RAPIDJSON_WRITER_H_ + +#include "stream.h" +#include "internal/meta.h" +#include "internal/stack.h" +#include "internal/strfunc.h" +#include "internal/dtoa.h" +#include "internal/itoa.h" +#include "stringbuffer.h" +#include // placement new + +#if defined(CEREAL_RAPIDJSON_SIMD) && defined(_MSC_VER) +#include +#pragma intrinsic(_BitScanForward) +#endif +#ifdef CEREAL_RAPIDJSON_SSE42 +#include +#elif defined(CEREAL_RAPIDJSON_SSE2) +#include +#elif defined(CEREAL_RAPIDJSON_NEON) +#include +#endif + +#ifdef __clang__ +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(padded) +CEREAL_RAPIDJSON_DIAG_OFF(unreachable-code) +CEREAL_RAPIDJSON_DIAG_OFF(c++98-compat) +#elif defined(_MSC_VER) +CEREAL_RAPIDJSON_DIAG_PUSH +CEREAL_RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant +#endif + +CEREAL_RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// WriteFlag + +/*! \def CEREAL_RAPIDJSON_WRITE_DEFAULT_FLAGS + \ingroup CEREAL_RAPIDJSON_CONFIG + \brief User-defined kWriteDefaultFlags definition. + + User can define this as any \c WriteFlag combinations. +*/ +#ifndef CEREAL_RAPIDJSON_WRITE_DEFAULT_FLAGS +#define CEREAL_RAPIDJSON_WRITE_DEFAULT_FLAGS kWriteNoFlags +#endif + +//! Combination of writeFlags +enum WriteFlag { + kWriteNoFlags = 0, //!< No flags are set. + kWriteValidateEncodingFlag = 1, //!< Validate encoding of JSON strings. + kWriteNanAndInfFlag = 2, //!< Allow writing of Infinity, -Infinity and NaN. + kWriteDefaultFlags = CEREAL_RAPIDJSON_WRITE_DEFAULT_FLAGS //!< Default write flags. Can be customized by defining CEREAL_RAPIDJSON_WRITE_DEFAULT_FLAGS +}; + +//! JSON writer +/*! Writer implements the concept Handler. + It generates JSON text by events to an output os. + + User may programmatically calls the functions of a writer to generate JSON text. + + On the other side, a writer can also be passed to objects that generates events, + + for example Reader::Parse() and Document::Accept(). + + \tparam OutputStream Type of output stream. + \tparam SourceEncoding Encoding of source string. + \tparam TargetEncoding Encoding of output stream. + \tparam StackAllocator Type of allocator for allocating memory of stack. + \note implements Handler concept +*/ +template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator, unsigned writeFlags = kWriteDefaultFlags> +class Writer { +public: + typedef typename SourceEncoding::Ch Ch; + + static const int kDefaultMaxDecimalPlaces = 324; + + //! Constructor + /*! \param os Output stream. + \param stackAllocator User supplied allocator. If it is null, it will create a private one. + \param levelDepth Initial capacity of stack. + */ + explicit + Writer(OutputStream& os, StackAllocator* stackAllocator = 0, size_t levelDepth = kDefaultLevelDepth) : + os_(&os), level_stack_(stackAllocator, levelDepth * sizeof(Level)), maxDecimalPlaces_(kDefaultMaxDecimalPlaces), hasRoot_(false) {} + + explicit + Writer(StackAllocator* allocator = 0, size_t levelDepth = kDefaultLevelDepth) : + os_(0), level_stack_(allocator, levelDepth * sizeof(Level)), maxDecimalPlaces_(kDefaultMaxDecimalPlaces), hasRoot_(false) {} + +#if CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS + Writer(Writer&& rhs) : + os_(rhs.os_), level_stack_(std::move(rhs.level_stack_)), maxDecimalPlaces_(rhs.maxDecimalPlaces_), hasRoot_(rhs.hasRoot_) { + rhs.os_ = 0; + } +#endif + + //! Reset the writer with a new stream. + /*! + This function reset the writer with a new stream and default settings, + in order to make a Writer object reusable for output multiple JSONs. + + \param os New output stream. + \code + Writer writer(os1); + writer.StartObject(); + // ... + writer.EndObject(); + + writer.Reset(os2); + writer.StartObject(); + // ... + writer.EndObject(); + \endcode + */ + void Reset(OutputStream& os) { + os_ = &os; + hasRoot_ = false; + level_stack_.Clear(); + } + + //! Checks whether the output is a complete JSON. + /*! + A complete JSON has a complete root object or array. + */ + bool IsComplete() const { + return hasRoot_ && level_stack_.Empty(); + } + + int GetMaxDecimalPlaces() const { + return maxDecimalPlaces_; + } + + //! Sets the maximum number of decimal places for double output. + /*! + This setting truncates the output with specified number of decimal places. + + For example, + + \code + writer.SetMaxDecimalPlaces(3); + writer.StartArray(); + writer.Double(0.12345); // "0.123" + writer.Double(0.0001); // "0.0" + writer.Double(1.234567890123456e30); // "1.234567890123456e30" (do not truncate significand for positive exponent) + writer.Double(1.23e-4); // "0.0" (do truncate significand for negative exponent) + writer.EndArray(); + \endcode + + The default setting does not truncate any decimal places. You can restore to this setting by calling + \code + writer.SetMaxDecimalPlaces(Writer::kDefaultMaxDecimalPlaces); + \endcode + */ + void SetMaxDecimalPlaces(int maxDecimalPlaces) { + maxDecimalPlaces_ = maxDecimalPlaces; + } + + /*!@name Implementation of Handler + \see Handler + */ + //@{ + + bool Null() { Prefix(kNullType); return EndValue(WriteNull()); } + bool Bool(bool b) { Prefix(b ? kTrueType : kFalseType); return EndValue(WriteBool(b)); } + bool Int(int i) { Prefix(kNumberType); return EndValue(WriteInt(i)); } + bool Uint(unsigned u) { Prefix(kNumberType); return EndValue(WriteUint(u)); } + bool Int64(int64_t i64) { Prefix(kNumberType); return EndValue(WriteInt64(i64)); } + bool Uint64(uint64_t u64) { Prefix(kNumberType); return EndValue(WriteUint64(u64)); } + + //! Writes the given \c double value to the stream + /*! + \param d The value to be written. + \return Whether it is succeed. + */ + bool Double(double d) { Prefix(kNumberType); return EndValue(WriteDouble(d)); } + + bool RawNumber(const Ch* str, SizeType length, bool copy = false) { + CEREAL_RAPIDJSON_ASSERT(str != 0); + (void)copy; + Prefix(kNumberType); + return EndValue(WriteString(str, length)); + } + + bool String(const Ch* str, SizeType length, bool copy = false) { + CEREAL_RAPIDJSON_ASSERT(str != 0); + (void)copy; + Prefix(kStringType); + return EndValue(WriteString(str, length)); + } + +#if CEREAL_RAPIDJSON_HAS_STDSTRING + bool String(const std::basic_string& str) { + return String(str.data(), SizeType(str.size())); + } +#endif + + bool StartObject() { + Prefix(kObjectType); + new (level_stack_.template Push()) Level(false); + return WriteStartObject(); + } + + bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } + +#if CEREAL_RAPIDJSON_HAS_STDSTRING + bool Key(const std::basic_string& str) + { + return Key(str.data(), SizeType(str.size())); + } +#endif + + bool EndObject(SizeType memberCount = 0) { + (void)memberCount; + CEREAL_RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); // not inside an Object + CEREAL_RAPIDJSON_ASSERT(!level_stack_.template Top()->inArray); // currently inside an Array, not Object + CEREAL_RAPIDJSON_ASSERT(0 == level_stack_.template Top()->valueCount % 2); // Object has a Key without a Value + level_stack_.template Pop(1); + return EndValue(WriteEndObject()); + } + + bool StartArray() { + Prefix(kArrayType); + new (level_stack_.template Push()) Level(true); + return WriteStartArray(); + } + + bool EndArray(SizeType elementCount = 0) { + (void)elementCount; + CEREAL_RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); + CEREAL_RAPIDJSON_ASSERT(level_stack_.template Top()->inArray); + level_stack_.template Pop(1); + return EndValue(WriteEndArray()); + } + //@} + + /*! @name Convenience extensions */ + //@{ + + //! Simpler but slower overload. + bool String(const Ch* const& str) { return String(str, internal::StrLen(str)); } + bool Key(const Ch* const& str) { return Key(str, internal::StrLen(str)); } + + //@} + + //! Write a raw JSON value. + /*! + For user to write a stringified JSON as a value. + + \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. + \param length Length of the json. + \param type Type of the root of json. + */ + bool RawValue(const Ch* json, size_t length, Type type) { + CEREAL_RAPIDJSON_ASSERT(json != 0); + Prefix(type); + return EndValue(WriteRawValue(json, length)); + } + + //! Flush the output stream. + /*! + Allows the user to flush the output stream immediately. + */ + void Flush() { + os_->Flush(); + } + +protected: + //! Information for each nested level + struct Level { + Level(bool inArray_) : valueCount(0), inArray(inArray_) {} + size_t valueCount; //!< number of values in this level + bool inArray; //!< true if in array, otherwise in object + }; + + static const size_t kDefaultLevelDepth = 32; + + bool WriteNull() { + PutReserve(*os_, 4); + PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 'l'); return true; + } + + bool WriteBool(bool b) { + if (b) { + PutReserve(*os_, 4); + PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'r'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'e'); + } + else { + PutReserve(*os_, 5); + PutUnsafe(*os_, 'f'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 's'); PutUnsafe(*os_, 'e'); + } + return true; + } + + bool WriteInt(int i) { + char buffer[11]; + const char* end = internal::i32toa(i, buffer); + PutReserve(*os_, static_cast(end - buffer)); + for (const char* p = buffer; p != end; ++p) + PutUnsafe(*os_, static_cast(*p)); + return true; + } + + bool WriteUint(unsigned u) { + char buffer[10]; + const char* end = internal::u32toa(u, buffer); + PutReserve(*os_, static_cast(end - buffer)); + for (const char* p = buffer; p != end; ++p) + PutUnsafe(*os_, static_cast(*p)); + return true; + } + + bool WriteInt64(int64_t i64) { + char buffer[21]; + const char* end = internal::i64toa(i64, buffer); + PutReserve(*os_, static_cast(end - buffer)); + for (const char* p = buffer; p != end; ++p) + PutUnsafe(*os_, static_cast(*p)); + return true; + } + + bool WriteUint64(uint64_t u64) { + char buffer[20]; + char* end = internal::u64toa(u64, buffer); + PutReserve(*os_, static_cast(end - buffer)); + for (char* p = buffer; p != end; ++p) + PutUnsafe(*os_, static_cast(*p)); + return true; + } + + bool WriteDouble(double d) { + if (internal::Double(d).IsNanOrInf()) { + if (!(writeFlags & kWriteNanAndInfFlag)) + return false; + if (internal::Double(d).IsNan()) { + PutReserve(*os_, 3); + PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); + return true; + } + if (internal::Double(d).Sign()) { + PutReserve(*os_, 9); + PutUnsafe(*os_, '-'); + } + else + PutReserve(*os_, 8); + PutUnsafe(*os_, 'I'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'f'); + PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'y'); + return true; + } + + char buffer[25]; + char* end = internal::dtoa(d, buffer, maxDecimalPlaces_); + PutReserve(*os_, static_cast(end - buffer)); + for (char* p = buffer; p != end; ++p) + PutUnsafe(*os_, static_cast(*p)); + return true; + } + + bool WriteString(const Ch* str, SizeType length) { + static const typename OutputStream::Ch hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; + static const char escape[256] = { +#define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + //0 1 2 3 4 5 6 7 8 9 A B C D E F + 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'b', 't', 'n', 'u', 'f', 'r', 'u', 'u', // 00 + 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', // 10 + 0, 0, '"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20 + Z16, Z16, // 30~4F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, // 50 + Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 // 60~FF +#undef Z16 + }; + + if (TargetEncoding::supportUnicode) + PutReserve(*os_, 2 + length * 6); // "\uxxxx..." + else + PutReserve(*os_, 2 + length * 12); // "\uxxxx\uyyyy..." + + PutUnsafe(*os_, '\"'); + GenericStringStream is(str); + while (ScanWriteUnescapedString(is, length)) { + const Ch c = is.Peek(); + if (!TargetEncoding::supportUnicode && static_cast(c) >= 0x80) { + // Unicode escaping + unsigned codepoint; + if (CEREAL_RAPIDJSON_UNLIKELY(!SourceEncoding::Decode(is, &codepoint))) + return false; + PutUnsafe(*os_, '\\'); + PutUnsafe(*os_, 'u'); + if (codepoint <= 0xD7FF || (codepoint >= 0xE000 && codepoint <= 0xFFFF)) { + PutUnsafe(*os_, hexDigits[(codepoint >> 12) & 15]); + PutUnsafe(*os_, hexDigits[(codepoint >> 8) & 15]); + PutUnsafe(*os_, hexDigits[(codepoint >> 4) & 15]); + PutUnsafe(*os_, hexDigits[(codepoint ) & 15]); + } + else { + CEREAL_RAPIDJSON_ASSERT(codepoint >= 0x010000 && codepoint <= 0x10FFFF); + // Surrogate pair + unsigned s = codepoint - 0x010000; + unsigned lead = (s >> 10) + 0xD800; + unsigned trail = (s & 0x3FF) + 0xDC00; + PutUnsafe(*os_, hexDigits[(lead >> 12) & 15]); + PutUnsafe(*os_, hexDigits[(lead >> 8) & 15]); + PutUnsafe(*os_, hexDigits[(lead >> 4) & 15]); + PutUnsafe(*os_, hexDigits[(lead ) & 15]); + PutUnsafe(*os_, '\\'); + PutUnsafe(*os_, 'u'); + PutUnsafe(*os_, hexDigits[(trail >> 12) & 15]); + PutUnsafe(*os_, hexDigits[(trail >> 8) & 15]); + PutUnsafe(*os_, hexDigits[(trail >> 4) & 15]); + PutUnsafe(*os_, hexDigits[(trail ) & 15]); + } + } + else if ((sizeof(Ch) == 1 || static_cast(c) < 256) && CEREAL_RAPIDJSON_UNLIKELY(escape[static_cast(c)])) { + is.Take(); + PutUnsafe(*os_, '\\'); + PutUnsafe(*os_, static_cast(escape[static_cast(c)])); + if (escape[static_cast(c)] == 'u') { + PutUnsafe(*os_, '0'); + PutUnsafe(*os_, '0'); + PutUnsafe(*os_, hexDigits[static_cast(c) >> 4]); + PutUnsafe(*os_, hexDigits[static_cast(c) & 0xF]); + } + } + else if (CEREAL_RAPIDJSON_UNLIKELY(!(writeFlags & kWriteValidateEncodingFlag ? + Transcoder::Validate(is, *os_) : + Transcoder::TranscodeUnsafe(is, *os_)))) + return false; + } + PutUnsafe(*os_, '\"'); + return true; + } + + bool ScanWriteUnescapedString(GenericStringStream& is, size_t length) { + return CEREAL_RAPIDJSON_LIKELY(is.Tell() < length); + } + + bool WriteStartObject() { os_->Put('{'); return true; } + bool WriteEndObject() { os_->Put('}'); return true; } + bool WriteStartArray() { os_->Put('['); return true; } + bool WriteEndArray() { os_->Put(']'); return true; } + + bool WriteRawValue(const Ch* json, size_t length) { + PutReserve(*os_, length); + GenericStringStream is(json); + while (CEREAL_RAPIDJSON_LIKELY(is.Tell() < length)) { + CEREAL_RAPIDJSON_ASSERT(is.Peek() != '\0'); + if (CEREAL_RAPIDJSON_UNLIKELY(!(writeFlags & kWriteValidateEncodingFlag ? + Transcoder::Validate(is, *os_) : + Transcoder::TranscodeUnsafe(is, *os_)))) + return false; + } + return true; + } + + void Prefix(Type type) { + (void)type; + if (CEREAL_RAPIDJSON_LIKELY(level_stack_.GetSize() != 0)) { // this value is not at root + Level* level = level_stack_.template Top(); + if (level->valueCount > 0) { + if (level->inArray) + os_->Put(','); // add comma if it is not the first element in array + else // in object + os_->Put((level->valueCount % 2 == 0) ? ',' : ':'); + } + if (!level->inArray && level->valueCount % 2 == 0) + CEREAL_RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name + level->valueCount++; + } + else { + CEREAL_RAPIDJSON_ASSERT(!hasRoot_); // Should only has one and only one root. + hasRoot_ = true; + } + } + + // Flush the value if it is the top level one. + bool EndValue(bool ret) { + if (CEREAL_RAPIDJSON_UNLIKELY(level_stack_.Empty())) // end of json text + Flush(); + return ret; + } + + OutputStream* os_; + internal::Stack level_stack_; + int maxDecimalPlaces_; + bool hasRoot_; + +private: + // Prohibit copy constructor & assignment operator. + Writer(const Writer&); + Writer& operator=(const Writer&); +}; + +// Full specialization for StringStream to prevent memory copying + +template<> +inline bool Writer::WriteInt(int i) { + char *buffer = os_->Push(11); + const char* end = internal::i32toa(i, buffer); + os_->Pop(static_cast(11 - (end - buffer))); + return true; +} + +template<> +inline bool Writer::WriteUint(unsigned u) { + char *buffer = os_->Push(10); + const char* end = internal::u32toa(u, buffer); + os_->Pop(static_cast(10 - (end - buffer))); + return true; +} + +template<> +inline bool Writer::WriteInt64(int64_t i64) { + char *buffer = os_->Push(21); + const char* end = internal::i64toa(i64, buffer); + os_->Pop(static_cast(21 - (end - buffer))); + return true; +} + +template<> +inline bool Writer::WriteUint64(uint64_t u) { + char *buffer = os_->Push(20); + const char* end = internal::u64toa(u, buffer); + os_->Pop(static_cast(20 - (end - buffer))); + return true; +} + +template<> +inline bool Writer::WriteDouble(double d) { + if (internal::Double(d).IsNanOrInf()) { + // Note: This code path can only be reached if (CEREAL_RAPIDJSON_WRITE_DEFAULT_FLAGS & kWriteNanAndInfFlag). + if (!(kWriteDefaultFlags & kWriteNanAndInfFlag)) + return false; + if (internal::Double(d).IsNan()) { + PutReserve(*os_, 3); + PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); + return true; + } + if (internal::Double(d).Sign()) { + PutReserve(*os_, 9); + PutUnsafe(*os_, '-'); + } + else + PutReserve(*os_, 8); + PutUnsafe(*os_, 'I'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'f'); + PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'y'); + return true; + } + + char *buffer = os_->Push(25); + char* end = internal::dtoa(d, buffer, maxDecimalPlaces_); + os_->Pop(static_cast(25 - (end - buffer))); + return true; +} + +#if defined(CEREAL_RAPIDJSON_SSE2) || defined(CEREAL_RAPIDJSON_SSE42) +template<> +inline bool Writer::ScanWriteUnescapedString(StringStream& is, size_t length) { + if (length < 16) + return CEREAL_RAPIDJSON_LIKELY(is.Tell() < length); + + if (!CEREAL_RAPIDJSON_LIKELY(is.Tell() < length)) + return false; + + const char* p = is.src_; + const char* end = is.head_ + length; + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + const char* endAligned = reinterpret_cast(reinterpret_cast(end) & static_cast(~15)); + if (nextAligned > end) + return true; + + while (p != nextAligned) + if (*p < 0x20 || *p == '\"' || *p == '\\') { + is.src_ = p; + return CEREAL_RAPIDJSON_LIKELY(is.Tell() < length); + } + else + os_->PutUnsafe(*p++); + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (; p != endAligned; p += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x1F) == 0x1F + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (CEREAL_RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped + SizeType len; +#ifdef _MSC_VER // Find the index of first escaped + unsigned long offset; + _BitScanForward(&offset, r); + len = offset; +#else + len = static_cast(__builtin_ffs(r) - 1); +#endif + char* q = reinterpret_cast(os_->PushUnsafe(len)); + for (size_t i = 0; i < len; i++) + q[i] = p[i]; + + p += len; + break; + } + _mm_storeu_si128(reinterpret_cast<__m128i *>(os_->PushUnsafe(16)), s); + } + + is.src_ = p; + return CEREAL_RAPIDJSON_LIKELY(is.Tell() < length); +} +#elif defined(CEREAL_RAPIDJSON_NEON) +template<> +inline bool Writer::ScanWriteUnescapedString(StringStream& is, size_t length) { + if (length < 16) + return CEREAL_RAPIDJSON_LIKELY(is.Tell() < length); + + if (!CEREAL_RAPIDJSON_LIKELY(is.Tell() < length)) + return false; + + const char* p = is.src_; + const char* end = is.head_ + length; + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + const char* endAligned = reinterpret_cast(reinterpret_cast(end) & static_cast(~15)); + if (nextAligned > end) + return true; + + while (p != nextAligned) + if (*p < 0x20 || *p == '\"' || *p == '\\') { + is.src_ = p; + return CEREAL_RAPIDJSON_LIKELY(is.Tell() < length); + } + else + os_->PutUnsafe(*p++); + + // The rest of string using SIMD + const uint8x16_t s0 = vmovq_n_u8('"'); + const uint8x16_t s1 = vmovq_n_u8('\\'); + const uint8x16_t s2 = vmovq_n_u8('\b'); + const uint8x16_t s3 = vmovq_n_u8(32); + + for (; p != endAligned; p += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, s0); + x = vorrq_u8(x, vceqq_u8(s, s1)); + x = vorrq_u8(x, vceqq_u8(s, s2)); + x = vorrq_u8(x, vcltq_u8(s, s3)); + + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + SizeType len = 0; + bool escaped = false; + if (low == 0) { + if (high != 0) { + unsigned lz = (unsigned)__builtin_clzll(high); + len = 8 + (lz >> 3); + escaped = true; + } + } else { + unsigned lz = (unsigned)__builtin_clzll(low); + len = lz >> 3; + escaped = true; + } + if (CEREAL_RAPIDJSON_UNLIKELY(escaped)) { // some of characters is escaped + char* q = reinterpret_cast(os_->PushUnsafe(len)); + for (size_t i = 0; i < len; i++) + q[i] = p[i]; + + p += len; + break; + } + vst1q_u8(reinterpret_cast(os_->PushUnsafe(16)), s); + } + + is.src_ = p; + return CEREAL_RAPIDJSON_LIKELY(is.Tell() < length); +} +#endif // CEREAL_RAPIDJSON_NEON + +CEREAL_RAPIDJSON_NAMESPACE_END + +#if defined(_MSC_VER) || defined(__clang__) +CEREAL_RAPIDJSON_DIAG_POP +#endif + +#endif // CEREAL_RAPIDJSON_CEREAL_RAPIDJSON_H_ diff --git a/third_party/cereal/external/rapidxml/license.txt b/third_party/cereal/external/rapidxml/license.txt new file mode 100755 index 0000000..0095bc7 --- /dev/null +++ b/third_party/cereal/external/rapidxml/license.txt @@ -0,0 +1,52 @@ +Use of this software is granted under one of the following two licenses, +to be chosen freely by the user. + +1. Boost Software License - Version 1.0 - August 17th, 2003 +=============================================================================== + +Copyright (c) 2006, 2007 Marcin Kalicinski + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +2. The MIT License +=============================================================================== + +Copyright (c) 2006, 2007 Marcin Kalicinski + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. diff --git a/third_party/cereal/external/rapidxml/manual.html b/third_party/cereal/external/rapidxml/manual.html new file mode 100755 index 0000000..4e6a5f1 --- /dev/null +++ b/third_party/cereal/external/rapidxml/manual.html @@ -0,0 +1,406 @@ +

RAPIDXML Manual

Version 1.13

Copyright (C) 2006, 2009 Marcin Kalicinski
See accompanying file license.txt for license information.

Table of Contents

1. What is RapidXml?
1.1 Dependencies And Compatibility
1.2 Character Types And Encodings
1.3 Error Handling
1.4 Memory Allocation
1.5 W3C Compliance
1.6 API Design
1.7 Reliability
1.8 Acknowledgements
2. Two Minute Tutorial
2.1 Parsing
2.2 Accessing The DOM Tree
2.3 Modifying The DOM Tree
2.4 Printing XML
3. Differences From Regular XML Parsers
3.1 Lifetime Of Source Text
3.2 Ownership Of Strings
3.3 Destructive Vs Non-Destructive Mode
4. Performance
4.1 Comparison With Other Parsers
5. Reference

1. What is RapidXml?

RapidXml is an attempt to create the fastest XML DOM parser possible, while retaining useability, portability and reasonable W3C compatibility. It is an in-situ parser written in C++, with parsing speed approaching that of strlen() function executed on the same data.

+ Entire parser is contained in a single header file, so no building or linking is neccesary. To use it you just need to copy rapidxml.hpp file to a convenient place (such as your project directory), and include it where needed. You may also want to use printing functions contained in header rapidxml_print.hpp.

1.1 Dependencies And Compatibility

RapidXml has no dependencies other than a very small subset of standard C++ library (<cassert>, <cstdlib>, <new> and <exception>, unless exceptions are disabled). It should compile on any reasonably conformant compiler, and was tested on Visual C++ 2003, Visual C++ 2005, Visual C++ 2008, gcc 3, gcc 4, and Comeau 4.3.3. Care was taken that no warnings are produced on these compilers, even with highest warning levels enabled.

1.2 Character Types And Encodings

RapidXml is character type agnostic, and can work both with narrow and wide characters. Current version does not fully support UTF-16 or UTF-32, so use of wide characters is somewhat incapacitated. However, it should succesfully parse wchar_t strings containing UTF-16 or UTF-32 if endianness of the data matches that of the machine. UTF-8 is fully supported, including all numeric character references, which are expanded into appropriate UTF-8 byte sequences (unless you enable parse_no_utf8 flag).

+ Note that RapidXml performs no decoding - strings returned by name() and value() functions will contain text encoded using the same encoding as source file. Rapidxml understands and expands the following character references: &apos; &amp; &quot; &lt; &gt; &#...; Other character references are not expanded.

1.3 Error Handling

By default, RapidXml uses C++ exceptions to report errors. If this behaviour is undesirable, RAPIDXML_NO_EXCEPTIONS can be defined to suppress exception code. See parse_error class and parse_error_handler() function for more information.

1.4 Memory Allocation

RapidXml uses a special memory pool object to allocate nodes and attributes, because direct allocation using new operator would be far too slow. Underlying memory allocations performed by the pool can be customized by use of memory_pool::set_allocator() function. See class memory_pool for more information.

1.5 W3C Compliance

RapidXml is not a W3C compliant parser, primarily because it ignores DOCTYPE declarations. There is a number of other, minor incompatibilities as well. Still, it can successfully parse and produce complete trees of all valid XML files in W3C conformance suite (over 1000 files specially designed to find flaws in XML processors). In destructive mode it performs whitespace normalization and character entity substitution for a small set of built-in entities.

1.6 API Design

RapidXml API is minimalistic, to reduce code size as much as possible, and facilitate use in embedded environments. Additional convenience functions are provided in separate headers: rapidxml_utils.hpp and rapidxml_print.hpp. Contents of these headers is not an essential part of the library, and is currently not documented (otherwise than with comments in code).

1.7 Reliability

RapidXml is very robust and comes with a large harness of unit tests. Special care has been taken to ensure stability of the parser no matter what source text is thrown at it. One of the unit tests produces 100,000 randomly corrupted variants of XML document, which (when uncorrupted) contains all constructs recognized by RapidXml. RapidXml passes this test when it correctly recognizes that errors have been introduced, and does not crash or loop indefinitely.

+ Another unit test puts RapidXml head-to-head with another, well estabilished XML parser, and verifies that their outputs match across a wide variety of small and large documents.

+ Yet another test feeds RapidXml with over 1000 test files from W3C compliance suite, and verifies that correct results are obtained. There are also additional tests that verify each API function separately, and test that various parsing modes work as expected.

1.8 Acknowledgements

I would like to thank Arseny Kapoulkine for his work on pugixml, which was an inspiration for this project. Additional thanks go to Kristen Wegner for creating pugxml, from which pugixml was derived. Janusz Wohlfeil kindly ran RapidXml speed tests on hardware that I did not have access to, allowing me to expand performance comparison table.

2. Two Minute Tutorial

2.1 Parsing

The following code causes RapidXml to parse a zero-terminated string named text: 
using namespace rapidxml;
+xml_document<> doc;    // character type defaults to char
+doc.parse<0>(text);    // 0 means default parse flags
+
doc object is now a root of DOM tree containing representation of the parsed XML. Because all RapidXml interface is contained inside namespace rapidxml, users must either bring contents of this namespace into scope, or fully qualify all the names. Class xml_document represents a root of the DOM hierarchy. By means of public inheritance, it is also an xml_node and a memory_pool. Template parameter of xml_document::parse() function is used to specify parsing flags, with which you can fine-tune behaviour of the parser. Note that flags must be a compile-time constant.

2.2 Accessing The DOM Tree

To access the DOM tree, use methods of xml_node and xml_attribute classes: 
cout << "Name of my first node is: " << doc.first_node()->name() << "\n";
+xml_node<> *node = doc.first_node("foobar");
+cout << "Node foobar has value " << node->value() << "\n";
+for (xml_attribute<> *attr = node->first_attribute();
+     attr; attr = attr->next_attribute())
+{
+    cout << "Node foobar has attribute " << attr->name() << " ";
+    cout << "with value " << attr->value() << "\n";
+}
+

2.3 Modifying The DOM Tree

DOM tree produced by the parser is fully modifiable. Nodes and attributes can be added/removed, and their contents changed. The below example creates a HTML document, whose sole contents is a link to google.com website: 
xml_document<> doc;
+xml_node<> *node = doc.allocate_node(node_element, "a", "Google");
+doc.append_node(node);
+xml_attribute<> *attr = doc.allocate_attribute("href", "google.com");
+node->append_attribute(attr);
+
One quirk is that nodes and attributes do not own the text of their names and values. This is because normally they only store pointers to the source text. So, when assigning a new name or value to the node, care must be taken to ensure proper lifetime of the string. The easiest way to achieve it is to allocate the string from the xml_document memory pool. In the above example this is not necessary, because we are only assigning character constants. But the code below uses memory_pool::allocate_string() function to allocate node name (which will have the same lifetime as the document), and assigns it to a new node: 
xml_document<> doc;
+char *node_name = doc.allocate_string(name);        // Allocate string and copy name into it
+xml_node<> *node = doc.allocate_node(node_element, node_name);  // Set node name to node_name
+
Check Reference section for description of the entire interface.

2.4 Printing XML

You can print xml_document and xml_node objects into an XML string. Use print() function or operator <<, which are defined in rapidxml_print.hpp header. 
using namespace rapidxml;
+xml_document<> doc;    // character type defaults to char
+// ... some code to fill the document
+
+// Print to stream using operator <<
+std::cout << doc;   
+
+// Print to stream using print function, specifying printing flags
+print(std::cout, doc, 0);   // 0 means default printing flags
+
+// Print to string using output iterator
+std::string s;
+print(std::back_inserter(s), doc, 0);
+
+// Print to memory buffer using output iterator
+char buffer[4096];                      // You are responsible for making the buffer large enough!
+char *end = print(buffer, doc, 0);      // end contains pointer to character after last printed character
+*end = 0;                               // Add string terminator after XML
+

3. Differences From Regular XML Parsers

RapidXml is an in-situ parser, which allows it to achieve very high parsing speed. In-situ means that parser does not make copies of strings. Instead, it places pointers to the source text in the DOM hierarchy.

3.1 Lifetime Of Source Text

In-situ parsing requires that source text lives at least as long as the document object. If source text is destroyed, names and values of nodes in DOM tree will become destroyed as well. Additionally, whitespace processing, character entity translation, and zero-termination of strings require that source text be modified during parsing (but see non-destructive mode). This makes the text useless for further processing once it was parsed by RapidXml.

+ In many cases however, these are not serious issues.

3.2 Ownership Of Strings

Nodes and attributes produced by RapidXml do not own their name and value strings. They merely hold the pointers to them. This means you have to be careful when setting these values manually, by using xml_base::name(const Ch *) or xml_base::value(const Ch *) functions. Care must be taken to ensure that lifetime of the string passed is at least as long as lifetime of the node/attribute. The easiest way to achieve it is to allocate the string from memory_pool owned by the document. Use memory_pool::allocate_string() function for this purpose.

3.3 Destructive Vs Non-Destructive Mode

By default, the parser modifies source text during the parsing process. This is required to achieve character entity translation, whitespace normalization, and zero-termination of strings.

+ In some cases this behaviour may be undesirable, for example if source text resides in read only memory, or is mapped to memory directly from file. By using appropriate parser flags (parse_non_destructive), source text modifications can be disabled. However, because RapidXml does in-situ parsing, it obviously has the following side-effects:

4. Performance

RapidXml achieves its speed through use of several techniques:
  • In-situ parsing. When building DOM tree, RapidXml does not make copies of string data, such as node names and values. Instead, it stores pointers to interior of the source text.
  • Use of template metaprogramming techniques. This allows it to move much of the work to compile time. Through magic of the templates, C++ compiler generates a separate copy of parsing code for any combination of parser flags you use. In each copy, all possible decisions are made at compile time and all unused code is omitted.
  • Extensive use of lookup tables for parsing.
  • Hand-tuned C++ with profiling done on several most popular CPUs.
This results in a very small and fast code: a parser which is custom tailored to exact needs with each invocation.

4.1 Comparison With Other Parsers

The table below compares speed of RapidXml to some other parsers, and to strlen() function executed on the same data. On a modern CPU (as of 2007), you can expect parsing throughput to be close to 1 GB/s. As a rule of thumb, parsing speed is about 50-100x faster than Xerces DOM, 30-60x faster than TinyXml, 3-12x faster than pugxml, and about 5% - 30% faster than pugixml, the fastest XML parser I know of.
  • The test file is a real-world, 50kB large, moderately dense XML file.
  • All timing is done by using RDTSC instruction present in Pentium-compatible CPUs.
  • No profile-guided optimizations are used.
  • All parsers are running in their fastest modes.
  • The results are given in CPU cycles per character, so frequency of CPUs is irrelevant.
  • The results are minimum values from a large number of runs, to minimize effects of operating system activity, task switching, interrupt handling etc.
  • A single parse of the test file takes about 1/10th of a millisecond, so with large number of runs there is a good chance of hitting at least one no-interrupt streak, and obtaining undisturbed results.
Platform
Compiler
strlen() RapidXml pugixml 0.3 pugxml TinyXml
Pentium 4
MSVC 8.0
2.5
5.4
7.0
61.7
298.8
Pentium 4
gcc 4.1.1
0.8
6.1
9.5
67.0
413.2
Core 2
MSVC 8.0
1.0
4.5
5.0
24.6
154.8
Core 2
gcc 4.1.1
0.6
4.6
5.4
28.3
229.3
Athlon XP
MSVC 8.0
3.1
7.7
8.0
25.5
182.6
Athlon XP
gcc 4.1.1
0.9
8.2
9.2
33.7
265.2
Pentium 3
MSVC 8.0
2.0
6.3
7.0
30.9
211.9
Pentium 3
gcc 4.1.1
1.0
6.7
8.9
35.3
316.0
(*) All results are in CPU cycles per character of source text

5. Reference

This section lists all classes, functions, constants etc. and describes them in detail.
class + template + rapidxml::memory_pool
+ constructor + memory_pool()
+ destructor + ~memory_pool()
function allocate_node(node_type type, const Ch *name=0, const Ch *value=0, std::size_t name_size=0, std::size_t value_size=0)
function allocate_attribute(const Ch *name=0, const Ch *value=0, std::size_t name_size=0, std::size_t value_size=0)
function allocate_string(const Ch *source=0, std::size_t size=0)
function clone_node(const xml_node< Ch > *source, xml_node< Ch > *result=0)
function clear()
function set_allocator(alloc_func *af, free_func *ff)

class rapidxml::parse_error
+ constructor + parse_error(const char *what, void *where)
function what() const
function where() const

class + template + rapidxml::xml_attribute
+ constructor + xml_attribute()
function document() const
function previous_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const
function next_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const

class + template + rapidxml::xml_base
+ constructor + xml_base()
function name() const
function name_size() const
function value() const
function value_size() const
function name(const Ch *name, std::size_t size)
function name(const Ch *name)
function value(const Ch *value, std::size_t size)
function value(const Ch *value)
function parent() const

class + template + rapidxml::xml_document
+ constructor + xml_document()
function parse(Ch *text)
function clear()

class + template + rapidxml::xml_node
+ constructor + xml_node(node_type type)
function type() const
function document() const
function first_node(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const
function last_node(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const
function previous_sibling(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const
function next_sibling(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const
function first_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const
function last_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const
function type(node_type type)
function prepend_node(xml_node< Ch > *child)
function append_node(xml_node< Ch > *child)
function insert_node(xml_node< Ch > *where, xml_node< Ch > *child)
function remove_first_node()
function remove_last_node()
function remove_node(xml_node< Ch > *where)
function remove_all_nodes()
function prepend_attribute(xml_attribute< Ch > *attribute)
function append_attribute(xml_attribute< Ch > *attribute)
function insert_attribute(xml_attribute< Ch > *where, xml_attribute< Ch > *attribute)
function remove_first_attribute()
function remove_last_attribute()
function remove_attribute(xml_attribute< Ch > *where)
function remove_all_attributes()

namespace rapidxml
enum node_type
function parse_error_handler(const char *what, void *where)
function print(OutIt out, const xml_node< Ch > &node, int flags=0)
function print(std::basic_ostream< Ch > &out, const xml_node< Ch > &node, int flags=0)
function operator<<(std::basic_ostream< Ch > &out, const xml_node< Ch > &node)
+ constant + parse_no_data_nodes
+ constant + parse_no_element_values
+ constant + parse_no_string_terminators
+ constant + parse_no_entity_translation
+ constant + parse_no_utf8
+ constant + parse_declaration_node
+ constant + parse_comment_nodes
+ constant + parse_doctype_node
+ constant + parse_pi_nodes
+ constant + parse_validate_closing_tags
+ constant + parse_trim_whitespace
+ constant + parse_normalize_whitespace
+ constant + parse_default
+ constant + parse_non_destructive
+ constant + parse_fastest
+ constant + parse_full
+ constant + print_no_indenting

class + template + rapidxml::memory_pool

+ + Defined in rapidxml.hpp
+ Base class for + xml_document

Description

This class is used by the parser to create new nodes and attributes, without overheads of dynamic memory allocation. In most cases, you will not need to use this class directly. However, if you need to create nodes manually or modify names/values of nodes, you are encouraged to use memory_pool of relevant xml_document to allocate the memory. Not only is this faster than allocating them by using new operator, but also their lifetime will be tied to the lifetime of document, possibly simplyfing memory management.

+ Call allocate_node() or allocate_attribute() functions to obtain new nodes or attributes from the pool. You can also call allocate_string() function to allocate strings. Such strings can then be used as names or values of nodes without worrying about their lifetime. Note that there is no free() function -- all allocations are freed at once when clear() function is called, or when the pool is destroyed.

+ It is also possible to create a standalone memory_pool, and use it to allocate nodes, whose lifetime will not be tied to any document.

+ Pool maintains RAPIDXML_STATIC_POOL_SIZE bytes of statically allocated memory. Until static memory is exhausted, no dynamic memory allocations are done. When static memory is exhausted, pool allocates additional blocks of memory of size RAPIDXML_DYNAMIC_POOL_SIZE each, by using global new[] and delete[] operators. This behaviour can be changed by setting custom allocation routines. Use set_allocator() function to set them.

+ Allocations for nodes, attributes and strings are aligned at RAPIDXML_ALIGNMENT bytes. This value defaults to the size of pointer on target architecture.

+ To obtain absolutely top performance from the parser, it is important that all nodes are allocated from a single, contiguous block of memory. Otherwise, cache misses when jumping between two (or more) disjoint blocks of memory can slow down parsing quite considerably. If required, you can tweak RAPIDXML_STATIC_POOL_SIZE, RAPIDXML_DYNAMIC_POOL_SIZE and RAPIDXML_ALIGNMENT to obtain best wasted memory to performance compromise. To do it, define their values before rapidxml.hpp file is included.

Parameters

Ch
Character type of created nodes.

+ constructor + memory_pool::memory_pool

Synopsis

memory_pool(); +

Description

Constructs empty pool with default allocator functions.

+ destructor + memory_pool::~memory_pool

Synopsis

~memory_pool(); +

Description

Destroys pool and frees all the memory. This causes memory occupied by nodes allocated by the pool to be freed. Nodes allocated from the pool are no longer valid.

function memory_pool::allocate_node

Synopsis

xml_node<Ch>* allocate_node(node_type type, const Ch *name=0, const Ch *value=0, std::size_t name_size=0, std::size_t value_size=0); +

Description

Allocates a new node from the pool, and optionally assigns name and value to it. If the allocation request cannot be accomodated, this function will throw std::bad_alloc. If exceptions are disabled by defining RAPIDXML_NO_EXCEPTIONS, this function will call rapidxml::parse_error_handler() function.

Parameters

type
Type of node to create.
name
Name to assign to the node, or 0 to assign no name.
value
Value to assign to the node, or 0 to assign no value.
name_size
Size of name to assign, or 0 to automatically calculate size from name string.
value_size
Size of value to assign, or 0 to automatically calculate size from value string.

Returns

Pointer to allocated node. This pointer will never be NULL.

function memory_pool::allocate_attribute

Synopsis

xml_attribute<Ch>* allocate_attribute(const Ch *name=0, const Ch *value=0, std::size_t name_size=0, std::size_t value_size=0); +

Description

Allocates a new attribute from the pool, and optionally assigns name and value to it. If the allocation request cannot be accomodated, this function will throw std::bad_alloc. If exceptions are disabled by defining RAPIDXML_NO_EXCEPTIONS, this function will call rapidxml::parse_error_handler() function.

Parameters

name
Name to assign to the attribute, or 0 to assign no name.
value
Value to assign to the attribute, or 0 to assign no value.
name_size
Size of name to assign, or 0 to automatically calculate size from name string.
value_size
Size of value to assign, or 0 to automatically calculate size from value string.

Returns

Pointer to allocated attribute. This pointer will never be NULL.

function memory_pool::allocate_string

Synopsis

Ch* allocate_string(const Ch *source=0, std::size_t size=0); +

Description

Allocates a char array of given size from the pool, and optionally copies a given string to it. If the allocation request cannot be accomodated, this function will throw std::bad_alloc. If exceptions are disabled by defining RAPIDXML_NO_EXCEPTIONS, this function will call rapidxml::parse_error_handler() function.

Parameters

source
String to initialize the allocated memory with, or 0 to not initialize it.
size
Number of characters to allocate, or zero to calculate it automatically from source string length; if size is 0, source string must be specified and null terminated.

Returns

Pointer to allocated char array. This pointer will never be NULL.

function memory_pool::clone_node

Synopsis

xml_node<Ch>* clone_node(const xml_node< Ch > *source, xml_node< Ch > *result=0); +

Description

Clones an xml_node and its hierarchy of child nodes and attributes. Nodes and attributes are allocated from this memory pool. Names and values are not cloned, they are shared between the clone and the source. Result node can be optionally specified as a second parameter, in which case its contents will be replaced with cloned source node. This is useful when you want to clone entire document.

Parameters

source
Node to clone.
result
Node to put results in, or 0 to automatically allocate result node

Returns

Pointer to cloned node. This pointer will never be NULL.

function memory_pool::clear

Synopsis

void clear(); +

Description

Clears the pool. This causes memory occupied by nodes allocated by the pool to be freed. Any nodes or strings allocated from the pool will no longer be valid.

function memory_pool::set_allocator

Synopsis

void set_allocator(alloc_func *af, free_func *ff); +

Description

Sets or resets the user-defined memory allocation functions for the pool. This can only be called when no memory is allocated from the pool yet, otherwise results are undefined. Allocation function must not return invalid pointer on failure. It should either throw, stop the program, or use longjmp() function to pass control to other place of program. If it returns invalid pointer, results are undefined.

+ User defined allocation functions must have the following forms:

+void *allocate(std::size_t size);
+void free(void *pointer);

Parameters

af
Allocation function, or 0 to restore default function
ff
Free function, or 0 to restore default function

class rapidxml::parse_error

+ + Defined in rapidxml.hpp

Description

Parse error exception. This exception is thrown by the parser when an error occurs. Use what() function to get human-readable error message. Use where() function to get a pointer to position within source text where error was detected.

+ If throwing exceptions by the parser is undesirable, it can be disabled by defining RAPIDXML_NO_EXCEPTIONS macro before rapidxml.hpp is included. This will cause the parser to call rapidxml::parse_error_handler() function instead of throwing an exception. This function must be defined by the user.

+ This class derives from std::exception class.

+ constructor + parse_error::parse_error

Synopsis

parse_error(const char *what, void *where); +

Description

Constructs parse error.

function parse_error::what

Synopsis

virtual const char* what() const; +

Description

Gets human readable description of error.

Returns

Pointer to null terminated description of the error.

function parse_error::where

Synopsis

Ch* where() const; +

Description

Gets pointer to character data where error happened. Ch should be the same as char type of xml_document that produced the error.

Returns

Pointer to location within the parsed string where error occured.

class + template + rapidxml::xml_attribute

+ + Defined in rapidxml.hpp
+ Inherits from + xml_base

Description

Class representing attribute node of XML document. Each attribute has name and value strings, which are available through name() and value() functions (inherited from xml_base). Note that after parse, both name and value of attribute will point to interior of source text used for parsing. Thus, this text must persist in memory for the lifetime of attribute.

Parameters

Ch
Character type to use.

+ constructor + xml_attribute::xml_attribute

Synopsis

xml_attribute(); +

Description

Constructs an empty attribute with the specified type. Consider using memory_pool of appropriate xml_document if allocating attributes manually.

function xml_attribute::document

Synopsis

xml_document<Ch>* document() const; +

Description

Gets document of which attribute is a child.

Returns

Pointer to document that contains this attribute, or 0 if there is no parent document.

function xml_attribute::previous_attribute

Synopsis

xml_attribute<Ch>* previous_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; +

Description

Gets previous attribute, optionally matching attribute name.

Parameters

name
Name of attribute to find, or 0 to return previous attribute regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found attribute, or 0 if not found.

function xml_attribute::next_attribute

Synopsis

xml_attribute<Ch>* next_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; +

Description

Gets next attribute, optionally matching attribute name.

Parameters

name
Name of attribute to find, or 0 to return next attribute regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found attribute, or 0 if not found.

class + template + rapidxml::xml_base

+ + Defined in rapidxml.hpp
+ Base class for + xml_attribute xml_node

Description

Base class for xml_node and xml_attribute implementing common functions: name(), name_size(), value(), value_size() and parent().

Parameters

Ch
Character type to use

+ constructor + xml_base::xml_base

Synopsis

xml_base(); +

function xml_base::name

Synopsis

Ch* name() const; +

Description

Gets name of the node. Interpretation of name depends on type of node. Note that name will not be zero-terminated if rapidxml::parse_no_string_terminators option was selected during parse.

+ Use name_size() function to determine length of the name.

Returns

Name of node, or empty string if node has no name.

function xml_base::name_size

Synopsis

std::size_t name_size() const; +

Description

Gets size of node name, not including terminator character. This function works correctly irrespective of whether name is or is not zero terminated.

Returns

Size of node name, in characters.

function xml_base::value

Synopsis

Ch* value() const; +

Description

Gets value of node. Interpretation of value depends on type of node. Note that value will not be zero-terminated if rapidxml::parse_no_string_terminators option was selected during parse.

+ Use value_size() function to determine length of the value.

Returns

Value of node, or empty string if node has no value.

function xml_base::value_size

Synopsis

std::size_t value_size() const; +

Description

Gets size of node value, not including terminator character. This function works correctly irrespective of whether value is or is not zero terminated.

Returns

Size of node value, in characters.

function xml_base::name

Synopsis

void name(const Ch *name, std::size_t size); +

Description

Sets name of node to a non zero-terminated string. See Ownership Of Strings .

+ Note that node does not own its name or value, it only stores a pointer to it. It will not delete or otherwise free the pointer on destruction. It is reponsibility of the user to properly manage lifetime of the string. The easiest way to achieve it is to use memory_pool of the document to allocate the string - on destruction of the document the string will be automatically freed.

+ Size of name must be specified separately, because name does not have to be zero terminated. Use name(const Ch *) function to have the length automatically calculated (string must be zero terminated).

Parameters

name
Name of node to set. Does not have to be zero terminated.
size
Size of name, in characters. This does not include zero terminator, if one is present.

function xml_base::name

Synopsis

void name(const Ch *name); +

Description

Sets name of node to a zero-terminated string. See also Ownership Of Strings and xml_node::name(const Ch *, std::size_t).

Parameters

name
Name of node to set. Must be zero terminated.

function xml_base::value

Synopsis

void value(const Ch *value, std::size_t size); +

Description

Sets value of node to a non zero-terminated string. See Ownership Of Strings .

+ Note that node does not own its name or value, it only stores a pointer to it. It will not delete or otherwise free the pointer on destruction. It is reponsibility of the user to properly manage lifetime of the string. The easiest way to achieve it is to use memory_pool of the document to allocate the string - on destruction of the document the string will be automatically freed.

+ Size of value must be specified separately, because it does not have to be zero terminated. Use value(const Ch *) function to have the length automatically calculated (string must be zero terminated).

+ If an element has a child node of type node_data, it will take precedence over element value when printing. If you want to manipulate data of elements using values, use parser flag rapidxml::parse_no_data_nodes to prevent creation of data nodes by the parser.

Parameters

value
value of node to set. Does not have to be zero terminated.
size
Size of value, in characters. This does not include zero terminator, if one is present.

function xml_base::value

Synopsis

void value(const Ch *value); +

Description

Sets value of node to a zero-terminated string. See also Ownership Of Strings and xml_node::value(const Ch *, std::size_t).

Parameters

value
Vame of node to set. Must be zero terminated.

function xml_base::parent

Synopsis

xml_node<Ch>* parent() const; +

Description

Gets node parent.

Returns

Pointer to parent node, or 0 if there is no parent.

class + template + rapidxml::xml_document

+ + Defined in rapidxml.hpp
+ Inherits from + xml_node memory_pool

Description

This class represents root of the DOM hierarchy. It is also an xml_node and a memory_pool through public inheritance. Use parse() function to build a DOM tree from a zero-terminated XML text string. parse() function allocates memory for nodes and attributes by using functions of xml_document, which are inherited from memory_pool. To access root node of the document, use the document itself, as if it was an xml_node.

Parameters

Ch
Character type to use.

+ constructor + xml_document::xml_document

Synopsis

xml_document(); +

Description

Constructs empty XML document.

function xml_document::parse

Synopsis

void parse(Ch *text); +

Description

Parses zero-terminated XML string according to given flags. Passed string will be modified by the parser, unless rapidxml::parse_non_destructive flag is used. The string must persist for the lifetime of the document. In case of error, rapidxml::parse_error exception will be thrown.

+ If you want to parse contents of a file, you must first load the file into the memory, and pass pointer to its beginning. Make sure that data is zero-terminated.

+ Document can be parsed into multiple times. Each new call to parse removes previous nodes and attributes (if any), but does not clear memory pool.

Parameters

text
XML data to parse; pointer is non-const to denote fact that this data may be modified by the parser.

function xml_document::clear

Synopsis

void clear(); +

Description

Clears the document by deleting all nodes and clearing the memory pool. All nodes owned by document pool are destroyed.

class + template + rapidxml::xml_node

+ + Defined in rapidxml.hpp
+ Inherits from + xml_base
+ Base class for + xml_document

Description

Class representing a node of XML document. Each node may have associated name and value strings, which are available through name() and value() functions. Interpretation of name and value depends on type of the node. Type of node can be determined by using type() function.

+ Note that after parse, both name and value of node, if any, will point interior of source text used for parsing. Thus, this text must persist in the memory for the lifetime of node.

Parameters

Ch
Character type to use.

+ constructor + xml_node::xml_node

Synopsis

xml_node(node_type type); +

Description

Constructs an empty node with the specified type. Consider using memory_pool of appropriate document to allocate nodes manually.

Parameters

type
Type of node to construct.

function xml_node::type

Synopsis

node_type type() const; +

Description

Gets type of node.

Returns

Type of node.

function xml_node::document

Synopsis

xml_document<Ch>* document() const; +

Description

Gets document of which node is a child.

Returns

Pointer to document that contains this node, or 0 if there is no parent document.

function xml_node::first_node

Synopsis

xml_node<Ch>* first_node(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; +

Description

Gets first child node, optionally matching node name.

Parameters

name
Name of child to find, or 0 to return first child regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found child, or 0 if not found.

function xml_node::last_node

Synopsis

xml_node<Ch>* last_node(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; +

Description

Gets last child node, optionally matching node name. Behaviour is undefined if node has no children. Use first_node() to test if node has children.

Parameters

name
Name of child to find, or 0 to return last child regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found child, or 0 if not found.

function xml_node::previous_sibling

Synopsis

xml_node<Ch>* previous_sibling(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; +

Description

Gets previous sibling node, optionally matching node name. Behaviour is undefined if node has no parent. Use parent() to test if node has a parent.

Parameters

name
Name of sibling to find, or 0 to return previous sibling regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found sibling, or 0 if not found.

function xml_node::next_sibling

Synopsis

xml_node<Ch>* next_sibling(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; +

Description

Gets next sibling node, optionally matching node name. Behaviour is undefined if node has no parent. Use parent() to test if node has a parent.

Parameters

name
Name of sibling to find, or 0 to return next sibling regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found sibling, or 0 if not found.

function xml_node::first_attribute

Synopsis

xml_attribute<Ch>* first_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; +

Description

Gets first attribute of node, optionally matching attribute name.

Parameters

name
Name of attribute to find, or 0 to return first attribute regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found attribute, or 0 if not found.

function xml_node::last_attribute

Synopsis

xml_attribute<Ch>* last_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; +

Description

Gets last attribute of node, optionally matching attribute name.

Parameters

name
Name of attribute to find, or 0 to return last attribute regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found attribute, or 0 if not found.

function xml_node::type

Synopsis

void type(node_type type); +

Description

Sets type of node.

Parameters

type
Type of node to set.

function xml_node::prepend_node

Synopsis

void prepend_node(xml_node< Ch > *child); +

Description

Prepends a new child node. The prepended child becomes the first child, and all existing children are moved one position back.

Parameters

child
Node to prepend.

function xml_node::append_node

Synopsis

void append_node(xml_node< Ch > *child); +

Description

Appends a new child node. The appended child becomes the last child.

Parameters

child
Node to append.

function xml_node::insert_node

Synopsis

void insert_node(xml_node< Ch > *where, xml_node< Ch > *child); +

Description

Inserts a new child node at specified place inside the node. All children after and including the specified node are moved one position back.

Parameters

where
Place where to insert the child, or 0 to insert at the back.
child
Node to insert.

function xml_node::remove_first_node

Synopsis

void remove_first_node(); +

Description

Removes first child node. If node has no children, behaviour is undefined. Use first_node() to test if node has children.

function xml_node::remove_last_node

Synopsis

void remove_last_node(); +

Description

Removes last child of the node. If node has no children, behaviour is undefined. Use first_node() to test if node has children.

function xml_node::remove_node

Synopsis

void remove_node(xml_node< Ch > *where); +

Description

Removes specified child from the node.

function xml_node::remove_all_nodes

Synopsis

void remove_all_nodes(); +

Description

Removes all child nodes (but not attributes).

function xml_node::prepend_attribute

Synopsis

void prepend_attribute(xml_attribute< Ch > *attribute); +

Description

Prepends a new attribute to the node.

Parameters

attribute
Attribute to prepend.

function xml_node::append_attribute

Synopsis

void append_attribute(xml_attribute< Ch > *attribute); +

Description

Appends a new attribute to the node.

Parameters

attribute
Attribute to append.

function xml_node::insert_attribute

Synopsis

void insert_attribute(xml_attribute< Ch > *where, xml_attribute< Ch > *attribute); +

Description

Inserts a new attribute at specified place inside the node. All attributes after and including the specified attribute are moved one position back.

Parameters

where
Place where to insert the attribute, or 0 to insert at the back.
attribute
Attribute to insert.

function xml_node::remove_first_attribute

Synopsis

void remove_first_attribute(); +

Description

Removes first attribute of the node. If node has no attributes, behaviour is undefined. Use first_attribute() to test if node has attributes.

function xml_node::remove_last_attribute

Synopsis

void remove_last_attribute(); +

Description

Removes last attribute of the node. If node has no attributes, behaviour is undefined. Use first_attribute() to test if node has attributes.

function xml_node::remove_attribute

Synopsis

void remove_attribute(xml_attribute< Ch > *where); +

Description

Removes specified attribute from node.

Parameters

where
Pointer to attribute to be removed.

function xml_node::remove_all_attributes

Synopsis

void remove_all_attributes(); +

Description

Removes all attributes of node.

enum node_type

Description

Enumeration listing all node types produced by the parser. Use xml_node::type() function to query node type.

Values

node_document
A document node. Name and value are empty.
node_element
An element node. Name contains element name. Value contains text of first data node.
node_data
A data node. Name is empty. Value contains data text.
node_cdata
A CDATA node. Name is empty. Value contains data text.
node_comment
A comment node. Name is empty. Value contains comment text.
node_declaration
A declaration node. Name and value are empty. Declaration parameters (version, encoding and standalone) are in node attributes.
node_doctype
A DOCTYPE node. Name is empty. Value contains DOCTYPE text.
node_pi
A PI node. Name contains target. Value contains instructions.

function parse_error_handler

Synopsis

void rapidxml::parse_error_handler(const char *what, void *where); +

Description

When exceptions are disabled by defining RAPIDXML_NO_EXCEPTIONS, this function is called to notify user about the error. It must be defined by the user.

+ This function cannot return. If it does, the results are undefined.

+ A very simple definition might look like that: + void rapidxml::parse_error_handler(const char *what, void *where) + { + std::cout << "Parse error: " << what << "\n"; + std::abort(); + } +

Parameters

what
Human readable description of the error.
where
Pointer to character data where error was detected.

function print

Synopsis

OutIt rapidxml::print(OutIt out, const xml_node< Ch > &node, int flags=0); +

Description

Prints XML to given output iterator.

Parameters

out
Output iterator to print to.
node
Node to be printed. Pass xml_document to print entire document.
flags
Flags controlling how XML is printed.

Returns

Output iterator pointing to position immediately after last character of printed text.

function print

Synopsis

std::basic_ostream<Ch>& rapidxml::print(std::basic_ostream< Ch > &out, const xml_node< Ch > &node, int flags=0); +

Description

Prints XML to given output stream.

Parameters

out
Output stream to print to.
node
Node to be printed. Pass xml_document to print entire document.
flags
Flags controlling how XML is printed.

Returns

Output stream.

function operator<<

Synopsis

std::basic_ostream<Ch>& rapidxml::operator<<(std::basic_ostream< Ch > &out, const xml_node< Ch > &node); +

Description

Prints formatted XML to given output stream. Uses default printing flags. Use print() function to customize printing process.

Parameters

out
Output stream to print to.
node
Node to be printed.

Returns

Output stream.

+ constant + parse_no_data_nodes

Synopsis

const int parse_no_data_nodes + = 0x1; +

Description

Parse flag instructing the parser to not create data nodes. Text of first data node will still be placed in value of parent element, unless rapidxml::parse_no_element_values flag is also specified. Can be combined with other flags by use of | operator.

+ See xml_document::parse() function.

+ constant + parse_no_element_values

Synopsis

const int parse_no_element_values + = 0x2; +

Description

Parse flag instructing the parser to not use text of first data node as a value of parent element. Can be combined with other flags by use of | operator. Note that child data nodes of element node take precendence over its value when printing. That is, if element has one or more child data nodes and a value, the value will be ignored. Use rapidxml::parse_no_data_nodes flag to prevent creation of data nodes if you want to manipulate data using values of elements.

+ See xml_document::parse() function.

+ constant + parse_no_string_terminators

Synopsis

const int parse_no_string_terminators + = 0x4; +

Description

Parse flag instructing the parser to not place zero terminators after strings in the source text. By default zero terminators are placed, modifying source text. Can be combined with other flags by use of | operator.

+ See xml_document::parse() function.

+ constant + parse_no_entity_translation

Synopsis

const int parse_no_entity_translation + = 0x8; +

Description

Parse flag instructing the parser to not translate entities in the source text. By default entities are translated, modifying source text. Can be combined with other flags by use of | operator.

+ See xml_document::parse() function.

+ constant + parse_no_utf8

Synopsis

const int parse_no_utf8 + = 0x10; +

Description

Parse flag instructing the parser to disable UTF-8 handling and assume plain 8 bit characters. By default, UTF-8 handling is enabled. Can be combined with other flags by use of | operator.

+ See xml_document::parse() function.

+ constant + parse_declaration_node

Synopsis

const int parse_declaration_node + = 0x20; +

Description

Parse flag instructing the parser to create XML declaration node. By default, declaration node is not created. Can be combined with other flags by use of | operator.

+ See xml_document::parse() function.

+ constant + parse_comment_nodes

Synopsis

const int parse_comment_nodes + = 0x40; +

Description

Parse flag instructing the parser to create comments nodes. By default, comment nodes are not created. Can be combined with other flags by use of | operator.

+ See xml_document::parse() function.

+ constant + parse_doctype_node

Synopsis

const int parse_doctype_node + = 0x80; +

Description

Parse flag instructing the parser to create DOCTYPE node. By default, doctype node is not created. Although W3C specification allows at most one DOCTYPE node, RapidXml will silently accept documents with more than one. Can be combined with other flags by use of | operator.

+ See xml_document::parse() function.

+ constant + parse_pi_nodes

Synopsis

const int parse_pi_nodes + = 0x100; +

Description

Parse flag instructing the parser to create PI nodes. By default, PI nodes are not created. Can be combined with other flags by use of | operator.

+ See xml_document::parse() function.

+ constant + parse_validate_closing_tags

Synopsis

const int parse_validate_closing_tags + = 0x200; +

Description

Parse flag instructing the parser to validate closing tag names. If not set, name inside closing tag is irrelevant to the parser. By default, closing tags are not validated. Can be combined with other flags by use of | operator.

+ See xml_document::parse() function.

+ constant + parse_trim_whitespace

Synopsis

const int parse_trim_whitespace + = 0x400; +

Description

Parse flag instructing the parser to trim all leading and trailing whitespace of data nodes. By default, whitespace is not trimmed. This flag does not cause the parser to modify source text. Can be combined with other flags by use of | operator.

+ See xml_document::parse() function.

+ constant + parse_normalize_whitespace

Synopsis

const int parse_normalize_whitespace + = 0x800; +

Description

Parse flag instructing the parser to condense all whitespace runs of data nodes to a single space character. Trimming of leading and trailing whitespace of data is controlled by rapidxml::parse_trim_whitespace flag. By default, whitespace is not normalized. If this flag is specified, source text will be modified. Can be combined with other flags by use of | operator.

+ See xml_document::parse() function.

+ constant + parse_default

Synopsis

const int parse_default + = 0; +

Description

Parse flags which represent default behaviour of the parser. This is always equal to 0, so that all other flags can be simply ored together. Normally there is no need to inconveniently disable flags by anding with their negated (~) values. This also means that meaning of each flag is a negation of the default setting. For example, if flag name is rapidxml::parse_no_utf8, it means that utf-8 is enabled by default, and using the flag will disable it.

+ See xml_document::parse() function.

+ constant + parse_non_destructive

Synopsis

const int parse_non_destructive + = parse_no_string_terminators | parse_no_entity_translation; +

Description

A combination of parse flags that forbids any modifications of the source text. This also results in faster parsing. However, note that the following will occur:
  • names and values of nodes will not be zero terminated, you have to use xml_base::name_size() and xml_base::value_size() functions to determine where name and value ends
  • entities will not be translated
  • whitespace will not be normalized
+See xml_document::parse() function.

+ constant + parse_fastest

Synopsis

const int parse_fastest + = parse_non_destructive | parse_no_data_nodes; +

Description

A combination of parse flags resulting in fastest possible parsing, without sacrificing important data.

+ See xml_document::parse() function.

+ constant + parse_full

Synopsis

const int parse_full + = parse_declaration_node | parse_comment_nodes | parse_doctype_node | parse_pi_nodes | parse_validate_closing_tags; +

Description

A combination of parse flags resulting in largest amount of data being extracted. This usually results in slowest parsing.

+ See xml_document::parse() function.

+ constant + print_no_indenting

Synopsis

const int print_no_indenting + = 0x1; +

Description

Printer flag instructing the printer to suppress indenting of XML. See print() function.

\ No newline at end of file diff --git a/third_party/cereal/external/rapidxml/rapidxml.hpp b/third_party/cereal/external/rapidxml/rapidxml.hpp new file mode 100755 index 0000000..d82893a --- /dev/null +++ b/third_party/cereal/external/rapidxml/rapidxml.hpp @@ -0,0 +1,2624 @@ +#ifndef CEREAL_RAPIDXML_HPP_INCLUDED +#define CEREAL_RAPIDXML_HPP_INCLUDED + +// Copyright (C) 2006, 2009 Marcin Kalicinski +// Version 1.13 +// Revision $DateTime: 2009/05/13 01:46:17 $ + +// If standard library is disabled, user must provide implementations of required functions and typedefs +#if !defined(CEREAL_RAPIDXML_NO_STDLIB) + #include // For std::size_t + #include // For assert + #include // For placement new +#endif + +// On MSVC, disable "conditional expression is constant" warning (level 4). +// This warning is almost impossible to avoid with certain types of templated code +#ifdef _MSC_VER + #pragma warning(push) + #pragma warning(disable:4127) // Conditional expression is constant + #pragma warning(disable:4100) // unreferenced formal parameter +#endif + +/////////////////////////////////////////////////////////////////////////// +// CEREAL_RAPIDXML_PARSE_ERROR + +#if defined(CEREAL_RAPIDXML_NO_EXCEPTIONS) + +#define CEREAL_RAPIDXML_PARSE_ERROR(what, where) { parse_error_handler(what, where); assert(0); } + +namespace cereal { +namespace rapidxml +{ + //! When exceptions are disabled by defining CEREAL_RAPIDXML_NO_EXCEPTIONS, + //! this function is called to notify user about the error. + //! It must be defined by the user. + //!

+ //! This function cannot return. If it does, the results are undefined. + //!

+ //! A very simple definition might look like that: + //! 

+    //! void %rapidxml::%parse_error_handler(const char *what, void *where)
+    //! {
+    //!     std::cout << "Parse error: " << what << "\n";
+    //!     std::abort();
+    //! }
+    //!
+ //! \param what Human readable description of the error. + //! \param where Pointer to character data where error was detected. + void parse_error_handler(const char *what, void *where); +} +} // end namespace cereal + +#else + +#include // For std::exception + +#define CEREAL_RAPIDXML_PARSE_ERROR(what, where) throw parse_error(what, where) + +namespace cereal { +namespace rapidxml +{ + + //! Parse error exception. + //! This exception is thrown by the parser when an error occurs. + //! Use what() function to get human-readable error message. + //! Use where() function to get a pointer to position within source text where error was detected. + //!

+ //! If throwing exceptions by the parser is undesirable, + //! it can be disabled by defining CEREAL_RAPIDXML_NO_EXCEPTIONS macro before rapidxml.hpp is included. + //! This will cause the parser to call rapidxml::parse_error_handler() function instead of throwing an exception. + //! This function must be defined by the user. + //!

+ //! This class derives from std::exception class. + class parse_error: public std::exception + { + + public: + + //! Constructs parse error + parse_error(const char *what_, void *where_) + : m_what(what_) + , m_where(where_) + { + } + + //! Gets human readable description of error. + //! \return Pointer to null terminated description of the error. + virtual const char *what() const CEREAL_NOEXCEPT override + { + return m_what; + } + + //! Gets pointer to character data where error happened. + //! Ch should be the same as char type of xml_document that produced the error. + //! \return Pointer to location within the parsed string where error occured. + template + Ch *where() const + { + return reinterpret_cast(m_where); + } + + private: + + const char *m_what; + void *m_where; + + }; +} +} // end namespace cereal + +#endif + +/////////////////////////////////////////////////////////////////////////// +// Pool sizes + +#ifndef CEREAL_RAPIDXML_STATIC_POOL_SIZE + // Size of static memory block of memory_pool. + // Define CEREAL_RAPIDXML_STATIC_POOL_SIZE before including rapidxml.hpp if you want to override the default value. + // No dynamic memory allocations are performed by memory_pool until static memory is exhausted. + #define CEREAL_RAPIDXML_STATIC_POOL_SIZE (64 * 1024) +#endif + +#ifndef CEREAL_RAPIDXML_DYNAMIC_POOL_SIZE + // Size of dynamic memory block of memory_pool. + // Define CEREAL_RAPIDXML_DYNAMIC_POOL_SIZE before including rapidxml.hpp if you want to override the default value. + // After the static block is exhausted, dynamic blocks with approximately this size are allocated by memory_pool. + #define CEREAL_RAPIDXML_DYNAMIC_POOL_SIZE (64 * 1024) +#endif + +#ifndef CEREAL_RAPIDXML_ALIGNMENT + // Memory allocation alignment. + // Define CEREAL_RAPIDXML_ALIGNMENT before including rapidxml.hpp if you want to override the default value, which is the size of pointer. + // All memory allocations for nodes, attributes and strings will be aligned to this value. + // This must be a power of 2 and at least 1, otherwise memory_pool will not work. + #define CEREAL_RAPIDXML_ALIGNMENT sizeof(void *) +#endif + +namespace cereal { +namespace rapidxml +{ + // Forward declarations + template class xml_node; + template class xml_attribute; + template class xml_document; + + //! Enumeration listing all node types produced by the parser. + //! Use xml_node::type() function to query node type. + enum node_type + { + node_document, //!< A document node. Name and value are empty. + node_element, //!< An element node. Name contains element name. Value contains text of first data node. + node_data, //!< A data node. Name is empty. Value contains data text. + node_cdata, //!< A CDATA node. Name is empty. Value contains data text. + node_comment, //!< A comment node. Name is empty. Value contains comment text. + node_declaration, //!< A declaration node. Name and value are empty. Declaration parameters (version, encoding and standalone) are in node attributes. + node_doctype, //!< A DOCTYPE node. Name is empty. Value contains DOCTYPE text. + node_pi //!< A PI node. Name contains target. Value contains instructions. + }; + + /////////////////////////////////////////////////////////////////////// + // Parsing flags + + //! Parse flag instructing the parser to not create data nodes. + //! Text of first data node will still be placed in value of parent element, unless rapidxml::parse_no_element_values flag is also specified. + //! Can be combined with other flags by use of | operator. + //!

+ //! See xml_document::parse() function. + const int parse_no_data_nodes = 0x1; + + //! Parse flag instructing the parser to not use text of first data node as a value of parent element. + //! Can be combined with other flags by use of | operator. + //! Note that child data nodes of element node take precendence over its value when printing. + //! That is, if element has one or more child data nodes and a value, the value will be ignored. + //! Use rapidxml::parse_no_data_nodes flag to prevent creation of data nodes if you want to manipulate data using values of elements. + //!

+ //! See xml_document::parse() function. + const int parse_no_element_values = 0x2; + + //! Parse flag instructing the parser to not place zero terminators after strings in the source text. + //! By default zero terminators are placed, modifying source text. + //! Can be combined with other flags by use of | operator. + //!

+ //! See xml_document::parse() function. + const int parse_no_string_terminators = 0x4; + + //! Parse flag instructing the parser to not translate entities in the source text. + //! By default entities are translated, modifying source text. + //! Can be combined with other flags by use of | operator. + //!

+ //! See xml_document::parse() function. + const int parse_no_entity_translation = 0x8; + + //! Parse flag instructing the parser to disable UTF-8 handling and assume plain 8 bit characters. + //! By default, UTF-8 handling is enabled. + //! Can be combined with other flags by use of | operator. + //!

+ //! See xml_document::parse() function. + const int parse_no_utf8 = 0x10; + + //! Parse flag instructing the parser to create XML declaration node. + //! By default, declaration node is not created. + //! Can be combined with other flags by use of | operator. + //!

+ //! See xml_document::parse() function. + const int parse_declaration_node = 0x20; + + //! Parse flag instructing the parser to create comments nodes. + //! By default, comment nodes are not created. + //! Can be combined with other flags by use of | operator. + //!

+ //! See xml_document::parse() function. + const int parse_comment_nodes = 0x40; + + //! Parse flag instructing the parser to create DOCTYPE node. + //! By default, doctype node is not created. + //! Although W3C specification allows at most one DOCTYPE node, RapidXml will silently accept documents with more than one. + //! Can be combined with other flags by use of | operator. + //!

+ //! See xml_document::parse() function. + const int parse_doctype_node = 0x80; + + //! Parse flag instructing the parser to create PI nodes. + //! By default, PI nodes are not created. + //! Can be combined with other flags by use of | operator. + //!

+ //! See xml_document::parse() function. + const int parse_pi_nodes = 0x100; + + //! Parse flag instructing the parser to validate closing tag names. + //! If not set, name inside closing tag is irrelevant to the parser. + //! By default, closing tags are not validated. + //! Can be combined with other flags by use of | operator. + //!

+ //! See xml_document::parse() function. + const int parse_validate_closing_tags = 0x200; + + //! Parse flag instructing the parser to trim all leading and trailing whitespace of data nodes. + //! By default, whitespace is not trimmed. + //! This flag does not cause the parser to modify source text. + //! Can be combined with other flags by use of | operator. + //!

+ //! See xml_document::parse() function. + const int parse_trim_whitespace = 0x400; + + //! Parse flag instructing the parser to condense all whitespace runs of data nodes to a single space character. + //! Trimming of leading and trailing whitespace of data is controlled by rapidxml::parse_trim_whitespace flag. + //! By default, whitespace is not normalized. + //! If this flag is specified, source text will be modified. + //! Can be combined with other flags by use of | operator. + //!

+ //! See xml_document::parse() function. + const int parse_normalize_whitespace = 0x800; + + // Compound flags + + //! Parse flags which represent default behaviour of the parser. + //! This is always equal to 0, so that all other flags can be simply ored together. + //! Normally there is no need to inconveniently disable flags by anding with their negated (~) values. + //! This also means that meaning of each flag is a negation of the default setting. + //! For example, if flag name is rapidxml::parse_no_utf8, it means that utf-8 is enabled by default, + //! and using the flag will disable it. + //!

+ //! See xml_document::parse() function. + const int parse_default = 0; + + //! A combination of parse flags that forbids any modifications of the source text. + //! This also results in faster parsing. However, note that the following will occur: + //!
    + //!
  • names and values of nodes will not be zero terminated, you have to use xml_base::name_size() and xml_base::value_size() functions to determine where name and value ends
    • + //!
  • entities will not be translated
    • + //!
  • whitespace will not be normalized
    • + //!
+ //! See xml_document::parse() function. + const int parse_non_destructive = parse_no_string_terminators | parse_no_entity_translation; + + //! A combination of parse flags resulting in fastest possible parsing, without sacrificing important data. + //!

+ //! See xml_document::parse() function. + const int parse_fastest = parse_non_destructive | parse_no_data_nodes; + + //! A combination of parse flags resulting in largest amount of data being extracted. + //! This usually results in slowest parsing. + //!

+ //! See xml_document::parse() function. + const int parse_full = parse_declaration_node | parse_comment_nodes | parse_doctype_node | parse_pi_nodes | parse_validate_closing_tags; + + /////////////////////////////////////////////////////////////////////// + // Internals + + //! \cond internal + namespace internal + { + + // Struct that contains lookup tables for the parser + // It must be a template to allow correct linking (because it has static data members, which are defined in a header file). + template + struct lookup_tables + { + static const unsigned char lookup_whitespace[256]; // Whitespace table + static const unsigned char lookup_node_name[256]; // Node name table + static const unsigned char lookup_text[256]; // Text table + static const unsigned char lookup_text_pure_no_ws[256]; // Text table + static const unsigned char lookup_text_pure_with_ws[256]; // Text table + static const unsigned char lookup_attribute_name[256]; // Attribute name table + static const unsigned char lookup_attribute_data_1[256]; // Attribute data table with single quote + static const unsigned char lookup_attribute_data_1_pure[256]; // Attribute data table with single quote + static const unsigned char lookup_attribute_data_2[256]; // Attribute data table with double quotes + static const unsigned char lookup_attribute_data_2_pure[256]; // Attribute data table with double quotes + static const unsigned char lookup_digits[256]; // Digits + static const unsigned char lookup_upcase[256]; // To uppercase conversion table for ASCII characters + }; + + // Find length of the string + template + inline std::size_t measure(const Ch *p) + { + const Ch *tmp = p; + while (*tmp) + ++tmp; + return static_cast(tmp - p); + } + + // Compare strings for equality + template + inline bool compare(const Ch *p1, std::size_t size1, const Ch *p2, std::size_t size2, bool case_sensitive) + { + if (size1 != size2) + return false; + if (case_sensitive) + { + for (const Ch *end = p1 + size1; p1 < end; ++p1, ++p2) + if (*p1 != *p2) + return false; + } + else + { + for (const Ch *end = p1 + size1; p1 < end; ++p1, ++p2) + if (lookup_tables<0>::lookup_upcase[static_cast(*p1)] != lookup_tables<0>::lookup_upcase[static_cast(*p2)]) + return false; + } + return true; + } + + template + inline bool preserve_space(xml_node* node) + { + const Ch preserve_value[] = { Ch('p'), Ch('r'), Ch('e'), Ch('s'), Ch('e'), Ch('r'), Ch('v'), Ch('e') }; + const xml_attribute* space = node->first_attribute("xml:space"); + return space && internal::compare(space->value(), space->value_size(), preserve_value, sizeof(preserve_value) / sizeof(Ch), true); + } + } + //! \endcond + + /////////////////////////////////////////////////////////////////////// + // Memory pool + + //! This class is used by the parser to create new nodes and attributes, without overheads of dynamic memory allocation. + //! In most cases, you will not need to use this class directly. + //! However, if you need to create nodes manually or modify names/values of nodes, + //! you are encouraged to use memory_pool of relevant xml_document to allocate the memory. + //! Not only is this faster than allocating them by using new operator, + //! but also their lifetime will be tied to the lifetime of document, + //! possibly simplyfing memory management. + //!

+ //! Call allocate_node() or allocate_attribute() functions to obtain new nodes or attributes from the pool. + //! You can also call allocate_string() function to allocate strings. + //! Such strings can then be used as names or values of nodes without worrying about their lifetime. + //! Note that there is no free() function -- all allocations are freed at once when clear() function is called, + //! or when the pool is destroyed. + //!

+ //! It is also possible to create a standalone memory_pool, and use it + //! to allocate nodes, whose lifetime will not be tied to any document. + //!

+ //! Pool maintains CEREAL_RAPIDXML_STATIC_POOL_SIZE bytes of statically allocated memory. + //! Until static memory is exhausted, no dynamic memory allocations are done. + //! When static memory is exhausted, pool allocates additional blocks of memory of size CEREAL_RAPIDXML_DYNAMIC_POOL_SIZE each, + //! by using global new[] and delete[] operators. + //! This behaviour can be changed by setting custom allocation routines. + //! Use set_allocator() function to set them. + //!

+ //! Allocations for nodes, attributes and strings are aligned at CEREAL_RAPIDXML_ALIGNMENT bytes. + //! This value defaults to the size of pointer on target architecture. + //!

+ //! To obtain absolutely top performance from the parser, + //! it is important that all nodes are allocated from a single, contiguous block of memory. + //! Otherwise, cache misses when jumping between two (or more) disjoint blocks of memory can slow down parsing quite considerably. + //! If required, you can tweak CEREAL_RAPIDXML_STATIC_POOL_SIZE, CEREAL_RAPIDXML_DYNAMIC_POOL_SIZE and CEREAL_RAPIDXML_ALIGNMENT + //! to obtain best wasted memory to performance compromise. + //! To do it, define their values before rapidxml.hpp file is included. + //! \tparam Ch Character type of created nodes. + template + class memory_pool + { + + public: + + //! \cond internal + typedef void *(alloc_func)(std::size_t); // Type of user-defined function used to allocate memory + typedef void (free_func)(void *); // Type of user-defined function used to free memory + //! \endcond + + //! Constructs empty pool with default allocator functions. + memory_pool() + : m_alloc_func(0) + , m_free_func(0) + { + init(); + } + + //! Destroys pool and frees all the memory. + //! This causes memory occupied by nodes allocated by the pool to be freed. + //! Nodes allocated from the pool are no longer valid. + ~memory_pool() + { + clear(); + } + + //! Allocates a new node from the pool, and optionally assigns name and value to it. + //! If the allocation request cannot be accomodated, this function will throw std::bad_alloc. + //! If exceptions are disabled by defining CEREAL_RAPIDXML_NO_EXCEPTIONS, this function + //! will call rapidxml::parse_error_handler() function. + //! \param type Type of node to create. + //! \param name Name to assign to the node, or 0 to assign no name. + //! \param value Value to assign to the node, or 0 to assign no value. + //! \param name_size Size of name to assign, or 0 to automatically calculate size from name string. + //! \param value_size Size of value to assign, or 0 to automatically calculate size from value string. + //! \return Pointer to allocated node. This pointer will never be NULL. + xml_node *allocate_node(node_type type, + const Ch *name = 0, const Ch *value = 0, + std::size_t name_size = 0, std::size_t value_size = 0) + { + void *memory = allocate_aligned(sizeof(xml_node)); + xml_node *node = new(memory) xml_node(type); + if (name) + { + if (name_size > 0) + node->name(name, name_size); + else + node->name(name); + } + if (value) + { + if (value_size > 0) + node->value(value, value_size); + else + node->value(value); + } + return node; + } + + //! Allocates a new attribute from the pool, and optionally assigns name and value to it. + //! If the allocation request cannot be accomodated, this function will throw std::bad_alloc. + //! If exceptions are disabled by defining CEREAL_RAPIDXML_NO_EXCEPTIONS, this function + //! will call rapidxml::parse_error_handler() function. + //! \param name Name to assign to the attribute, or 0 to assign no name. + //! \param value Value to assign to the attribute, or 0 to assign no value. + //! \param name_size Size of name to assign, or 0 to automatically calculate size from name string. + //! \param value_size Size of value to assign, or 0 to automatically calculate size from value string. + //! \return Pointer to allocated attribute. This pointer will never be NULL. + xml_attribute *allocate_attribute(const Ch *name = 0, const Ch *value = 0, + std::size_t name_size = 0, std::size_t value_size = 0) + { + void *memory = allocate_aligned(sizeof(xml_attribute)); + xml_attribute *attribute = new(memory) xml_attribute; + if (name) + { + if (name_size > 0) + attribute->name(name, name_size); + else + attribute->name(name); + } + if (value) + { + if (value_size > 0) + attribute->value(value, value_size); + else + attribute->value(value); + } + return attribute; + } + + //! Allocates a char array of given size from the pool, and optionally copies a given string to it. + //! If the allocation request cannot be accomodated, this function will throw std::bad_alloc. + //! If exceptions are disabled by defining CEREAL_RAPIDXML_NO_EXCEPTIONS, this function + //! will call rapidxml::parse_error_handler() function. + //! \param source String to initialize the allocated memory with, or 0 to not initialize it. + //! \param size Number of characters to allocate, or zero to calculate it automatically from source string length; if size is 0, source string must be specified and null terminated. + //! \return Pointer to allocated char array. This pointer will never be NULL. + Ch *allocate_string(const Ch *source = 0, std::size_t size = 0) + { + assert(source || size); // Either source or size (or both) must be specified + if (size == 0) + size = internal::measure(source) + 1; + Ch *result = static_cast(allocate_aligned(size * sizeof(Ch))); + if (source) + for (std::size_t i = 0; i < size; ++i) + result[i] = source[i]; + return result; + } + + //! Clones an xml_node and its hierarchy of child nodes and attributes. + //! Nodes and attributes are allocated from this memory pool. + //! Names and values are not cloned, they are shared between the clone and the source. + //! Result node can be optionally specified as a second parameter, + //! in which case its contents will be replaced with cloned source node. + //! This is useful when you want to clone entire document. + //! \param source Node to clone. + //! \param result Node to put results in, or 0 to automatically allocate result node + //! \return Pointer to cloned node. This pointer will never be NULL. + xml_node *clone_node(const xml_node *source, xml_node *result = 0) + { + // Prepare result node + if (result) + { + result->remove_all_attributes(); + result->remove_all_nodes(); + result->type(source->type()); + } + else + result = allocate_node(source->type()); + + // Clone name and value + result->name(source->name(), source->name_size()); + result->value(source->value(), source->value_size()); + + // Clone child nodes and attributes + for (xml_node *child = source->first_node(); child; child = child->next_sibling()) + result->append_node(clone_node(child)); + for (xml_attribute *attr = source->first_attribute(); attr; attr = attr->next_attribute()) + result->append_attribute(allocate_attribute(attr->name(), attr->value(), attr->name_size(), attr->value_size())); + + return result; + } + + //! Clears the pool. + //! This causes memory occupied by nodes allocated by the pool to be freed. + //! Any nodes or strings allocated from the pool will no longer be valid. + void clear() + { + while (m_begin != m_static_memory) + { + char *previous_begin = reinterpret_cast
(align(m_begin))->previous_begin; + if (m_free_func) + m_free_func(m_begin); + else + delete[] m_begin; + m_begin = previous_begin; + } + init(); + } + + //! Sets or resets the user-defined memory allocation functions for the pool. + //! This can only be called when no memory is allocated from the pool yet, otherwise results are undefined. + //! Allocation function must not return invalid pointer on failure. It should either throw, + //! stop the program, or use longjmp() function to pass control to other place of program. + //! If it returns invalid pointer, results are undefined. + //!

+ //! User defined allocation functions must have the following forms: + //!
+ //!
void *allocate(std::size_t size); + //!
void free(void *pointer); + //!
+ //! \param af Allocation function, or 0 to restore default function + //! \param ff Free function, or 0 to restore default function + void set_allocator(alloc_func *af, free_func *ff) + { + assert(m_begin == m_static_memory && m_ptr == align(m_begin)); // Verify that no memory is allocated yet + m_alloc_func = af; + m_free_func = ff; + } + + private: + + struct header + { + char *previous_begin; + }; + + void init() + { + m_begin = m_static_memory; + m_ptr = align(m_begin); + m_end = m_static_memory + sizeof(m_static_memory); + } + + char *align(char *ptr) + { + std::size_t alignment = ((CEREAL_RAPIDXML_ALIGNMENT - (std::size_t(ptr) & (CEREAL_RAPIDXML_ALIGNMENT - 1))) & (CEREAL_RAPIDXML_ALIGNMENT - 1)); + return ptr + alignment; + } + + char *allocate_raw(std::size_t size) + { + // Allocate + void *memory; + if (m_alloc_func) // Allocate memory using either user-specified allocation function or global operator new[] + { + memory = m_alloc_func(size); + assert(memory); // Allocator is not allowed to return 0, on failure it must either throw, stop the program or use longjmp + } + else + { + memory = new char[size]; +#ifdef CEREAL_RAPIDXML_NO_EXCEPTIONS + if (!memory) // If exceptions are disabled, verify memory allocation, because new will not be able to throw bad_alloc + CEREAL_RAPIDXML_PARSE_ERROR("out of memory", 0); +#endif + } + return static_cast(memory); + } + + void *allocate_aligned(std::size_t size) + { + // Calculate aligned pointer + char *result = align(m_ptr); + + // If not enough memory left in current pool, allocate a new pool + if (result + size > m_end) + { + // Calculate required pool size (may be bigger than CEREAL_RAPIDXML_DYNAMIC_POOL_SIZE) + std::size_t pool_size = CEREAL_RAPIDXML_DYNAMIC_POOL_SIZE; + if (pool_size < size) + pool_size = size; + + // Allocate + std::size_t alloc_size = sizeof(header) + (2 * CEREAL_RAPIDXML_ALIGNMENT - 2) + pool_size; // 2 alignments required in worst case: one for header, one for actual allocation + char *raw_memory = allocate_raw(alloc_size); + + // Setup new pool in allocated memory + char *pool = align(raw_memory); + header *new_header = reinterpret_cast
(pool); + new_header->previous_begin = m_begin; + m_begin = raw_memory; + m_ptr = pool + sizeof(header); + m_end = raw_memory + alloc_size; + + // Calculate aligned pointer again using new pool + result = align(m_ptr); + } + + // Update pool and return aligned pointer + m_ptr = result + size; + return result; + } + + char *m_begin; // Start of raw memory making up current pool + char *m_ptr; // First free byte in current pool + char *m_end; // One past last available byte in current pool + char m_static_memory[CEREAL_RAPIDXML_STATIC_POOL_SIZE]; // Static raw memory + alloc_func *m_alloc_func; // Allocator function, or 0 if default is to be used + free_func *m_free_func; // Free function, or 0 if default is to be used + }; + + /////////////////////////////////////////////////////////////////////////// + // XML base + + //! Base class for xml_node and xml_attribute implementing common functions: + //! name(), name_size(), value(), value_size() and parent(). + //! \tparam Ch Character type to use + template + class xml_base + { + + public: + + /////////////////////////////////////////////////////////////////////////// + // Construction & destruction + + // Construct a base with empty name, value and parent + xml_base() + : m_name(0) + , m_value(0) + , m_parent(0) + { + } + + /////////////////////////////////////////////////////////////////////////// + // Node data access + + //! Gets name of the node. + //! Interpretation of name depends on type of node. + //! Note that name will not be zero-terminated if rapidxml::parse_no_string_terminators option was selected during parse. + //!

+ //! Use name_size() function to determine length of the name. + //! \return Name of node, or empty string if node has no name. + Ch *name() const + { + return m_name ? m_name : nullstr(); + } + + //! Gets size of node name, not including terminator character. + //! This function works correctly irrespective of whether name is or is not zero terminated. + //! \return Size of node name, in characters. + std::size_t name_size() const + { + return m_name ? m_name_size : 0; + } + + //! Gets value of node. + //! Interpretation of value depends on type of node. + //! Note that value will not be zero-terminated if rapidxml::parse_no_string_terminators option was selected during parse. + //!

+ //! Use value_size() function to determine length of the value. + //! \return Value of node, or empty string if node has no value. + Ch *value() const + { + return m_value ? m_value : nullstr(); + } + + //! Gets size of node value, not including terminator character. + //! This function works correctly irrespective of whether value is or is not zero terminated. + //! \return Size of node value, in characters. + std::size_t value_size() const + { + return m_value ? m_value_size : 0; + } + + /////////////////////////////////////////////////////////////////////////// + // Node modification + + //! Sets name of node to a non zero-terminated string. + //! See \ref ownership_of_strings. + //!

+ //! Note that node does not own its name or value, it only stores a pointer to it. + //! It will not delete or otherwise free the pointer on destruction. + //! It is reponsibility of the user to properly manage lifetime of the string. + //! The easiest way to achieve it is to use memory_pool of the document to allocate the string - + //! on destruction of the document the string will be automatically freed. + //!

+ //! Size of name must be specified separately, because name does not have to be zero terminated. + //! Use name(const Ch *) function to have the length automatically calculated (string must be zero terminated). + //! \param name_ Name of node to set. Does not have to be zero terminated. + //! \param size Size of name, in characters. This does not include zero terminator, if one is present. + void name(const Ch *name_, std::size_t size) + { + m_name = const_cast(name_); + m_name_size = size; + } + + //! Sets name of node to a zero-terminated string. + //! See also \ref ownership_of_strings and xml_node::name(const Ch *, std::size_t). + //! \param name_ Name of node to set. Must be zero terminated. + void name(const Ch *name_) + { + this->name(name_, internal::measure(name_)); + } + + //! Sets value of node to a non zero-terminated string. + //! See \ref ownership_of_strings. + //!

+ //! Note that node does not own its name or value, it only stores a pointer to it. + //! It will not delete or otherwise free the pointer on destruction. + //! It is reponsibility of the user to properly manage lifetime of the string. + //! The easiest way to achieve it is to use memory_pool of the document to allocate the string - + //! on destruction of the document the string will be automatically freed. + //!

+ //! Size of value must be specified separately, because it does not have to be zero terminated. + //! Use value(const Ch *) function to have the length automatically calculated (string must be zero terminated). + //!

+ //! If an element has a child node of type node_data, it will take precedence over element value when printing. + //! If you want to manipulate data of elements using values, use parser flag rapidxml::parse_no_data_nodes to prevent creation of data nodes by the parser. + //! \param value_ value of node to set. Does not have to be zero terminated. + //! \param size Size of value, in characters. This does not include zero terminator, if one is present. + void value(const Ch *value_, std::size_t size) + { + m_value = const_cast(value_); + m_value_size = size; + } + + //! Sets value of node to a zero-terminated string. + //! See also \ref ownership_of_strings and xml_node::value(const Ch *, std::size_t). + //! \param value_ Vame of node to set. Must be zero terminated. + void value(const Ch *value_) + { + this->value(value_, internal::measure(value_)); + } + + /////////////////////////////////////////////////////////////////////////// + // Related nodes access + + //! Gets node parent. + //! \return Pointer to parent node, or 0 if there is no parent. + xml_node *parent() const + { + return m_parent; + } + + protected: + + // Return empty string + static Ch *nullstr() + { + static Ch zero = Ch('\0'); + return &zero; + } + + Ch *m_name; // Name of node, or 0 if no name + Ch *m_value; // Value of node, or 0 if no value + std::size_t m_name_size; // Length of node name, or undefined of no name + std::size_t m_value_size; // Length of node value, or undefined if no value + xml_node *m_parent; // Pointer to parent node, or 0 if none + + }; + + //! Class representing attribute node of XML document. + //! Each attribute has name and value strings, which are available through name() and value() functions (inherited from xml_base). + //! Note that after parse, both name and value of attribute will point to interior of source text used for parsing. + //! Thus, this text must persist in memory for the lifetime of attribute. + //! \tparam Ch Character type to use. + template + class xml_attribute: public xml_base + { + + friend class xml_node; + + public: + + /////////////////////////////////////////////////////////////////////////// + // Construction & destruction + + //! Constructs an empty attribute with the specified type. + //! Consider using memory_pool of appropriate xml_document if allocating attributes manually. + xml_attribute() + { + } + + /////////////////////////////////////////////////////////////////////////// + // Related nodes access + + //! Gets document of which attribute is a child. + //! \return Pointer to document that contains this attribute, or 0 if there is no parent document. + xml_document *document() const + { + if (xml_node *node = this->parent()) + { + while (node->parent()) + node = node->parent(); + return node->type() == node_document ? static_cast *>(node) : 0; + } + else + return 0; + } + + //! Gets previous attribute, optionally matching attribute name. + //! \param name Name of attribute to find, or 0 to return previous attribute regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero + //! \param name_size Size of name, in characters, or 0 to have size calculated automatically from string + //! \param case_sensitive Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters + //! \return Pointer to found attribute, or 0 if not found. + xml_attribute *previous_attribute(const Ch *name = 0, std::size_t name_size = 0, bool case_sensitive = true) const + { + if (name) + { + if (name_size == 0) + name_size = internal::measure(name); + for (xml_attribute *attribute = m_prev_attribute; attribute; attribute = attribute->m_prev_attribute) + if (internal::compare(attribute->name(), attribute->name_size(), name, name_size, case_sensitive)) + return attribute; + return 0; + } + else + return this->m_parent ? m_prev_attribute : 0; + } + + //! Gets next attribute, optionally matching attribute name. + //! \param name_ Name of attribute to find, or 0 to return next attribute regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero + //! \param name_size_ Size of name, in characters, or 0 to have size calculated automatically from string + //! \param case_sensitive Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters + //! \return Pointer to found attribute, or 0 if not found. + xml_attribute *next_attribute(const Ch *name_ = 0, std::size_t name_size_ = 0, bool case_sensitive = true) const + { + if (name_) + { + if (name_size_ == 0) + name_size_ = internal::measure(name_); + for (xml_attribute *attribute = m_next_attribute; attribute; attribute = attribute->m_next_attribute) + if (internal::compare(attribute->name(), attribute->name_size(), name_, name_size_, case_sensitive)) + return attribute; + return 0; + } + else + return this->m_parent ? m_next_attribute : 0; + } + + private: + + xml_attribute *m_prev_attribute; // Pointer to previous sibling of attribute, or 0 if none; only valid if parent is non-zero + xml_attribute *m_next_attribute; // Pointer to next sibling of attribute, or 0 if none; only valid if parent is non-zero + + }; + + /////////////////////////////////////////////////////////////////////////// + // XML node + + //! Class representing a node of XML document. + //! Each node may have associated name and value strings, which are available through name() and value() functions. + //! Interpretation of name and value depends on type of the node. + //! Type of node can be determined by using type() function. + //!

+ //! Note that after parse, both name and value of node, if any, will point interior of source text used for parsing. + //! Thus, this text must persist in the memory for the lifetime of node. + //! \tparam Ch Character type to use. + template + class xml_node: public xml_base + { + + public: + + /////////////////////////////////////////////////////////////////////////// + // Construction & destruction + + //! Constructs an empty node with the specified type. + //! Consider using memory_pool of appropriate document to allocate nodes manually. + //! \param type_ Type of node to construct. + xml_node(node_type type_) + : m_type(type_) + , m_first_node(0) + , m_first_attribute(0) + { + } + + /////////////////////////////////////////////////////////////////////////// + // Node data access + + //! Gets type of node. + //! \return Type of node. + node_type type() const + { + return m_type; + } + + /////////////////////////////////////////////////////////////////////////// + // Related nodes access + + //! Gets document of which node is a child. + //! \return Pointer to document that contains this node, or 0 if there is no parent document. + xml_document *document() const + { + xml_node *node = const_cast *>(this); + while (node->parent()) + node = node->parent(); + return node->type() == node_document ? static_cast *>(node) : 0; + } + + //! Gets first child node, optionally matching node name. + //! \param name_ Name of child to find, or 0 to return first child regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero + //! \param name_size_ Size of name, in characters, or 0 to have size calculated automatically from string + //! \param case_sensitive Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters + //! \return Pointer to found child, or 0 if not found. + xml_node *first_node(const Ch *name_ = 0, std::size_t name_size_ = 0, bool case_sensitive = true) const + { + if (name_) + { + if (name_size_ == 0) + name_size_ = internal::measure(name_); + for (xml_node *child = m_first_node; child; child = child->next_sibling()) + if (internal::compare(child->name(), child->name_size(), name_, name_size_, case_sensitive)) + return child; + return 0; + } + else + return m_first_node; + } + + //! Gets last child node, optionally matching node name. + //! Behaviour is undefined if node has no children. + //! Use first_node() to test if node has children. + //! \param name Name of child to find, or 0 to return last child regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero + //! \param name_size Size of name, in characters, or 0 to have size calculated automatically from string + //! \param case_sensitive Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters + //! \return Pointer to found child, or 0 if not found. + xml_node *last_node(const Ch *name = 0, std::size_t name_size = 0, bool case_sensitive = true) const + { + assert(m_first_node); // Cannot query for last child if node has no children + if (name) + { + if (name_size == 0) + name_size = internal::measure(name); + for (xml_node *child = m_last_node; child; child = child->previous_sibling()) + if (internal::compare(child->name(), child->name_size(), name, name_size, case_sensitive)) + return child; + return 0; + } + else + return m_last_node; + } + + //! Gets previous sibling node, optionally matching node name. + //! Behaviour is undefined if node has no parent. + //! Use parent() to test if node has a parent. + //! \param name Name of sibling to find, or 0 to return previous sibling regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero + //! \param name_size Size of name, in characters, or 0 to have size calculated automatically from string + //! \param case_sensitive Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters + //! \return Pointer to found sibling, or 0 if not found. + xml_node *previous_sibling(const Ch *name = 0, std::size_t name_size = 0, bool case_sensitive = true) const + { + assert(this->m_parent); // Cannot query for siblings if node has no parent + if (name) + { + if (name_size == 0) + name_size = internal::measure(name); + for (xml_node *sibling = m_prev_sibling; sibling; sibling = sibling->m_prev_sibling) + if (internal::compare(sibling->name(), sibling->name_size(), name, name_size, case_sensitive)) + return sibling; + return 0; + } + else + return m_prev_sibling; + } + + //! Gets next sibling node, optionally matching node name. + //! Behaviour is undefined if node has no parent. + //! Use parent() to test if node has a parent. + //! \param name_ Name of sibling to find, or 0 to return next sibling regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero + //! \param name_size_ Size of name, in characters, or 0 to have size calculated automatically from string + //! \param case_sensitive Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters + //! \return Pointer to found sibling, or 0 if not found. + xml_node *next_sibling(const Ch *name_ = 0, std::size_t name_size_ = 0, bool case_sensitive = true) const + { + assert(this->m_parent); // Cannot query for siblings if node has no parent + if (name_) + { + if (name_size_ == 0) + name_size_ = internal::measure(name_); + for (xml_node *sibling = m_next_sibling; sibling; sibling = sibling->m_next_sibling) + if (internal::compare(sibling->name(), sibling->name_size(), name_, name_size_, case_sensitive)) + return sibling; + return 0; + } + else + return m_next_sibling; + } + + //! Gets first attribute of node, optionally matching attribute name. + //! \param name_ Name of attribute to find, or 0 to return first attribute regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero + //! \param name_size_ Size of name, in characters, or 0 to have size calculated automatically from string + //! \param case_sensitive Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters + //! \return Pointer to found attribute, or 0 if not found. + xml_attribute *first_attribute(const Ch *name_ = 0, std::size_t name_size_ = 0, bool case_sensitive = true) const + { + if (name_) + { + if (name_size_ == 0) + name_size_ = internal::measure(name_); + for (xml_attribute *attribute = m_first_attribute; attribute; attribute = attribute->m_next_attribute) + if (internal::compare(attribute->name(), attribute->name_size(), name_, name_size_, case_sensitive)) + return attribute; + return 0; + } + else + return m_first_attribute; + } + + //! Gets last attribute of node, optionally matching attribute name. + //! \param name Name of attribute to find, or 0 to return last attribute regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero + //! \param name_size Size of name, in characters, or 0 to have size calculated automatically from string + //! \param case_sensitive Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters + //! \return Pointer to found attribute, or 0 if not found. + xml_attribute *last_attribute(const Ch *name = 0, std::size_t name_size = 0, bool case_sensitive = true) const + { + if (name) + { + if (name_size == 0) + name_size = internal::measure(name); + for (xml_attribute *attribute = m_last_attribute; attribute; attribute = attribute->m_prev_attribute) + if (internal::compare(attribute->name(), attribute->name_size(), name, name_size, case_sensitive)) + return attribute; + return 0; + } + else + return m_first_attribute ? m_last_attribute : 0; + } + + /////////////////////////////////////////////////////////////////////////// + // Node modification + + //! Sets type of node. + //! \param type_ Type of node to set. + void type(node_type type_) + { + m_type = type_; + } + + /////////////////////////////////////////////////////////////////////////// + // Node manipulation + + //! Prepends a new child node. + //! The prepended child becomes the first child, and all existing children are moved one position back. + //! \param child Node to prepend. + void prepend_node(xml_node *child) + { + assert(child && !child->parent() && child->type() != node_document); + if (first_node()) + { + child->m_next_sibling = m_first_node; + m_first_node->m_prev_sibling = child; + } + else + { + child->m_next_sibling = 0; + m_last_node = child; + } + m_first_node = child; + child->m_parent = this; + child->m_prev_sibling = 0; + } + + //! Appends a new child node. + //! The appended child becomes the last child. + //! \param child Node to append. + void append_node(xml_node *child) + { + assert(child && !child->parent() && child->type() != node_document); + if (first_node()) + { + child->m_prev_sibling = m_last_node; + m_last_node->m_next_sibling = child; + } + else + { + child->m_prev_sibling = 0; + m_first_node = child; + } + m_last_node = child; + child->m_parent = this; + child->m_next_sibling = 0; + } + + //! Inserts a new child node at specified place inside the node. + //! All children after and including the specified node are moved one position back. + //! \param where Place where to insert the child, or 0 to insert at the back. + //! \param child Node to insert. + void insert_node(xml_node *where, xml_node *child) + { + assert(!where || where->parent() == this); + assert(child && !child->parent() && child->type() != node_document); + if (where == m_first_node) + prepend_node(child); + else if (where == 0) + append_node(child); + else + { + child->m_prev_sibling = where->m_prev_sibling; + child->m_next_sibling = where; + where->m_prev_sibling->m_next_sibling = child; + where->m_prev_sibling = child; + child->m_parent = this; + } + } + + //! Removes first child node. + //! If node has no children, behaviour is undefined. + //! Use first_node() to test if node has children. + void remove_first_node() + { + assert(first_node()); + xml_node *child = m_first_node; + m_first_node = child->m_next_sibling; + if (child->m_next_sibling) + child->m_next_sibling->m_prev_sibling = 0; + else + m_last_node = 0; + child->m_parent = 0; + } + + //! Removes last child of the node. + //! If node has no children, behaviour is undefined. + //! Use first_node() to test if node has children. + void remove_last_node() + { + assert(first_node()); + xml_node *child = m_last_node; + if (child->m_prev_sibling) + { + m_last_node = child->m_prev_sibling; + child->m_prev_sibling->m_next_sibling = 0; + } + else + m_first_node = 0; + child->m_parent = 0; + } + + //! Removes specified child from the node + // \param where Pointer to child to be removed. + void remove_node(xml_node *where) + { + assert(where && where->parent() == this); + assert(first_node()); + if (where == m_first_node) + remove_first_node(); + else if (where == m_last_node) + remove_last_node(); + else + { + where->m_prev_sibling->m_next_sibling = where->m_next_sibling; + where->m_next_sibling->m_prev_sibling = where->m_prev_sibling; + where->m_parent = 0; + } + } + + //! Removes all child nodes (but not attributes). + void remove_all_nodes() + { + for (xml_node *node = first_node(); node; node = node->m_next_sibling) + node->m_parent = 0; + m_first_node = 0; + } + + //! Prepends a new attribute to the node. + //! \param attribute Attribute to prepend. + void prepend_attribute(xml_attribute *attribute) + { + assert(attribute && !attribute->parent()); + if (first_attribute()) + { + attribute->m_next_attribute = m_first_attribute; + m_first_attribute->m_prev_attribute = attribute; + } + else + { + attribute->m_next_attribute = 0; + m_last_attribute = attribute; + } + m_first_attribute = attribute; + attribute->m_parent = this; + attribute->m_prev_attribute = 0; + } + + //! Appends a new attribute to the node. + //! \param attribute Attribute to append. + void append_attribute(xml_attribute *attribute) + { + assert(attribute && !attribute->parent()); + if (first_attribute()) + { + attribute->m_prev_attribute = m_last_attribute; + m_last_attribute->m_next_attribute = attribute; + } + else + { + attribute->m_prev_attribute = 0; + m_first_attribute = attribute; + } + m_last_attribute = attribute; + attribute->m_parent = this; + attribute->m_next_attribute = 0; + } + + //! Inserts a new attribute at specified place inside the node. + //! All attributes after and including the specified attribute are moved one position back. + //! \param where Place where to insert the attribute, or 0 to insert at the back. + //! \param attribute Attribute to insert. + void insert_attribute(xml_attribute *where, xml_attribute *attribute) + { + assert(!where || where->parent() == this); + assert(attribute && !attribute->parent()); + if (where == m_first_attribute) + prepend_attribute(attribute); + else if (where == 0) + append_attribute(attribute); + else + { + attribute->m_prev_attribute = where->m_prev_attribute; + attribute->m_next_attribute = where; + where->m_prev_attribute->m_next_attribute = attribute; + where->m_prev_attribute = attribute; + attribute->m_parent = this; + } + } + + //! Removes first attribute of the node. + //! If node has no attributes, behaviour is undefined. + //! Use first_attribute() to test if node has attributes. + void remove_first_attribute() + { + assert(first_attribute()); + xml_attribute *attribute = m_first_attribute; + if (attribute->m_next_attribute) + { + attribute->m_next_attribute->m_prev_attribute = 0; + } + else + m_last_attribute = 0; + attribute->m_parent = 0; + m_first_attribute = attribute->m_next_attribute; + } + + //! Removes last attribute of the node. + //! If node has no attributes, behaviour is undefined. + //! Use first_attribute() to test if node has attributes. + void remove_last_attribute() + { + assert(first_attribute()); + xml_attribute *attribute = m_last_attribute; + if (attribute->m_prev_attribute) + { + attribute->m_prev_attribute->m_next_attribute = 0; + m_last_attribute = attribute->m_prev_attribute; + } + else + m_first_attribute = 0; + attribute->m_parent = 0; + } + + //! Removes specified attribute from node. + //! \param where Pointer to attribute to be removed. + void remove_attribute(xml_attribute *where) + { + assert(first_attribute() && where->parent() == this); + if (where == m_first_attribute) + remove_first_attribute(); + else if (where == m_last_attribute) + remove_last_attribute(); + else + { + where->m_prev_attribute->m_next_attribute = where->m_next_attribute; + where->m_next_attribute->m_prev_attribute = where->m_prev_attribute; + where->m_parent = 0; + } + } + + //! Removes all attributes of node. + void remove_all_attributes() + { + for (xml_attribute *attribute = first_attribute(); attribute; attribute = attribute->m_next_attribute) + attribute->m_parent = 0; + m_first_attribute = 0; + } + + private: + + /////////////////////////////////////////////////////////////////////////// + // Restrictions + + // No copying + xml_node(const xml_node &); + void operator =(const xml_node &); + + /////////////////////////////////////////////////////////////////////////// + // Data members + + // Note that some of the pointers below have UNDEFINED values if certain other pointers are 0. + // This is required for maximum performance, as it allows the parser to omit initialization of + // unneded/redundant values. + // + // The rules are as follows: + // 1. first_node and first_attribute contain valid pointers, or 0 if node has no children/attributes respectively + // 2. last_node and last_attribute are valid only if node has at least one child/attribute respectively, otherwise they contain garbage + // 3. prev_sibling and next_sibling are valid only if node has a parent, otherwise they contain garbage + + node_type m_type; // Type of node; always valid + xml_node *m_first_node; // Pointer to first child node, or 0 if none; always valid + xml_node *m_last_node; // Pointer to last child node, or 0 if none; this value is only valid if m_first_node is non-zero + xml_attribute *m_first_attribute; // Pointer to first attribute of node, or 0 if none; always valid + xml_attribute *m_last_attribute; // Pointer to last attribute of node, or 0 if none; this value is only valid if m_first_attribute is non-zero + xml_node *m_prev_sibling; // Pointer to previous sibling of node, or 0 if none; this value is only valid if m_parent is non-zero + xml_node *m_next_sibling; // Pointer to next sibling of node, or 0 if none; this value is only valid if m_parent is non-zero + + }; + + /////////////////////////////////////////////////////////////////////////// + // XML document + + //! This class represents root of the DOM hierarchy. + //! It is also an xml_node and a memory_pool through public inheritance. + //! Use parse() function to build a DOM tree from a zero-terminated XML text string. + //! parse() function allocates memory for nodes and attributes by using functions of xml_document, + //! which are inherited from memory_pool. + //! To access root node of the document, use the document itself, as if it was an xml_node. + //! \tparam Ch Character type to use. + template + class xml_document: public xml_node, public memory_pool + { + + public: + + //! Constructs empty XML document + xml_document() + : xml_node(node_document) + { + } + + //! Parses zero-terminated XML string according to given flags. + //! Passed string will be modified by the parser, unless rapidxml::parse_non_destructive flag is used. + //! The string must persist for the lifetime of the document. + //! In case of error, rapidxml::parse_error exception will be thrown. + //!

+ //! If you want to parse contents of a file, you must first load the file into the memory, and pass pointer to its beginning. + //! Make sure that data is zero-terminated. + //!

+ //! Document can be parsed into multiple times. + //! Each new call to parse removes previous nodes and attributes (if any), but does not clear memory pool. + //! \param text XML data to parse; pointer is non-const to denote fact that this data may be modified by the parser. + template + void parse(Ch *text) + { + assert(text); + + // Remove current contents + this->remove_all_nodes(); + this->remove_all_attributes(); + + // Parse BOM, if any + parse_bom(text); + + // Parse children + while (1) + { + // Skip whitespace before node + skip(text); + if (*text == 0) + break; + + // Parse and append new child + if (*text == Ch('<')) + { + ++text; // Skip '<' + if (xml_node *node = parse_node(text)) + this->append_node(node); + } + else + CEREAL_RAPIDXML_PARSE_ERROR("expected <", text); + } + + } + + //! Clears the document by deleting all nodes and clearing the memory pool. + //! All nodes owned by document pool are destroyed. + void clear() + { + this->remove_all_nodes(); + this->remove_all_attributes(); + memory_pool::clear(); + } + + private: + + /////////////////////////////////////////////////////////////////////// + // Internal character utility functions + + // Detect whitespace character + struct whitespace_pred + { + static unsigned char test(Ch ch) + { + return internal::lookup_tables<0>::lookup_whitespace[static_cast(ch)]; + } + }; + + // Detect node name character + struct node_name_pred + { + static unsigned char test(Ch ch) + { + return internal::lookup_tables<0>::lookup_node_name[static_cast(ch)]; + } + }; + + // Detect attribute name character + struct attribute_name_pred + { + static unsigned char test(Ch ch) + { + return internal::lookup_tables<0>::lookup_attribute_name[static_cast(ch)]; + } + }; + + // Detect text character (PCDATA) + struct text_pred + { + static unsigned char test(Ch ch) + { + return internal::lookup_tables<0>::lookup_text[static_cast(ch)]; + } + }; + + // Detect text character (PCDATA) that does not require processing + struct text_pure_no_ws_pred + { + static unsigned char test(Ch ch) + { + return internal::lookup_tables<0>::lookup_text_pure_no_ws[static_cast(ch)]; + } + }; + + // Detect text character (PCDATA) that does not require processing + struct text_pure_with_ws_pred + { + static unsigned char test(Ch ch) + { + return internal::lookup_tables<0>::lookup_text_pure_with_ws[static_cast(ch)]; + } + }; + + // Detect attribute value character + template + struct attribute_value_pred + { + static unsigned char test(Ch ch) + { + if (Quote == Ch('\'')) + return internal::lookup_tables<0>::lookup_attribute_data_1[static_cast(ch)]; + if (Quote == Ch('\"')) + return internal::lookup_tables<0>::lookup_attribute_data_2[static_cast(ch)]; + return 0; // Should never be executed, to avoid warnings on Comeau + } + }; + + // Detect attribute value character + template + struct attribute_value_pure_pred + { + static unsigned char test(Ch ch) + { + if (Quote == Ch('\'')) + return internal::lookup_tables<0>::lookup_attribute_data_1_pure[static_cast(ch)]; + if (Quote == Ch('\"')) + return internal::lookup_tables<0>::lookup_attribute_data_2_pure[static_cast(ch)]; + return 0; // Should never be executed, to avoid warnings on Comeau + } + }; + + // Insert coded character, using UTF8 or 8-bit ASCII + template + static void insert_coded_character(Ch *&text, unsigned long code) + { + if (Flags & parse_no_utf8) + { + // Insert 8-bit ASCII character + // Todo: possibly verify that code is less than 256 and use replacement char otherwise? + text[0] = static_cast(code); + text += 1; + } + else + { + // Insert UTF8 sequence + if (code < 0x80) // 1 byte sequence + { + text[0] = static_cast(code); + text += 1; + } + else if (code < 0x800) // 2 byte sequence + { + text[1] = static_cast((code | 0x80) & 0xBF); code >>= 6; + text[0] = static_cast(code | 0xC0); + text += 2; + } + else if (code < 0x10000) // 3 byte sequence + { + text[2] = static_cast((code | 0x80) & 0xBF); code >>= 6; + text[1] = static_cast((code | 0x80) & 0xBF); code >>= 6; + text[0] = static_cast(code | 0xE0); + text += 3; + } + else if (code < 0x110000) // 4 byte sequence + { + text[3] = static_cast((code | 0x80) & 0xBF); code >>= 6; + text[2] = static_cast((code | 0x80) & 0xBF); code >>= 6; + text[1] = static_cast((code | 0x80) & 0xBF); code >>= 6; + text[0] = static_cast(code | 0xF0); + text += 4; + } + else // Invalid, only codes up to 0x10FFFF are allowed in Unicode + { + CEREAL_RAPIDXML_PARSE_ERROR("invalid numeric character entity", text); + } + } + } + + // Skip characters until predicate evaluates to true + template + static void skip(Ch *&text) + { + Ch *tmp = text; + while (StopPred::test(*tmp)) + ++tmp; + text = tmp; + } + + // Skip characters until predicate evaluates to true while doing the following: + // - replacing XML character entity references with proper characters (' & " < > &#...;) + // - condensing whitespace sequences to single space character + template + static Ch *skip_and_expand_character_refs(Ch *&text, bool preserve_space) + { + // If entity translation, whitespace condense and whitespace trimming is disabled, use plain skip + if (Flags & parse_no_entity_translation && + !(Flags & parse_normalize_whitespace) && + !(Flags & parse_trim_whitespace)) + { + skip(text); + return text; + } + + // Use simple skip until first modification is detected + skip(text); + + // Use translation skip + Ch *src = text; + Ch *dest = src; + while (StopPred::test(*src)) + { + // If entity translation is enabled + if (!(Flags & parse_no_entity_translation)) + { + // Test if replacement is needed + if (src[0] == Ch('&')) + { + switch (src[1]) + { + + // & ' + case Ch('a'): + if (src[2] == Ch('m') && src[3] == Ch('p') && src[4] == Ch(';')) + { + *dest = Ch('&'); + ++dest; + src += 5; + continue; + } + if (src[2] == Ch('p') && src[3] == Ch('o') && src[4] == Ch('s') && src[5] == Ch(';')) + { + *dest = Ch('\''); + ++dest; + src += 6; + continue; + } + break; + + // " + case Ch('q'): + if (src[2] == Ch('u') && src[3] == Ch('o') && src[4] == Ch('t') && src[5] == Ch(';')) + { + *dest = Ch('"'); + ++dest; + src += 6; + continue; + } + break; + + // > + case Ch('g'): + if (src[2] == Ch('t') && src[3] == Ch(';')) + { + *dest = Ch('>'); + ++dest; + src += 4; + continue; + } + break; + + // < + case Ch('l'): + if (src[2] == Ch('t') && src[3] == Ch(';')) + { + *dest = Ch('<'); + ++dest; + src += 4; + continue; + } + break; + + // &#...; - assumes ASCII + case Ch('#'): + if (src[2] == Ch('x')) + { + unsigned long code = 0; + src += 3; // Skip &#x + while (1) + { + unsigned char digit = internal::lookup_tables<0>::lookup_digits[static_cast(*src)]; + if (digit == 0xFF) + break; + code = code * 16 + digit; + ++src; + } + insert_coded_character(dest, code); // Put character in output + } + else + { + unsigned long code = 0; + src += 2; // Skip &# + while (1) + { + unsigned char digit = internal::lookup_tables<0>::lookup_digits[static_cast(*src)]; + if (digit == 0xFF) + break; + code = code * 10 + digit; + ++src; + } + insert_coded_character(dest, code); // Put character in output + } + if (*src == Ch(';')) + ++src; + else + CEREAL_RAPIDXML_PARSE_ERROR("expected ;", src); + continue; + + // Something else + default: + // Ignore, just copy '&' verbatim + break; + + } + } + } + + // If whitespace condensing is enabled + if ((Flags & parse_normalize_whitespace) && !preserve_space) + { + // Test if condensing is needed + if (whitespace_pred::test(*src)) + { + *dest = Ch(' '); ++dest; // Put single space in dest + ++src; // Skip first whitespace char + // Skip remaining whitespace chars + while (whitespace_pred::test(*src)) + ++src; + continue; + } + } + + // No replacement, only copy character + *dest++ = *src++; + + } + + // Return new end + text = src; + return dest; + + } + + /////////////////////////////////////////////////////////////////////// + // Internal parsing functions + + // Parse BOM, if any + template + void parse_bom(Ch *&text) + { + // UTF-8? + if (static_cast(text[0]) == 0xEF && + static_cast(text[1]) == 0xBB && + static_cast(text[2]) == 0xBF) + { + text += 3; // Skup utf-8 bom + } + } + + // Parse XML declaration ( + xml_node *parse_xml_declaration(Ch *&text) + { + // If parsing of declaration is disabled + if (!(Flags & parse_declaration_node)) + { + // Skip until end of declaration + while (text[0] != Ch('?') || text[1] != Ch('>')) + { + if (!text[0]) + CEREAL_RAPIDXML_PARSE_ERROR("unexpected end of data", text); + ++text; + } + text += 2; // Skip '?>' + return 0; + } + + // Create declaration + xml_node *declaration = this->allocate_node(node_declaration); + + // Skip whitespace before attributes or ?> + skip(text); + + // Parse declaration attributes + parse_node_attributes(text, declaration); + + // Skip ?> + if (text[0] != Ch('?') || text[1] != Ch('>')) + CEREAL_RAPIDXML_PARSE_ERROR("expected ?>", text); + text += 2; + + return declaration; + } + + // Parse XML comment (' + return 0; // Do not produce comment node + } + + // Remember value start + Ch *value_ = text; + + // Skip until end of comment + while (text[0] != Ch('-') || text[1] != Ch('-') || text[2] != Ch('>')) + { + if (!text[0]) + CEREAL_RAPIDXML_PARSE_ERROR("unexpected end of data", text); + ++text; + } + + // Create comment node + xml_node *comment = this->allocate_node(node_comment); + comment->value(value_, static_cast(text - value_)); + + // Place zero terminator after comment value + if (!(Flags & parse_no_string_terminators)) + *text = Ch('\0'); + + text += 3; // Skip '-->' + return comment; + } + + // Parse DOCTYPE + template + xml_node *parse_doctype(Ch *&text) + { + // Remember value start + Ch *value_ = text; + + // Skip to > + while (*text != Ch('>')) + { + // Determine character type + switch (*text) + { + + // If '[' encountered, scan for matching ending ']' using naive algorithm with depth + // This works for all W3C test files except for 2 most wicked + case Ch('['): + { + ++text; // Skip '[' + int depth = 1; + while (depth > 0) + { + switch (*text) + { + case Ch('['): ++depth; break; + case Ch(']'): --depth; break; + case 0: CEREAL_RAPIDXML_PARSE_ERROR("unexpected end of data", text); + } + ++text; + } + break; + } + + // Error on end of text + case Ch('\0'): + CEREAL_RAPIDXML_PARSE_ERROR("unexpected end of data", text); + + // Other character, skip it + default: + ++text; + + } + } + + // If DOCTYPE nodes enabled + if (Flags & parse_doctype_node) + { + // Create a new doctype node + xml_node *doctype = this->allocate_node(node_doctype); + doctype->value(value_, static_cast(text - value_)); + + // Place zero terminator after value + if (!(Flags & parse_no_string_terminators)) + *text = Ch('\0'); + + text += 1; // skip '>' + return doctype; + } + else + { + text += 1; // skip '>' + return 0; + } + + } + + // Parse PI + template + xml_node *parse_pi(Ch *&text) + { + // If creation of PI nodes is enabled + if (Flags & parse_pi_nodes) + { + // Create pi node + xml_node *pi = this->allocate_node(node_pi); + + // Extract PI target name + Ch *name_ = text; + skip(text); + if (text == name_) + CEREAL_RAPIDXML_PARSE_ERROR("expected PI target", text); + pi->name(name_, static_cast(text - name_)); + + // Skip whitespace between pi target and pi + skip(text); + + // Remember start of pi + Ch *value_ = text; + + // Skip to '?>' + while (text[0] != Ch('?') || text[1] != Ch('>')) + { + if (*text == Ch('\0')) + CEREAL_RAPIDXML_PARSE_ERROR("unexpected end of data", text); + ++text; + } + + // Set pi value (verbatim, no entity expansion or whitespace normalization) + pi->value(value_, static_cast(text - value_)); + + // Place zero terminator after name and value + if (!(Flags & parse_no_string_terminators)) + { + pi->name()[pi->name_size()] = Ch('\0'); + pi->value()[pi->value_size()] = Ch('\0'); + } + + text += 2; // Skip '?>' + return pi; + } + else + { + // Skip to '?>' + while (text[0] != Ch('?') || text[1] != Ch('>')) + { + if (*text == Ch('\0')) + CEREAL_RAPIDXML_PARSE_ERROR("unexpected end of data", text); + ++text; + } + text += 2; // Skip '?>' + return 0; + } + } + + // Parse and append data + // Return character that ends data. + // This is necessary because this character might have been overwritten by a terminating 0 + template + Ch parse_and_append_data(xml_node *node, Ch *&text, Ch *contents_start) + { + // Backup to contents start if whitespace trimming is disabled + if (!(Flags & parse_trim_whitespace)) + text = contents_start; + + const bool preserve_space = internal::preserve_space(node); + + // Skip until end of data + Ch *value_ = text, *end; + if ((Flags & parse_normalize_whitespace) && !preserve_space) + end = skip_and_expand_character_refs(text, false); + else + end = skip_and_expand_character_refs(text, preserve_space); + + // Trim trailing whitespace if flag is set; leading was already trimmed by whitespace skip after > + if ((Flags & parse_trim_whitespace) && !preserve_space) + { + if (Flags & parse_normalize_whitespace) + { + // Whitespace is already condensed to single space characters by skipping function, so just trim 1 char off the end + if (*(end - 1) == Ch(' ')) + --end; + } + else + { + // Backup until non-whitespace character is found + while (whitespace_pred::test(*(end - 1))) + --end; + } + } + + // If characters are still left between end and value (this test is only necessary if normalization is enabled) + // Create new data node + if (!(Flags & parse_no_data_nodes)) + { + xml_node *data = this->allocate_node(node_data); + data->value(value_, static_cast(end - value_)); + node->append_node(data); + } + + // Add data to parent node if no data exists yet + if (!(Flags & parse_no_element_values)) + if (*node->value() == Ch('\0')) + node->value(value_, static_cast(end - value_)); + + // Place zero terminator after value + if (!(Flags & parse_no_string_terminators)) + { + Ch ch = *text; + *end = Ch('\0'); + return ch; // Return character that ends data; this is required because zero terminator overwritten it + } + + // Return character that ends data + return *text; + } + + // Parse CDATA + template + xml_node *parse_cdata(Ch *&text) + { + // If CDATA is disabled + if (Flags & parse_no_data_nodes) + { + // Skip until end of cdata + while (text[0] != Ch(']') || text[1] != Ch(']') || text[2] != Ch('>')) + { + if (!text[0]) + CEREAL_RAPIDXML_PARSE_ERROR("unexpected end of data", text); + ++text; + } + text += 3; // Skip ]]> + return 0; // Do not produce CDATA node + } + + // Skip until end of cdata + Ch *value_ = text; + while (text[0] != Ch(']') || text[1] != Ch(']') || text[2] != Ch('>')) + { + if (!text[0]) + CEREAL_RAPIDXML_PARSE_ERROR("unexpected end of data", text); + ++text; + } + + // Create new cdata node + xml_node *cdata = this->allocate_node(node_cdata); + cdata->value(value_, static_cast(text - value_)); + + // Place zero terminator after value + if (!(Flags & parse_no_string_terminators)) + *text = Ch('\0'); + + text += 3; // Skip ]]> + return cdata; + } + + // Parse element node + template + xml_node *parse_element(Ch *&text) + { + // Create element node + xml_node *element = this->allocate_node(node_element); + + // Extract element name + Ch *name_ = text; + skip(text); + if (text == name_) + CEREAL_RAPIDXML_PARSE_ERROR("expected element name", text); + element->name(name_, static_cast(text - name_)); + + // Skip whitespace between element name and attributes or > + skip(text); + + // Parse attributes, if any + parse_node_attributes(text, element); + + // Determine ending type + if (*text == Ch('>')) + { + ++text; + parse_node_contents(text, element); + } + else if (*text == Ch('/')) + { + ++text; + if (*text != Ch('>')) + CEREAL_RAPIDXML_PARSE_ERROR("expected >", text); + ++text; + } + else + CEREAL_RAPIDXML_PARSE_ERROR("expected >", text); + + // Place zero terminator after name + if (!(Flags & parse_no_string_terminators)) + element->name()[element->name_size()] = Ch('\0'); + + // Return parsed element + return element; + } + + // Determine node type, and parse it + template + xml_node *parse_node(Ch *&text) + { + // Parse proper node type + switch (text[0]) + { + + // <... + default: + // Parse and append element node + return parse_element(text); + + // (text); + } + else + { + // Parse PI + return parse_pi(text); + } + + // (text); + } + break; + + // (text); + } + break; + + // (text); + } + + } // switch + + // Attempt to skip other, unrecognized node types starting with ')) + { + if (*text == 0) + CEREAL_RAPIDXML_PARSE_ERROR("unexpected end of data", text); + ++text; + } + ++text; // Skip '>' + return 0; // No node recognized + + } + } + + // Parse contents of the node - children, data etc. + template + void parse_node_contents(Ch *&text, xml_node *node) + { + // For all children and text + while (1) + { + // Skip whitespace between > and node contents + Ch *contents_start = text; // Store start of node contents before whitespace is skipped + skip(text); + Ch next_char = *text; + + // After data nodes, instead of continuing the loop, control jumps here. + // This is because zero termination inside parse_and_append_data() function + // would wreak havoc with the above code. + // Also, skipping whitespace after data nodes is unnecessary. + after_data_node: + + // Determine what comes next: node closing, child node, data node, or 0? + switch (next_char) + { + + // Node closing or child node + case Ch('<'): + if (text[1] == Ch('/')) + { + Ch *contents_end = 0; + if (internal::preserve_space(node)) + { + contents_end = text; + } + + // Node closing + text += 2; // Skip '(text); + if (!internal::compare(node->name(), node->name_size(), closing_name, static_cast(text - closing_name), true)) + CEREAL_RAPIDXML_PARSE_ERROR("invalid closing tag name", text); + } + else + { + // No validation, just skip name + skip(text); + } + // Skip remaining whitespace after node name + skip(text); + if (*text != Ch('>')) + CEREAL_RAPIDXML_PARSE_ERROR("expected >", text); + ++text; // Skip '>' + + if (contents_end && contents_end != contents_start) + { + node->value(contents_start, static_cast(contents_end - contents_start)); + node->value()[node->value_size()] = Ch('\0'); + } + return; // Node closed, finished parsing contents + } + else + { + // Child node + ++text; // Skip '<' + if (xml_node *child = parse_node(text)) + node->append_node(child); + } + break; + + // End of data - error + case Ch('\0'): + CEREAL_RAPIDXML_PARSE_ERROR("unexpected end of data", text); + + // Data node + default: + next_char = parse_and_append_data(node, text, contents_start); + goto after_data_node; // Bypass regular processing after data nodes + + } + } + } + + // Parse XML attributes of the node + template + void parse_node_attributes(Ch *&text, xml_node *node) + { + // For all attributes + while (attribute_name_pred::test(*text)) + { + // Extract attribute name + Ch *name_ = text; + ++text; // Skip first character of attribute name + skip(text); + if (text == name_) + CEREAL_RAPIDXML_PARSE_ERROR("expected attribute name", name_); + + // Create new attribute + xml_attribute *attribute = this->allocate_attribute(); + attribute->name(name_, static_cast(text - name_)); + node->append_attribute(attribute); + + // Skip whitespace after attribute name + skip(text); + + // Skip = + if (*text != Ch('=')) + CEREAL_RAPIDXML_PARSE_ERROR("expected =", text); + ++text; + + // Add terminating zero after name + if (!(Flags & parse_no_string_terminators)) + attribute->name()[attribute->name_size()] = 0; + + // Skip whitespace after = + skip(text); + + // Skip quote and remember if it was ' or " + Ch quote = *text; + if (quote != Ch('\'') && quote != Ch('"')) + CEREAL_RAPIDXML_PARSE_ERROR("expected ' or \"", text); + ++text; + + // Extract attribute value and expand char refs in it + Ch *value_ = text, *end; + const int AttFlags = Flags & ~parse_normalize_whitespace; // No whitespace normalization in attributes + if (quote == Ch('\'')) + end = skip_and_expand_character_refs, attribute_value_pure_pred, AttFlags>(text, false); + else + end = skip_and_expand_character_refs, attribute_value_pure_pred, AttFlags>(text, false); + + // Set attribute value + attribute->value(value_, static_cast(end - value_)); + + // Make sure that end quote is present + if (*text != quote) + CEREAL_RAPIDXML_PARSE_ERROR("expected ' or \"", text); + ++text; // Skip quote + + // Add terminating zero after value + if (!(Flags & parse_no_string_terminators)) + attribute->value()[attribute->value_size()] = 0; + + // Skip whitespace after attribute value + skip(text); + } + } + + }; + + //! \cond internal + namespace internal + { + + // Whitespace (space \n \r \t) + template + const unsigned char lookup_tables::lookup_whitespace[256] = + { + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, // 0 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1 + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 2 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 3 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 4 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 5 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 6 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 7 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 8 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 9 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // A + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // B + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // C + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // D + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // E + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 // F + }; + + // Node name (anything but space \n \r \t / > ? \0) + template + const unsigned char lookup_tables::lookup_node_name[256] = + { + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, // 0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 1 + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, // 2 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, // 3 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 4 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 5 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 6 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 7 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 8 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 9 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // A + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // B + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // C + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // D + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // E + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // F + }; + + // Text (i.e. PCDATA) (anything but < \0) + template + const unsigned char lookup_tables::lookup_text[256] = + { + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 1 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 2 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, // 3 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 4 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 5 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 6 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 7 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 8 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 9 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // A + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // B + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // C + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // D + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // E + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // F + }; + + // Text (i.e. PCDATA) that does not require processing when ws normalization is disabled + // (anything but < \0 &) + template + const unsigned char lookup_tables::lookup_text_pure_no_ws[256] = + { + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 1 + 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 2 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, // 3 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 4 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 5 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 6 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 7 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 8 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 9 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // A + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // B + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // C + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // D + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // E + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // F + }; + + // Text (i.e. PCDATA) that does not require processing when ws normalizationis is enabled + // (anything but < \0 & space \n \r \t) + template + const unsigned char lookup_tables::lookup_text_pure_with_ws[256] = + { + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, // 0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 1 + 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 2 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, // 3 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 4 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 5 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 6 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 7 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 8 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 9 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // A + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // B + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // C + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // D + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // E + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // F + }; + + // Attribute name (anything but space \n \r \t / < > = ? ! \0) + template + const unsigned char lookup_tables::lookup_attribute_name[256] = + { + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, // 0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 1 + 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, // 2 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, // 3 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 4 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 5 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 6 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 7 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 8 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 9 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // A + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // B + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // C + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // D + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // E + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // F + }; + + // Attribute data with single quote (anything but ' \0) + template + const unsigned char lookup_tables::lookup_attribute_data_1[256] = + { + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 1 + 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, // 2 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 3 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 4 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 5 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 6 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 7 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 8 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 9 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // A + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // B + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // C + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // D + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // E + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // F + }; + + // Attribute data with single quote that does not require processing (anything but ' \0 &) + template + const unsigned char lookup_tables::lookup_attribute_data_1_pure[256] = + { + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 1 + 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, // 2 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 3 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 4 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 5 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 6 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 7 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 8 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 9 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // A + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // B + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // C + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // D + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // E + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // F + }; + + // Attribute data with double quote (anything but " \0) + template + const unsigned char lookup_tables::lookup_attribute_data_2[256] = + { + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 1 + 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 2 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 3 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 4 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 5 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 6 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 7 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 8 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 9 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // A + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // B + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // C + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // D + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // E + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // F + }; + + // Attribute data with double quote that does not require processing (anything but " \0 &) + template + const unsigned char lookup_tables::lookup_attribute_data_2_pure[256] = + { + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 1 + 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 2 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 3 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 4 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 5 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 6 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 7 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 8 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 9 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // A + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // B + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // C + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // D + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // E + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // F + }; + + // Digits (dec and hex, 255 denotes end of numeric character reference) + template + const unsigned char lookup_tables::lookup_digits[256] = + { + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, // 0 + 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, // 1 + 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, // 2 + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,255,255,255,255,255,255, // 3 + 255, 10, 11, 12, 13, 14, 15,255,255,255,255,255,255,255,255,255, // 4 + 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, // 5 + 255, 10, 11, 12, 13, 14, 15,255,255,255,255,255,255,255,255,255, // 6 + 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, // 7 + 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, // 8 + 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, // 9 + 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, // A + 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, // B + 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, // C + 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, // D + 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, // E + 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255 // F + }; + + // Upper case conversion + template + const unsigned char lookup_tables::lookup_upcase[256] = + { + // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, A B C D E F + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // 0 + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, // 1 + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, // 2 + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, // 3 + 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, // 4 + 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, // 5 + 96, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, // 6 + 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 123,124,125,126,127, // 7 + 128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143, // 8 + 144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159, // 9 + 160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175, // A + 176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191, // B + 192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207, // C + 208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223, // D + 224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239, // E + 240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255 // F + }; + } + //! \endcond + +} +} // end namespace cereal + +// Undefine internal macros +#undef CEREAL_RAPIDXML_PARSE_ERROR + +// On MSVC, restore warnings state +#ifdef _MSC_VER + #pragma warning(pop) +#endif + +#endif diff --git a/third_party/cereal/external/rapidxml/rapidxml_iterators.hpp b/third_party/cereal/external/rapidxml/rapidxml_iterators.hpp new file mode 100755 index 0000000..736d46c --- /dev/null +++ b/third_party/cereal/external/rapidxml/rapidxml_iterators.hpp @@ -0,0 +1,175 @@ +#ifndef CEREAL_RAPIDXML_ITERATORS_HPP_INCLUDED +#define CEREAL_RAPIDXML_ITERATORS_HPP_INCLUDED + +// Copyright (C) 2006, 2009 Marcin Kalicinski +// Version 1.13 +// Revision $DateTime: 2009/05/13 01:46:17 $ + +#include "rapidxml.hpp" + +namespace cereal { +namespace rapidxml +{ + + //! Iterator of child nodes of xml_node + template + class node_iterator + { + + public: + + typedef typename xml_node value_type; + typedef typename xml_node &reference; + typedef typename xml_node *pointer; + typedef std::ptrdiff_t difference_type; + typedef std::bidirectional_iterator_tag iterator_category; + + node_iterator() + : m_node(0) + { + } + + node_iterator(xml_node *node) + : m_node(node->first_node()) + { + } + + reference operator *() const + { + assert(m_node); + return *m_node; + } + + pointer operator->() const + { + assert(m_node); + return m_node; + } + + node_iterator& operator++() + { + assert(m_node); + m_node = m_node->next_sibling(); + return *this; + } + + node_iterator operator++(int) + { + node_iterator tmp = *this; + ++this; + return tmp; + } + + node_iterator& operator--() + { + assert(m_node && m_node->previous_sibling()); + m_node = m_node->previous_sibling(); + return *this; + } + + node_iterator operator--(int) + { + node_iterator tmp = *this; + ++this; + return tmp; + } + + bool operator ==(const node_iterator &rhs) + { + return m_node == rhs.m_node; + } + + bool operator !=(const node_iterator &rhs) + { + return m_node != rhs.m_node; + } + + private: + + xml_node *m_node; + + }; + + //! Iterator of child attributes of xml_node + template + class attribute_iterator + { + + public: + + typedef typename xml_attribute value_type; + typedef typename xml_attribute &reference; + typedef typename xml_attribute *pointer; + typedef std::ptrdiff_t difference_type; + typedef std::bidirectional_iterator_tag iterator_category; + + attribute_iterator() + : m_attribute(0) + { + } + + attribute_iterator(xml_node *node) + : m_attribute(node->first_attribute()) + { + } + + reference operator *() const + { + assert(m_attribute); + return *m_attribute; + } + + pointer operator->() const + { + assert(m_attribute); + return m_attribute; + } + + attribute_iterator& operator++() + { + assert(m_attribute); + m_attribute = m_attribute->next_attribute(); + return *this; + } + + attribute_iterator operator++(int) + { + attribute_iterator tmp = *this; + ++this; + return tmp; + } + + attribute_iterator& operator--() + { + assert(m_attribute && m_attribute->previous_attribute()); + m_attribute = m_attribute->previous_attribute(); + return *this; + } + + attribute_iterator operator--(int) + { + attribute_iterator tmp = *this; + ++this; + return tmp; + } + + bool operator ==(const attribute_iterator &rhs) + { + return m_attribute == rhs.m_attribute; + } + + bool operator !=(const attribute_iterator &rhs) + { + return m_attribute != rhs.m_attribute; + } + + private: + + xml_attribute *m_attribute; + + }; + +} +} // namespace cereal + +#endif diff --git a/third_party/cereal/external/rapidxml/rapidxml_print.hpp b/third_party/cereal/external/rapidxml/rapidxml_print.hpp new file mode 100755 index 0000000..7fbb962 --- /dev/null +++ b/third_party/cereal/external/rapidxml/rapidxml_print.hpp @@ -0,0 +1,428 @@ +#ifndef CEREAL_RAPIDXML_PRINT_HPP_INCLUDED +#define CEREAL_RAPIDXML_PRINT_HPP_INCLUDED + +// Copyright (C) 2006, 2009 Marcin Kalicinski +// Version 1.13 +// Revision $DateTime: 2009/05/13 01:46:17 $ + +#include "rapidxml.hpp" + +// Only include streams if not disabled +#ifndef CEREAL_RAPIDXML_NO_STREAMS + #include + #include +#endif + +namespace cereal { +namespace rapidxml +{ + + /////////////////////////////////////////////////////////////////////// + // Printing flags + + const int print_no_indenting = 0x1; //!< Printer flag instructing the printer to suppress indenting of XML. See print() function. + + /////////////////////////////////////////////////////////////////////// + // Internal + + //! \cond internal + namespace internal + { + + /////////////////////////////////////////////////////////////////////////// + // Internal character operations + + // Copy characters from given range to given output iterator + template + inline OutIt copy_chars(const Ch *begin, const Ch *end, OutIt out) + { + while (begin != end) + *out++ = *begin++; + return out; + } + + // Copy characters from given range to given output iterator and expand + // characters into references (< > ' " &) + template + inline OutIt copy_and_expand_chars(const Ch *begin, const Ch *end, Ch noexpand, OutIt out) + { + while (begin != end) + { + if (*begin == noexpand) + { + *out++ = *begin; // No expansion, copy character + } + else + { + switch (*begin) + { + case Ch('<'): + *out++ = Ch('&'); *out++ = Ch('l'); *out++ = Ch('t'); *out++ = Ch(';'); + break; + case Ch('>'): + *out++ = Ch('&'); *out++ = Ch('g'); *out++ = Ch('t'); *out++ = Ch(';'); + break; + case Ch('\''): + *out++ = Ch('&'); *out++ = Ch('a'); *out++ = Ch('p'); *out++ = Ch('o'); *out++ = Ch('s'); *out++ = Ch(';'); + break; + case Ch('"'): + *out++ = Ch('&'); *out++ = Ch('q'); *out++ = Ch('u'); *out++ = Ch('o'); *out++ = Ch('t'); *out++ = Ch(';'); + break; + case Ch('&'): + *out++ = Ch('&'); *out++ = Ch('a'); *out++ = Ch('m'); *out++ = Ch('p'); *out++ = Ch(';'); + break; + default: + *out++ = *begin; // No expansion, copy character + } + } + ++begin; // Step to next character + } + return out; + } + + // Fill given output iterator with repetitions of the same character + template + inline OutIt fill_chars(OutIt out, int n, Ch ch) + { + for (int i = 0; i < n; ++i) + *out++ = ch; + return out; + } + + // Find character + template + inline bool find_char(const Ch *begin, const Ch *end) + { + while (begin != end) + if (*begin++ == ch) + return true; + return false; + } + + /////////////////////////////////////////////////////////////////////////// + // Internal printing operations + + // Print node + template + inline OutIt print_node(OutIt out, const xml_node *node, int flags, int indent); + + // Print children of the node + template + inline OutIt print_children(OutIt out, const xml_node *node, int flags, int indent) + { + for (xml_node *child = node->first_node(); child; child = child->next_sibling()) + out = print_node(out, child, flags, indent); + return out; + } + + // Print attributes of the node + template + inline OutIt print_attributes(OutIt out, const xml_node *node, int /*flags*/) + { + for (xml_attribute *attribute = node->first_attribute(); attribute; attribute = attribute->next_attribute()) + { + if (attribute->name() && attribute->value()) + { + // Print attribute name + *out = Ch(' '), ++out; + out = copy_chars(attribute->name(), attribute->name() + attribute->name_size(), out); + *out = Ch('='), ++out; + // Print attribute value using appropriate quote type + if (find_char(attribute->value(), attribute->value() + attribute->value_size())) + { + *out = Ch('\''), ++out; + out = copy_and_expand_chars(attribute->value(), attribute->value() + attribute->value_size(), Ch('"'), out); + *out = Ch('\''), ++out; + } + else + { + *out = Ch('"'), ++out; + out = copy_and_expand_chars(attribute->value(), attribute->value() + attribute->value_size(), Ch('\''), out); + *out = Ch('"'), ++out; + } + } + } + return out; + } + + // Print data node + template + inline OutIt print_data_node(OutIt out, const xml_node *node, int flags, int indent) + { + assert(node->type() == node_data); + if (!(flags & print_no_indenting)) + out = fill_chars(out, indent, Ch('\t')); + out = copy_and_expand_chars(node->value(), node->value() + node->value_size(), Ch(0), out); + return out; + } + + // Print data node + template + inline OutIt print_cdata_node(OutIt out, const xml_node *node, int flags, int indent) + { + assert(node->type() == node_cdata); + if (!(flags & print_no_indenting)) + out = fill_chars(out, indent, Ch('\t')); + *out = Ch('<'); ++out; + *out = Ch('!'); ++out; + *out = Ch('['); ++out; + *out = Ch('C'); ++out; + *out = Ch('D'); ++out; + *out = Ch('A'); ++out; + *out = Ch('T'); ++out; + *out = Ch('A'); ++out; + *out = Ch('['); ++out; + out = copy_chars(node->value(), node->value() + node->value_size(), out); + *out = Ch(']'); ++out; + *out = Ch(']'); ++out; + *out = Ch('>'); ++out; + return out; + } + + // Print element node + template + inline OutIt print_element_node(OutIt out, const xml_node *node, int flags, int indent) + { + assert(node->type() == node_element); + + // Print element name and attributes, if any + if (!(flags & print_no_indenting)) + out = fill_chars(out, indent, Ch('\t')); + *out = Ch('<'), ++out; + out = copy_chars(node->name(), node->name() + node->name_size(), out); + out = print_attributes(out, node, flags); + + // If node is childless + if (node->value_size() == 0 && !node->first_node()) + { + // Print childless node tag ending + *out = Ch('/'), ++out; + *out = Ch('>'), ++out; + } + else + { + // Print normal node tag ending + *out = Ch('>'), ++out; + + // Test if node contains a single data node only (and no other nodes) + xml_node *child = node->first_node(); + if (!child) + { + // If node has no children, only print its value without indenting + out = copy_and_expand_chars(node->value(), node->value() + node->value_size(), Ch(0), out); + } + else if (child->next_sibling() == 0 && child->type() == node_data) + { + // If node has a sole data child, only print its value without indenting + out = copy_and_expand_chars(child->value(), child->value() + child->value_size(), Ch(0), out); + } + else + { + // Print all children with full indenting + if (!(flags & print_no_indenting)) + *out = Ch('\n'), ++out; + out = print_children(out, node, flags, indent + 1); + if (!(flags & print_no_indenting)) + out = fill_chars(out, indent, Ch('\t')); + } + + // Print node end + *out = Ch('<'), ++out; + *out = Ch('/'), ++out; + out = copy_chars(node->name(), node->name() + node->name_size(), out); + *out = Ch('>'), ++out; + } + return out; + } + + // Print declaration node + template + inline OutIt print_declaration_node(OutIt out, const xml_node *node, int flags, int indent) + { + // Print declaration start + if (!(flags & print_no_indenting)) + out = fill_chars(out, indent, Ch('\t')); + *out = Ch('<'), ++out; + *out = Ch('?'), ++out; + *out = Ch('x'), ++out; + *out = Ch('m'), ++out; + *out = Ch('l'), ++out; + + // Print attributes + out = print_attributes(out, node, flags); + + // Print declaration end + *out = Ch('?'), ++out; + *out = Ch('>'), ++out; + + return out; + } + + // Print comment node + template + inline OutIt print_comment_node(OutIt out, const xml_node *node, int flags, int indent) + { + assert(node->type() == node_comment); + if (!(flags & print_no_indenting)) + out = fill_chars(out, indent, Ch('\t')); + *out = Ch('<'), ++out; + *out = Ch('!'), ++out; + *out = Ch('-'), ++out; + *out = Ch('-'), ++out; + out = copy_chars(node->value(), node->value() + node->value_size(), out); + *out = Ch('-'), ++out; + *out = Ch('-'), ++out; + *out = Ch('>'), ++out; + return out; + } + + // Print doctype node + template + inline OutIt print_doctype_node(OutIt out, const xml_node *node, int flags, int indent) + { + assert(node->type() == node_doctype); + if (!(flags & print_no_indenting)) + out = fill_chars(out, indent, Ch('\t')); + *out = Ch('<'), ++out; + *out = Ch('!'), ++out; + *out = Ch('D'), ++out; + *out = Ch('O'), ++out; + *out = Ch('C'), ++out; + *out = Ch('T'), ++out; + *out = Ch('Y'), ++out; + *out = Ch('P'), ++out; + *out = Ch('E'), ++out; + *out = Ch(' '), ++out; + out = copy_chars(node->value(), node->value() + node->value_size(), out); + *out = Ch('>'), ++out; + return out; + } + + // Print pi node + template + inline OutIt print_pi_node(OutIt out, const xml_node *node, int flags, int indent) + { + assert(node->type() == node_pi); + if (!(flags & print_no_indenting)) + out = fill_chars(out, indent, Ch('\t')); + *out = Ch('<'), ++out; + *out = Ch('?'), ++out; + out = copy_chars(node->name(), node->name() + node->name_size(), out); + *out = Ch(' '), ++out; + out = copy_chars(node->value(), node->value() + node->value_size(), out); + *out = Ch('?'), ++out; + *out = Ch('>'), ++out; + return out; + } + + // Print node + template + inline OutIt print_node(OutIt out, const xml_node *node, int flags, int indent) + { + // Print proper node type + switch (node->type()) + { + + // Document + case node_document: + out = print_children(out, node, flags, indent); + break; + + // Element + case node_element: + out = print_element_node(out, node, flags, indent); + break; + + // Data + case node_data: + out = print_data_node(out, node, flags, indent); + break; + + // CDATA + case node_cdata: + out = print_cdata_node(out, node, flags, indent); + break; + + // Declaration + case node_declaration: + out = print_declaration_node(out, node, flags, indent); + break; + + // Comment + case node_comment: + out = print_comment_node(out, node, flags, indent); + break; + + // Doctype + case node_doctype: + out = print_doctype_node(out, node, flags, indent); + break; + + // Pi + case node_pi: + out = print_pi_node(out, node, flags, indent); + break; + +#ifndef __GNUC__ + // Unknown + default: + assert(0); + break; +#endif + } + + // If indenting not disabled, add line break after node + if (!(flags & print_no_indenting)) + *out = Ch('\n'), ++out; + + // Return modified iterator + return out; + } + + } + //! \endcond + + /////////////////////////////////////////////////////////////////////////// + // Printing + + //! Prints XML to given output iterator. + //! \param out Output iterator to print to. + //! \param node Node to be printed. Pass xml_document to print entire document. + //! \param flags Flags controlling how XML is printed. + //! \return Output iterator pointing to position immediately after last character of printed text. + template + inline OutIt print(OutIt out, const xml_node &node, int flags = 0) + { + return internal::print_node(out, &node, flags, 0); + } + +#ifndef CEREAL_RAPIDXML_NO_STREAMS + + //! Prints XML to given output stream. + //! \param out Output stream to print to. + //! \param node Node to be printed. Pass xml_document to print entire document. + //! \param flags Flags controlling how XML is printed. + //! \return Output stream. + template + inline std::basic_ostream &print(std::basic_ostream &out, const xml_node &node, int flags = 0) + { + print(std::ostream_iterator(out), node, flags); + return out; + } + + //! Prints formatted XML to given output stream. Uses default printing flags. Use print() function to customize printing process. + //! \param out Output stream to print to. + //! \param node Node to be printed. + //! \return Output stream. + template + inline std::basic_ostream &operator <<(std::basic_ostream &out, const xml_node &node) + { + return print(out, node); + } + +#endif + +} +} // namespace cereal + +#endif diff --git a/third_party/cereal/external/rapidxml/rapidxml_utils.hpp b/third_party/cereal/external/rapidxml/rapidxml_utils.hpp new file mode 100755 index 0000000..e11ecf3 --- /dev/null +++ b/third_party/cereal/external/rapidxml/rapidxml_utils.hpp @@ -0,0 +1,123 @@ +#ifndef CEREAL_RAPIDXML_UTILS_HPP_INCLUDED +#define CEREAL_RAPIDXML_UTILS_HPP_INCLUDED + +// Copyright (C) 2006, 2009 Marcin Kalicinski +// Version 1.13 +// Revision $DateTime: 2009/05/13 01:46:17 $ +//! in certain simple scenarios. They should probably not be used if maximizing performance is the main objective. + +#include "rapidxml.hpp" +#include +#include +#include +#include + +namespace cereal { +namespace rapidxml +{ + + //! Represents data loaded from a file + template + class file + { + + public: + + //! Loads file into the memory. Data will be automatically destroyed by the destructor. + //! \param filename Filename to load. + file(const char *filename) + { + using namespace std; + + // Open stream + basic_ifstream stream(filename, ios::binary); + if (!stream) + throw runtime_error(string("cannot open file ") + filename); + stream.unsetf(ios::skipws); + + // Determine stream size + stream.seekg(0, ios::end); + size_t size = stream.tellg(); + stream.seekg(0); + + // Load data and add terminating 0 + m_data.resize(size + 1); + stream.read(&m_data.front(), static_cast(size)); + m_data[size] = 0; + } + + //! Loads file into the memory. Data will be automatically destroyed by the destructor + //! \param stream Stream to load from + file(std::basic_istream &stream) + { + using namespace std; + + // Load data and add terminating 0 + stream.unsetf(ios::skipws); + m_data.assign(istreambuf_iterator(stream), istreambuf_iterator()); + if (stream.fail() || stream.bad()) + throw runtime_error("error reading stream"); + m_data.push_back(0); + } + + //! Gets file data. + //! \return Pointer to data of file. + Ch *data() + { + return &m_data.front(); + } + + //! Gets file data. + //! \return Pointer to data of file. + const Ch *data() const + { + return &m_data.front(); + } + + //! Gets file data size. + //! \return Size of file data, in characters. + std::size_t size() const + { + return m_data.size(); + } + + private: + + std::vector m_data; // File data + + }; + + //! Counts children of node. Time complexity is O(n). + //! \return Number of children of node + template + inline std::size_t count_children(xml_node *node) + { + xml_node *child = node->first_node(); + std::size_t count = 0; + while (child) + { + ++count; + child = child->next_sibling(); + } + return count; + } + + //! Counts attributes of node. Time complexity is O(n). + //! \return Number of attributes of node + template + inline std::size_t count_attributes(xml_node *node) + { + xml_attribute *attr = node->first_attribute(); + std::size_t count = 0; + while (attr) + { + ++count; + attr = attr->next_attribute(); + } + return count; + } + +} +} // namespace cereal + +#endif diff --git a/third_party/cereal/macros.hpp b/third_party/cereal/macros.hpp new file mode 100755 index 0000000..85cf9b5 --- /dev/null +++ b/third_party/cereal/macros.hpp @@ -0,0 +1,156 @@ +/*! \file macros.hpp + \brief Preprocessor macros that can customise the cereal library + + By default, cereal looks for serialization functions with very + specific names, that is: serialize, load, save, load_minimal, + or save_minimal. + + This file allows an advanced user to change these names to conform + to some other style or preference. This is implemented using + preprocessor macros. + + As a result of this, in internal cereal code you will see macros + used for these function names. In user code, you should name + the functions like you normally would and not use the macros + to improve readability. + \ingroup utility */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef CEREAL_MACROS_HPP_ +#define CEREAL_MACROS_HPP_ + +#ifndef CEREAL_THREAD_SAFE +//! Whether cereal should be compiled for a threaded environment +/*! This macro causes cereal to use mutexes to control access to + global internal state in a thread safe manner. + + Note that even with this enabled you must still ensure that + archives are accessed by only one thread at a time; it is safe + to use multiple archives in parallel, but not to access one archive + from many places simultaneously. */ +#define CEREAL_THREAD_SAFE 0 +#endif // CEREAL_THREAD_SAFE + +#ifndef CEREAL_SIZE_TYPE +//! Determines the data type used for size_type +/*! cereal uses size_type to ensure that the serialized size of + dynamic containers is compatible across different architectures + (e.g. 32 vs 64 bit), which may use different underlying types for + std::size_t. + + More information can be found in cereal/details/helpers.hpp. + + If you choose to modify this type, ensure that you use a fixed + size type (e.g. uint32_t). */ +#define CEREAL_SIZE_TYPE uint64_t +#endif // CEREAL_SIZE_TYPE + +// ###################################################################### +#ifndef CEREAL_SERIALIZE_FUNCTION_NAME +//! The serialization/deserialization function name to search for. +/*! You can define @c CEREAL_SERIALIZE_FUNCTION_NAME to be different assuming + you do so before this file is included. */ +#define CEREAL_SERIALIZE_FUNCTION_NAME serialize +#endif // CEREAL_SERIALIZE_FUNCTION_NAME + +#ifndef CEREAL_LOAD_FUNCTION_NAME +//! The deserialization (load) function name to search for. +/*! You can define @c CEREAL_LOAD_FUNCTION_NAME to be different assuming you do so + before this file is included. */ +#define CEREAL_LOAD_FUNCTION_NAME load +#endif // CEREAL_LOAD_FUNCTION_NAME + +#ifndef CEREAL_SAVE_FUNCTION_NAME +//! The serialization (save) function name to search for. +/*! You can define @c CEREAL_SAVE_FUNCTION_NAME to be different assuming you do so + before this file is included. */ +#define CEREAL_SAVE_FUNCTION_NAME save +#endif // CEREAL_SAVE_FUNCTION_NAME + +#ifndef CEREAL_LOAD_MINIMAL_FUNCTION_NAME +//! The deserialization (load_minimal) function name to search for. +/*! You can define @c CEREAL_LOAD_MINIMAL_FUNCTION_NAME to be different assuming you do so + before this file is included. */ +#define CEREAL_LOAD_MINIMAL_FUNCTION_NAME load_minimal +#endif // CEREAL_LOAD_MINIMAL_FUNCTION_NAME + +#ifndef CEREAL_SAVE_MINIMAL_FUNCTION_NAME +//! The serialization (save_minimal) function name to search for. +/*! You can define @c CEREAL_SAVE_MINIMAL_FUNCTION_NAME to be different assuming you do so + before this file is included. */ +#define CEREAL_SAVE_MINIMAL_FUNCTION_NAME save_minimal +#endif // CEREAL_SAVE_MINIMAL_FUNCTION_NAME + +// ###################################################################### +//! Defines the CEREAL_NOEXCEPT macro to use instead of noexcept +/*! If a compiler we support does not support noexcept, this macro + will detect this and define CEREAL_NOEXCEPT as a no-op + @internal */ +#if !defined(CEREAL_HAS_NOEXCEPT) + #if defined(__clang__) + #if __has_feature(cxx_noexcept) + #define CEREAL_HAS_NOEXCEPT + #endif + #else // NOT clang + #if defined(__GXX_EXPERIMENTAL_CXX0X__) && __GNUC__ * 10 + __GNUC_MINOR__ >= 46 || \ + defined(_MSC_FULL_VER) && _MSC_FULL_VER >= 190023026 + #define CEREAL_HAS_NOEXCEPT + #endif // end GCC/MSVC check + #endif // end NOT clang block + + #ifndef CEREAL_NOEXCEPT + #ifdef CEREAL_HAS_NOEXCEPT + #define CEREAL_NOEXCEPT noexcept + #else + #define CEREAL_NOEXCEPT + #endif // end CEREAL_HAS_NOEXCEPT + #endif // end !defined(CEREAL_HAS_NOEXCEPT) +#endif // ifndef CEREAL_NOEXCEPT + +// ###################################################################### +//! Checks if C++17 is available +//! NOTE: clang v5 has a bug with inline variables, so disable C++17 on that compiler +#if (__cplusplus >= 201703L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L)) \ + && (!defined(__clang__) || __clang_major__ > 5) +#define CEREAL_HAS_CPP17 +#endif + +//! Checks if C++14 is available +#if __cplusplus >= 201402L +#define CEREAL_HAS_CPP14 +#endif + +// ###################################################################### +//! Defines the CEREAL_ALIGNOF macro to use instead of alignof +#if defined(_MSC_VER) && _MSC_VER < 1900 +#define CEREAL_ALIGNOF __alignof +#else // not MSVC 2013 or older +#define CEREAL_ALIGNOF alignof +#endif // end MSVC check + +#endif // CEREAL_MACROS_HPP_ diff --git a/third_party/cereal/specialize.hpp b/third_party/cereal/specialize.hpp new file mode 100755 index 0000000..da9c9a5 --- /dev/null +++ b/third_party/cereal/specialize.hpp @@ -0,0 +1,139 @@ +/*! \file specialize.hpp + \brief Serialization disambiguation */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef CEREAL_SPECIALIZE_HPP_ +#define CEREAL_SPECIALIZE_HPP_ + +namespace cereal +{ + // Forward declaration of access class that users can become friends with + class access; + + // ###################################################################### + //! A specifier used in conjunction with cereal::specialize to disambiguate + //! serialization in special cases + /*! @relates specialize + @ingroup Access */ + enum class specialization + { + member_serialize, //!< Force the use of a member serialize function + member_load_save, //!< Force the use of a member load/save pair + member_load_save_minimal, //!< Force the use of a member minimal load/save pair + non_member_serialize, //!< Force the use of a non-member serialize function + non_member_load_save, //!< Force the use of a non-member load/save pair + non_member_load_save_minimal //!< Force the use of a non-member minimal load/save pair + }; + + //! A class used to disambiguate cases where cereal cannot detect a unique way of serializing a class + /*! cereal attempts to figure out which method of serialization (member vs. non-member serialize + or load/save pair) at compile time. If for some reason cereal cannot find a non-ambiguous way + of serializing a type, it will produce a static assertion complaining about this. + + This can happen because you have both a serialize and load/save pair, or even because a base + class has a serialize (public or private with friend access) and a derived class does not + overwrite this due to choosing some other serialization type. + + Specializing this class will tell cereal to explicitly use the serialization type you specify + and it will not complain about ambiguity in its compile time selection. However, if cereal detects + an ambiguity in specializations, it will continue to issue a static assertion. + + @code{.cpp} + class MyParent + { + friend class cereal::access; + template + void serialize( Archive & ar ) {} + }; + + // Although serialize is private in MyParent, to cereal::access it will look public, + // even through MyDerived + class MyDerived : public MyParent + { + public: + template + void load( Archive & ar ) {} + + template + void save( Archive & ar ) {} + }; + + // The load/save pair in MyDerived is ambiguous because serialize in MyParent can + // be accessed from cereal::access. This looks the same as making serialize public + // in MyParent, making it seem as though MyDerived has both a serialize and a load/save pair. + // cereal will complain about this at compile time unless we disambiguate: + + namespace cereal + { + // This struct specialization will tell cereal which is the right way to serialize the ambiguity + template struct specialize {}; + + // If we only had a disambiguation for a specific archive type, it would look something like this + template <> struct specialize {}; + } + @endcode + + You can also choose to use the macros CEREAL_SPECIALIZE_FOR_ALL_ARCHIVES or + CEREAL_SPECIALIZE_FOR_ARCHIVE if you want to type a little bit less. + + @tparam T The type to specialize the serialization for + @tparam S The specialization type to use for T + @ingroup Access */ + template + struct specialize : public std::false_type {}; + + //! Convenient macro for performing specialization for all archive types + /*! This performs specialization for the specific type for all types of archives. + This macro should be placed at the global namespace. + + @code{cpp} + struct MyType {}; + CEREAL_SPECIALIZE_FOR_ALL_ARCHIVES( MyType, cereal::specialization::member_load_save ); + @endcode + + @relates specialize + @ingroup Access */ + #define CEREAL_SPECIALIZE_FOR_ALL_ARCHIVES( Type, Specialization ) \ + namespace cereal { template struct specialize {}; } + + //! Convenient macro for performing specialization for a single archive type + /*! This performs specialization for the specific type for a single type of archive. + This macro should be placed at the global namespace. + + @code{cpp} + struct MyType {}; + CEREAL_SPECIALIZE_FOR_ARCHIVE( cereal::XMLInputArchive, MyType, cereal::specialization::member_load_save ); + @endcode + + @relates specialize + @ingroup Access */ + #define CEREAL_SPECIALIZE_FOR_ARCHIVE( Archive, Type, Specialization ) \ + namespace cereal { template <> struct specialize {}; } +} + +#endif diff --git a/third_party/cereal/types/array.hpp b/third_party/cereal/types/array.hpp new file mode 100755 index 0000000..e3b3775 --- /dev/null +++ b/third_party/cereal/types/array.hpp @@ -0,0 +1,79 @@ +/*! \file array.hpp + \brief Support for types found in \ + \ingroup STLSupport */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_ARRAY_HPP_ +#define CEREAL_TYPES_ARRAY_HPP_ + +#include "../cereal.hpp" +#include + +namespace cereal +{ + //! Saving for std::array primitive types + //! using binary serialization, if supported + template inline + typename std::enable_if, Archive>::value + && std::is_arithmetic::value, void>::type + CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::array const & array ) + { + ar( binary_data( array.data(), sizeof(array) ) ); + } + + //! Loading for std::array primitive types + //! using binary serialization, if supported + template inline + typename std::enable_if, Archive>::value + && std::is_arithmetic::value, void>::type + CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::array & array ) + { + ar( binary_data( array.data(), sizeof(array) ) ); + } + + //! Saving for std::array all other types + template inline + typename std::enable_if, Archive>::value + || !std::is_arithmetic::value, void>::type + CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::array const & array ) + { + for( auto const & i : array ) + ar( i ); + } + + //! Loading for std::array all other types + template inline + typename std::enable_if, Archive>::value + || !std::is_arithmetic::value, void>::type + CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::array & array ) + { + for( auto & i : array ) + ar( i ); + } +} // namespace cereal + +#endif // CEREAL_TYPES_ARRAY_HPP_ diff --git a/third_party/cereal/types/atomic.hpp b/third_party/cereal/types/atomic.hpp new file mode 100755 index 0000000..e5ed5d9 --- /dev/null +++ b/third_party/cereal/types/atomic.hpp @@ -0,0 +1,55 @@ +/*! \file atomic.hpp + \brief Support for types found in \ + \ingroup STLSupport */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_ATOMIC_HPP_ +#define CEREAL_TYPES_ATOMIC_HPP_ + +#include +#include + +namespace cereal +{ + //! Serializing (save) for std::atomic + template inline + void CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::atomic const & a ) + { + ar( CEREAL_NVP_("atomic_data", a.load()) ); + } + + //! Serializing (load) for std::atomic + template inline + void CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::atomic & a ) + { + T tmp; + ar( CEREAL_NVP_("atomic_data", tmp) ); + a.store( tmp ); + } +} // namespace cereal + +#endif // CEREAL_TYPES_ATOMIC_HPP_ diff --git a/third_party/cereal/types/base_class.hpp b/third_party/cereal/types/base_class.hpp new file mode 100755 index 0000000..c5186f6 --- /dev/null +++ b/third_party/cereal/types/base_class.hpp @@ -0,0 +1,203 @@ +/*! \file base_class.hpp + \brief Support for base classes (virtual and non-virtual) + \ingroup OtherTypes */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_BASE_CLASS_HPP_ +#define CEREAL_TYPES_BASE_CLASS_HPP_ + +#include "../details/traits.hpp" +#include "../details/polymorphic_impl_fwd.hpp" + +namespace cereal +{ + namespace base_class_detail + { + //! Used to register polymorphic relations and avoid the need to include + //! polymorphic.hpp when no polymorphism is used + /*! @internal */ + template ::value> + struct RegisterPolymorphicBaseClass + { + static void bind() + { } + }; + + //! Polymorphic version + /*! @internal */ + template + struct RegisterPolymorphicBaseClass + { + static void bind() + { detail::RegisterPolymorphicCaster::bind(); } + }; + } + + //! Casts a derived class to its non-virtual base class in a way that safely supports abstract classes + /*! This should be used in cases when a derived type needs to serialize its base type. This is better than directly + using static_cast, as it allows for serialization of pure virtual (abstract) base classes. + + This also automatically registers polymorphic relation between the base and derived class, assuming they + are indeed polymorphic. Note this is not the same as polymorphic type registration. For more information + see the documentation on polymorphism. If using a polymorphic class, be sure to include support for + polymorphism (cereal/types/polymorphic.hpp). + + \sa virtual_base_class + + @code{.cpp} + struct MyBase + { + int x; + + virtual void foo() = 0; + + template + void serialize( Archive & ar ) + { + ar( x ); + } + }; + + struct MyDerived : public MyBase //<-- Note non-virtual inheritance + { + int y; + + virtual void foo() {}; + + template + void serialize( Archive & ar ) + { + ar( cereal::base_class(this) ); + ar( y ); + } + }; + @endcode */ + template + struct base_class : private traits::detail::BaseCastBase + { + template + base_class(Derived const * derived) : + base_ptr(const_cast(static_cast(derived))) + { + static_assert( std::is_base_of::value, "Can only use base_class on a valid base class" ); + base_class_detail::RegisterPolymorphicBaseClass::bind(); + } + + Base * base_ptr; + }; + + //! Casts a derived class to its virtual base class in a way that allows cereal to track inheritance + /*! This should be used in cases when a derived type features virtual inheritance from some + base type. This allows cereal to track the inheritance and to avoid making duplicate copies + during serialization. + + It is safe to use virtual_base_class in all circumstances for serializing base classes, even in cases + where virtual inheritance does not take place, though it may be slightly faster to utilize + cereal::base_class<> if you do not need to worry about virtual inheritance. + + This also automatically registers polymorphic relation between the base and derived class, assuming they + are indeed polymorphic. Note this is not the same as polymorphic type registration. For more information + see the documentation on polymorphism. If using a polymorphic class, be sure to include support for + polymorphism (cereal/types/polymorphic.hpp). + + \sa base_class + + @code{.cpp} + struct MyBase + { + int x; + + template + void serialize( Archive & ar ) + { + ar( x ); + } + }; + + struct MyLeft : virtual MyBase //<-- Note the virtual inheritance + { + int y; + + template + void serialize( Archive & ar ) + { + ar( cereal::virtual_base_class( this ) ); + ar( y ); + } + }; + + struct MyRight : virtual MyBase + { + int z; + + template + void serialize( Archive & ar ) + { + ar( cereal::virtual_base_clas( this ) ); + ar( z ); + } + }; + + // diamond virtual inheritance; contains one copy of each base class + struct MyDerived : virtual MyLeft, virtual MyRight + { + int a; + + template + void serialize( Archive & ar ) + { + ar( cereal::virtual_base_class( this ) ); // safely serialize data members in MyLeft + ar( cereal::virtual_base_class( this ) ); // safely serialize data members in MyRight + ar( a ); + + // Because we used virtual_base_class, cereal will ensure that only one instance of MyBase is + // serialized as we traverse the inheritance heirarchy. This means that there will be one copy + // each of the variables x, y, z, and a + + // If we had chosen to use static_cast<> instead, cereal would perform no tracking and + // assume that every base class should be serialized (in this case leading to a duplicate + // serialization of MyBase due to diamond inheritance + }; + } + @endcode */ + template + struct virtual_base_class : private traits::detail::BaseCastBase + { + template + virtual_base_class(Derived const * derived) : + base_ptr(const_cast(static_cast(derived))) + { + static_assert( std::is_base_of::value, "Can only use virtual_base_class on a valid base class" ); + base_class_detail::RegisterPolymorphicBaseClass::bind(); + } + + Base * base_ptr; + }; + +} // namespace cereal + +#endif // CEREAL_TYPES_BASE_CLASS_HPP_ diff --git a/third_party/cereal/types/bitset.hpp b/third_party/cereal/types/bitset.hpp new file mode 100755 index 0000000..6b63817 --- /dev/null +++ b/third_party/cereal/types/bitset.hpp @@ -0,0 +1,176 @@ +/*! \file bitset.hpp + \brief Support for types found in \ + \ingroup STLSupport */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_BITSET_HPP_ +#define CEREAL_TYPES_BITSET_HPP_ + +#include "../cereal.hpp" +#include "../types/string.hpp" +#include + +namespace cereal +{ + namespace bitset_detail + { + //! The type the bitset is encoded with + /*! @internal */ + enum class type : uint8_t + { + ulong, + ullong, + string, + bits + }; + } + + //! Serializing (save) for std::bitset when BinaryData optimization supported + template , Archive>::value> + = traits::sfinae> inline + void CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::bitset const & bits ) + { + ar( CEREAL_NVP_("type", bitset_detail::type::bits) ); + + // Serialize 8 bit chunks + std::uint8_t chunk = 0; + std::uint8_t mask = 0x80; + + // Set each chunk using a rotating mask for the current bit + for( std::size_t i = 0; i < N; ++i ) + { + if( bits[i] ) + chunk |= mask; + + mask = static_cast(mask >> 1); + + // output current chunk when mask is empty (8 bits) + if( mask == 0 ) + { + ar( chunk ); + chunk = 0; + mask = 0x80; + } + } + + // serialize remainder, if it exists + if( mask != 0x80 ) + ar( chunk ); + } + + //! Serializing (save) for std::bitset when BinaryData is not supported + template , Archive>::value> + = traits::sfinae> inline + void CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::bitset const & bits ) + { + try + { + auto const b = bits.to_ulong(); + ar( CEREAL_NVP_("type", bitset_detail::type::ulong) ); + ar( CEREAL_NVP_("data", b) ); + } + catch( std::overflow_error const & ) + { + try + { + auto const b = bits.to_ullong(); + ar( CEREAL_NVP_("type", bitset_detail::type::ullong) ); + ar( CEREAL_NVP_("data", b) ); + } + catch( std::overflow_error const & ) + { + ar( CEREAL_NVP_("type", bitset_detail::type::string) ); + ar( CEREAL_NVP_("data", bits.to_string()) ); + } + } + } + + //! Serializing (load) for std::bitset + template inline + void CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::bitset & bits ) + { + bitset_detail::type t; + ar( CEREAL_NVP_("type", t) ); + + switch( t ) + { + case bitset_detail::type::ulong: + { + unsigned long b; + ar( CEREAL_NVP_("data", b) ); + bits = std::bitset( b ); + break; + } + case bitset_detail::type::ullong: + { + unsigned long long b; + ar( CEREAL_NVP_("data", b) ); + bits = std::bitset( b ); + break; + } + case bitset_detail::type::string: + { + std::string b; + ar( CEREAL_NVP_("data", b) ); + bits = std::bitset( b ); + break; + } + case bitset_detail::type::bits: + { + // Normally we would use BinaryData to route this at compile time, + // but doing this at runtime doesn't break any old serialization + std::uint8_t chunk = 0; + std::uint8_t mask = 0; + + bits.reset(); + + // Load one chunk at a time, rotating through the chunk + // to set bits in the bitset + for( std::size_t i = 0; i < N; ++i ) + { + if( mask == 0 ) + { + ar( chunk ); + mask = 0x80; + } + + if( chunk & mask ) + bits[i] = 1; + + mask = static_cast(mask >> 1); + } + break; + } + default: + throw Exception("Invalid bitset data representation"); + } + } +} // namespace cereal + +#endif // CEREAL_TYPES_BITSET_HPP_ diff --git a/third_party/cereal/types/boost_variant.hpp b/third_party/cereal/types/boost_variant.hpp new file mode 100755 index 0000000..6bbcca1 --- /dev/null +++ b/third_party/cereal/types/boost_variant.hpp @@ -0,0 +1,164 @@ +/*! \file boost_variant.hpp + \brief Support for boost::variant + \ingroup OtherTypes */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_BOOST_VARIANT_HPP_ +#define CEREAL_TYPES_BOOST_VARIANT_HPP_ + +//! @internal +#if defined(_MSC_VER) && _MSC_VER < 1911 +#define CEREAL_CONSTEXPR_LAMBDA +#else // MSVC 2017 or newer, all other compilers +#define CEREAL_CONSTEXPR_LAMBDA constexpr +#endif + +#include "../cereal.hpp" +#include +#include + +namespace cereal +{ + namespace boost_variant_detail + { + //! @internal + template + struct variant_save_visitor : boost::static_visitor<> + { + variant_save_visitor(Archive & ar_) : ar(ar_) {} + + template + void operator()(T const & value) const + { + ar( CEREAL_NVP_("data", value) ); + } + + Archive & ar; + }; + + //! @internal + template + struct LoadAndConstructLoadWrapper + { + using ST = typename std::aligned_storage::type; + + LoadAndConstructLoadWrapper() : + construct( reinterpret_cast( &st ) ) + { } + + ~LoadAndConstructLoadWrapper() + { + if (construct.itsValid) + { + construct->~T(); + } + } + + void CEREAL_SERIALIZE_FUNCTION_NAME( Archive & ar ) + { + ::cereal::detail::Construct::load_andor_construct( ar, construct ); + } + + ST st; + ::cereal::construct construct; + }; + + //! @internal + template struct load_variant_wrapper; + + //! Avoid serializing variant void_ type + /*! @internal */ + template <> + struct load_variant_wrapper + { + template + static void load_variant( Archive &, Variant & ) + { } + }; + + //! @internal + template + struct load_variant_wrapper + { + // default constructible + template + static void load_variant_impl( Archive & ar, Variant & variant, std::true_type ) + { + T value; + ar( CEREAL_NVP_("data", value) ); + variant = std::move(value); + } + + // not default constructible + template + static void load_variant_impl(Archive & ar, Variant & variant, std::false_type ) + { + LoadAndConstructLoadWrapper loadWrapper; + + ar( CEREAL_NVP_("data", loadWrapper) ); + variant = std::move(*loadWrapper.construct.ptr()); + } + + //! @internal + template + static void load_variant(Archive & ar, Variant & variant) + { + load_variant_impl( ar, variant, typename std::is_default_constructible::type() ); + } + }; + } // namespace boost_variant_detail + + //! Saving for boost::variant + template inline + void CEREAL_SAVE_FUNCTION_NAME( Archive & ar, boost::variant const & variant ) + { + int32_t which = variant.which(); + ar( CEREAL_NVP_("which", which) ); + boost_variant_detail::variant_save_visitor visitor(ar); + variant.apply_visitor(visitor); + } + + //! Loading for boost::variant + template inline + void CEREAL_LOAD_FUNCTION_NAME( Archive & ar, boost::variant & variant ) + { + int32_t which; + ar( CEREAL_NVP_("which", which) ); + + using LoadFuncType = void(*)(Archive &, boost::variant &); + CEREAL_CONSTEXPR_LAMBDA LoadFuncType loadFuncArray[] = {&boost_variant_detail::load_variant_wrapper::load_variant...}; + + if(which >= int32_t(sizeof(loadFuncArray)/sizeof(loadFuncArray[0]))) + throw Exception("Invalid 'which' selector when deserializing boost::variant"); + + loadFuncArray[which](ar, variant); + } +} // namespace cereal + +#undef CEREAL_CONSTEXPR_LAMBDA + +#endif // CEREAL_TYPES_BOOST_VARIANT_HPP_ diff --git a/third_party/cereal/types/chrono.hpp b/third_party/cereal/types/chrono.hpp new file mode 100755 index 0000000..da1ef42 --- /dev/null +++ b/third_party/cereal/types/chrono.hpp @@ -0,0 +1,72 @@ +/*! \file chrono.hpp + \brief Support for types found in \ + \ingroup STLSupport */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_CHRONO_HPP_ +#define CEREAL_TYPES_CHRONO_HPP_ + +#include + +namespace cereal +{ + //! Saving std::chrono::duration + template inline + void CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::chrono::duration const & dur ) + { + ar( CEREAL_NVP_("count", dur.count()) ); + } + + //! Loading std::chrono::duration + template inline + void CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::chrono::duration & dur ) + { + R count; + ar( CEREAL_NVP_("count", count) ); + + dur = std::chrono::duration{count}; + } + + //! Saving std::chrono::time_point + template inline + void CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::chrono::time_point const & dur ) + { + ar( CEREAL_NVP_("time_since_epoch", dur.time_since_epoch()) ); + } + + //! Loading std::chrono::time_point + template inline + void CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::chrono::time_point & dur ) + { + D elapsed; + ar( CEREAL_NVP_("time_since_epoch", elapsed) ); + + dur = std::chrono::time_point{elapsed}; + } +} // namespace cereal + +#endif // CEREAL_TYPES_CHRONO_HPP_ diff --git a/third_party/cereal/types/common.hpp b/third_party/cereal/types/common.hpp new file mode 100755 index 0000000..fc194bd --- /dev/null +++ b/third_party/cereal/types/common.hpp @@ -0,0 +1,129 @@ +/*! \file common.hpp + \brief Support common types - always included automatically + \ingroup OtherTypes */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_COMMON_HPP_ +#define CEREAL_TYPES_COMMON_HPP_ + +#include "../cereal.hpp" + +namespace cereal +{ + namespace common_detail + { + //! Serialization for arrays if BinaryData is supported and we are arithmetic + /*! @internal */ + template inline + void serializeArray( Archive & ar, T & array, std::true_type /* binary_supported */ ) + { + ar( binary_data( array, sizeof(array) ) ); + } + + //! Serialization for arrays if BinaryData is not supported or we are not arithmetic + /*! @internal */ + template inline + void serializeArray( Archive & ar, T & array, std::false_type /* binary_supported */ ) + { + for( auto & i : array ) + ar( i ); + } + + namespace + { + //! Gets the underlying type of an enum + /*! @internal */ + template + struct enum_underlying_type : std::false_type {}; + + //! Gets the underlying type of an enum + /*! Specialization for when we actually have an enum + @internal */ + template + struct enum_underlying_type { using type = typename std::underlying_type::type; }; + } // anon namespace + + //! Checks if a type is an enum + /*! This is needed over simply calling std::is_enum because the type + traits checking at compile time will attempt to call something like + load_minimal with a special NoConvertRef struct that wraps up the true type. + + This will strip away any of that and also expose the true underlying type. + @internal */ + template + class is_enum + { + private: + using DecayedT = typename std::decay::type; + using StrippedT = typename ::cereal::traits::strip_minimal::type; + + public: + static const bool value = std::is_enum::value; + using type = StrippedT; + using base_type = typename enum_underlying_type::type; + }; + } + + //! Saving for enum types + template inline + typename std::enable_if::value, + typename common_detail::is_enum::base_type>::type + CEREAL_SAVE_MINIMAL_FUNCTION_NAME( Archive const &, T const & t ) + { + return static_cast::base_type>(t); + } + + //! Loading for enum types + template inline + typename std::enable_if::value, void>::type + CEREAL_LOAD_MINIMAL_FUNCTION_NAME( Archive const &, T && t, + typename common_detail::is_enum::base_type const & value ) + { + t = reinterpret_cast::type const &>( value ); + } + + //! Serialization for raw pointers + /*! This exists only to throw a static_assert to let users know we don't support raw pointers. */ + template inline + void CEREAL_SERIALIZE_FUNCTION_NAME( Archive &, T * & ) + { + static_assert(cereal::traits::detail::delay_static_assert::value, + "Cereal does not support serializing raw pointers - please use a smart pointer"); + } + + //! Serialization for C style arrays + template inline + typename std::enable_if::value, void>::type + CEREAL_SERIALIZE_FUNCTION_NAME(Archive & ar, T & array) + { + common_detail::serializeArray( ar, array, + std::integral_constant, Archive>::value && + std::is_arithmetic::type>::value>() ); + } +} // namespace cereal + +#endif // CEREAL_TYPES_COMMON_HPP_ diff --git a/third_party/cereal/types/complex.hpp b/third_party/cereal/types/complex.hpp new file mode 100755 index 0000000..b9b8691 --- /dev/null +++ b/third_party/cereal/types/complex.hpp @@ -0,0 +1,56 @@ +/*! \file complex.hpp + \brief Support for types found in \ + \ingroup STLSupport */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_COMPLEX_HPP_ +#define CEREAL_TYPES_COMPLEX_HPP_ + +#include + +namespace cereal +{ + //! Serializing (save) for std::complex + template inline + void CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::complex const & comp ) + { + ar( CEREAL_NVP_("real", comp.real()), + CEREAL_NVP_("imag", comp.imag()) ); + } + + //! Serializing (load) for std::complex + template inline + void CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::complex & bits ) + { + T real, imag; + ar( CEREAL_NVP_("real", real), + CEREAL_NVP_("imag", imag) ); + bits = {real, imag}; + } +} // namespace cereal + +#endif // CEREAL_TYPES_COMPLEX_HPP_ diff --git a/third_party/cereal/types/concepts/pair_associative_container.hpp b/third_party/cereal/types/concepts/pair_associative_container.hpp new file mode 100755 index 0000000..95356cf --- /dev/null +++ b/third_party/cereal/types/concepts/pair_associative_container.hpp @@ -0,0 +1,73 @@ +/*! \file pair_associative_container.hpp + \brief Support for the PairAssociativeContainer refinement of the + AssociativeContainer concept. + \ingroup TypeConcepts */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_CONCEPTS_PAIR_ASSOCIATIVE_CONTAINER_HPP_ +#define CEREAL_CONCEPTS_PAIR_ASSOCIATIVE_CONTAINER_HPP_ + +#include "../../cereal.hpp" + +namespace cereal +{ + //! Saving for std-like pair associative containers + template class Map, typename... Args, typename = typename Map::mapped_type> inline + void CEREAL_SAVE_FUNCTION_NAME( Archive & ar, Map const & map ) + { + ar( make_size_tag( static_cast(map.size()) ) ); + + for( const auto & i : map ) + ar( make_map_item(i.first, i.second) ); + } + + //! Loading for std-like pair associative containers + template class Map, typename... Args, typename = typename Map::mapped_type> inline + void CEREAL_LOAD_FUNCTION_NAME( Archive & ar, Map & map ) + { + size_type size; + ar( make_size_tag( size ) ); + + map.clear(); + + auto hint = map.begin(); + for( size_t i = 0; i < size; ++i ) + { + typename Map::key_type key; + typename Map::mapped_type value; + + ar( make_map_item(key, value) ); + #ifdef CEREAL_OLDER_GCC + hint = map.insert( hint, std::make_pair(std::move(key), std::move(value)) ); + #else // NOT CEREAL_OLDER_GCC + hint = map.emplace_hint( hint, std::move( key ), std::move( value ) ); + #endif // NOT CEREAL_OLDER_GCC + } + } +} // namespace cereal + +#endif // CEREAL_CONCEPTS_PAIR_ASSOCIATIVE_CONTAINER_HPP_ diff --git a/third_party/cereal/types/deque.hpp b/third_party/cereal/types/deque.hpp new file mode 100755 index 0000000..74ce3a7 --- /dev/null +++ b/third_party/cereal/types/deque.hpp @@ -0,0 +1,62 @@ +/*! \file deque.hpp + \brief Support for types found in \ + \ingroup STLSupport */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_DEQUE_HPP_ +#define CEREAL_TYPES_DEQUE_HPP_ + +#include "../cereal.hpp" +#include + +namespace cereal +{ + //! Saving for std::deque + template inline + void CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::deque const & deque ) + { + ar( make_size_tag( static_cast(deque.size()) ) ); + + for( auto const & i : deque ) + ar( i ); + } + + //! Loading for std::deque + template inline + void CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::deque & deque ) + { + size_type size; + ar( make_size_tag( size ) ); + + deque.resize( static_cast( size ) ); + + for( auto & i : deque ) + ar( i ); + } +} // namespace cereal + +#endif // CEREAL_TYPES_DEQUE_HPP_ diff --git a/third_party/cereal/types/forward_list.hpp b/third_party/cereal/types/forward_list.hpp new file mode 100755 index 0000000..493728d --- /dev/null +++ b/third_party/cereal/types/forward_list.hpp @@ -0,0 +1,68 @@ +/*! \file forward_list.hpp + \brief Support for types found in \ + \ingroup STLSupport */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_FORWARD_LIST_HPP_ +#define CEREAL_TYPES_FORWARD_LIST_HPP_ + +#include "../cereal.hpp" +#include + +namespace cereal +{ + //! Saving for std::forward_list all other types + template inline + void CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::forward_list const & forward_list ) + { + // write the size - note that this is slow because we need to traverse + // the entire list. there are ways we could avoid this but this was chosen + // since it works in the most general fashion with any archive type + size_type const size = std::distance( forward_list.begin(), forward_list.end() ); + + ar( make_size_tag( size ) ); + + // write the list + for( const auto & i : forward_list ) + ar( i ); + } + + //! Loading for std::forward_list all other types from + template + void CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::forward_list & forward_list ) + { + size_type size; + ar( make_size_tag( size ) ); + + forward_list.resize( static_cast( size ) ); + + for( auto & i : forward_list ) + ar( i ); + } +} // namespace cereal + +#endif // CEREAL_TYPES_FORWARD_LIST_HPP_ diff --git a/third_party/cereal/types/functional.hpp b/third_party/cereal/types/functional.hpp new file mode 100755 index 0000000..0b9cdf3 --- /dev/null +++ b/third_party/cereal/types/functional.hpp @@ -0,0 +1,43 @@ +/*! \file functional.hpp + \brief Support for types found in \ + \ingroup STLSupport */ +/* + Copyright (c) 2016, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_FUNCTIONAL_HPP_ +#define CEREAL_TYPES_FUNCTIONAL_HPP_ + +#include + +namespace cereal +{ + //! Saving for std::less + template inline + void serialize( Archive &, std::less & ) + { } +} // namespace cereal + +#endif // CEREAL_TYPES_FUNCTIONAL_HPP_ diff --git a/third_party/cereal/types/list.hpp b/third_party/cereal/types/list.hpp new file mode 100755 index 0000000..a1a9eca --- /dev/null +++ b/third_party/cereal/types/list.hpp @@ -0,0 +1,62 @@ +/*! \file list.hpp + \brief Support for types found in \ + \ingroup STLSupport */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_LIST_HPP_ +#define CEREAL_TYPES_LIST_HPP_ + +#include "../cereal.hpp" +#include + +namespace cereal +{ + //! Saving for std::list + template inline + void CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::list const & list ) + { + ar( make_size_tag( static_cast(list.size()) ) ); + + for( auto const & i : list ) + ar( i ); + } + + //! Loading for std::list + template inline + void CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::list & list ) + { + size_type size; + ar( make_size_tag( size ) ); + + list.resize( static_cast( size ) ); + + for( auto & i : list ) + ar( i ); + } +} // namespace cereal + +#endif // CEREAL_TYPES_LIST_HPP_ diff --git a/third_party/cereal/types/map.hpp b/third_party/cereal/types/map.hpp new file mode 100755 index 0000000..d2e7d26 --- /dev/null +++ b/third_party/cereal/types/map.hpp @@ -0,0 +1,36 @@ +/*! \file map.hpp + \brief Support for types found in \ + \ingroup STLSupport */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_MAP_HPP_ +#define CEREAL_TYPES_MAP_HPP_ + +#include "../types/concepts/pair_associative_container.hpp" +#include + +#endif // CEREAL_TYPES_MAP_HPP_ diff --git a/third_party/cereal/types/memory.hpp b/third_party/cereal/types/memory.hpp new file mode 100755 index 0000000..d353b00 --- /dev/null +++ b/third_party/cereal/types/memory.hpp @@ -0,0 +1,425 @@ +/*! \file memory.hpp + \brief Support for types found in \ + \ingroup STLSupport */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_SHARED_PTR_HPP_ +#define CEREAL_TYPES_SHARED_PTR_HPP_ + +#include "../cereal.hpp" +#include +#include + +namespace cereal +{ + namespace memory_detail + { + //! A wrapper class to notify cereal that it is ok to serialize the contained pointer + /*! This mechanism allows us to intercept and properly handle polymorphic pointers + @internal */ + template + struct PtrWrapper + { + PtrWrapper(T && p) : ptr(std::forward(p)) {} + T & ptr; + + PtrWrapper( PtrWrapper const & ) = default; + PtrWrapper & operator=( PtrWrapper const & ) = delete; + }; + + //! Make a PtrWrapper + /*! @internal */ + template inline + PtrWrapper make_ptr_wrapper(T && t) + { + return {std::forward(t)}; + } + + //! A struct that acts as a wrapper around calling load_andor_construct + /*! The purpose of this is to allow a load_and_construct call to properly enter into the + 'data' NVP of the ptr_wrapper + @internal */ + template + struct LoadAndConstructLoadWrapper + { + LoadAndConstructLoadWrapper( T * ptr ) : + construct( ptr ) + { } + + //! Constructor for embedding an early call for restoring shared_from_this + template + LoadAndConstructLoadWrapper( T * ptr, F && sharedFromThisFunc ) : + construct( ptr, sharedFromThisFunc ) + { } + + inline void CEREAL_SERIALIZE_FUNCTION_NAME( Archive & ar ) + { + ::cereal::detail::Construct::load_andor_construct( ar, construct ); + } + + ::cereal::construct construct; + }; + + //! A helper struct for saving and restoring the state of types that derive from + //! std::enable_shared_from_this + /*! This special struct is necessary because when a user uses load_and_construct, + the weak_ptr (or whatever implementation defined variant) that allows + enable_shared_from_this to function correctly will not be initialized properly. + + This internal weak_ptr can also be modified by the shared_ptr that is created + during the serialization of a polymorphic pointer, where cereal creates a + wrapper shared_ptr out of a void pointer to the real data. + + In the case of load_and_construct, this happens because it is the allocation + of shared_ptr that perform this initialization, which we let happen on a buffer + of memory (aligned_storage). This buffer is then used for placement new + later on, effectively overwriting any initialized weak_ptr with a default + initialized one, eventually leading to issues when the user calls shared_from_this. + + To get around these issues, we will store the memory for the enable_shared_from_this + portion of the class and replace it after whatever happens to modify it (e.g. the + user performing construction or the wrapper shared_ptr in saving). + + Note that this goes into undefined behavior territory, but as of the initial writing + of this, all standard library implementations of std::enable_shared_from_this are + compatible with this memory manipulation. It is entirely possible that this may someday + break or may not work with convoluted use cases. + + Example usage: + + @code{.cpp} + T * myActualPointer; + { + EnableSharedStateHelper helper( myActualPointer ); // save the state + std::shared_ptr myPtr( myActualPointer ); // modifies the internal weak_ptr + // helper restores state when it goes out of scope + } + @endcode + + When possible, this is designed to be used in an RAII fashion - it will save state on + construction and restore it on destruction. The restore can be done at an earlier time + (e.g. after construct() is called in load_and_construct) in which case the destructor will + do nothing. Performing the restore immediately following construct() allows a user to call + shared_from_this within their load_and_construct function. + + @tparam T Type pointed to by shared_ptr + @internal */ + template + class EnableSharedStateHelper + { + // typedefs for parent type and storage type + using BaseType = typename ::cereal::traits::get_shared_from_this_base::type; + using ParentType = std::enable_shared_from_this; + using StorageType = typename std::aligned_storage::type; + + public: + //! Saves the state of some type inheriting from enable_shared_from_this + /*! @param ptr The raw pointer held by the shared_ptr */ + inline EnableSharedStateHelper( T * ptr ) : + itsPtr( static_cast( ptr ) ), + itsState(), + itsRestored( false ) + { + std::memcpy( &itsState, itsPtr, sizeof(ParentType) ); + } + + //! Restores the state of the held pointer (can only be done once) + inline void restore() + { + if( !itsRestored ) + { + // void * cast needed when type has no trivial copy-assignment + std::memcpy( static_cast(itsPtr), &itsState, sizeof(ParentType) ); + itsRestored = true; + } + } + + //! Restores the state of the held pointer if not done previously + inline ~EnableSharedStateHelper() + { + restore(); + } + + private: + ParentType * itsPtr; + StorageType itsState; + bool itsRestored; + }; // end EnableSharedStateHelper + + //! Performs loading and construction for a shared pointer that is derived from + //! std::enable_shared_from_this + /*! @param ar The archive + @param ptr Raw pointer held by the shared_ptr + @internal */ + template inline + void loadAndConstructSharedPtr( Archive & ar, T * ptr, std::true_type /* has_shared_from_this */ ) + { + memory_detail::EnableSharedStateHelper state( ptr ); + memory_detail::LoadAndConstructLoadWrapper loadWrapper( ptr, [&](){ state.restore(); } ); + + // let the user perform their initialization, shared state will be restored as soon as construct() + // is called + ar( CEREAL_NVP_("data", loadWrapper) ); + } + + //! Performs loading and construction for a shared pointer that is NOT derived from + //! std::enable_shared_from_this + /*! This is the typical case, where we simply pass the load wrapper to the + archive. + + @param ar The archive + @param ptr Raw pointer held by the shared_ptr + @internal */ + template inline + void loadAndConstructSharedPtr( Archive & ar, T * ptr, std::false_type /* has_shared_from_this */ ) + { + memory_detail::LoadAndConstructLoadWrapper loadWrapper( ptr ); + ar( CEREAL_NVP_("data", loadWrapper) ); + } + } // end namespace memory_detail + + //! Saving std::shared_ptr for non polymorphic types + template inline + typename std::enable_if::value, void>::type + CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::shared_ptr const & ptr ) + { + ar( CEREAL_NVP_("ptr_wrapper", memory_detail::make_ptr_wrapper( ptr )) ); + } + + //! Loading std::shared_ptr, case when no user load and construct for non polymorphic types + template inline + typename std::enable_if::value, void>::type + CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::shared_ptr & ptr ) + { + ar( CEREAL_NVP_("ptr_wrapper", memory_detail::make_ptr_wrapper( ptr )) ); + } + + //! Saving std::weak_ptr for non polymorphic types + template inline + typename std::enable_if::value, void>::type + CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::weak_ptr const & ptr ) + { + auto const sptr = ptr.lock(); + ar( CEREAL_NVP_("ptr_wrapper", memory_detail::make_ptr_wrapper( sptr )) ); + } + + //! Loading std::weak_ptr for non polymorphic types + template inline + typename std::enable_if::value, void>::type + CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::weak_ptr & ptr ) + { + std::shared_ptr sptr; + ar( CEREAL_NVP_("ptr_wrapper", memory_detail::make_ptr_wrapper( sptr )) ); + ptr = sptr; + } + + //! Saving std::unique_ptr for non polymorphic types + template inline + typename std::enable_if::value, void>::type + CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::unique_ptr const & ptr ) + { + ar( CEREAL_NVP_("ptr_wrapper", memory_detail::make_ptr_wrapper( ptr )) ); + } + + //! Loading std::unique_ptr, case when user provides load_and_construct for non polymorphic types + template inline + typename std::enable_if::value, void>::type + CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::unique_ptr & ptr ) + { + ar( CEREAL_NVP_("ptr_wrapper", memory_detail::make_ptr_wrapper( ptr )) ); + } + + // ###################################################################### + // Pointer wrapper implementations follow below + + //! Saving std::shared_ptr (wrapper implementation) + /*! @internal */ + template inline + void CEREAL_SAVE_FUNCTION_NAME( Archive & ar, memory_detail::PtrWrapper const &> const & wrapper ) + { + auto & ptr = wrapper.ptr; + + uint32_t id = ar.registerSharedPointer( ptr ); + ar( CEREAL_NVP_("id", id) ); + + if( id & detail::msb_32bit ) + { + ar( CEREAL_NVP_("data", *ptr) ); + } + } + + //! Loading std::shared_ptr, case when user load and construct (wrapper implementation) + /*! @internal */ + template inline + typename std::enable_if::value, void>::type + CEREAL_LOAD_FUNCTION_NAME( Archive & ar, memory_detail::PtrWrapper &> & wrapper ) + { + uint32_t id; + + ar( CEREAL_NVP_("id", id) ); + + if( id & detail::msb_32bit ) + { + // Storage type for the pointer - since we can't default construct this type, + // we'll allocate it using std::aligned_storage and use a custom deleter + using ST = typename std::aligned_storage::type; + + // Valid flag - set to true once construction finishes + // This prevents us from calling the destructor on + // uninitialized data. + auto valid = std::make_shared( false ); + + // Allocate our storage, which we will treat as + // uninitialized until initialized with placement new + using NonConstT = typename std::remove_const::type; + std::shared_ptr ptr(reinterpret_cast(new ST()), + [=]( NonConstT * t ) + { + if( *valid ) + t->~T(); + + delete reinterpret_cast( t ); + } ); + + // Register the pointer + ar.registerSharedPointer( id, ptr ); + + // Perform the actual loading and allocation + memory_detail::loadAndConstructSharedPtr( ar, ptr.get(), typename ::cereal::traits::has_shared_from_this::type() ); + + // Mark pointer as valid (initialized) + *valid = true; + wrapper.ptr = std::move(ptr); + } + else + wrapper.ptr = std::static_pointer_cast(ar.getSharedPointer(id)); + } + + //! Loading std::shared_ptr, case when no user load and construct (wrapper implementation) + /*! @internal */ + template inline + typename std::enable_if::value, void>::type + CEREAL_LOAD_FUNCTION_NAME( Archive & ar, memory_detail::PtrWrapper &> & wrapper ) + { + uint32_t id; + + ar( CEREAL_NVP_("id", id) ); + + if( id & detail::msb_32bit ) + { + using NonConstT = typename std::remove_const::type; + std::shared_ptr ptr( detail::Construct::load_andor_construct() ); + ar.registerSharedPointer( id, ptr ); + ar( CEREAL_NVP_("data", *ptr) ); + wrapper.ptr = std::move(ptr); + } + else + wrapper.ptr = std::static_pointer_cast(ar.getSharedPointer(id)); + } + + //! Saving std::unique_ptr (wrapper implementation) + /*! @internal */ + template inline + void CEREAL_SAVE_FUNCTION_NAME( Archive & ar, memory_detail::PtrWrapper const &> const & wrapper ) + { + auto & ptr = wrapper.ptr; + + // unique_ptr get one byte of metadata which signifies whether they were a nullptr + // 0 == nullptr + // 1 == not null + + if( !ptr ) + ar( CEREAL_NVP_("valid", uint8_t(0)) ); + else + { + ar( CEREAL_NVP_("valid", uint8_t(1)) ); + ar( CEREAL_NVP_("data", *ptr) ); + } + } + + //! Loading std::unique_ptr, case when user provides load_and_construct (wrapper implementation) + /*! @internal */ + template inline + typename std::enable_if::value, void>::type + CEREAL_LOAD_FUNCTION_NAME( Archive & ar, memory_detail::PtrWrapper &> & wrapper ) + { + uint8_t isValid; + ar( CEREAL_NVP_("valid", isValid) ); + + auto & ptr = wrapper.ptr; + + if( isValid ) + { + using NonConstT = typename std::remove_const::type; + // Storage type for the pointer - since we can't default construct this type, + // we'll allocate it using std::aligned_storage + using ST = typename std::aligned_storage::type; + + // Allocate storage - note the ST type so that deleter is correct if + // an exception is thrown before we are initialized + std::unique_ptr stPtr( new ST() ); + + // Use wrapper to enter into "data" nvp of ptr_wrapper + memory_detail::LoadAndConstructLoadWrapper loadWrapper( reinterpret_cast( stPtr.get() ) ); + + // Initialize storage + ar( CEREAL_NVP_("data", loadWrapper) ); + + // Transfer ownership to correct unique_ptr type + ptr.reset( reinterpret_cast( stPtr.release() ) ); + } + else + ptr.reset( nullptr ); + } + + //! Loading std::unique_ptr, case when no load_and_construct (wrapper implementation) + /*! @internal */ + template inline + typename std::enable_if::value, void>::type + CEREAL_LOAD_FUNCTION_NAME( Archive & ar, memory_detail::PtrWrapper &> & wrapper ) + { + uint8_t isValid; + ar( CEREAL_NVP_("valid", isValid) ); + + if( isValid ) + { + using NonConstT = typename std::remove_const::type; + std::unique_ptr ptr( detail::Construct::load_andor_construct() ); + ar( CEREAL_NVP_( "data", *ptr ) ); + wrapper.ptr = std::move(ptr); + } + else + { + wrapper.ptr.reset( nullptr ); + } + } +} // namespace cereal + +// automatically include polymorphic support +#include "../types/polymorphic.hpp" + +#endif // CEREAL_TYPES_SHARED_PTR_HPP_ diff --git a/third_party/cereal/types/optional.hpp b/third_party/cereal/types/optional.hpp new file mode 100755 index 0000000..b925c2f --- /dev/null +++ b/third_party/cereal/types/optional.hpp @@ -0,0 +1,65 @@ +/*! \file optional.hpp + \brief Support for std::optional + \ingroup STLSupport */ +/* + Copyright (c) 2017, Juan Pedro Bolivar Puente + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_STD_OPTIONAL_ +#define CEREAL_TYPES_STD_OPTIONAL_ + +#include "../cereal.hpp" +#include + +namespace cereal { + //! Saving for std::optional + template inline + void CEREAL_SAVE_FUNCTION_NAME(Archive& ar, const std::optional& optional) + { + if(!optional) { + ar(CEREAL_NVP_("nullopt", true)); + } else { + ar(CEREAL_NVP_("nullopt", false), + CEREAL_NVP_("data", *optional)); + } + } + + //! Loading for std::optional + template inline + void CEREAL_LOAD_FUNCTION_NAME(Archive& ar, std::optional& optional) + { + bool nullopt; + ar(CEREAL_NVP_("nullopt", nullopt)); + + if (nullopt) { + optional = std::nullopt; + } else { + optional.emplace(); + ar(CEREAL_NVP_("data", *optional)); + } + } +} // namespace cereal + +#endif // CEREAL_TYPES_STD_OPTIONAL_ diff --git a/third_party/cereal/types/polymorphic.hpp b/third_party/cereal/types/polymorphic.hpp new file mode 100755 index 0000000..41a56e2 --- /dev/null +++ b/third_party/cereal/types/polymorphic.hpp @@ -0,0 +1,483 @@ +/*! \file polymorphic.hpp + \brief Support for pointers to polymorphic base classes + \ingroup OtherTypes */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_POLYMORPHIC_HPP_ +#define CEREAL_TYPES_POLYMORPHIC_HPP_ + +#include "../cereal.hpp" +#include "../types/memory.hpp" + +#include "../details/util.hpp" +#include "../details/helpers.hpp" +#include "../details/traits.hpp" +#include "../details/polymorphic_impl.hpp" + +#if defined(_MSC_VER) && _MSC_VER < 1916 +#define CEREAL_STATIC_CONSTEXPR static +#else +#define CEREAL_STATIC_CONSTEXPR static constexpr +#endif + +//! Registers a derived polymorphic type with cereal +/*! Polymorphic types must be registered before smart + pointers to them can be serialized. Note that base + classes do not need to be registered. + + Registering a type lets cereal know how to properly + serialize it when a smart pointer to a base object is + used in conjunction with a derived class. + + This assumes that all relevant archives have also + previously been registered. Registration for archives + is usually done in the header file in which they are + defined. This means that type registration needs to + happen after specific archives to be used are included. + + It is recommended that type registration be done in + the header file in which the type is declared. + + Registration can also be placed in a source file, + but this may require the use of the + CEREAL_REGISTER_DYNAMIC_INIT macro (see below). + + Registration may be called repeatedly for the same + type in different translation units to add support + for additional archives if they are not initially + available (included and registered). + + When building serialization support as a DLL on + Windows, registration must happen in the header file. + On Linux and Mac things should still work properly + if placed in a source file, but see the above comments + on registering in source files. + + Polymorphic support in cereal requires RTTI to be + enabled */ +#define CEREAL_REGISTER_TYPE(...) \ + namespace cereal { \ + namespace detail { \ + template <> \ + struct binding_name<__VA_ARGS__> \ + { \ + CEREAL_STATIC_CONSTEXPR char const * name() { return #__VA_ARGS__; } \ + }; \ + } } /* end namespaces */ \ + CEREAL_BIND_TO_ARCHIVES(__VA_ARGS__) + +//! Registers a polymorphic type with cereal, giving it a +//! user defined name +/*! In some cases the default name used with + CEREAL_REGISTER_TYPE (the name of the type) may not be + suitable. This macro allows any name to be associated + with the type. The name should be unique */ +#define CEREAL_REGISTER_TYPE_WITH_NAME(T, Name) \ + namespace cereal { \ + namespace detail { \ + template <> \ + struct binding_name \ + { CEREAL_STATIC_CONSTEXPR char const * name() { return Name; } }; \ + } } /* end namespaces */ \ + CEREAL_BIND_TO_ARCHIVES(T) + +//! Registers the base-derived relationship for a polymorphic type +/*! When polymorphic serialization occurs, cereal needs to know how to + properly cast between derived and base types for the polymorphic + type. Normally this happens automatically whenever cereal::base_class + or cereal::virtual_base_class are used to serialize a base class. In + cases where neither of these is ever called but a base class still + exists, this explicit registration is required. + + The Derived class should be the most derived type that will be serialized, + and the Base type any possible base that has not been covered under a base + class serialization that will be used to store a Derived pointer. + + Placement of this is the same as for CEREAL_REGISTER_TYPE. */ +#define CEREAL_REGISTER_POLYMORPHIC_RELATION(Base, Derived) \ + namespace cereal { \ + namespace detail { \ + template <> \ + struct PolymorphicRelation \ + { static void bind() { RegisterPolymorphicCaster::bind(); } }; \ + } } /* end namespaces */ + +//! Adds a way to force initialization of a translation unit containing +//! calls to CEREAL_REGISTER_TYPE +/*! In C++, dynamic initialization of non-local variables of a translation + unit may be deferred until "the first odr-use of any function or variable + defined in the same translation unit as the variable to be initialized." + + Informally, odr-use means that your program takes the address of or binds + a reference directly to an object, which must have a definition. + + Since polymorphic type support in cereal relies on the dynamic + initialization of certain global objects happening before + serialization is performed, it is important to ensure that something + from files that call CEREAL_REGISTER_TYPE is odr-used before serialization + occurs, otherwise the registration will never take place. This may often + be the case when serialization is built as a shared library external from + your main program. + + This macro, with any name of your choosing, should be placed into the + source file that contains calls to CEREAL_REGISTER_TYPE. + + Its counterpart, CEREAL_FORCE_DYNAMIC_INIT, should be placed in its + associated header file such that it is included in the translation units + (source files) in which you want the registration to appear. + + @relates CEREAL_FORCE_DYNAMIC_INIT + */ +#define CEREAL_REGISTER_DYNAMIC_INIT(LibName) \ + namespace cereal { \ + namespace detail { \ + void CEREAL_DLL_EXPORT dynamic_init_dummy_##LibName() {} \ + } } /* end namespaces */ + +//! Forces dynamic initialization of polymorphic support in a +//! previously registered source file +/*! @sa CEREAL_REGISTER_DYNAMIC_INIT + + See CEREAL_REGISTER_DYNAMIC_INIT for detailed explanation + of how this macro should be used. The name used should + match that for CEREAL_REGISTER_DYNAMIC_INIT. */ +#define CEREAL_FORCE_DYNAMIC_INIT(LibName) \ + namespace cereal { \ + namespace detail { \ + void CEREAL_DLL_EXPORT dynamic_init_dummy_##LibName(); \ + } /* end detail */ \ + } /* end cereal */ \ + namespace { \ + struct dynamic_init_##LibName { \ + dynamic_init_##LibName() { \ + ::cereal::detail::dynamic_init_dummy_##LibName(); \ + } \ + } dynamic_init_instance_##LibName; \ + } /* end anonymous namespace */ + +namespace cereal +{ + namespace polymorphic_detail + { + //! Error message used for unregistered polymorphic types + /*! @internal */ + #define UNREGISTERED_POLYMORPHIC_EXCEPTION(LoadSave, Name) \ + throw cereal::Exception("Trying to " #LoadSave " an unregistered polymorphic type (" + Name + ").\n" \ + "Make sure your type is registered with CEREAL_REGISTER_TYPE and that the archive " \ + "you are using was included (and registered with CEREAL_REGISTER_ARCHIVE) prior to calling CEREAL_REGISTER_TYPE.\n" \ + "If your type is already registered and you still see this error, you may need to use CEREAL_REGISTER_DYNAMIC_INIT."); + + //! Get an input binding from the given archive by deserializing the type meta data + /*! @internal */ + template inline + typename ::cereal::detail::InputBindingMap::Serializers getInputBinding(Archive & ar, std::uint32_t const nameid) + { + // If the nameid is zero, we serialized a null pointer + if(nameid == 0) + { + typename ::cereal::detail::InputBindingMap::Serializers emptySerializers; + emptySerializers.shared_ptr = [](void*, std::shared_ptr & ptr, std::type_info const &) { ptr.reset(); }; + emptySerializers.unique_ptr = [](void*, std::unique_ptr> & ptr, std::type_info const &) { ptr.reset( nullptr ); }; + return emptySerializers; + } + + std::string name; + if(nameid & detail::msb_32bit) + { + ar( CEREAL_NVP_("polymorphic_name", name) ); + ar.registerPolymorphicName(nameid, name); + } + else + name = ar.getPolymorphicName(nameid); + + auto const & bindingMap = detail::StaticObject>::getInstance().map; + + auto binding = bindingMap.find(name); + if(binding == bindingMap.end()) + UNREGISTERED_POLYMORPHIC_EXCEPTION(load, name) + return binding->second; + } + + //! Serialize a shared_ptr if the 2nd msb in the nameid is set, and if we can actually construct the pointee + /*! This check lets us try and skip doing polymorphic machinery if we can get away with + using the derived class serialize function + + Note that on MSVC 2013 preview, is_default_constructible returns true for abstract classes with + default constructors, but on clang/gcc this will return false. So we also need to check for that here. + @internal */ + template inline + typename std::enable_if<(traits::is_default_constructible::value + || traits::has_load_and_construct::value) + && !std::is_abstract::value, bool>::type + serialize_wrapper(Archive & ar, std::shared_ptr & ptr, std::uint32_t const nameid) + { + if(nameid & detail::msb2_32bit) + { + ar( CEREAL_NVP_("ptr_wrapper", memory_detail::make_ptr_wrapper(ptr)) ); + return true; + } + return false; + } + + //! Serialize a unique_ptr if the 2nd msb in the nameid is set, and if we can actually construct the pointee + /*! This check lets us try and skip doing polymorphic machinery if we can get away with + using the derived class serialize function + @internal */ + template inline + typename std::enable_if<(traits::is_default_constructible::value + || traits::has_load_and_construct::value) + && !std::is_abstract::value, bool>::type + serialize_wrapper(Archive & ar, std::unique_ptr & ptr, std::uint32_t const nameid) + { + if(nameid & detail::msb2_32bit) + { + ar( CEREAL_NVP_("ptr_wrapper", memory_detail::make_ptr_wrapper(ptr)) ); + return true; + } + return false; + } + + //! Serialize a shared_ptr if the 2nd msb in the nameid is set, and if we can actually construct the pointee + /*! This case is for when we can't actually construct the shared pointer. Normally this would be caught + as the pointer itself is serialized, but since this is a polymorphic pointer, if we tried to serialize + the pointer we'd end up back here recursively. So we have to catch the error here as well, if + this was a polymorphic type serialized by its proper pointer type + @internal */ + template inline + typename std::enable_if<(!traits::is_default_constructible::value + && !traits::has_load_and_construct::value) + || std::is_abstract::value, bool>::type + serialize_wrapper(Archive &, std::shared_ptr &, std::uint32_t const nameid) + { + if(nameid & detail::msb2_32bit) + throw cereal::Exception("Cannot load a polymorphic type that is not default constructable and does not have a load_and_construct function"); + return false; + } + + //! Serialize a unique_ptr if the 2nd msb in the nameid is set, and if we can actually construct the pointee + /*! This case is for when we can't actually construct the unique pointer. Normally this would be caught + as the pointer itself is serialized, but since this is a polymorphic pointer, if we tried to serialize + the pointer we'd end up back here recursively. So we have to catch the error here as well, if + this was a polymorphic type serialized by its proper pointer type + @internal */ + template inline + typename std::enable_if<(!traits::is_default_constructible::value + && !traits::has_load_and_construct::value) + || std::is_abstract::value, bool>::type + serialize_wrapper(Archive &, std::unique_ptr &, std::uint32_t const nameid) + { + if(nameid & detail::msb2_32bit) + throw cereal::Exception("Cannot load a polymorphic type that is not default constructable and does not have a load_and_construct function"); + return false; + } + } // polymorphic_detail + + // ###################################################################### + // Pointer serialization for polymorphic types + + //! Saving std::shared_ptr for polymorphic types, abstract + template inline + typename std::enable_if::value && std::is_abstract::value, void>::type + CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::shared_ptr const & ptr ) + { + if(!ptr) + { + // same behavior as nullptr in memory implementation + ar( CEREAL_NVP_("polymorphic_id", std::uint32_t(0)) ); + return; + } + + std::type_info const & ptrinfo = typeid(*ptr.get()); + static std::type_info const & tinfo = typeid(T); + // ptrinfo can never be equal to T info since we can't have an instance + // of an abstract object + // this implies we need to do the lookup + + auto const & bindingMap = detail::StaticObject>::getInstance().map; + + auto binding = bindingMap.find(std::type_index(ptrinfo)); + if(binding == bindingMap.end()) + UNREGISTERED_POLYMORPHIC_EXCEPTION(save, cereal::util::demangle(ptrinfo.name())) + + binding->second.shared_ptr(&ar, ptr.get(), tinfo); + } + + //! Saving std::shared_ptr for polymorphic types, not abstract + template inline + typename std::enable_if::value && !std::is_abstract::value, void>::type + CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::shared_ptr const & ptr ) + { + if(!ptr) + { + // same behavior as nullptr in memory implementation + ar( CEREAL_NVP_("polymorphic_id", std::uint32_t(0)) ); + return; + } + + std::type_info const & ptrinfo = typeid(*ptr.get()); + static std::type_info const & tinfo = typeid(T); + + if(ptrinfo == tinfo) + { + // The 2nd msb signals that the following pointer does not need to be + // cast with our polymorphic machinery + ar( CEREAL_NVP_("polymorphic_id", detail::msb2_32bit) ); + + ar( CEREAL_NVP_("ptr_wrapper", memory_detail::make_ptr_wrapper(ptr)) ); + + return; + } + + auto const & bindingMap = detail::StaticObject>::getInstance().map; + + auto binding = bindingMap.find(std::type_index(ptrinfo)); + if(binding == bindingMap.end()) + UNREGISTERED_POLYMORPHIC_EXCEPTION(save, cereal::util::demangle(ptrinfo.name())) + + binding->second.shared_ptr(&ar, ptr.get(), tinfo); + } + + //! Loading std::shared_ptr for polymorphic types + template inline + typename std::enable_if::value, void>::type + CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::shared_ptr & ptr ) + { + std::uint32_t nameid; + ar( CEREAL_NVP_("polymorphic_id", nameid) ); + + // Check to see if we can skip all of this polymorphism business + if(polymorphic_detail::serialize_wrapper(ar, ptr, nameid)) + return; + + auto binding = polymorphic_detail::getInputBinding(ar, nameid); + std::shared_ptr result; + binding.shared_ptr(&ar, result, typeid(T)); + ptr = std::static_pointer_cast(result); + } + + //! Saving std::weak_ptr for polymorphic types + template inline + typename std::enable_if::value, void>::type + CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::weak_ptr const & ptr ) + { + auto const sptr = ptr.lock(); + ar( CEREAL_NVP_("locked_ptr", sptr) ); + } + + //! Loading std::weak_ptr for polymorphic types + template inline + typename std::enable_if::value, void>::type + CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::weak_ptr & ptr ) + { + std::shared_ptr sptr; + ar( CEREAL_NVP_("locked_ptr", sptr) ); + ptr = sptr; + } + + //! Saving std::unique_ptr for polymorphic types that are abstract + template inline + typename std::enable_if::value && std::is_abstract::value, void>::type + CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::unique_ptr const & ptr ) + { + if(!ptr) + { + // same behavior as nullptr in memory implementation + ar( CEREAL_NVP_("polymorphic_id", std::uint32_t(0)) ); + return; + } + + std::type_info const & ptrinfo = typeid(*ptr.get()); + static std::type_info const & tinfo = typeid(T); + // ptrinfo can never be equal to T info since we can't have an instance + // of an abstract object + // this implies we need to do the lookup + + auto const & bindingMap = detail::StaticObject>::getInstance().map; + + auto binding = bindingMap.find(std::type_index(ptrinfo)); + if(binding == bindingMap.end()) + UNREGISTERED_POLYMORPHIC_EXCEPTION(save, cereal::util::demangle(ptrinfo.name())) + + binding->second.unique_ptr(&ar, ptr.get(), tinfo); + } + + //! Saving std::unique_ptr for polymorphic types, not abstract + template inline + typename std::enable_if::value && !std::is_abstract::value, void>::type + CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::unique_ptr const & ptr ) + { + if(!ptr) + { + // same behavior as nullptr in memory implementation + ar( CEREAL_NVP_("polymorphic_id", std::uint32_t(0)) ); + return; + } + + std::type_info const & ptrinfo = typeid(*ptr.get()); + static std::type_info const & tinfo = typeid(T); + + if(ptrinfo == tinfo) + { + // The 2nd msb signals that the following pointer does not need to be + // cast with our polymorphic machinery + ar( CEREAL_NVP_("polymorphic_id", detail::msb2_32bit) ); + + ar( CEREAL_NVP_("ptr_wrapper", memory_detail::make_ptr_wrapper(ptr)) ); + + return; + } + + auto const & bindingMap = detail::StaticObject>::getInstance().map; + + auto binding = bindingMap.find(std::type_index(ptrinfo)); + if(binding == bindingMap.end()) + UNREGISTERED_POLYMORPHIC_EXCEPTION(save, cereal::util::demangle(ptrinfo.name())) + + binding->second.unique_ptr(&ar, ptr.get(), tinfo); + } + + //! Loading std::unique_ptr, case when user provides load_and_construct for polymorphic types + template inline + typename std::enable_if::value, void>::type + CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::unique_ptr & ptr ) + { + std::uint32_t nameid; + ar( CEREAL_NVP_("polymorphic_id", nameid) ); + + // Check to see if we can skip all of this polymorphism business + if(polymorphic_detail::serialize_wrapper(ar, ptr, nameid)) + return; + + auto binding = polymorphic_detail::getInputBinding(ar, nameid); + std::unique_ptr> result; + binding.unique_ptr(&ar, result, typeid(T)); + ptr.reset(static_cast(result.release())); + } + + #undef UNREGISTERED_POLYMORPHIC_EXCEPTION +} // namespace cereal +#endif // CEREAL_TYPES_POLYMORPHIC_HPP_ diff --git a/third_party/cereal/types/queue.hpp b/third_party/cereal/types/queue.hpp new file mode 100755 index 0000000..edf76f5 --- /dev/null +++ b/third_party/cereal/types/queue.hpp @@ -0,0 +1,132 @@ +/*! \file queue.hpp + \brief Support for types found in \ + \ingroup STLSupport */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_QUEUE_HPP_ +#define CEREAL_TYPES_QUEUE_HPP_ + +#include "../details/helpers.hpp" +#include + +// The default container for queue is deque, so let's include that too +#include "../types/deque.hpp" +// The default comparator for queue is less +#include "../types/functional.hpp" + +namespace cereal +{ + namespace queue_detail + { + //! Allows access to the protected container in queue + /*! @internal */ + template inline + C const & container( std::queue const & queue ) + { + struct H : public std::queue + { + static C const & get( std::queue const & q ) + { + return q.*(&H::c); + } + }; + + return H::get( queue ); + } + + //! Allows access to the protected container in priority queue + /*! @internal */ + template inline + C const & container( std::priority_queue const & priority_queue ) + { + struct H : public std::priority_queue + { + static C const & get( std::priority_queue const & pq ) + { + return pq.*(&H::c); + } + }; + + return H::get( priority_queue ); + } + + //! Allows access to the protected comparator in priority queue + /*! @internal */ + template inline + Comp const & comparator( std::priority_queue const & priority_queue ) + { + struct H : public std::priority_queue + { + static Comp const & get( std::priority_queue const & pq ) + { + return pq.*(&H::comp); + } + }; + + return H::get( priority_queue ); + } + } + + //! Saving for std::queue + template inline + void CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::queue const & queue ) + { + ar( CEREAL_NVP_("container", queue_detail::container( queue )) ); + } + + //! Loading for std::queue + template inline + void CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::queue & queue ) + { + C container; + ar( CEREAL_NVP_("container", container) ); + queue = std::queue( std::move( container ) ); + } + + //! Saving for std::priority_queue + template inline + void CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::priority_queue const & priority_queue ) + { + ar( CEREAL_NVP_("comparator", queue_detail::comparator( priority_queue )) ); + ar( CEREAL_NVP_("container", queue_detail::container( priority_queue )) ); + } + + //! Loading for std::priority_queue + template inline + void CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::priority_queue & priority_queue ) + { + Comp comparator; + ar( CEREAL_NVP_("comparator", comparator) ); + + C container; + ar( CEREAL_NVP_("container", container) ); + + priority_queue = std::priority_queue( comparator, std::move( container ) ); + } +} // namespace cereal + +#endif // CEREAL_TYPES_QUEUE_HPP_ diff --git a/third_party/cereal/types/set.hpp b/third_party/cereal/types/set.hpp new file mode 100755 index 0000000..2709e51 --- /dev/null +++ b/third_party/cereal/types/set.hpp @@ -0,0 +1,103 @@ +/*! \file set.hpp + \brief Support for types found in \ + \ingroup STLSupport */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_SET_HPP_ +#define CEREAL_TYPES_SET_HPP_ + +#include "../cereal.hpp" +#include + +namespace cereal +{ + namespace set_detail + { + //! @internal + template inline + void save( Archive & ar, SetT const & set ) + { + ar( make_size_tag( static_cast(set.size()) ) ); + + for( const auto & i : set ) + ar( i ); + } + + //! @internal + template inline + void load( Archive & ar, SetT & set ) + { + size_type size; + ar( make_size_tag( size ) ); + + set.clear(); + + auto hint = set.begin(); + for( size_type i = 0; i < size; ++i ) + { + typename SetT::key_type key; + + ar( key ); + #ifdef CEREAL_OLDER_GCC + hint = set.insert( hint, std::move( key ) ); + #else // NOT CEREAL_OLDER_GCC + hint = set.emplace_hint( hint, std::move( key ) ); + #endif // NOT CEREAL_OLDER_GCC + } + } + } + + //! Saving for std::set + template inline + void CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::set const & set ) + { + set_detail::save( ar, set ); + } + + //! Loading for std::set + template inline + void CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::set & set ) + { + set_detail::load( ar, set ); + } + + //! Saving for std::multiset + template inline + void CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::multiset const & multiset ) + { + set_detail::save( ar, multiset ); + } + + //! Loading for std::multiset + template inline + void CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::multiset & multiset ) + { + set_detail::load( ar, multiset ); + } +} // namespace cereal + +#endif // CEREAL_TYPES_SET_HPP_ diff --git a/third_party/cereal/types/stack.hpp b/third_party/cereal/types/stack.hpp new file mode 100755 index 0000000..06844f0 --- /dev/null +++ b/third_party/cereal/types/stack.hpp @@ -0,0 +1,76 @@ +/*! \file stack.hpp + \brief Support for types found in \ + \ingroup STLSupport */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_STACK_HPP_ +#define CEREAL_TYPES_STACK_HPP_ + +#include "../cereal.hpp" +#include + +// The default container for stack is deque, so let's include that too +#include "../types/deque.hpp" + +namespace cereal +{ + namespace stack_detail + { + //! Allows access to the protected container in stack + template inline + C const & container( std::stack const & stack ) + { + struct H : public std::stack + { + static C const & get( std::stack const & s ) + { + return s.*(&H::c); + } + }; + + return H::get( stack ); + } + } + + //! Saving for std::stack + template inline + void CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::stack const & stack ) + { + ar( CEREAL_NVP_("container", stack_detail::container( stack )) ); + } + + //! Loading for std::stack + template inline + void CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::stack & stack ) + { + C container; + ar( CEREAL_NVP_("container", container) ); + stack = std::stack( std::move( container ) ); + } +} // namespace cereal + +#endif // CEREAL_TYPES_STACK_HPP_ diff --git a/third_party/cereal/types/string.hpp b/third_party/cereal/types/string.hpp new file mode 100755 index 0000000..0f9b337 --- /dev/null +++ b/third_party/cereal/types/string.hpp @@ -0,0 +1,61 @@ +/*! \file string.hpp + \brief Support for types found in \ + \ingroup STLSupport */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_STRING_HPP_ +#define CEREAL_TYPES_STRING_HPP_ + +#include "../cereal.hpp" +#include + +namespace cereal +{ + //! Serialization for basic_string types, if binary data is supported + template inline + typename std::enable_if, Archive>::value, void>::type + CEREAL_SAVE_FUNCTION_NAME(Archive & ar, std::basic_string const & str) + { + // Save number of chars + the data + ar( make_size_tag( static_cast(str.size()) ) ); + ar( binary_data( str.data(), str.size() * sizeof(CharT) ) ); + } + + //! Serialization for basic_string types, if binary data is supported + template inline + typename std::enable_if, Archive>::value, void>::type + CEREAL_LOAD_FUNCTION_NAME(Archive & ar, std::basic_string & str) + { + size_type size; + ar( make_size_tag( size ) ); + str.resize(static_cast(size)); + ar( binary_data( const_cast( str.data() ), static_cast(size) * sizeof(CharT) ) ); + } +} // namespace cereal + +#endif // CEREAL_TYPES_STRING_HPP_ + diff --git a/third_party/cereal/types/tuple.hpp b/third_party/cereal/types/tuple.hpp new file mode 100755 index 0000000..e040cae --- /dev/null +++ b/third_party/cereal/types/tuple.hpp @@ -0,0 +1,123 @@ +/*! \file tuple.hpp + \brief Support for types found in \ + \ingroup STLSupport */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_TUPLE_HPP_ +#define CEREAL_TYPES_TUPLE_HPP_ + +#include "../cereal.hpp" +#include + +namespace cereal +{ + namespace tuple_detail + { + //! Creates a c string from a sequence of characters + /*! The c string created will always be prefixed by "tuple_element" + Based on code from: http://stackoverflow/a/20973438/710791 + @internal */ + template + struct char_seq_to_c_str + { + static const int size = 14;// Size of array for the word: tuple_element + typedef const char (&arr_type)[sizeof...(Cs) + size]; + static const char str[sizeof...(Cs) + size]; + }; + + // the word tuple_element plus a number + //! @internal + template + const char char_seq_to_c_str::str[sizeof...(Cs) + size] = + {'t','u','p','l','e','_','e','l','e','m','e','n','t', Cs..., '\0'}; + + //! Converts a number into a sequence of characters + /*! @tparam Q The quotient of dividing the original number by 10 + @tparam R The remainder of dividing the original number by 10 + @tparam C The sequence built so far + @internal */ + template + struct to_string_impl + { + using type = typename to_string_impl(R+std::size_t{'0'}), C...>::type; + }; + + //! Base case with no quotient + /*! @internal */ + template + struct to_string_impl<0, R, C...> + { + using type = char_seq_to_c_str(R+std::size_t{'0'}), C...>; + }; + + //! Generates a c string for a given index of a tuple + /*! Example use: + @code{cpp} + tuple_element_name<3>::c_str();// returns "tuple_element3" + @endcode + @internal */ + template + struct tuple_element_name + { + using type = typename to_string_impl::type; + static const typename type::arr_type c_str(){ return type::str; } + }; + + // unwinds a tuple to save it + //! @internal + template + struct serialize + { + template inline + static void apply( Archive & ar, std::tuple & tuple ) + { + serialize::template apply( ar, tuple ); + ar( CEREAL_NVP_(tuple_element_name::c_str(), + std::get( tuple )) ); + } + }; + + // Zero height specialization - nothing to do here + //! @internal + template <> + struct serialize<0> + { + template inline + static void apply( Archive &, std::tuple & ) + { } + }; + } + + //! Serializing for std::tuple + template inline + void CEREAL_SERIALIZE_FUNCTION_NAME( Archive & ar, std::tuple & tuple ) + { + tuple_detail::serialize>::value>::template apply( ar, tuple ); + } +} // namespace cereal + +#endif // CEREAL_TYPES_TUPLE_HPP_ diff --git a/third_party/cereal/types/unordered_map.hpp b/third_party/cereal/types/unordered_map.hpp new file mode 100755 index 0000000..8581436 --- /dev/null +++ b/third_party/cereal/types/unordered_map.hpp @@ -0,0 +1,36 @@ +/*! \file unordered_map.hpp + \brief Support for types found in \ + \ingroup STLSupport */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_UNORDERED_MAP_HPP_ +#define CEREAL_TYPES_UNORDERED_MAP_HPP_ + +#include "../types/concepts/pair_associative_container.hpp" +#include + +#endif // CEREAL_TYPES_UNORDERED_MAP_HPP_ diff --git a/third_party/cereal/types/unordered_set.hpp b/third_party/cereal/types/unordered_set.hpp new file mode 100755 index 0000000..9d199fe --- /dev/null +++ b/third_party/cereal/types/unordered_set.hpp @@ -0,0 +1,99 @@ +/*! \file unordered_set.hpp + \brief Support for types found in \ + \ingroup STLSupport */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_UNORDERED_SET_HPP_ +#define CEREAL_TYPES_UNORDERED_SET_HPP_ + +#include "../cereal.hpp" +#include + +namespace cereal +{ + namespace unordered_set_detail + { + //! @internal + template inline + void save( Archive & ar, SetT const & set ) + { + ar( make_size_tag( static_cast(set.size()) ) ); + + for( const auto & i : set ) + ar( i ); + } + + //! @internal + template inline + void load( Archive & ar, SetT & set ) + { + size_type size; + ar( make_size_tag( size ) ); + + set.clear(); + set.reserve( static_cast( size ) ); + + for( size_type i = 0; i < size; ++i ) + { + typename SetT::key_type key; + + ar( key ); + set.emplace( std::move( key ) ); + } + } + } + + //! Saving for std::unordered_set + template inline + void CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::unordered_set const & unordered_set ) + { + unordered_set_detail::save( ar, unordered_set ); + } + + //! Loading for std::unordered_set + template inline + void CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::unordered_set & unordered_set ) + { + unordered_set_detail::load( ar, unordered_set ); + } + + //! Saving for std::unordered_multiset + template inline + void CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::unordered_multiset const & unordered_multiset ) + { + unordered_set_detail::save( ar, unordered_multiset ); + } + + //! Loading for std::unordered_multiset + template inline + void CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::unordered_multiset & unordered_multiset ) + { + unordered_set_detail::load( ar, unordered_multiset ); + } +} // namespace cereal + +#endif // CEREAL_TYPES_UNORDERED_SET_HPP_ diff --git a/third_party/cereal/types/utility.hpp b/third_party/cereal/types/utility.hpp new file mode 100755 index 0000000..e56eb83 --- /dev/null +++ b/third_party/cereal/types/utility.hpp @@ -0,0 +1,47 @@ +/*! \file utility.hpp + \brief Support for types found in \ + \ingroup STLSupport */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_UTILITY_HPP_ +#define CEREAL_TYPES_UTILITY_HPP_ + +#include "../cereal.hpp" +#include + +namespace cereal +{ + //! Serializing for std::pair + template inline + void CEREAL_SERIALIZE_FUNCTION_NAME( Archive & ar, std::pair & pair ) + { + ar( CEREAL_NVP_("first", pair.first), + CEREAL_NVP_("second", pair.second) ); + } +} // namespace cereal + +#endif // CEREAL_TYPES_UTILITY_HPP_ diff --git a/third_party/cereal/types/valarray.hpp b/third_party/cereal/types/valarray.hpp new file mode 100755 index 0000000..b17a186 --- /dev/null +++ b/third_party/cereal/types/valarray.hpp @@ -0,0 +1,89 @@ +/*! \file valarray.hpp +\brief Support for types found in \ +\ingroup STLSupport */ + +/* +Copyright (c) 2014, Randolph Voorhies, Shane Grant +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +* Neither the name of cereal nor the +names of its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef CEREAL_TYPES_VALARRAY_HPP_ +#define CEREAL_TYPES_VALARRAY_HPP_ + +#include "../cereal.hpp" +#include + +namespace cereal +{ + //! Saving for std::valarray arithmetic types, using binary serialization, if supported + template inline + typename std::enable_if, Archive>::value + && std::is_arithmetic::value, void>::type + CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::valarray const & valarray ) + { + ar( make_size_tag( static_cast(valarray.size()) ) ); // number of elements + ar( binary_data( &valarray[0], valarray.size() * sizeof(T) ) ); // &valarray[0] ok since guaranteed contiguous + } + + //! Loading for std::valarray arithmetic types, using binary serialization, if supported + template inline + typename std::enable_if, Archive>::value + && std::is_arithmetic::value, void>::type + CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::valarray & valarray ) + { + size_type valarraySize; + ar( make_size_tag( valarraySize ) ); + + valarray.resize( static_cast( valarraySize ) ); + ar( binary_data( &valarray[0], static_cast( valarraySize ) * sizeof(T) ) ); + } + + //! Saving for std::valarray all other types + template inline + typename std::enable_if, Archive>::value + || !std::is_arithmetic::value, void>::type + CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::valarray const & valarray ) + { + ar( make_size_tag( static_cast(valarray.size()) ) ); // number of elements + for(auto && v : valarray) + ar(v); + } + + //! Loading for std::valarray all other types + template inline + typename std::enable_if, Archive>::value + || !std::is_arithmetic::value, void>::type + CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::valarray & valarray ) + { + size_type valarraySize; + ar( make_size_tag( valarraySize ) ); + + valarray.resize( static_cast( valarraySize ) ); + for(auto && v : valarray) + ar(v); + } +} // namespace cereal + +#endif // CEREAL_TYPES_VALARRAY_HPP_ diff --git a/third_party/cereal/types/variant.hpp b/third_party/cereal/types/variant.hpp new file mode 100755 index 0000000..a17a301 --- /dev/null +++ b/third_party/cereal/types/variant.hpp @@ -0,0 +1,108 @@ +/*! \file variant.hpp + \brief Support for std::variant + \ingroup STLSupport */ +/* + Copyright (c) 2014, 2017, Randolph Voorhies, Shane Grant, Juan Pedro + Bolivar Puente. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_STD_VARIANT_HPP_ +#define CEREAL_TYPES_STD_VARIANT_HPP_ + +#include "../cereal.hpp" +#include +#include + +namespace cereal +{ + namespace variant_detail + { + //! @internal + template + struct variant_save_visitor + { + variant_save_visitor(Archive & ar_) : ar(ar_) {} + + template + void operator()(T const & value) const + { + ar( CEREAL_NVP_("data", value) ); + } + + Archive & ar; + }; + + //! @internal + template + typename std::enable_if, void>::type + load_variant(Archive & /*ar*/, int /*target*/, Variant & /*variant*/) + { + throw ::cereal::Exception("Error traversing variant during load"); + } + //! @internal + template + typename std::enable_if, void>::type + load_variant(Archive & ar, int target, Variant & variant) + { + if(N == target) + { + variant.template emplace(); + ar( CEREAL_NVP_("data", std::get(variant)) ); + } + else + load_variant(ar, target, variant); + } + + } // namespace variant_detail + + //! Saving for std::variant + template inline + void CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::variant const & variant ) + { + std::int32_t index = static_cast(variant.index()); + ar( CEREAL_NVP_("index", index) ); + variant_detail::variant_save_visitor visitor(ar); + std::visit(visitor, variant); + } + + //! Loading for std::variant + template inline + void CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::variant & variant ) + { + using variant_t = typename std::variant; + + std::int32_t index; + ar( CEREAL_NVP_("index", index) ); + if(index >= static_cast(std::variant_size_v)) + throw Exception("Invalid 'index' selector when deserializing std::variant"); + + variant_detail::load_variant<0>(ar, index, variant); + } + + //! Serializing a std::monostate + template + void CEREAL_SERIALIZE_FUNCTION_NAME( Archive &, std::monostate const & ) {} +} // namespace cereal + +#endif // CEREAL_TYPES_STD_VARIANT_HPP_ diff --git a/third_party/cereal/types/vector.hpp b/third_party/cereal/types/vector.hpp new file mode 100755 index 0000000..a45f3d9 --- /dev/null +++ b/third_party/cereal/types/vector.hpp @@ -0,0 +1,112 @@ +/*! \file vector.hpp + \brief Support for types found in \ + \ingroup STLSupport */ +/* + Copyright (c) 2014, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_TYPES_VECTOR_HPP_ +#define CEREAL_TYPES_VECTOR_HPP_ + +#include "../cereal.hpp" +#include + +namespace cereal +{ + //! Serialization for std::vectors of arithmetic (but not bool) using binary serialization, if supported + template inline + typename std::enable_if, Archive>::value + && std::is_arithmetic::value && !std::is_same::value, void>::type + CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::vector const & vector ) + { + ar( make_size_tag( static_cast(vector.size()) ) ); // number of elements + ar( binary_data( vector.data(), vector.size() * sizeof(T) ) ); + } + + //! Serialization for std::vectors of arithmetic (but not bool) using binary serialization, if supported + template inline + typename std::enable_if, Archive>::value + && std::is_arithmetic::value && !std::is_same::value, void>::type + CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::vector & vector ) + { + size_type vectorSize; + ar( make_size_tag( vectorSize ) ); + + vector.resize( static_cast( vectorSize ) ); + ar( binary_data( vector.data(), static_cast( vectorSize ) * sizeof(T) ) ); + } + + //! Serialization for non-arithmetic vector types + template inline + typename std::enable_if<(!traits::is_output_serializable, Archive>::value + || !std::is_arithmetic::value) && !std::is_same::value, void>::type + CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::vector const & vector ) + { + ar( make_size_tag( static_cast(vector.size()) ) ); // number of elements + for(auto && v : vector) + ar( v ); + } + + //! Serialization for non-arithmetic vector types + template inline + typename std::enable_if<(!traits::is_input_serializable, Archive>::value + || !std::is_arithmetic::value) && !std::is_same::value, void>::type + CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::vector & vector ) + { + size_type size; + ar( make_size_tag( size ) ); + + vector.resize( static_cast( size ) ); + for(auto && v : vector) + ar( v ); + } + + //! Serialization for bool vector types + template inline + void CEREAL_SAVE_FUNCTION_NAME( Archive & ar, std::vector const & vector ) + { + ar( make_size_tag( static_cast(vector.size()) ) ); // number of elements + for(const auto v : vector) + ar( static_cast(v) ); + } + + //! Serialization for bool vector types + template inline + void CEREAL_LOAD_FUNCTION_NAME( Archive & ar, std::vector & vector ) + { + size_type size; + ar( make_size_tag( size ) ); + + vector.resize( static_cast( size ) ); + for(auto && v : vector) + { + bool b; + ar( b ); + v = b; + } + } +} // namespace cereal + +#endif // CEREAL_TYPES_VECTOR_HPP_ diff --git a/third_party/cereal/version.hpp b/third_party/cereal/version.hpp new file mode 100755 index 0000000..0ad7d41 --- /dev/null +++ b/third_party/cereal/version.hpp @@ -0,0 +1,52 @@ +/*! \file version.hpp + \brief Macros to detect cereal version + + These macros can assist in determining the version of cereal. Be + warned that cereal is not guaranteed to be compatible across + different versions. For more information on releases of cereal, + see https://github.com/USCiLab/cereal/releases. + + \ingroup utility */ +/* + Copyright (c) 2018, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef CEREAL_VERSION_HPP_ +#define CEREAL_VERSION_HPP_ + +//! The major version +#define CEREAL_VERSION_MAJOR 1 +//! The minor version +#define CEREAL_VERSION_MINOR 3 +//! The patch version +#define CEREAL_VERSION_PATCH 2 + +//! The full version as a single number +#define CEREAL_VERSION (CEREAL_VERSION_MAJOR * 10000 \ + + CEREAL_VERSION_MINOR * 100 \ + + CEREAL_VERSION_PATCH) + +#endif // CEREAL_VERSION_HPP_ diff --git a/third_party/cpp_base64/CMakeLists.txt b/third_party/cpp_base64/CMakeLists.txt new file mode 100644 index 0000000..2a9cf82 --- /dev/null +++ b/third_party/cpp_base64/CMakeLists.txt @@ -0,0 +1,17 @@ +cmake_minimum_required(VERSION 3.10) +project(base64_test VERSION 1.0) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_BUILD_TYPE Debug) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fPIC -w -fdiagnostics-color=always -pthread") + +# OpenCV required +find_package(OpenCV REQUIRED) +message(STATUS "OpenCV library status:") +message(STATUS " version: ${OpenCV_VERSION}") +message(STATUS " libraries: ${OpenCV_LIBS}") +message(STATUS " include path: ${OpenCV_INCLUDE_DIRS}") +include_directories(${OpenCV_INCLUDE_DIRS}) + +add_executable(base64_test "base64_test.cpp") +target_link_libraries(base64_test ${OpenCV_LIBS}) diff --git a/third_party/cpp_base64/README.md b/third_party/cpp_base64/README.md new file mode 100644 index 0000000..99e508d --- /dev/null +++ b/third_party/cpp_base64/README.md @@ -0,0 +1,14 @@ +base64 library from github: https://github.com/ReneNyffenegger/cpp-base64, merged .cpp and .h files together. + +**NOTE** + +used for encoding image to base64 string when interact with mLLM in VidepPipe. + +test file `base64_test.cpp`, which convert cv::Mat to base64 string and convert back to cv::Mat +``` +mkdir build +cd build +cmake .. +make -j8 +./base64_test +``` \ No newline at end of file diff --git a/third_party/cpp_base64/base64.h b/third_party/cpp_base64/base64.h new file mode 100644 index 0000000..9f0e92e --- /dev/null +++ b/third_party/cpp_base64/base64.h @@ -0,0 +1,317 @@ +// +// base64 encoding and decoding with C++. +// Version: 2.rc.09 (release candidate) +// + +#ifndef BASE64_H_C0CE2A47_D10E_42C9_A27C_C883944E704A +#define BASE64_H_C0CE2A47_D10E_42C9_A27C_C883944E704A + +#include + +#if __cplusplus >= 201703L +#include +#endif // __cplusplus >= 201703L + +std::string base64_encode (std::string const& s, bool url = false); +std::string base64_encode_pem (std::string const& s); +std::string base64_encode_mime(std::string const& s); + +std::string base64_decode(std::string const& s, bool remove_linebreaks = false); +std::string base64_encode(unsigned char const*, size_t len, bool url = false); + +#if __cplusplus >= 201703L +// +// Interface with std::string_view rather than const std::string& +// Requires C++17 +// Provided by Yannic Bonenberger (https://github.com/Yannic) +// +std::string base64_encode (std::string_view s, bool url = false); +std::string base64_encode_pem (std::string_view s); +std::string base64_encode_mime(std::string_view s); + +std::string base64_decode(std::string_view s, bool remove_linebreaks = false); +#endif // __cplusplus >= 201703L + + +/* + base64.cpp and base64.h + + base64 encoding and decoding with C++. + More information at + https://renenyffenegger.ch/notes/development/Base64/Encoding-and-decoding-base-64-with-cpp + + Version: 2.rc.09 (release candidate) + + Copyright (C) 2004-2017, 2020-2022 René Nyffenegger + + This source code is provided 'as-is', without any express or implied + warranty. In no event will the author be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this source code must not be misrepresented; you must not + claim that you wrote the original source code. If you use this source code + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original source code. + + 3. This notice may not be removed or altered from any source distribution. + + René Nyffenegger rene.nyffenegger@adp-gmbh.ch + +*/ + +#include +#include + + // + // Depending on the url parameter in base64_chars, one of + // two sets of base64 characters needs to be chosen. + // They differ in their last two characters. + // +static const char* base64_chars[2] = { + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789" + "+/", + + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789" + "-_"}; + +static unsigned int pos_of_char(const unsigned char chr) { + // + // Return the position of chr within base64_encode() + // + + if (chr >= 'A' && chr <= 'Z') return chr - 'A'; + else if (chr >= 'a' && chr <= 'z') return chr - 'a' + ('Z' - 'A') + 1; + else if (chr >= '0' && chr <= '9') return chr - '0' + ('Z' - 'A') + ('z' - 'a') + 2; + else if (chr == '+' || chr == '-') return 62; // Be liberal with input and accept both url ('-') and non-url ('+') base 64 characters ( + else if (chr == '/' || chr == '_') return 63; // Ditto for '/' and '_' + else + // + // 2020-10-23: Throw std::exception rather than const char* + //(Pablo Martin-Gomez, https://github.com/Bouska) + // + throw std::runtime_error("Input is not valid base64-encoded data."); +} + +static std::string insert_linebreaks(std::string str, size_t distance) { + // + // Provided by https://github.com/JomaCorpFX, adapted by me. + // + if (!str.length()) { + return ""; + } + + size_t pos = distance; + + while (pos < str.size()) { + str.insert(pos, "\n"); + pos += distance + 1; + } + + return str; +} + +template +static std::string encode_with_line_breaks(String s) { + return insert_linebreaks(base64_encode(s, false), line_length); +} + +template +static std::string encode_pem(String s) { + return encode_with_line_breaks(s); +} + +template +static std::string encode_mime(String s) { + return encode_with_line_breaks(s); +} + +template +static std::string encode(String s, bool url) { + return base64_encode(reinterpret_cast(s.data()), s.length(), url); +} + +std::string base64_encode(unsigned char const* bytes_to_encode, size_t in_len, bool url) { + + size_t len_encoded = (in_len +2) / 3 * 4; + + unsigned char trailing_char = url ? '.' : '='; + + // + // Choose set of base64 characters. They differ + // for the last two positions, depending on the url + // parameter. + // A bool (as is the parameter url) is guaranteed + // to evaluate to either 0 or 1 in C++ therefore, + // the correct character set is chosen by subscripting + // base64_chars with url. + // + const char* base64_chars_ = base64_chars[url]; + + std::string ret; + ret.reserve(len_encoded); + + unsigned int pos = 0; + + while (pos < in_len) { + ret.push_back(base64_chars_[(bytes_to_encode[pos + 0] & 0xfc) >> 2]); + + if (pos+1 < in_len) { + ret.push_back(base64_chars_[((bytes_to_encode[pos + 0] & 0x03) << 4) + ((bytes_to_encode[pos + 1] & 0xf0) >> 4)]); + + if (pos+2 < in_len) { + ret.push_back(base64_chars_[((bytes_to_encode[pos + 1] & 0x0f) << 2) + ((bytes_to_encode[pos + 2] & 0xc0) >> 6)]); + ret.push_back(base64_chars_[ bytes_to_encode[pos + 2] & 0x3f]); + } + else { + ret.push_back(base64_chars_[(bytes_to_encode[pos + 1] & 0x0f) << 2]); + ret.push_back(trailing_char); + } + } + else { + + ret.push_back(base64_chars_[(bytes_to_encode[pos + 0] & 0x03) << 4]); + ret.push_back(trailing_char); + ret.push_back(trailing_char); + } + + pos += 3; + } + + + return ret; +} + +template +static std::string decode(String const& encoded_string, bool remove_linebreaks) { + // + // decode(…) is templated so that it can be used with String = const std::string& + // or std::string_view (requires at least C++17) + // + + if (encoded_string.empty()) return std::string(); + + if (remove_linebreaks) { + + std::string copy(encoded_string); + + copy.erase(std::remove(copy.begin(), copy.end(), '\n'), copy.end()); + + return base64_decode(copy, false); + } + + size_t length_of_string = encoded_string.length(); + size_t pos = 0; + + // + // The approximate length (bytes) of the decoded string might be one or + // two bytes smaller, depending on the amount of trailing equal signs + // in the encoded string. This approximation is needed to reserve + // enough space in the string to be returned. + // + size_t approx_length_of_decoded_string = length_of_string / 4 * 3; + std::string ret; + ret.reserve(approx_length_of_decoded_string); + + while (pos < length_of_string) { + // + // Iterate over encoded input string in chunks. The size of all + // chunks except the last one is 4 bytes. + // + // The last chunk might be padded with equal signs or dots + // in order to make it 4 bytes in size as well, but this + // is not required as per RFC 2045. + // + // All chunks except the last one produce three output bytes. + // + // The last chunk produces at least one and up to three bytes. + // + + size_t pos_of_char_1 = pos_of_char(encoded_string.at(pos+1) ); + + // + // Emit the first output byte that is produced in each chunk: + // + ret.push_back(static_cast( ( (pos_of_char(encoded_string.at(pos+0)) ) << 2 ) + ( (pos_of_char_1 & 0x30 ) >> 4))); + + if ( ( pos + 2 < length_of_string ) && // Check for data that is not padded with equal signs (which is allowed by RFC 2045) + encoded_string.at(pos+2) != '=' && + encoded_string.at(pos+2) != '.' // accept URL-safe base 64 strings, too, so check for '.' also. + ) + { + // + // Emit a chunk's second byte (which might not be produced in the last chunk). + // + unsigned int pos_of_char_2 = pos_of_char(encoded_string.at(pos+2) ); + ret.push_back(static_cast( (( pos_of_char_1 & 0x0f) << 4) + (( pos_of_char_2 & 0x3c) >> 2))); + + if ( ( pos + 3 < length_of_string ) && + encoded_string.at(pos+3) != '=' && + encoded_string.at(pos+3) != '.' + ) + { + // + // Emit a chunk's third byte (which might not be produced in the last chunk). + // + ret.push_back(static_cast( ( (pos_of_char_2 & 0x03 ) << 6 ) + pos_of_char(encoded_string.at(pos+3)) )); + } + } + + pos += 4; + } + + return ret; +} + +std::string base64_decode(std::string const& s, bool remove_linebreaks) { + return decode(s, remove_linebreaks); +} + +std::string base64_encode(std::string const& s, bool url) { + return encode(s, url); +} + +std::string base64_encode_pem (std::string const& s) { + return encode_pem(s); +} + +std::string base64_encode_mime(std::string const& s) { + return encode_mime(s); +} + +#if __cplusplus >= 201703L +// +// Interface with std::string_view rather than const std::string& +// Requires C++17 +// Provided by Yannic Bonenberger (https://github.com/Yannic) +// + +std::string base64_encode(std::string_view s, bool url) { + return encode(s, url); +} + +std::string base64_encode_pem(std::string_view s) { + return encode_pem(s); +} + +std::string base64_encode_mime(std::string_view s) { + return encode_mime(s); +} + +std::string base64_decode(std::string_view s, bool remove_linebreaks) { + return decode(s, remove_linebreaks); +} + +#endif // __cplusplus >= 201703L + +#endif /* BASE64_H_C0CE2A47_D10E_42C9_A27C_C883944E704A */ \ No newline at end of file diff --git a/third_party/cpp_base64/base64_test.cpp b/third_party/cpp_base64/base64_test.cpp new file mode 100644 index 0000000..9261b12 --- /dev/null +++ b/third_party/cpp_base64/base64_test.cpp @@ -0,0 +1,42 @@ +#include +#include +#include +#include "base64.h" + +/* + * convert base64 string to cv::Mat +*/ +cv::Mat base64ToMat(const std::string& base64_data) { + std::string decoded_data = base64_decode(base64_data); + std::vector data(decoded_data.begin(), decoded_data.end()); + cv::Mat img = cv::imdecode(data, cv::IMREAD_UNCHANGED); + return img; +} + +/* + * convert cv::Mat to base64 string +*/ +std::string matToBase64(const cv::Mat& img, const std::string& ext = ".jpg") { + std::vector buf; + cv::imencode(ext, img, buf); + std::string encoded = base64_encode(buf.data(), buf.size()); + return encoded; +} + +/* + * read image into mat, convert it to base64 string and convert back to mat + */ +int main() { + auto ori_mat = cv::imread("/windows2/zhzhi/github/vp_data/test_images/vehicle/0.jpg"); + auto base64_str = matToBase64(ori_mat); + auto decoded_mat = base64ToMat(base64_str); + + std::cout << "based64_str: " << base64_str << std::endl; + std::cout << "based64_str's length: " << base64_str.size() << std::endl; + std::cout << "ori mat's byte size: " << ori_mat.cols * ori_mat.rows * ori_mat.elemSize() << std::endl; + + cv::imshow("ori_mat", ori_mat); + cv::imshow("decoded_mat", decoded_mat); + + cv::waitKey(0); +} \ No newline at end of file diff --git a/third_party/cpp_httplib/README.md b/third_party/cpp_httplib/README.md new file mode 100644 index 0000000..2dcb5b6 --- /dev/null +++ b/third_party/cpp_httplib/README.md @@ -0,0 +1,14 @@ +a header-only http library from github: https://github.com/yhirose/cpp-httplib + +**NOTE** + +support online LLM REST API calling based on OpenAI protocols or local deployment such as Ollama/vLLM in VideoPipe. + +``` +compile test: +g++ httplib_test.cpp -o httplib_test -O2 + +and run: +./httplib_test + +``` \ No newline at end of file diff --git a/third_party/cpp_httplib/httplib.h b/third_party/cpp_httplib/httplib.h new file mode 100644 index 0000000..c697691 --- /dev/null +++ b/third_party/cpp_httplib/httplib.h @@ -0,0 +1,11882 @@ +// +// httplib.h +// +// Copyright (c) 2025 Yuji Hirose. All rights reserved. +// MIT License +// + +#ifndef CPPHTTPLIB_HTTPLIB_H +#define CPPHTTPLIB_HTTPLIB_H + +#define CPPHTTPLIB_VERSION "0.24.0" +#define CPPHTTPLIB_VERSION_NUM "0x001800" + +/* + * Platform compatibility check + */ + +#if defined(_WIN32) && !defined(_WIN64) +#error \ + "cpp-httplib doesn't support 32-bit Windows. Please use a 64-bit compiler." +#elif defined(__SIZEOF_POINTER__) && __SIZEOF_POINTER__ < 8 +#warning \ + "cpp-httplib doesn't support 32-bit platforms. Please use a 64-bit compiler." +#elif defined(__SIZEOF_SIZE_T__) && __SIZEOF_SIZE_T__ < 8 +#warning \ + "cpp-httplib doesn't support platforms where size_t is less than 64 bits." +#endif + +#ifdef _WIN32 +#if defined(_WIN32_WINNT) && _WIN32_WINNT < 0x0A00 +#error \ + "cpp-httplib doesn't support Windows 8 or lower. Please use Windows 10 or later." +#endif +#endif + +/* + * Configuration + */ + +#ifndef CPPHTTPLIB_KEEPALIVE_TIMEOUT_SECOND +#define CPPHTTPLIB_KEEPALIVE_TIMEOUT_SECOND 5 +#endif + +#ifndef CPPHTTPLIB_KEEPALIVE_TIMEOUT_CHECK_INTERVAL_USECOND +#define CPPHTTPLIB_KEEPALIVE_TIMEOUT_CHECK_INTERVAL_USECOND 10000 +#endif + +#ifndef CPPHTTPLIB_KEEPALIVE_MAX_COUNT +#define CPPHTTPLIB_KEEPALIVE_MAX_COUNT 100 +#endif + +#ifndef CPPHTTPLIB_CONNECTION_TIMEOUT_SECOND +#define CPPHTTPLIB_CONNECTION_TIMEOUT_SECOND 300 +#endif + +#ifndef CPPHTTPLIB_CONNECTION_TIMEOUT_USECOND +#define CPPHTTPLIB_CONNECTION_TIMEOUT_USECOND 0 +#endif + +#ifndef CPPHTTPLIB_SERVER_READ_TIMEOUT_SECOND +#define CPPHTTPLIB_SERVER_READ_TIMEOUT_SECOND 5 +#endif + +#ifndef CPPHTTPLIB_SERVER_READ_TIMEOUT_USECOND +#define CPPHTTPLIB_SERVER_READ_TIMEOUT_USECOND 0 +#endif + +#ifndef CPPHTTPLIB_SERVER_WRITE_TIMEOUT_SECOND +#define CPPHTTPLIB_SERVER_WRITE_TIMEOUT_SECOND 5 +#endif + +#ifndef CPPHTTPLIB_SERVER_WRITE_TIMEOUT_USECOND +#define CPPHTTPLIB_SERVER_WRITE_TIMEOUT_USECOND 0 +#endif + +#ifndef CPPHTTPLIB_CLIENT_READ_TIMEOUT_SECOND +#define CPPHTTPLIB_CLIENT_READ_TIMEOUT_SECOND 300 +#endif + +#ifndef CPPHTTPLIB_CLIENT_READ_TIMEOUT_USECOND +#define CPPHTTPLIB_CLIENT_READ_TIMEOUT_USECOND 0 +#endif + +#ifndef CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_SECOND +#define CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_SECOND 5 +#endif + +#ifndef CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_USECOND +#define CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_USECOND 0 +#endif + +#ifndef CPPHTTPLIB_CLIENT_MAX_TIMEOUT_MSECOND +#define CPPHTTPLIB_CLIENT_MAX_TIMEOUT_MSECOND 0 +#endif + +#ifndef CPPHTTPLIB_IDLE_INTERVAL_SECOND +#define CPPHTTPLIB_IDLE_INTERVAL_SECOND 0 +#endif + +#ifndef CPPHTTPLIB_IDLE_INTERVAL_USECOND +#ifdef _WIN64 +#define CPPHTTPLIB_IDLE_INTERVAL_USECOND 1000 +#else +#define CPPHTTPLIB_IDLE_INTERVAL_USECOND 0 +#endif +#endif + +#ifndef CPPHTTPLIB_REQUEST_URI_MAX_LENGTH +#define CPPHTTPLIB_REQUEST_URI_MAX_LENGTH 8192 +#endif + +#ifndef CPPHTTPLIB_HEADER_MAX_LENGTH +#define CPPHTTPLIB_HEADER_MAX_LENGTH 8192 +#endif + +#ifndef CPPHTTPLIB_HEADER_MAX_COUNT +#define CPPHTTPLIB_HEADER_MAX_COUNT 100 +#endif + +#ifndef CPPHTTPLIB_REDIRECT_MAX_COUNT +#define CPPHTTPLIB_REDIRECT_MAX_COUNT 20 +#endif + +#ifndef CPPHTTPLIB_MULTIPART_FORM_DATA_FILE_MAX_COUNT +#define CPPHTTPLIB_MULTIPART_FORM_DATA_FILE_MAX_COUNT 1024 +#endif + +#ifndef CPPHTTPLIB_PAYLOAD_MAX_LENGTH +#define CPPHTTPLIB_PAYLOAD_MAX_LENGTH ((std::numeric_limits::max)()) +#endif + +#ifndef CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH +#define CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH 8192 +#endif + +#ifndef CPPHTTPLIB_RANGE_MAX_COUNT +#define CPPHTTPLIB_RANGE_MAX_COUNT 1024 +#endif + +#ifndef CPPHTTPLIB_TCP_NODELAY +#define CPPHTTPLIB_TCP_NODELAY false +#endif + +#ifndef CPPHTTPLIB_IPV6_V6ONLY +#define CPPHTTPLIB_IPV6_V6ONLY false +#endif + +#ifndef CPPHTTPLIB_RECV_BUFSIZ +#define CPPHTTPLIB_RECV_BUFSIZ size_t(16384u) +#endif + +#ifndef CPPHTTPLIB_SEND_BUFSIZ +#define CPPHTTPLIB_SEND_BUFSIZ size_t(16384u) +#endif + +#ifndef CPPHTTPLIB_COMPRESSION_BUFSIZ +#define CPPHTTPLIB_COMPRESSION_BUFSIZ size_t(16384u) +#endif + +#ifndef CPPHTTPLIB_THREAD_POOL_COUNT +#define CPPHTTPLIB_THREAD_POOL_COUNT \ + ((std::max)(8u, std::thread::hardware_concurrency() > 0 \ + ? std::thread::hardware_concurrency() - 1 \ + : 0)) +#endif + +#ifndef CPPHTTPLIB_RECV_FLAGS +#define CPPHTTPLIB_RECV_FLAGS 0 +#endif + +#ifndef CPPHTTPLIB_SEND_FLAGS +#define CPPHTTPLIB_SEND_FLAGS 0 +#endif + +#ifndef CPPHTTPLIB_LISTEN_BACKLOG +#define CPPHTTPLIB_LISTEN_BACKLOG 5 +#endif + +#ifndef CPPHTTPLIB_MAX_LINE_LENGTH +#define CPPHTTPLIB_MAX_LINE_LENGTH 32768 +#endif + +/* + * Headers + */ + +#ifdef _WIN64 +#ifndef _CRT_SECURE_NO_WARNINGS +#define _CRT_SECURE_NO_WARNINGS +#endif //_CRT_SECURE_NO_WARNINGS + +#ifndef _CRT_NONSTDC_NO_DEPRECATE +#define _CRT_NONSTDC_NO_DEPRECATE +#endif //_CRT_NONSTDC_NO_DEPRECATE + +#if defined(_MSC_VER) +#if _MSC_VER < 1900 +#error Sorry, Visual Studio versions prior to 2015 are not supported +#endif + +#pragma comment(lib, "ws2_32.lib") + +using ssize_t = __int64; +#endif // _MSC_VER + +#ifndef S_ISREG +#define S_ISREG(m) (((m) & S_IFREG) == S_IFREG) +#endif // S_ISREG + +#ifndef S_ISDIR +#define S_ISDIR(m) (((m) & S_IFDIR) == S_IFDIR) +#endif // S_ISDIR + +#ifndef NOMINMAX +#define NOMINMAX +#endif // NOMINMAX + +#include +#include +#include + +#if defined(__has_include) +#if __has_include() +// afunix.h uses types declared in winsock2.h, so has to be included after it. +#include +#define CPPHTTPLIB_HAVE_AFUNIX_H 1 +#endif +#endif + +#ifndef WSA_FLAG_NO_HANDLE_INHERIT +#define WSA_FLAG_NO_HANDLE_INHERIT 0x80 +#endif + +using nfds_t = unsigned long; +using socket_t = SOCKET; +using socklen_t = int; + +#else // not _WIN64 + +#include +#if !defined(_AIX) && !defined(__MVS__) +#include +#endif +#ifdef __MVS__ +#include +#ifndef NI_MAXHOST +#define NI_MAXHOST 1025 +#endif +#endif +#include +#include +#include +#ifdef __linux__ +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include + +using socket_t = int; +#ifndef INVALID_SOCKET +#define INVALID_SOCKET (-1) +#endif +#endif //_WIN64 + +#if defined(__APPLE__) +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(CPPHTTPLIB_USE_NON_BLOCKING_GETADDRINFO) || \ + defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) +#if TARGET_OS_OSX +#include +#include +#endif +#endif // CPPHTTPLIB_USE_NON_BLOCKING_GETADDRINFO or + // CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +#ifdef _WIN64 +#include + +// these are defined in wincrypt.h and it breaks compilation if BoringSSL is +// used +#undef X509_NAME +#undef X509_CERT_PAIR +#undef X509_EXTENSIONS +#undef PKCS7_SIGNER_INFO + +#ifdef _MSC_VER +#pragma comment(lib, "crypt32.lib") +#endif +#endif // _WIN64 + +#if defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) +#if TARGET_OS_OSX +#include +#endif +#endif // CPPHTTPLIB_USE_NON_BLOCKING_GETADDRINFO + +#include +#include +#include +#include + +#if defined(_WIN64) && defined(OPENSSL_USE_APPLINK) +#include +#endif + +#include +#include + +#if defined(OPENSSL_IS_BORINGSSL) || defined(LIBRESSL_VERSION_NUMBER) +#if OPENSSL_VERSION_NUMBER < 0x1010107f +#error Please use OpenSSL or a current version of BoringSSL +#endif +#define SSL_get1_peer_certificate SSL_get_peer_certificate +#elif OPENSSL_VERSION_NUMBER < 0x30000000L +#error Sorry, OpenSSL versions prior to 3.0.0 are not supported +#endif + +#endif // CPPHTTPLIB_OPENSSL_SUPPORT + +#ifdef CPPHTTPLIB_ZLIB_SUPPORT +#include +#endif + +#ifdef CPPHTTPLIB_BROTLI_SUPPORT +#include +#include +#endif + +#ifdef CPPHTTPLIB_ZSTD_SUPPORT +#include +#endif + +/* + * Declaration + */ +namespace httplib { + +namespace detail { + +/* + * Backport std::make_unique from C++14. + * + * NOTE: This code came up with the following stackoverflow post: + * https://stackoverflow.com/questions/10149840/c-arrays-and-make-unique + * + */ + +template +typename std::enable_if::value, std::unique_ptr>::type +make_unique(Args &&...args) { + return std::unique_ptr(new T(std::forward(args)...)); +} + +template +typename std::enable_if::value, std::unique_ptr>::type +make_unique(std::size_t n) { + typedef typename std::remove_extent::type RT; + return std::unique_ptr(new RT[n]); +} + +namespace case_ignore { + +inline unsigned char to_lower(int c) { + const static unsigned char table[256] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, + 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, + 122, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, + 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, + 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, + 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 224, 225, 226, + 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, + 242, 243, 244, 245, 246, 215, 248, 249, 250, 251, 252, 253, 254, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, + 255, + }; + return table[(unsigned char)(char)c]; +} + +inline bool equal(const std::string &a, const std::string &b) { + return a.size() == b.size() && + std::equal(a.begin(), a.end(), b.begin(), [](char ca, char cb) { + return to_lower(ca) == to_lower(cb); + }); +} + +struct equal_to { + bool operator()(const std::string &a, const std::string &b) const { + return equal(a, b); + } +}; + +struct hash { + size_t operator()(const std::string &key) const { + return hash_core(key.data(), key.size(), 0); + } + + size_t hash_core(const char *s, size_t l, size_t h) const { + return (l == 0) ? h + : hash_core(s + 1, l - 1, + // Unsets the 6 high bits of h, therefore no + // overflow happens + (((std::numeric_limits::max)() >> 6) & + h * 33) ^ + static_cast(to_lower(*s))); + } +}; + +template +using unordered_set = std::unordered_set; + +} // namespace case_ignore + +// This is based on +// "http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2014/n4189". + +struct scope_exit { + explicit scope_exit(std::function &&f) + : exit_function(std::move(f)), execute_on_destruction{true} {} + + scope_exit(scope_exit &&rhs) noexcept + : exit_function(std::move(rhs.exit_function)), + execute_on_destruction{rhs.execute_on_destruction} { + rhs.release(); + } + + ~scope_exit() { + if (execute_on_destruction) { this->exit_function(); } + } + + void release() { this->execute_on_destruction = false; } + +private: + scope_exit(const scope_exit &) = delete; + void operator=(const scope_exit &) = delete; + scope_exit &operator=(scope_exit &&) = delete; + + std::function exit_function; + bool execute_on_destruction; +}; + +} // namespace detail + +enum SSLVerifierResponse { + // no decision has been made, use the built-in certificate verifier + NoDecisionMade, + // connection certificate is verified and accepted + CertificateAccepted, + // connection certificate was processed but is rejected + CertificateRejected +}; + +enum StatusCode { + // Information responses + Continue_100 = 100, + SwitchingProtocol_101 = 101, + Processing_102 = 102, + EarlyHints_103 = 103, + + // Successful responses + OK_200 = 200, + Created_201 = 201, + Accepted_202 = 202, + NonAuthoritativeInformation_203 = 203, + NoContent_204 = 204, + ResetContent_205 = 205, + PartialContent_206 = 206, + MultiStatus_207 = 207, + AlreadyReported_208 = 208, + IMUsed_226 = 226, + + // Redirection messages + MultipleChoices_300 = 300, + MovedPermanently_301 = 301, + Found_302 = 302, + SeeOther_303 = 303, + NotModified_304 = 304, + UseProxy_305 = 305, + unused_306 = 306, + TemporaryRedirect_307 = 307, + PermanentRedirect_308 = 308, + + // Client error responses + BadRequest_400 = 400, + Unauthorized_401 = 401, + PaymentRequired_402 = 402, + Forbidden_403 = 403, + NotFound_404 = 404, + MethodNotAllowed_405 = 405, + NotAcceptable_406 = 406, + ProxyAuthenticationRequired_407 = 407, + RequestTimeout_408 = 408, + Conflict_409 = 409, + Gone_410 = 410, + LengthRequired_411 = 411, + PreconditionFailed_412 = 412, + PayloadTooLarge_413 = 413, + UriTooLong_414 = 414, + UnsupportedMediaType_415 = 415, + RangeNotSatisfiable_416 = 416, + ExpectationFailed_417 = 417, + ImATeapot_418 = 418, + MisdirectedRequest_421 = 421, + UnprocessableContent_422 = 422, + Locked_423 = 423, + FailedDependency_424 = 424, + TooEarly_425 = 425, + UpgradeRequired_426 = 426, + PreconditionRequired_428 = 428, + TooManyRequests_429 = 429, + RequestHeaderFieldsTooLarge_431 = 431, + UnavailableForLegalReasons_451 = 451, + + // Server error responses + InternalServerError_500 = 500, + NotImplemented_501 = 501, + BadGateway_502 = 502, + ServiceUnavailable_503 = 503, + GatewayTimeout_504 = 504, + HttpVersionNotSupported_505 = 505, + VariantAlsoNegotiates_506 = 506, + InsufficientStorage_507 = 507, + LoopDetected_508 = 508, + NotExtended_510 = 510, + NetworkAuthenticationRequired_511 = 511, +}; + +using Headers = + std::unordered_multimap; + +using Params = std::multimap; +using Match = std::smatch; + +using DownloadProgress = std::function; +using UploadProgress = std::function; + +struct Response; +using ResponseHandler = std::function; + +struct FormData { + std::string name; + std::string content; + std::string filename; + std::string content_type; + Headers headers; +}; + +struct FormField { + std::string name; + std::string content; + Headers headers; +}; +using FormFields = std::multimap; + +using FormFiles = std::multimap; + +struct MultipartFormData { + FormFields fields; // Text fields from multipart + FormFiles files; // Files from multipart + + // Text field access + std::string get_field(const std::string &key, size_t id = 0) const; + std::vector get_fields(const std::string &key) const; + bool has_field(const std::string &key) const; + size_t get_field_count(const std::string &key) const; + + // File access + FormData get_file(const std::string &key, size_t id = 0) const; + std::vector get_files(const std::string &key) const; + bool has_file(const std::string &key) const; + size_t get_file_count(const std::string &key) const; +}; + +struct UploadFormData { + std::string name; + std::string content; + std::string filename; + std::string content_type; +}; +using UploadFormDataItems = std::vector; + +class DataSink { +public: + DataSink() : os(&sb_), sb_(*this) {} + + DataSink(const DataSink &) = delete; + DataSink &operator=(const DataSink &) = delete; + DataSink(DataSink &&) = delete; + DataSink &operator=(DataSink &&) = delete; + + std::function write; + std::function is_writable; + std::function done; + std::function done_with_trailer; + std::ostream os; + +private: + class data_sink_streambuf final : public std::streambuf { + public: + explicit data_sink_streambuf(DataSink &sink) : sink_(sink) {} + + protected: + std::streamsize xsputn(const char *s, std::streamsize n) override { + sink_.write(s, static_cast(n)); + return n; + } + + private: + DataSink &sink_; + }; + + data_sink_streambuf sb_; +}; + +using ContentProvider = + std::function; + +using ContentProviderWithoutLength = + std::function; + +using ContentProviderResourceReleaser = std::function; + +struct FormDataProvider { + std::string name; + ContentProviderWithoutLength provider; + std::string filename; + std::string content_type; +}; +using FormDataProviderItems = std::vector; + +using ContentReceiverWithProgress = std::function; + +using ContentReceiver = + std::function; + +using FormDataHeader = std::function; + +class ContentReader { +public: + using Reader = std::function; + using FormDataReader = + std::function; + + ContentReader(Reader reader, FormDataReader multipart_reader) + : reader_(std::move(reader)), + formdata_reader_(std::move(multipart_reader)) {} + + bool operator()(FormDataHeader header, ContentReceiver receiver) const { + return formdata_reader_(std::move(header), std::move(receiver)); + } + + bool operator()(ContentReceiver receiver) const { + return reader_(std::move(receiver)); + } + + Reader reader_; + FormDataReader formdata_reader_; +}; + +using Range = std::pair; +using Ranges = std::vector; + +struct Request { + std::string method; + std::string path; + std::string matched_route; + Params params; + Headers headers; + Headers trailers; + std::string body; + + std::string remote_addr; + int remote_port = -1; + std::string local_addr; + int local_port = -1; + + // for server + std::string version; + std::string target; + MultipartFormData form; + Ranges ranges; + Match matches; + std::unordered_map path_params; + std::function is_connection_closed = []() { return true; }; + + // for client + std::vector accept_content_types; + ResponseHandler response_handler; + ContentReceiverWithProgress content_receiver; + DownloadProgress download_progress; + UploadProgress upload_progress; +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + const SSL *ssl = nullptr; +#endif + + bool has_header(const std::string &key) const; + std::string get_header_value(const std::string &key, const char *def = "", + size_t id = 0) const; + size_t get_header_value_u64(const std::string &key, size_t def = 0, + size_t id = 0) const; + size_t get_header_value_count(const std::string &key) const; + void set_header(const std::string &key, const std::string &val); + + bool has_trailer(const std::string &key) const; + std::string get_trailer_value(const std::string &key, size_t id = 0) const; + size_t get_trailer_value_count(const std::string &key) const; + + bool has_param(const std::string &key) const; + std::string get_param_value(const std::string &key, size_t id = 0) const; + size_t get_param_value_count(const std::string &key) const; + + bool is_multipart_form_data() const; + + // private members... + size_t redirect_count_ = CPPHTTPLIB_REDIRECT_MAX_COUNT; + size_t content_length_ = 0; + ContentProvider content_provider_; + bool is_chunked_content_provider_ = false; + size_t authorization_count_ = 0; + std::chrono::time_point start_time_ = + (std::chrono::steady_clock::time_point::min)(); +}; + +struct Response { + std::string version; + int status = -1; + std::string reason; + Headers headers; + Headers trailers; + std::string body; + std::string location; // Redirect location + + bool has_header(const std::string &key) const; + std::string get_header_value(const std::string &key, const char *def = "", + size_t id = 0) const; + size_t get_header_value_u64(const std::string &key, size_t def = 0, + size_t id = 0) const; + size_t get_header_value_count(const std::string &key) const; + void set_header(const std::string &key, const std::string &val); + + bool has_trailer(const std::string &key) const; + std::string get_trailer_value(const std::string &key, size_t id = 0) const; + size_t get_trailer_value_count(const std::string &key) const; + + void set_redirect(const std::string &url, int status = StatusCode::Found_302); + void set_content(const char *s, size_t n, const std::string &content_type); + void set_content(const std::string &s, const std::string &content_type); + void set_content(std::string &&s, const std::string &content_type); + + void set_content_provider( + size_t length, const std::string &content_type, ContentProvider provider, + ContentProviderResourceReleaser resource_releaser = nullptr); + + void set_content_provider( + const std::string &content_type, ContentProviderWithoutLength provider, + ContentProviderResourceReleaser resource_releaser = nullptr); + + void set_chunked_content_provider( + const std::string &content_type, ContentProviderWithoutLength provider, + ContentProviderResourceReleaser resource_releaser = nullptr); + + void set_file_content(const std::string &path, + const std::string &content_type); + void set_file_content(const std::string &path); + + Response() = default; + Response(const Response &) = default; + Response &operator=(const Response &) = default; + Response(Response &&) = default; + Response &operator=(Response &&) = default; + ~Response() { + if (content_provider_resource_releaser_) { + content_provider_resource_releaser_(content_provider_success_); + } + } + + // private members... + size_t content_length_ = 0; + ContentProvider content_provider_; + ContentProviderResourceReleaser content_provider_resource_releaser_; + bool is_chunked_content_provider_ = false; + bool content_provider_success_ = false; + std::string file_content_path_; + std::string file_content_content_type_; +}; + +class Stream { +public: + virtual ~Stream() = default; + + virtual bool is_readable() const = 0; + virtual bool wait_readable() const = 0; + virtual bool wait_writable() const = 0; + + virtual ssize_t read(char *ptr, size_t size) = 0; + virtual ssize_t write(const char *ptr, size_t size) = 0; + virtual void get_remote_ip_and_port(std::string &ip, int &port) const = 0; + virtual void get_local_ip_and_port(std::string &ip, int &port) const = 0; + virtual socket_t socket() const = 0; + + virtual time_t duration() const = 0; + + ssize_t write(const char *ptr); + ssize_t write(const std::string &s); +}; + +class TaskQueue { +public: + TaskQueue() = default; + virtual ~TaskQueue() = default; + + virtual bool enqueue(std::function fn) = 0; + virtual void shutdown() = 0; + + virtual void on_idle() {} +}; + +class ThreadPool final : public TaskQueue { +public: + explicit ThreadPool(size_t n, size_t mqr = 0) + : shutdown_(false), max_queued_requests_(mqr) { + while (n) { + threads_.emplace_back(worker(*this)); + n--; + } + } + + ThreadPool(const ThreadPool &) = delete; + ~ThreadPool() override = default; + + bool enqueue(std::function fn) override { + { + std::unique_lock lock(mutex_); + if (max_queued_requests_ > 0 && jobs_.size() >= max_queued_requests_) { + return false; + } + jobs_.push_back(std::move(fn)); + } + + cond_.notify_one(); + return true; + } + + void shutdown() override { + // Stop all worker threads... + { + std::unique_lock lock(mutex_); + shutdown_ = true; + } + + cond_.notify_all(); + + // Join... + for (auto &t : threads_) { + t.join(); + } + } + +private: + struct worker { + explicit worker(ThreadPool &pool) : pool_(pool) {} + + void operator()() { + for (;;) { + std::function fn; + { + std::unique_lock lock(pool_.mutex_); + + pool_.cond_.wait( + lock, [&] { return !pool_.jobs_.empty() || pool_.shutdown_; }); + + if (pool_.shutdown_ && pool_.jobs_.empty()) { break; } + + fn = pool_.jobs_.front(); + pool_.jobs_.pop_front(); + } + + assert(true == static_cast(fn)); + fn(); + } + +#if defined(CPPHTTPLIB_OPENSSL_SUPPORT) && !defined(OPENSSL_IS_BORINGSSL) && \ + !defined(LIBRESSL_VERSION_NUMBER) + OPENSSL_thread_stop(); +#endif + } + + ThreadPool &pool_; + }; + friend struct worker; + + std::vector threads_; + std::list> jobs_; + + bool shutdown_; + size_t max_queued_requests_ = 0; + + std::condition_variable cond_; + std::mutex mutex_; +}; + +using Logger = std::function; + +// Forward declaration for Error type +enum class Error; +using ErrorLogger = std::function; + +using SocketOptions = std::function; + +namespace detail { + +bool set_socket_opt_impl(socket_t sock, int level, int optname, + const void *optval, socklen_t optlen); +bool set_socket_opt(socket_t sock, int level, int optname, int opt); +bool set_socket_opt_time(socket_t sock, int level, int optname, time_t sec, + time_t usec); + +} // namespace detail + +void default_socket_options(socket_t sock); + +const char *status_message(int status); + +std::string get_bearer_token_auth(const Request &req); + +namespace detail { + +class MatcherBase { +public: + MatcherBase(std::string pattern) : pattern_(pattern) {} + virtual ~MatcherBase() = default; + + const std::string &pattern() const { return pattern_; } + + // Match request path and populate its matches and + virtual bool match(Request &request) const = 0; + +private: + std::string pattern_; +}; + +/** + * Captures parameters in request path and stores them in Request::path_params + * + * Capture name is a substring of a pattern from : to /. + * The rest of the pattern is matched against the request path directly + * Parameters are captured starting from the next character after + * the end of the last matched static pattern fragment until the next /. + * + * Example pattern: + * "/path/fragments/:capture/more/fragments/:second_capture" + * Static fragments: + * "/path/fragments/", "more/fragments/" + * + * Given the following request path: + * "/path/fragments/:1/more/fragments/:2" + * the resulting capture will be + * {{"capture", "1"}, {"second_capture", "2"}} + */ +class PathParamsMatcher final : public MatcherBase { +public: + PathParamsMatcher(const std::string &pattern); + + bool match(Request &request) const override; + +private: + // Treat segment separators as the end of path parameter capture + // Does not need to handle query parameters as they are parsed before path + // matching + static constexpr char separator = '/'; + + // Contains static path fragments to match against, excluding the '/' after + // path params + // Fragments are separated by path params + std::vector static_fragments_; + // Stores the names of the path parameters to be used as keys in the + // Request::path_params map + std::vector param_names_; +}; + +/** + * Performs std::regex_match on request path + * and stores the result in Request::matches + * + * Note that regex match is performed directly on the whole request. + * This means that wildcard patterns may match multiple path segments with /: + * "/begin/(.*)/end" will match both "/begin/middle/end" and "/begin/1/2/end". + */ +class RegexMatcher final : public MatcherBase { +public: + RegexMatcher(const std::string &pattern) + : MatcherBase(pattern), regex_(pattern) {} + + bool match(Request &request) const override; + +private: + std::regex regex_; +}; + +ssize_t write_headers(Stream &strm, const Headers &headers); + +} // namespace detail + +class Server { +public: + using Handler = std::function; + + using ExceptionHandler = + std::function; + + enum class HandlerResponse { + Handled, + Unhandled, + }; + using HandlerWithResponse = + std::function; + + using HandlerWithContentReader = std::function; + + using Expect100ContinueHandler = + std::function; + + Server(); + + virtual ~Server(); + + virtual bool is_valid() const; + + Server &Get(const std::string &pattern, Handler handler); + Server &Post(const std::string &pattern, Handler handler); + Server &Post(const std::string &pattern, HandlerWithContentReader handler); + Server &Put(const std::string &pattern, Handler handler); + Server &Put(const std::string &pattern, HandlerWithContentReader handler); + Server &Patch(const std::string &pattern, Handler handler); + Server &Patch(const std::string &pattern, HandlerWithContentReader handler); + Server &Delete(const std::string &pattern, Handler handler); + Server &Delete(const std::string &pattern, HandlerWithContentReader handler); + Server &Options(const std::string &pattern, Handler handler); + + bool set_base_dir(const std::string &dir, + const std::string &mount_point = std::string()); + bool set_mount_point(const std::string &mount_point, const std::string &dir, + Headers headers = Headers()); + bool remove_mount_point(const std::string &mount_point); + Server &set_file_extension_and_mimetype_mapping(const std::string &ext, + const std::string &mime); + Server &set_default_file_mimetype(const std::string &mime); + Server &set_file_request_handler(Handler handler); + + template + Server &set_error_handler(ErrorHandlerFunc &&handler) { + return set_error_handler_core( + std::forward(handler), + std::is_convertible{}); + } + + Server &set_exception_handler(ExceptionHandler handler); + + Server &set_pre_routing_handler(HandlerWithResponse handler); + Server &set_post_routing_handler(Handler handler); + + Server &set_pre_request_handler(HandlerWithResponse handler); + + Server &set_expect_100_continue_handler(Expect100ContinueHandler handler); + Server &set_logger(Logger logger); + Server &set_pre_compression_logger(Logger logger); + Server &set_error_logger(ErrorLogger error_logger); + + Server &set_address_family(int family); + Server &set_tcp_nodelay(bool on); + Server &set_ipv6_v6only(bool on); + Server &set_socket_options(SocketOptions socket_options); + + Server &set_default_headers(Headers headers); + Server & + set_header_writer(std::function const &writer); + + Server &set_keep_alive_max_count(size_t count); + Server &set_keep_alive_timeout(time_t sec); + + Server &set_read_timeout(time_t sec, time_t usec = 0); + template + Server &set_read_timeout(const std::chrono::duration &duration); + + Server &set_write_timeout(time_t sec, time_t usec = 0); + template + Server &set_write_timeout(const std::chrono::duration &duration); + + Server &set_idle_interval(time_t sec, time_t usec = 0); + template + Server &set_idle_interval(const std::chrono::duration &duration); + + Server &set_payload_max_length(size_t length); + + bool bind_to_port(const std::string &host, int port, int socket_flags = 0); + int bind_to_any_port(const std::string &host, int socket_flags = 0); + bool listen_after_bind(); + + bool listen(const std::string &host, int port, int socket_flags = 0); + + bool is_running() const; + void wait_until_ready() const; + void stop(); + void decommission(); + + std::function new_task_queue; + +protected: + bool process_request(Stream &strm, const std::string &remote_addr, + int remote_port, const std::string &local_addr, + int local_port, bool close_connection, + bool &connection_closed, + const std::function &setup_request); + + std::atomic svr_sock_{INVALID_SOCKET}; + size_t keep_alive_max_count_ = CPPHTTPLIB_KEEPALIVE_MAX_COUNT; + time_t keep_alive_timeout_sec_ = CPPHTTPLIB_KEEPALIVE_TIMEOUT_SECOND; + time_t read_timeout_sec_ = CPPHTTPLIB_SERVER_READ_TIMEOUT_SECOND; + time_t read_timeout_usec_ = CPPHTTPLIB_SERVER_READ_TIMEOUT_USECOND; + time_t write_timeout_sec_ = CPPHTTPLIB_SERVER_WRITE_TIMEOUT_SECOND; + time_t write_timeout_usec_ = CPPHTTPLIB_SERVER_WRITE_TIMEOUT_USECOND; + time_t idle_interval_sec_ = CPPHTTPLIB_IDLE_INTERVAL_SECOND; + time_t idle_interval_usec_ = CPPHTTPLIB_IDLE_INTERVAL_USECOND; + size_t payload_max_length_ = CPPHTTPLIB_PAYLOAD_MAX_LENGTH; + +private: + using Handlers = + std::vector, Handler>>; + using HandlersForContentReader = + std::vector, + HandlerWithContentReader>>; + + static std::unique_ptr + make_matcher(const std::string &pattern); + + Server &set_error_handler_core(HandlerWithResponse handler, std::true_type); + Server &set_error_handler_core(Handler handler, std::false_type); + + socket_t create_server_socket(const std::string &host, int port, + int socket_flags, + SocketOptions socket_options) const; + int bind_internal(const std::string &host, int port, int socket_flags); + bool listen_internal(); + + bool routing(Request &req, Response &res, Stream &strm); + bool handle_file_request(const Request &req, Response &res); + bool dispatch_request(Request &req, Response &res, + const Handlers &handlers) const; + bool dispatch_request_for_content_reader( + Request &req, Response &res, ContentReader content_reader, + const HandlersForContentReader &handlers) const; + + bool parse_request_line(const char *s, Request &req) const; + void apply_ranges(const Request &req, Response &res, + std::string &content_type, std::string &boundary) const; + bool write_response(Stream &strm, bool close_connection, Request &req, + Response &res); + bool write_response_with_content(Stream &strm, bool close_connection, + const Request &req, Response &res); + bool write_response_core(Stream &strm, bool close_connection, + const Request &req, Response &res, + bool need_apply_ranges); + bool write_content_with_provider(Stream &strm, const Request &req, + Response &res, const std::string &boundary, + const std::string &content_type); + bool read_content(Stream &strm, Request &req, Response &res); + bool read_content_with_content_receiver(Stream &strm, Request &req, + Response &res, + ContentReceiver receiver, + FormDataHeader multipart_header, + ContentReceiver multipart_receiver); + bool read_content_core(Stream &strm, Request &req, Response &res, + ContentReceiver receiver, + FormDataHeader multipart_header, + ContentReceiver multipart_receiver) const; + + virtual bool process_and_close_socket(socket_t sock); + + void output_log(const Request &req, const Response &res) const; + void output_pre_compression_log(const Request &req, + const Response &res) const; + void output_error_log(const Error &err, const Request *req) const; + + std::atomic is_running_{false}; + std::atomic is_decommissioned{false}; + + struct MountPointEntry { + std::string mount_point; + std::string base_dir; + Headers headers; + }; + std::vector base_dirs_; + std::map file_extension_and_mimetype_map_; + std::string default_file_mimetype_ = "application/octet-stream"; + Handler file_request_handler_; + + Handlers get_handlers_; + Handlers post_handlers_; + HandlersForContentReader post_handlers_for_content_reader_; + Handlers put_handlers_; + HandlersForContentReader put_handlers_for_content_reader_; + Handlers patch_handlers_; + HandlersForContentReader patch_handlers_for_content_reader_; + Handlers delete_handlers_; + HandlersForContentReader delete_handlers_for_content_reader_; + Handlers options_handlers_; + + HandlerWithResponse error_handler_; + ExceptionHandler exception_handler_; + HandlerWithResponse pre_routing_handler_; + Handler post_routing_handler_; + HandlerWithResponse pre_request_handler_; + Expect100ContinueHandler expect_100_continue_handler_; + + mutable std::mutex logger_mutex_; + Logger logger_; + Logger pre_compression_logger_; + ErrorLogger error_logger_; + + int address_family_ = AF_UNSPEC; + bool tcp_nodelay_ = CPPHTTPLIB_TCP_NODELAY; + bool ipv6_v6only_ = CPPHTTPLIB_IPV6_V6ONLY; + SocketOptions socket_options_ = default_socket_options; + + Headers default_headers_; + std::function header_writer_ = + detail::write_headers; +}; + +enum class Error { + Success = 0, + Unknown, + Connection, + BindIPAddress, + Read, + Write, + ExceedRedirectCount, + Canceled, + SSLConnection, + SSLLoadingCerts, + SSLServerVerification, + SSLServerHostnameVerification, + UnsupportedMultipartBoundaryChars, + Compression, + ConnectionTimeout, + ProxyConnection, + ResourceExhaustion, + TooManyFormDataFiles, + ExceedMaxPayloadSize, + ExceedUriMaxLength, + ExceedMaxSocketDescriptorCount, + InvalidRequestLine, + InvalidHTTPMethod, + InvalidHTTPVersion, + InvalidHeaders, + MultipartParsing, + OpenFile, + Listen, + GetSockName, + UnsupportedAddressFamily, + HTTPParsing, + InvalidRangeHeader, + + // For internal use only + SSLPeerCouldBeClosed_, +}; + +std::string to_string(Error error); + +std::ostream &operator<<(std::ostream &os, const Error &obj); + +class Result { +public: + Result() = default; + Result(std::unique_ptr &&res, Error err, + Headers &&request_headers = Headers{}) + : res_(std::move(res)), err_(err), + request_headers_(std::move(request_headers)) {} +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + Result(std::unique_ptr &&res, Error err, Headers &&request_headers, + int ssl_error) + : res_(std::move(res)), err_(err), + request_headers_(std::move(request_headers)), ssl_error_(ssl_error) {} + Result(std::unique_ptr &&res, Error err, Headers &&request_headers, + int ssl_error, unsigned long ssl_openssl_error) + : res_(std::move(res)), err_(err), + request_headers_(std::move(request_headers)), ssl_error_(ssl_error), + ssl_openssl_error_(ssl_openssl_error) {} +#endif + // Response + operator bool() const { return res_ != nullptr; } + bool operator==(std::nullptr_t) const { return res_ == nullptr; } + bool operator!=(std::nullptr_t) const { return res_ != nullptr; } + const Response &value() const { return *res_; } + Response &value() { return *res_; } + const Response &operator*() const { return *res_; } + Response &operator*() { return *res_; } + const Response *operator->() const { return res_.get(); } + Response *operator->() { return res_.get(); } + + // Error + Error error() const { return err_; } + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + // SSL Error + int ssl_error() const { return ssl_error_; } + // OpenSSL Error + unsigned long ssl_openssl_error() const { return ssl_openssl_error_; } +#endif + + // Request Headers + bool has_request_header(const std::string &key) const; + std::string get_request_header_value(const std::string &key, + const char *def = "", + size_t id = 0) const; + size_t get_request_header_value_u64(const std::string &key, size_t def = 0, + size_t id = 0) const; + size_t get_request_header_value_count(const std::string &key) const; + +private: + std::unique_ptr res_; + Error err_ = Error::Unknown; + Headers request_headers_; +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + int ssl_error_ = 0; + unsigned long ssl_openssl_error_ = 0; +#endif +}; + +class ClientImpl { +public: + explicit ClientImpl(const std::string &host); + + explicit ClientImpl(const std::string &host, int port); + + explicit ClientImpl(const std::string &host, int port, + const std::string &client_cert_path, + const std::string &client_key_path); + + virtual ~ClientImpl(); + + virtual bool is_valid() const; + + // clang-format off + Result Get(const std::string &path, DownloadProgress progress = nullptr); + Result Get(const std::string &path, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Get(const std::string &path, ResponseHandler response_handler, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Headers &headers, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Headers &headers, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Headers &headers, ResponseHandler response_handler, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Params ¶ms, const Headers &headers, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Params ¶ms, const Headers &headers, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Params ¶ms, const Headers &headers, ResponseHandler response_handler, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + + Result Head(const std::string &path); + Result Head(const std::string &path, const Headers &headers); + + Result Post(const std::string &path); + Result Post(const std::string &path, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Params ¶ms); + Result Post(const std::string &path, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers); + Result Post(const std::string &path, const Headers &headers, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, const Params ¶ms); + Result Post(const std::string &path, const Headers &headers, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const std::string &boundary, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const FormDataProviderItems &provider_items, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + + Result Put(const std::string &path); + Result Put(const std::string &path, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Params ¶ms); + Result Put(const std::string &path, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers); + Result Put(const std::string &path, const Headers &headers, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, const Params ¶ms); + Result Put(const std::string &path, const Headers &headers, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const std::string &boundary, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const FormDataProviderItems &provider_items, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + + Result Patch(const std::string &path); + Result Patch(const std::string &path, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Params ¶ms); + Result Patch(const std::string &path, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, const Params ¶ms); + Result Patch(const std::string &path, const Headers &headers, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const std::string &boundary, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const FormDataProviderItems &provider_items, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + + Result Delete(const std::string &path, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const char *body, size_t content_length, const std::string &content_type, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const std::string &body, const std::string &content_type, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const Params ¶ms, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const Headers &headers, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const Headers &headers, const char *body, size_t content_length, const std::string &content_type, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const Headers &headers, const Params ¶ms, DownloadProgress progress = nullptr); + + Result Options(const std::string &path); + Result Options(const std::string &path, const Headers &headers); + // clang-format on + + bool send(Request &req, Response &res, Error &error); + Result send(const Request &req); + + void stop(); + + std::string host() const; + int port() const; + + size_t is_socket_open() const; + socket_t socket() const; + + void set_hostname_addr_map(std::map addr_map); + + void set_default_headers(Headers headers); + + void + set_header_writer(std::function const &writer); + + void set_address_family(int family); + void set_tcp_nodelay(bool on); + void set_ipv6_v6only(bool on); + void set_socket_options(SocketOptions socket_options); + + void set_connection_timeout(time_t sec, time_t usec = 0); + template + void + set_connection_timeout(const std::chrono::duration &duration); + + void set_read_timeout(time_t sec, time_t usec = 0); + template + void set_read_timeout(const std::chrono::duration &duration); + + void set_write_timeout(time_t sec, time_t usec = 0); + template + void set_write_timeout(const std::chrono::duration &duration); + + void set_max_timeout(time_t msec); + template + void set_max_timeout(const std::chrono::duration &duration); + + void set_basic_auth(const std::string &username, const std::string &password); + void set_bearer_token_auth(const std::string &token); +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + void set_digest_auth(const std::string &username, + const std::string &password); +#endif + + void set_keep_alive(bool on); + void set_follow_location(bool on); + + void set_path_encode(bool on); + + void set_compress(bool on); + + void set_decompress(bool on); + + void set_interface(const std::string &intf); + + void set_proxy(const std::string &host, int port); + void set_proxy_basic_auth(const std::string &username, + const std::string &password); + void set_proxy_bearer_token_auth(const std::string &token); +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + void set_proxy_digest_auth(const std::string &username, + const std::string &password); +#endif + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + void set_ca_cert_path(const std::string &ca_cert_file_path, + const std::string &ca_cert_dir_path = std::string()); + void set_ca_cert_store(X509_STORE *ca_cert_store); + X509_STORE *create_ca_cert_store(const char *ca_cert, std::size_t size) const; +#endif + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + void enable_server_certificate_verification(bool enabled); + void enable_server_hostname_verification(bool enabled); + void set_server_certificate_verifier( + std::function verifier); +#endif + + void set_logger(Logger logger); + void set_error_logger(ErrorLogger error_logger); + +protected: + struct Socket { + socket_t sock = INVALID_SOCKET; +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + SSL *ssl = nullptr; +#endif + + bool is_open() const { return sock != INVALID_SOCKET; } + }; + + virtual bool create_and_connect_socket(Socket &socket, Error &error); + + // All of: + // shutdown_ssl + // shutdown_socket + // close_socket + // should ONLY be called when socket_mutex_ is locked. + // Also, shutdown_ssl and close_socket should also NOT be called concurrently + // with a DIFFERENT thread sending requests using that socket. + virtual void shutdown_ssl(Socket &socket, bool shutdown_gracefully); + void shutdown_socket(Socket &socket) const; + void close_socket(Socket &socket); + + bool process_request(Stream &strm, Request &req, Response &res, + bool close_connection, Error &error); + + bool write_content_with_provider(Stream &strm, const Request &req, + Error &error) const; + + void copy_settings(const ClientImpl &rhs); + + void output_log(const Request &req, const Response &res) const; + void output_error_log(const Error &err, const Request *req) const; + + // Socket endpoint information + const std::string host_; + const int port_; + const std::string host_and_port_; + + // Current open socket + Socket socket_; + mutable std::mutex socket_mutex_; + std::recursive_mutex request_mutex_; + + // These are all protected under socket_mutex + size_t socket_requests_in_flight_ = 0; + std::thread::id socket_requests_are_from_thread_ = std::thread::id(); + bool socket_should_be_closed_when_request_is_done_ = false; + + // Hostname-IP map + std::map addr_map_; + + // Default headers + Headers default_headers_; + + // Header writer + std::function header_writer_ = + detail::write_headers; + + // Settings + std::string client_cert_path_; + std::string client_key_path_; + + time_t connection_timeout_sec_ = CPPHTTPLIB_CONNECTION_TIMEOUT_SECOND; + time_t connection_timeout_usec_ = CPPHTTPLIB_CONNECTION_TIMEOUT_USECOND; + time_t read_timeout_sec_ = CPPHTTPLIB_CLIENT_READ_TIMEOUT_SECOND; + time_t read_timeout_usec_ = CPPHTTPLIB_CLIENT_READ_TIMEOUT_USECOND; + time_t write_timeout_sec_ = CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_SECOND; + time_t write_timeout_usec_ = CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_USECOND; + time_t max_timeout_msec_ = CPPHTTPLIB_CLIENT_MAX_TIMEOUT_MSECOND; + + std::string basic_auth_username_; + std::string basic_auth_password_; + std::string bearer_token_auth_token_; +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + std::string digest_auth_username_; + std::string digest_auth_password_; +#endif + + bool keep_alive_ = false; + bool follow_location_ = false; + + bool path_encode_ = true; + + int address_family_ = AF_UNSPEC; + bool tcp_nodelay_ = CPPHTTPLIB_TCP_NODELAY; + bool ipv6_v6only_ = CPPHTTPLIB_IPV6_V6ONLY; + SocketOptions socket_options_ = nullptr; + + bool compress_ = false; + bool decompress_ = true; + + std::string interface_; + + std::string proxy_host_; + int proxy_port_ = -1; + + std::string proxy_basic_auth_username_; + std::string proxy_basic_auth_password_; + std::string proxy_bearer_token_auth_token_; +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + std::string proxy_digest_auth_username_; + std::string proxy_digest_auth_password_; +#endif + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + std::string ca_cert_file_path_; + std::string ca_cert_dir_path_; + + X509_STORE *ca_cert_store_ = nullptr; +#endif + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + bool server_certificate_verification_ = true; + bool server_hostname_verification_ = true; + std::function server_certificate_verifier_; +#endif + + mutable std::mutex logger_mutex_; + Logger logger_; + ErrorLogger error_logger_; + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + int last_ssl_error_ = 0; + unsigned long last_openssl_error_ = 0; +#endif + +private: + bool send_(Request &req, Response &res, Error &error); + Result send_(Request &&req); + + socket_t create_client_socket(Error &error) const; + bool read_response_line(Stream &strm, const Request &req, + Response &res) const; + bool write_request(Stream &strm, Request &req, bool close_connection, + Error &error); + bool redirect(Request &req, Response &res, Error &error); + bool create_redirect_client(const std::string &scheme, + const std::string &host, int port, Request &req, + Response &res, const std::string &path, + const std::string &location, Error &error); + template void setup_redirect_client(ClientType &client); + bool handle_request(Stream &strm, Request &req, Response &res, + bool close_connection, Error &error); + std::unique_ptr send_with_content_provider( + Request &req, const char *body, size_t content_length, + ContentProvider content_provider, + ContentProviderWithoutLength content_provider_without_length, + const std::string &content_type, Error &error); + Result send_with_content_provider( + const std::string &method, const std::string &path, + const Headers &headers, const char *body, size_t content_length, + ContentProvider content_provider, + ContentProviderWithoutLength content_provider_without_length, + const std::string &content_type, UploadProgress progress); + ContentProviderWithoutLength get_multipart_content_provider( + const std::string &boundary, const UploadFormDataItems &items, + const FormDataProviderItems &provider_items) const; + + std::string adjust_host_string(const std::string &host) const; + + virtual bool + process_socket(const Socket &socket, + std::chrono::time_point start_time, + std::function callback); + virtual bool is_ssl() const; +}; + +class Client { +public: + // Universal interface + explicit Client(const std::string &scheme_host_port); + + explicit Client(const std::string &scheme_host_port, + const std::string &client_cert_path, + const std::string &client_key_path); + + // HTTP only interface + explicit Client(const std::string &host, int port); + + explicit Client(const std::string &host, int port, + const std::string &client_cert_path, + const std::string &client_key_path); + + Client(Client &&) = default; + Client &operator=(Client &&) = default; + + ~Client(); + + bool is_valid() const; + + // clang-format off + Result Get(const std::string &path, DownloadProgress progress = nullptr); + Result Get(const std::string &path, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Get(const std::string &path, ResponseHandler response_handler, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Headers &headers, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Headers &headers, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Headers &headers, ResponseHandler response_handler, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Params ¶ms, const Headers &headers, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Params ¶ms, const Headers &headers, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + Result Get(const std::string &path, const Params ¶ms, const Headers &headers, ResponseHandler response_handler, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + + Result Head(const std::string &path); + Result Head(const std::string &path, const Headers &headers); + + Result Post(const std::string &path); + Result Post(const std::string &path, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Params ¶ms); + Result Post(const std::string &path, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers); + Result Post(const std::string &path, const Headers &headers, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, const Params ¶ms); + Result Post(const std::string &path, const Headers &headers, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const std::string &boundary, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const FormDataProviderItems &provider_items, UploadProgress progress = nullptr); + Result Post(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + + Result Put(const std::string &path); + Result Put(const std::string &path, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Params ¶ms); + Result Put(const std::string &path, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers); + Result Put(const std::string &path, const Headers &headers, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, const Params ¶ms); + Result Put(const std::string &path, const Headers &headers, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const std::string &boundary, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const FormDataProviderItems &provider_items, UploadProgress progress = nullptr); + Result Put(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + + Result Patch(const std::string &path); + Result Patch(const std::string &path, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Params ¶ms); + Result Patch(const std::string &path, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers); + Result Patch(const std::string &path, const Headers &headers, const char *body, size_t content_length, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, size_t content_length, ContentProvider content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, ContentProviderWithoutLength content_provider, const std::string &content_type, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, const Params ¶ms); + Result Patch(const std::string &path, const Headers &headers, const UploadFormDataItems &items, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const std::string &boundary, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, const UploadFormDataItems &items, const FormDataProviderItems &provider_items, UploadProgress progress = nullptr); + Result Patch(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, ContentReceiver content_receiver, DownloadProgress progress = nullptr); + + Result Delete(const std::string &path, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const char *body, size_t content_length, const std::string &content_type, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const std::string &body, const std::string &content_type, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const Params ¶ms, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const Headers &headers, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const Headers &headers, const char *body, size_t content_length, const std::string &content_type, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const Headers &headers, const std::string &body, const std::string &content_type, DownloadProgress progress = nullptr); + Result Delete(const std::string &path, const Headers &headers, const Params ¶ms, DownloadProgress progress = nullptr); + + Result Options(const std::string &path); + Result Options(const std::string &path, const Headers &headers); + // clang-format on + + bool send(Request &req, Response &res, Error &error); + Result send(const Request &req); + + void stop(); + + std::string host() const; + int port() const; + + size_t is_socket_open() const; + socket_t socket() const; + + void set_hostname_addr_map(std::map addr_map); + + void set_default_headers(Headers headers); + + void + set_header_writer(std::function const &writer); + + void set_address_family(int family); + void set_tcp_nodelay(bool on); + void set_socket_options(SocketOptions socket_options); + + void set_connection_timeout(time_t sec, time_t usec = 0); + template + void + set_connection_timeout(const std::chrono::duration &duration); + + void set_read_timeout(time_t sec, time_t usec = 0); + template + void set_read_timeout(const std::chrono::duration &duration); + + void set_write_timeout(time_t sec, time_t usec = 0); + template + void set_write_timeout(const std::chrono::duration &duration); + + void set_max_timeout(time_t msec); + template + void set_max_timeout(const std::chrono::duration &duration); + + void set_basic_auth(const std::string &username, const std::string &password); + void set_bearer_token_auth(const std::string &token); +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + void set_digest_auth(const std::string &username, + const std::string &password); +#endif + + void set_keep_alive(bool on); + void set_follow_location(bool on); + + void set_path_encode(bool on); + void set_url_encode(bool on); + + void set_compress(bool on); + + void set_decompress(bool on); + + void set_interface(const std::string &intf); + + void set_proxy(const std::string &host, int port); + void set_proxy_basic_auth(const std::string &username, + const std::string &password); + void set_proxy_bearer_token_auth(const std::string &token); +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + void set_proxy_digest_auth(const std::string &username, + const std::string &password); +#endif + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + void enable_server_certificate_verification(bool enabled); + void enable_server_hostname_verification(bool enabled); + void set_server_certificate_verifier( + std::function verifier); +#endif + + void set_logger(Logger logger); + void set_error_logger(ErrorLogger error_logger); + + // SSL +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + void set_ca_cert_path(const std::string &ca_cert_file_path, + const std::string &ca_cert_dir_path = std::string()); + + void set_ca_cert_store(X509_STORE *ca_cert_store); + void load_ca_cert_store(const char *ca_cert, std::size_t size); + + long get_openssl_verify_result() const; + + SSL_CTX *ssl_context() const; +#endif + +private: + std::unique_ptr cli_; + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + bool is_ssl_ = false; +#endif +}; + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +class SSLServer : public Server { +public: + SSLServer(const char *cert_path, const char *private_key_path, + const char *client_ca_cert_file_path = nullptr, + const char *client_ca_cert_dir_path = nullptr, + const char *private_key_password = nullptr); + + SSLServer(X509 *cert, EVP_PKEY *private_key, + X509_STORE *client_ca_cert_store = nullptr); + + SSLServer( + const std::function &setup_ssl_ctx_callback); + + ~SSLServer() override; + + bool is_valid() const override; + + SSL_CTX *ssl_context() const; + + void update_certs(X509 *cert, EVP_PKEY *private_key, + X509_STORE *client_ca_cert_store = nullptr); + +private: + bool process_and_close_socket(socket_t sock) override; + + SSL_CTX *ctx_; + std::mutex ctx_mutex_; +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + int last_ssl_error_ = 0; +#endif +}; + +class SSLClient final : public ClientImpl { +public: + explicit SSLClient(const std::string &host); + + explicit SSLClient(const std::string &host, int port); + + explicit SSLClient(const std::string &host, int port, + const std::string &client_cert_path, + const std::string &client_key_path, + const std::string &private_key_password = std::string()); + + explicit SSLClient(const std::string &host, int port, X509 *client_cert, + EVP_PKEY *client_key, + const std::string &private_key_password = std::string()); + + ~SSLClient() override; + + bool is_valid() const override; + + void set_ca_cert_store(X509_STORE *ca_cert_store); + void load_ca_cert_store(const char *ca_cert, std::size_t size); + + long get_openssl_verify_result() const; + + SSL_CTX *ssl_context() const; + +private: + bool create_and_connect_socket(Socket &socket, Error &error) override; + void shutdown_ssl(Socket &socket, bool shutdown_gracefully) override; + void shutdown_ssl_impl(Socket &socket, bool shutdown_gracefully); + + bool + process_socket(const Socket &socket, + std::chrono::time_point start_time, + std::function callback) override; + bool is_ssl() const override; + + bool connect_with_proxy( + Socket &sock, + std::chrono::time_point start_time, + Response &res, bool &success, Error &error); + bool initialize_ssl(Socket &socket, Error &error); + + bool load_certs(); + + bool verify_host(X509 *server_cert) const; + bool verify_host_with_subject_alt_name(X509 *server_cert) const; + bool verify_host_with_common_name(X509 *server_cert) const; + bool check_host_name(const char *pattern, size_t pattern_len) const; + + SSL_CTX *ctx_; + std::mutex ctx_mutex_; + std::once_flag initialize_cert_; + + std::vector host_components_; + + long verify_result_ = 0; + + friend class ClientImpl; +}; +#endif + +/* + * Implementation of template methods. + */ + +namespace detail { + +template +inline void duration_to_sec_and_usec(const T &duration, U callback) { + auto sec = std::chrono::duration_cast(duration).count(); + auto usec = std::chrono::duration_cast( + duration - std::chrono::seconds(sec)) + .count(); + callback(static_cast(sec), static_cast(usec)); +} + +template inline constexpr size_t str_len(const char (&)[N]) { + return N - 1; +} + +inline bool is_numeric(const std::string &str) { + return !str.empty() && + std::all_of(str.cbegin(), str.cend(), + [](unsigned char c) { return std::isdigit(c); }); +} + +inline size_t get_header_value_u64(const Headers &headers, + const std::string &key, size_t def, + size_t id, bool &is_invalid_value) { + is_invalid_value = false; + auto rng = headers.equal_range(key); + auto it = rng.first; + std::advance(it, static_cast(id)); + if (it != rng.second) { + if (is_numeric(it->second)) { + return std::strtoull(it->second.data(), nullptr, 10); + } else { + is_invalid_value = true; + } + } + return def; +} + +inline size_t get_header_value_u64(const Headers &headers, + const std::string &key, size_t def, + size_t id) { + auto dummy = false; + return get_header_value_u64(headers, key, def, id, dummy); +} + +} // namespace detail + +inline size_t Request::get_header_value_u64(const std::string &key, size_t def, + size_t id) const { + return detail::get_header_value_u64(headers, key, def, id); +} + +inline size_t Response::get_header_value_u64(const std::string &key, size_t def, + size_t id) const { + return detail::get_header_value_u64(headers, key, def, id); +} + +namespace detail { + +inline bool set_socket_opt_impl(socket_t sock, int level, int optname, + const void *optval, socklen_t optlen) { + return setsockopt(sock, level, optname, +#ifdef _WIN64 + reinterpret_cast(optval), +#else + optval, +#endif + optlen) == 0; +} + +inline bool set_socket_opt(socket_t sock, int level, int optname, int optval) { + return set_socket_opt_impl(sock, level, optname, &optval, sizeof(optval)); +} + +inline bool set_socket_opt_time(socket_t sock, int level, int optname, + time_t sec, time_t usec) { +#ifdef _WIN64 + auto timeout = static_cast(sec * 1000 + usec / 1000); +#else + timeval timeout; + timeout.tv_sec = static_cast(sec); + timeout.tv_usec = static_cast(usec); +#endif + return set_socket_opt_impl(sock, level, optname, &timeout, sizeof(timeout)); +} + +} // namespace detail + +inline void default_socket_options(socket_t sock) { + detail::set_socket_opt(sock, SOL_SOCKET, +#ifdef SO_REUSEPORT + SO_REUSEPORT, +#else + SO_REUSEADDR, +#endif + 1); +} + +inline const char *status_message(int status) { + switch (status) { + case StatusCode::Continue_100: return "Continue"; + case StatusCode::SwitchingProtocol_101: return "Switching Protocol"; + case StatusCode::Processing_102: return "Processing"; + case StatusCode::EarlyHints_103: return "Early Hints"; + case StatusCode::OK_200: return "OK"; + case StatusCode::Created_201: return "Created"; + case StatusCode::Accepted_202: return "Accepted"; + case StatusCode::NonAuthoritativeInformation_203: + return "Non-Authoritative Information"; + case StatusCode::NoContent_204: return "No Content"; + case StatusCode::ResetContent_205: return "Reset Content"; + case StatusCode::PartialContent_206: return "Partial Content"; + case StatusCode::MultiStatus_207: return "Multi-Status"; + case StatusCode::AlreadyReported_208: return "Already Reported"; + case StatusCode::IMUsed_226: return "IM Used"; + case StatusCode::MultipleChoices_300: return "Multiple Choices"; + case StatusCode::MovedPermanently_301: return "Moved Permanently"; + case StatusCode::Found_302: return "Found"; + case StatusCode::SeeOther_303: return "See Other"; + case StatusCode::NotModified_304: return "Not Modified"; + case StatusCode::UseProxy_305: return "Use Proxy"; + case StatusCode::unused_306: return "unused"; + case StatusCode::TemporaryRedirect_307: return "Temporary Redirect"; + case StatusCode::PermanentRedirect_308: return "Permanent Redirect"; + case StatusCode::BadRequest_400: return "Bad Request"; + case StatusCode::Unauthorized_401: return "Unauthorized"; + case StatusCode::PaymentRequired_402: return "Payment Required"; + case StatusCode::Forbidden_403: return "Forbidden"; + case StatusCode::NotFound_404: return "Not Found"; + case StatusCode::MethodNotAllowed_405: return "Method Not Allowed"; + case StatusCode::NotAcceptable_406: return "Not Acceptable"; + case StatusCode::ProxyAuthenticationRequired_407: + return "Proxy Authentication Required"; + case StatusCode::RequestTimeout_408: return "Request Timeout"; + case StatusCode::Conflict_409: return "Conflict"; + case StatusCode::Gone_410: return "Gone"; + case StatusCode::LengthRequired_411: return "Length Required"; + case StatusCode::PreconditionFailed_412: return "Precondition Failed"; + case StatusCode::PayloadTooLarge_413: return "Payload Too Large"; + case StatusCode::UriTooLong_414: return "URI Too Long"; + case StatusCode::UnsupportedMediaType_415: return "Unsupported Media Type"; + case StatusCode::RangeNotSatisfiable_416: return "Range Not Satisfiable"; + case StatusCode::ExpectationFailed_417: return "Expectation Failed"; + case StatusCode::ImATeapot_418: return "I'm a teapot"; + case StatusCode::MisdirectedRequest_421: return "Misdirected Request"; + case StatusCode::UnprocessableContent_422: return "Unprocessable Content"; + case StatusCode::Locked_423: return "Locked"; + case StatusCode::FailedDependency_424: return "Failed Dependency"; + case StatusCode::TooEarly_425: return "Too Early"; + case StatusCode::UpgradeRequired_426: return "Upgrade Required"; + case StatusCode::PreconditionRequired_428: return "Precondition Required"; + case StatusCode::TooManyRequests_429: return "Too Many Requests"; + case StatusCode::RequestHeaderFieldsTooLarge_431: + return "Request Header Fields Too Large"; + case StatusCode::UnavailableForLegalReasons_451: + return "Unavailable For Legal Reasons"; + case StatusCode::NotImplemented_501: return "Not Implemented"; + case StatusCode::BadGateway_502: return "Bad Gateway"; + case StatusCode::ServiceUnavailable_503: return "Service Unavailable"; + case StatusCode::GatewayTimeout_504: return "Gateway Timeout"; + case StatusCode::HttpVersionNotSupported_505: + return "HTTP Version Not Supported"; + case StatusCode::VariantAlsoNegotiates_506: return "Variant Also Negotiates"; + case StatusCode::InsufficientStorage_507: return "Insufficient Storage"; + case StatusCode::LoopDetected_508: return "Loop Detected"; + case StatusCode::NotExtended_510: return "Not Extended"; + case StatusCode::NetworkAuthenticationRequired_511: + return "Network Authentication Required"; + + default: + case StatusCode::InternalServerError_500: return "Internal Server Error"; + } +} + +inline std::string get_bearer_token_auth(const Request &req) { + if (req.has_header("Authorization")) { + constexpr auto bearer_header_prefix_len = detail::str_len("Bearer "); + return req.get_header_value("Authorization") + .substr(bearer_header_prefix_len); + } + return ""; +} + +template +inline Server & +Server::set_read_timeout(const std::chrono::duration &duration) { + detail::duration_to_sec_and_usec( + duration, [&](time_t sec, time_t usec) { set_read_timeout(sec, usec); }); + return *this; +} + +template +inline Server & +Server::set_write_timeout(const std::chrono::duration &duration) { + detail::duration_to_sec_and_usec( + duration, [&](time_t sec, time_t usec) { set_write_timeout(sec, usec); }); + return *this; +} + +template +inline Server & +Server::set_idle_interval(const std::chrono::duration &duration) { + detail::duration_to_sec_and_usec( + duration, [&](time_t sec, time_t usec) { set_idle_interval(sec, usec); }); + return *this; +} + +inline std::string to_string(const Error error) { + switch (error) { + case Error::Success: return "Success (no error)"; + case Error::Unknown: return "Unknown"; + case Error::Connection: return "Could not establish connection"; + case Error::BindIPAddress: return "Failed to bind IP address"; + case Error::Read: return "Failed to read connection"; + case Error::Write: return "Failed to write connection"; + case Error::ExceedRedirectCount: return "Maximum redirect count exceeded"; + case Error::Canceled: return "Connection handling canceled"; + case Error::SSLConnection: return "SSL connection failed"; + case Error::SSLLoadingCerts: return "SSL certificate loading failed"; + case Error::SSLServerVerification: return "SSL server verification failed"; + case Error::SSLServerHostnameVerification: + return "SSL server hostname verification failed"; + case Error::UnsupportedMultipartBoundaryChars: + return "Unsupported HTTP multipart boundary characters"; + case Error::Compression: return "Compression failed"; + case Error::ConnectionTimeout: return "Connection timed out"; + case Error::ProxyConnection: return "Proxy connection failed"; + case Error::ResourceExhaustion: return "Resource exhaustion"; + case Error::TooManyFormDataFiles: return "Too many form data files"; + case Error::ExceedMaxPayloadSize: return "Exceeded maximum payload size"; + case Error::ExceedUriMaxLength: return "Exceeded maximum URI length"; + case Error::ExceedMaxSocketDescriptorCount: + return "Exceeded maximum socket descriptor count"; + case Error::InvalidRequestLine: return "Invalid request line"; + case Error::InvalidHTTPMethod: return "Invalid HTTP method"; + case Error::InvalidHTTPVersion: return "Invalid HTTP version"; + case Error::InvalidHeaders: return "Invalid headers"; + case Error::MultipartParsing: return "Multipart parsing failed"; + case Error::OpenFile: return "Failed to open file"; + case Error::Listen: return "Failed to listen on socket"; + case Error::GetSockName: return "Failed to get socket name"; + case Error::UnsupportedAddressFamily: return "Unsupported address family"; + case Error::HTTPParsing: return "HTTP parsing failed"; + case Error::InvalidRangeHeader: return "Invalid Range header"; + default: break; + } + + return "Invalid"; +} + +inline std::ostream &operator<<(std::ostream &os, const Error &obj) { + os << to_string(obj); + os << " (" << static_cast::type>(obj) << ')'; + return os; +} + +inline size_t Result::get_request_header_value_u64(const std::string &key, + size_t def, + size_t id) const { + return detail::get_header_value_u64(request_headers_, key, def, id); +} + +template +inline void ClientImpl::set_connection_timeout( + const std::chrono::duration &duration) { + detail::duration_to_sec_and_usec(duration, [&](time_t sec, time_t usec) { + set_connection_timeout(sec, usec); + }); +} + +template +inline void ClientImpl::set_read_timeout( + const std::chrono::duration &duration) { + detail::duration_to_sec_and_usec( + duration, [&](time_t sec, time_t usec) { set_read_timeout(sec, usec); }); +} + +template +inline void ClientImpl::set_write_timeout( + const std::chrono::duration &duration) { + detail::duration_to_sec_and_usec( + duration, [&](time_t sec, time_t usec) { set_write_timeout(sec, usec); }); +} + +template +inline void ClientImpl::set_max_timeout( + const std::chrono::duration &duration) { + auto msec = + std::chrono::duration_cast(duration).count(); + set_max_timeout(msec); +} + +template +inline void Client::set_connection_timeout( + const std::chrono::duration &duration) { + cli_->set_connection_timeout(duration); +} + +template +inline void +Client::set_read_timeout(const std::chrono::duration &duration) { + cli_->set_read_timeout(duration); +} + +template +inline void +Client::set_write_timeout(const std::chrono::duration &duration) { + cli_->set_write_timeout(duration); +} + +inline void Client::set_max_timeout(time_t msec) { + cli_->set_max_timeout(msec); +} + +template +inline void +Client::set_max_timeout(const std::chrono::duration &duration) { + cli_->set_max_timeout(duration); +} + +/* + * Forward declarations and types that will be part of the .h file if split into + * .h + .cc. + */ + +std::string hosted_at(const std::string &hostname); + +void hosted_at(const std::string &hostname, std::vector &addrs); + +// JavaScript-style URL encoding/decoding functions +std::string encode_uri_component(const std::string &value); +std::string encode_uri(const std::string &value); +std::string decode_uri_component(const std::string &value); +std::string decode_uri(const std::string &value); + +// RFC 3986 compliant URL component encoding/decoding functions +std::string encode_path_component(const std::string &component); +std::string decode_path_component(const std::string &component); +std::string encode_query_component(const std::string &component, + bool space_as_plus = true); +std::string decode_query_component(const std::string &component, + bool plus_as_space = true); + +std::string append_query_params(const std::string &path, const Params ¶ms); + +std::pair make_range_header(const Ranges &ranges); + +std::pair +make_basic_authentication_header(const std::string &username, + const std::string &password, + bool is_proxy = false); + +namespace detail { + +#if defined(_WIN64) +inline std::wstring u8string_to_wstring(const char *s) { + std::wstring ws; + auto len = static_cast(strlen(s)); + auto wlen = ::MultiByteToWideChar(CP_UTF8, 0, s, len, nullptr, 0); + if (wlen > 0) { + ws.resize(wlen); + wlen = ::MultiByteToWideChar( + CP_UTF8, 0, s, len, + const_cast(reinterpret_cast(ws.data())), wlen); + if (wlen != static_cast(ws.size())) { ws.clear(); } + } + return ws; +} +#endif + +struct FileStat { + FileStat(const std::string &path); + bool is_file() const; + bool is_dir() const; + +private: +#if defined(_WIN64) + struct _stat st_; +#else + struct stat st_; +#endif + int ret_ = -1; +}; + +std::string trim_copy(const std::string &s); + +void divide( + const char *data, std::size_t size, char d, + std::function + fn); + +void divide( + const std::string &str, char d, + std::function + fn); + +void split(const char *b, const char *e, char d, + std::function fn); + +void split(const char *b, const char *e, char d, size_t m, + std::function fn); + +bool process_client_socket( + socket_t sock, time_t read_timeout_sec, time_t read_timeout_usec, + time_t write_timeout_sec, time_t write_timeout_usec, + time_t max_timeout_msec, + std::chrono::time_point start_time, + std::function callback); + +socket_t create_client_socket(const std::string &host, const std::string &ip, + int port, int address_family, bool tcp_nodelay, + bool ipv6_v6only, SocketOptions socket_options, + time_t connection_timeout_sec, + time_t connection_timeout_usec, + time_t read_timeout_sec, time_t read_timeout_usec, + time_t write_timeout_sec, + time_t write_timeout_usec, + const std::string &intf, Error &error); + +const char *get_header_value(const Headers &headers, const std::string &key, + const char *def, size_t id); + +std::string params_to_query_str(const Params ¶ms); + +void parse_query_text(const char *data, std::size_t size, Params ¶ms); + +void parse_query_text(const std::string &s, Params ¶ms); + +bool parse_multipart_boundary(const std::string &content_type, + std::string &boundary); + +bool parse_range_header(const std::string &s, Ranges &ranges); + +bool parse_accept_header(const std::string &s, + std::vector &content_types); + +int close_socket(socket_t sock); + +ssize_t send_socket(socket_t sock, const void *ptr, size_t size, int flags); + +ssize_t read_socket(socket_t sock, void *ptr, size_t size, int flags); + +enum class EncodingType { None = 0, Gzip, Brotli, Zstd }; + +EncodingType encoding_type(const Request &req, const Response &res); + +class BufferStream final : public Stream { +public: + BufferStream() = default; + ~BufferStream() override = default; + + bool is_readable() const override; + bool wait_readable() const override; + bool wait_writable() const override; + ssize_t read(char *ptr, size_t size) override; + ssize_t write(const char *ptr, size_t size) override; + void get_remote_ip_and_port(std::string &ip, int &port) const override; + void get_local_ip_and_port(std::string &ip, int &port) const override; + socket_t socket() const override; + time_t duration() const override; + + const std::string &get_buffer() const; + +private: + std::string buffer; + size_t position = 0; +}; + +class compressor { +public: + virtual ~compressor() = default; + + typedef std::function Callback; + virtual bool compress(const char *data, size_t data_length, bool last, + Callback callback) = 0; +}; + +class decompressor { +public: + virtual ~decompressor() = default; + + virtual bool is_valid() const = 0; + + typedef std::function Callback; + virtual bool decompress(const char *data, size_t data_length, + Callback callback) = 0; +}; + +class nocompressor final : public compressor { +public: + ~nocompressor() override = default; + + bool compress(const char *data, size_t data_length, bool /*last*/, + Callback callback) override; +}; + +#ifdef CPPHTTPLIB_ZLIB_SUPPORT +class gzip_compressor final : public compressor { +public: + gzip_compressor(); + ~gzip_compressor() override; + + bool compress(const char *data, size_t data_length, bool last, + Callback callback) override; + +private: + bool is_valid_ = false; + z_stream strm_; +}; + +class gzip_decompressor final : public decompressor { +public: + gzip_decompressor(); + ~gzip_decompressor() override; + + bool is_valid() const override; + + bool decompress(const char *data, size_t data_length, + Callback callback) override; + +private: + bool is_valid_ = false; + z_stream strm_; +}; +#endif + +#ifdef CPPHTTPLIB_BROTLI_SUPPORT +class brotli_compressor final : public compressor { +public: + brotli_compressor(); + ~brotli_compressor(); + + bool compress(const char *data, size_t data_length, bool last, + Callback callback) override; + +private: + BrotliEncoderState *state_ = nullptr; +}; + +class brotli_decompressor final : public decompressor { +public: + brotli_decompressor(); + ~brotli_decompressor(); + + bool is_valid() const override; + + bool decompress(const char *data, size_t data_length, + Callback callback) override; + +private: + BrotliDecoderResult decoder_r; + BrotliDecoderState *decoder_s = nullptr; +}; +#endif + +#ifdef CPPHTTPLIB_ZSTD_SUPPORT +class zstd_compressor : public compressor { +public: + zstd_compressor(); + ~zstd_compressor(); + + bool compress(const char *data, size_t data_length, bool last, + Callback callback) override; + +private: + ZSTD_CCtx *ctx_ = nullptr; +}; + +class zstd_decompressor : public decompressor { +public: + zstd_decompressor(); + ~zstd_decompressor(); + + bool is_valid() const override; + + bool decompress(const char *data, size_t data_length, + Callback callback) override; + +private: + ZSTD_DCtx *ctx_ = nullptr; +}; +#endif + +// NOTE: until the read size reaches `fixed_buffer_size`, use `fixed_buffer` +// to store data. The call can set memory on stack for performance. +class stream_line_reader { +public: + stream_line_reader(Stream &strm, char *fixed_buffer, + size_t fixed_buffer_size); + const char *ptr() const; + size_t size() const; + bool end_with_crlf() const; + bool getline(); + +private: + void append(char c); + + Stream &strm_; + char *fixed_buffer_; + const size_t fixed_buffer_size_; + size_t fixed_buffer_used_size_ = 0; + std::string growable_buffer_; +}; + +class mmap { +public: + mmap(const char *path); + ~mmap(); + + bool open(const char *path); + void close(); + + bool is_open() const; + size_t size() const; + const char *data() const; + +private: +#if defined(_WIN64) + HANDLE hFile_ = NULL; + HANDLE hMapping_ = NULL; +#else + int fd_ = -1; +#endif + size_t size_ = 0; + void *addr_ = nullptr; + bool is_open_empty_file = false; +}; + +// NOTE: https://www.rfc-editor.org/rfc/rfc9110#section-5 +namespace fields { + +inline bool is_token_char(char c) { + return std::isalnum(c) || c == '!' || c == '#' || c == '$' || c == '%' || + c == '&' || c == '\'' || c == '*' || c == '+' || c == '-' || + c == '.' || c == '^' || c == '_' || c == '`' || c == '|' || c == '~'; +} + +inline bool is_token(const std::string &s) { + if (s.empty()) { return false; } + for (auto c : s) { + if (!is_token_char(c)) { return false; } + } + return true; +} + +inline bool is_field_name(const std::string &s) { return is_token(s); } + +inline bool is_vchar(char c) { return c >= 33 && c <= 126; } + +inline bool is_obs_text(char c) { return 128 <= static_cast(c); } + +inline bool is_field_vchar(char c) { return is_vchar(c) || is_obs_text(c); } + +inline bool is_field_content(const std::string &s) { + if (s.empty()) { return true; } + + if (s.size() == 1) { + return is_field_vchar(s[0]); + } else if (s.size() == 2) { + return is_field_vchar(s[0]) && is_field_vchar(s[1]); + } else { + size_t i = 0; + + if (!is_field_vchar(s[i])) { return false; } + i++; + + while (i < s.size() - 1) { + auto c = s[i++]; + if (c == ' ' || c == '\t' || is_field_vchar(c)) { + } else { + return false; + } + } + + return is_field_vchar(s[i]); + } +} + +inline bool is_field_value(const std::string &s) { return is_field_content(s); } + +} // namespace fields + +} // namespace detail + +// ---------------------------------------------------------------------------- + +/* + * Implementation that will be part of the .cc file if split into .h + .cc. + */ + +namespace detail { + +inline bool is_hex(char c, int &v) { + if (0x20 <= c && isdigit(c)) { + v = c - '0'; + return true; + } else if ('A' <= c && c <= 'F') { + v = c - 'A' + 10; + return true; + } else if ('a' <= c && c <= 'f') { + v = c - 'a' + 10; + return true; + } + return false; +} + +inline bool from_hex_to_i(const std::string &s, size_t i, size_t cnt, + int &val) { + if (i >= s.size()) { return false; } + + val = 0; + for (; cnt; i++, cnt--) { + if (!s[i]) { return false; } + auto v = 0; + if (is_hex(s[i], v)) { + val = val * 16 + v; + } else { + return false; + } + } + return true; +} + +inline std::string from_i_to_hex(size_t n) { + static const auto charset = "0123456789abcdef"; + std::string ret; + do { + ret = charset[n & 15] + ret; + n >>= 4; + } while (n > 0); + return ret; +} + +inline size_t to_utf8(int code, char *buff) { + if (code < 0x0080) { + buff[0] = static_cast(code & 0x7F); + return 1; + } else if (code < 0x0800) { + buff[0] = static_cast(0xC0 | ((code >> 6) & 0x1F)); + buff[1] = static_cast(0x80 | (code & 0x3F)); + return 2; + } else if (code < 0xD800) { + buff[0] = static_cast(0xE0 | ((code >> 12) & 0xF)); + buff[1] = static_cast(0x80 | ((code >> 6) & 0x3F)); + buff[2] = static_cast(0x80 | (code & 0x3F)); + return 3; + } else if (code < 0xE000) { // D800 - DFFF is invalid... + return 0; + } else if (code < 0x10000) { + buff[0] = static_cast(0xE0 | ((code >> 12) & 0xF)); + buff[1] = static_cast(0x80 | ((code >> 6) & 0x3F)); + buff[2] = static_cast(0x80 | (code & 0x3F)); + return 3; + } else if (code < 0x110000) { + buff[0] = static_cast(0xF0 | ((code >> 18) & 0x7)); + buff[1] = static_cast(0x80 | ((code >> 12) & 0x3F)); + buff[2] = static_cast(0x80 | ((code >> 6) & 0x3F)); + buff[3] = static_cast(0x80 | (code & 0x3F)); + return 4; + } + + // NOTREACHED + return 0; +} + +// NOTE: This code came up with the following stackoverflow post: +// https://stackoverflow.com/questions/180947/base64-decode-snippet-in-c +inline std::string base64_encode(const std::string &in) { + static const auto lookup = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + std::string out; + out.reserve(in.size()); + + auto val = 0; + auto valb = -6; + + for (auto c : in) { + val = (val << 8) + static_cast(c); + valb += 8; + while (valb >= 0) { + out.push_back(lookup[(val >> valb) & 0x3F]); + valb -= 6; + } + } + + if (valb > -6) { out.push_back(lookup[((val << 8) >> (valb + 8)) & 0x3F]); } + + while (out.size() % 4) { + out.push_back('='); + } + + return out; +} + +inline bool is_valid_path(const std::string &path) { + size_t level = 0; + size_t i = 0; + + // Skip slash + while (i < path.size() && path[i] == '/') { + i++; + } + + while (i < path.size()) { + // Read component + auto beg = i; + while (i < path.size() && path[i] != '/') { + if (path[i] == '\0') { + return false; + } else if (path[i] == '\\') { + return false; + } + i++; + } + + auto len = i - beg; + assert(len > 0); + + if (!path.compare(beg, len, ".")) { + ; + } else if (!path.compare(beg, len, "..")) { + if (level == 0) { return false; } + level--; + } else { + level++; + } + + // Skip slash + while (i < path.size() && path[i] == '/') { + i++; + } + } + + return true; +} + +inline FileStat::FileStat(const std::string &path) { +#if defined(_WIN64) + auto wpath = u8string_to_wstring(path.c_str()); + ret_ = _wstat(wpath.c_str(), &st_); +#else + ret_ = stat(path.c_str(), &st_); +#endif +} +inline bool FileStat::is_file() const { + return ret_ >= 0 && S_ISREG(st_.st_mode); +} +inline bool FileStat::is_dir() const { + return ret_ >= 0 && S_ISDIR(st_.st_mode); +} + +inline std::string encode_path(const std::string &s) { + std::string result; + result.reserve(s.size()); + + for (size_t i = 0; s[i]; i++) { + switch (s[i]) { + case ' ': result += "%20"; break; + case '+': result += "%2B"; break; + case '\r': result += "%0D"; break; + case '\n': result += "%0A"; break; + case '\'': result += "%27"; break; + case ',': result += "%2C"; break; + // case ':': result += "%3A"; break; // ok? probably... + case ';': result += "%3B"; break; + default: + auto c = static_cast(s[i]); + if (c >= 0x80) { + result += '%'; + char hex[4]; + auto len = snprintf(hex, sizeof(hex) - 1, "%02X", c); + assert(len == 2); + result.append(hex, static_cast(len)); + } else { + result += s[i]; + } + break; + } + } + + return result; +} + +inline std::string file_extension(const std::string &path) { + std::smatch m; + thread_local auto re = std::regex("\\.([a-zA-Z0-9]+)$"); + if (std::regex_search(path, m, re)) { return m[1].str(); } + return std::string(); +} + +inline bool is_space_or_tab(char c) { return c == ' ' || c == '\t'; } + +inline std::pair trim(const char *b, const char *e, size_t left, + size_t right) { + while (b + left < e && is_space_or_tab(b[left])) { + left++; + } + while (right > 0 && is_space_or_tab(b[right - 1])) { + right--; + } + return std::make_pair(left, right); +} + +inline std::string trim_copy(const std::string &s) { + auto r = trim(s.data(), s.data() + s.size(), 0, s.size()); + return s.substr(r.first, r.second - r.first); +} + +inline std::string trim_double_quotes_copy(const std::string &s) { + if (s.length() >= 2 && s.front() == '"' && s.back() == '"') { + return s.substr(1, s.size() - 2); + } + return s; +} + +inline void +divide(const char *data, std::size_t size, char d, + std::function + fn) { + const auto it = std::find(data, data + size, d); + const auto found = static_cast(it != data + size); + const auto lhs_data = data; + const auto lhs_size = static_cast(it - data); + const auto rhs_data = it + found; + const auto rhs_size = size - lhs_size - found; + + fn(lhs_data, lhs_size, rhs_data, rhs_size); +} + +inline void +divide(const std::string &str, char d, + std::function + fn) { + divide(str.data(), str.size(), d, std::move(fn)); +} + +inline void split(const char *b, const char *e, char d, + std::function fn) { + return split(b, e, d, (std::numeric_limits::max)(), std::move(fn)); +} + +inline void split(const char *b, const char *e, char d, size_t m, + std::function fn) { + size_t i = 0; + size_t beg = 0; + size_t count = 1; + + while (e ? (b + i < e) : (b[i] != '\0')) { + if (b[i] == d && count < m) { + auto r = trim(b, e, beg, i); + if (r.first < r.second) { fn(&b[r.first], &b[r.second]); } + beg = i + 1; + count++; + } + i++; + } + + if (i) { + auto r = trim(b, e, beg, i); + if (r.first < r.second) { fn(&b[r.first], &b[r.second]); } + } +} + +inline stream_line_reader::stream_line_reader(Stream &strm, char *fixed_buffer, + size_t fixed_buffer_size) + : strm_(strm), fixed_buffer_(fixed_buffer), + fixed_buffer_size_(fixed_buffer_size) {} + +inline const char *stream_line_reader::ptr() const { + if (growable_buffer_.empty()) { + return fixed_buffer_; + } else { + return growable_buffer_.data(); + } +} + +inline size_t stream_line_reader::size() const { + if (growable_buffer_.empty()) { + return fixed_buffer_used_size_; + } else { + return growable_buffer_.size(); + } +} + +inline bool stream_line_reader::end_with_crlf() const { + auto end = ptr() + size(); + return size() >= 2 && end[-2] == '\r' && end[-1] == '\n'; +} + +inline bool stream_line_reader::getline() { + fixed_buffer_used_size_ = 0; + growable_buffer_.clear(); + +#ifndef CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR + char prev_byte = 0; +#endif + + for (size_t i = 0;; i++) { + if (size() >= CPPHTTPLIB_MAX_LINE_LENGTH) { + // Treat exceptionally long lines as an error to + // prevent infinite loops/memory exhaustion + return false; + } + char byte; + auto n = strm_.read(&byte, 1); + + if (n < 0) { + return false; + } else if (n == 0) { + if (i == 0) { + return false; + } else { + break; + } + } + + append(byte); + +#ifdef CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR + if (byte == '\n') { break; } +#else + if (prev_byte == '\r' && byte == '\n') { break; } + prev_byte = byte; +#endif + } + + return true; +} + +inline void stream_line_reader::append(char c) { + if (fixed_buffer_used_size_ < fixed_buffer_size_ - 1) { + fixed_buffer_[fixed_buffer_used_size_++] = c; + fixed_buffer_[fixed_buffer_used_size_] = '\0'; + } else { + if (growable_buffer_.empty()) { + assert(fixed_buffer_[fixed_buffer_used_size_] == '\0'); + growable_buffer_.assign(fixed_buffer_, fixed_buffer_used_size_); + } + growable_buffer_ += c; + } +} + +inline mmap::mmap(const char *path) { open(path); } + +inline mmap::~mmap() { close(); } + +inline bool mmap::open(const char *path) { + close(); + +#if defined(_WIN64) + auto wpath = u8string_to_wstring(path); + if (wpath.empty()) { return false; } + + hFile_ = ::CreateFile2(wpath.c_str(), GENERIC_READ, FILE_SHARE_READ, + OPEN_EXISTING, NULL); + + if (hFile_ == INVALID_HANDLE_VALUE) { return false; } + + LARGE_INTEGER size{}; + if (!::GetFileSizeEx(hFile_, &size)) { return false; } + // If the following line doesn't compile due to QuadPart, update Windows SDK. + // See: + // https://github.com/yhirose/cpp-httplib/issues/1903#issuecomment-2316520721 + if (static_cast(size.QuadPart) > + (std::numeric_limits::max)()) { + // `size_t` might be 32-bits, on 32-bits Windows. + return false; + } + size_ = static_cast(size.QuadPart); + + hMapping_ = + ::CreateFileMappingFromApp(hFile_, NULL, PAGE_READONLY, size_, NULL); + + // Special treatment for an empty file... + if (hMapping_ == NULL && size_ == 0) { + close(); + is_open_empty_file = true; + return true; + } + + if (hMapping_ == NULL) { + close(); + return false; + } + + addr_ = ::MapViewOfFileFromApp(hMapping_, FILE_MAP_READ, 0, 0); + + if (addr_ == nullptr) { + close(); + return false; + } +#else + fd_ = ::open(path, O_RDONLY); + if (fd_ == -1) { return false; } + + struct stat sb; + if (fstat(fd_, &sb) == -1) { + close(); + return false; + } + size_ = static_cast(sb.st_size); + + addr_ = ::mmap(NULL, size_, PROT_READ, MAP_PRIVATE, fd_, 0); + + // Special treatment for an empty file... + if (addr_ == MAP_FAILED && size_ == 0) { + close(); + is_open_empty_file = true; + return false; + } +#endif + + return true; +} + +inline bool mmap::is_open() const { + return is_open_empty_file ? true : addr_ != nullptr; +} + +inline size_t mmap::size() const { return size_; } + +inline const char *mmap::data() const { + return is_open_empty_file ? "" : static_cast(addr_); +} + +inline void mmap::close() { +#if defined(_WIN64) + if (addr_) { + ::UnmapViewOfFile(addr_); + addr_ = nullptr; + } + + if (hMapping_) { + ::CloseHandle(hMapping_); + hMapping_ = NULL; + } + + if (hFile_ != INVALID_HANDLE_VALUE) { + ::CloseHandle(hFile_); + hFile_ = INVALID_HANDLE_VALUE; + } + + is_open_empty_file = false; +#else + if (addr_ != nullptr) { + munmap(addr_, size_); + addr_ = nullptr; + } + + if (fd_ != -1) { + ::close(fd_); + fd_ = -1; + } +#endif + size_ = 0; +} +inline int close_socket(socket_t sock) { +#ifdef _WIN64 + return closesocket(sock); +#else + return close(sock); +#endif +} + +template inline ssize_t handle_EINTR(T fn) { + ssize_t res = 0; + while (true) { + res = fn(); + if (res < 0 && errno == EINTR) { + std::this_thread::sleep_for(std::chrono::microseconds{1}); + continue; + } + break; + } + return res; +} + +inline ssize_t read_socket(socket_t sock, void *ptr, size_t size, int flags) { + return handle_EINTR([&]() { + return recv(sock, +#ifdef _WIN64 + static_cast(ptr), static_cast(size), +#else + ptr, size, +#endif + flags); + }); +} + +inline ssize_t send_socket(socket_t sock, const void *ptr, size_t size, + int flags) { + return handle_EINTR([&]() { + return send(sock, +#ifdef _WIN64 + static_cast(ptr), static_cast(size), +#else + ptr, size, +#endif + flags); + }); +} + +inline int poll_wrapper(struct pollfd *fds, nfds_t nfds, int timeout) { +#ifdef _WIN64 + return ::WSAPoll(fds, nfds, timeout); +#else + return ::poll(fds, nfds, timeout); +#endif +} + +template +inline ssize_t select_impl(socket_t sock, time_t sec, time_t usec) { +#ifdef __APPLE__ + if (sock >= FD_SETSIZE) { return -1; } + + fd_set fds, *rfds, *wfds; + FD_ZERO(&fds); + FD_SET(sock, &fds); + rfds = (Read ? &fds : nullptr); + wfds = (Read ? nullptr : &fds); + + timeval tv; + tv.tv_sec = static_cast(sec); + tv.tv_usec = static_cast(usec); + + return handle_EINTR([&]() { + return select(static_cast(sock + 1), rfds, wfds, nullptr, &tv); + }); +#else + struct pollfd pfd; + pfd.fd = sock; + pfd.events = (Read ? POLLIN : POLLOUT); + + auto timeout = static_cast(sec * 1000 + usec / 1000); + + return handle_EINTR([&]() { return poll_wrapper(&pfd, 1, timeout); }); +#endif +} + +inline ssize_t select_read(socket_t sock, time_t sec, time_t usec) { + return select_impl(sock, sec, usec); +} + +inline ssize_t select_write(socket_t sock, time_t sec, time_t usec) { + return select_impl(sock, sec, usec); +} + +inline Error wait_until_socket_is_ready(socket_t sock, time_t sec, + time_t usec) { +#ifdef __APPLE__ + if (sock >= FD_SETSIZE) { return Error::Connection; } + + fd_set fdsr, fdsw; + FD_ZERO(&fdsr); + FD_ZERO(&fdsw); + FD_SET(sock, &fdsr); + FD_SET(sock, &fdsw); + + timeval tv; + tv.tv_sec = static_cast(sec); + tv.tv_usec = static_cast(usec); + + auto ret = handle_EINTR([&]() { + return select(static_cast(sock + 1), &fdsr, &fdsw, nullptr, &tv); + }); + + if (ret == 0) { return Error::ConnectionTimeout; } + + if (ret > 0 && (FD_ISSET(sock, &fdsr) || FD_ISSET(sock, &fdsw))) { + auto error = 0; + socklen_t len = sizeof(error); + auto res = getsockopt(sock, SOL_SOCKET, SO_ERROR, + reinterpret_cast(&error), &len); + auto successful = res >= 0 && !error; + return successful ? Error::Success : Error::Connection; + } + + return Error::Connection; +#else + struct pollfd pfd_read; + pfd_read.fd = sock; + pfd_read.events = POLLIN | POLLOUT; + + auto timeout = static_cast(sec * 1000 + usec / 1000); + + auto poll_res = + handle_EINTR([&]() { return poll_wrapper(&pfd_read, 1, timeout); }); + + if (poll_res == 0) { return Error::ConnectionTimeout; } + + if (poll_res > 0 && pfd_read.revents & (POLLIN | POLLOUT)) { + auto error = 0; + socklen_t len = sizeof(error); + auto res = getsockopt(sock, SOL_SOCKET, SO_ERROR, + reinterpret_cast(&error), &len); + auto successful = res >= 0 && !error; + return successful ? Error::Success : Error::Connection; + } + + return Error::Connection; +#endif +} + +inline bool is_socket_alive(socket_t sock) { + const auto val = detail::select_read(sock, 0, 0); + if (val == 0) { + return true; + } else if (val < 0 && errno == EBADF) { + return false; + } + char buf[1]; + return detail::read_socket(sock, &buf[0], sizeof(buf), MSG_PEEK) > 0; +} + +class SocketStream final : public Stream { +public: + SocketStream(socket_t sock, time_t read_timeout_sec, time_t read_timeout_usec, + time_t write_timeout_sec, time_t write_timeout_usec, + time_t max_timeout_msec = 0, + std::chrono::time_point start_time = + (std::chrono::steady_clock::time_point::min)()); + ~SocketStream() override; + + bool is_readable() const override; + bool wait_readable() const override; + bool wait_writable() const override; + ssize_t read(char *ptr, size_t size) override; + ssize_t write(const char *ptr, size_t size) override; + void get_remote_ip_and_port(std::string &ip, int &port) const override; + void get_local_ip_and_port(std::string &ip, int &port) const override; + socket_t socket() const override; + time_t duration() const override; + +private: + socket_t sock_; + time_t read_timeout_sec_; + time_t read_timeout_usec_; + time_t write_timeout_sec_; + time_t write_timeout_usec_; + time_t max_timeout_msec_; + const std::chrono::time_point start_time_; + + std::vector read_buff_; + size_t read_buff_off_ = 0; + size_t read_buff_content_size_ = 0; + + static const size_t read_buff_size_ = 1024l * 4; +}; + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +class SSLSocketStream final : public Stream { +public: + SSLSocketStream( + socket_t sock, SSL *ssl, time_t read_timeout_sec, + time_t read_timeout_usec, time_t write_timeout_sec, + time_t write_timeout_usec, time_t max_timeout_msec = 0, + std::chrono::time_point start_time = + (std::chrono::steady_clock::time_point::min)()); + ~SSLSocketStream() override; + + bool is_readable() const override; + bool wait_readable() const override; + bool wait_writable() const override; + ssize_t read(char *ptr, size_t size) override; + ssize_t write(const char *ptr, size_t size) override; + void get_remote_ip_and_port(std::string &ip, int &port) const override; + void get_local_ip_and_port(std::string &ip, int &port) const override; + socket_t socket() const override; + time_t duration() const override; + +private: + socket_t sock_; + SSL *ssl_; + time_t read_timeout_sec_; + time_t read_timeout_usec_; + time_t write_timeout_sec_; + time_t write_timeout_usec_; + time_t max_timeout_msec_; + const std::chrono::time_point start_time_; +}; +#endif + +inline bool keep_alive(const std::atomic &svr_sock, socket_t sock, + time_t keep_alive_timeout_sec) { + using namespace std::chrono; + + const auto interval_usec = + CPPHTTPLIB_KEEPALIVE_TIMEOUT_CHECK_INTERVAL_USECOND; + + // Avoid expensive `steady_clock::now()` call for the first time + if (select_read(sock, 0, interval_usec) > 0) { return true; } + + const auto start = steady_clock::now() - microseconds{interval_usec}; + const auto timeout = seconds{keep_alive_timeout_sec}; + + while (true) { + if (svr_sock == INVALID_SOCKET) { + break; // Server socket is closed + } + + auto val = select_read(sock, 0, interval_usec); + if (val < 0) { + break; // Ssocket error + } else if (val == 0) { + if (steady_clock::now() - start > timeout) { + break; // Timeout + } + } else { + return true; // Ready for read + } + } + + return false; +} + +template +inline bool +process_server_socket_core(const std::atomic &svr_sock, socket_t sock, + size_t keep_alive_max_count, + time_t keep_alive_timeout_sec, T callback) { + assert(keep_alive_max_count > 0); + auto ret = false; + auto count = keep_alive_max_count; + while (count > 0 && keep_alive(svr_sock, sock, keep_alive_timeout_sec)) { + auto close_connection = count == 1; + auto connection_closed = false; + ret = callback(close_connection, connection_closed); + if (!ret || connection_closed) { break; } + count--; + } + return ret; +} + +template +inline bool +process_server_socket(const std::atomic &svr_sock, socket_t sock, + size_t keep_alive_max_count, + time_t keep_alive_timeout_sec, time_t read_timeout_sec, + time_t read_timeout_usec, time_t write_timeout_sec, + time_t write_timeout_usec, T callback) { + return process_server_socket_core( + svr_sock, sock, keep_alive_max_count, keep_alive_timeout_sec, + [&](bool close_connection, bool &connection_closed) { + SocketStream strm(sock, read_timeout_sec, read_timeout_usec, + write_timeout_sec, write_timeout_usec); + return callback(strm, close_connection, connection_closed); + }); +} + +inline bool process_client_socket( + socket_t sock, time_t read_timeout_sec, time_t read_timeout_usec, + time_t write_timeout_sec, time_t write_timeout_usec, + time_t max_timeout_msec, + std::chrono::time_point start_time, + std::function callback) { + SocketStream strm(sock, read_timeout_sec, read_timeout_usec, + write_timeout_sec, write_timeout_usec, max_timeout_msec, + start_time); + return callback(strm); +} + +inline int shutdown_socket(socket_t sock) { +#ifdef _WIN64 + return shutdown(sock, SD_BOTH); +#else + return shutdown(sock, SHUT_RDWR); +#endif +} + +inline std::string escape_abstract_namespace_unix_domain(const std::string &s) { + if (s.size() > 1 && s[0] == '\0') { + auto ret = s; + ret[0] = '@'; + return ret; + } + return s; +} + +inline std::string +unescape_abstract_namespace_unix_domain(const std::string &s) { + if (s.size() > 1 && s[0] == '@') { + auto ret = s; + ret[0] = '\0'; + return ret; + } + return s; +} + +inline int getaddrinfo_with_timeout(const char *node, const char *service, + const struct addrinfo *hints, + struct addrinfo **res, time_t timeout_sec) { +#ifdef CPPHTTPLIB_USE_NON_BLOCKING_GETADDRINFO + if (timeout_sec <= 0) { + // No timeout specified, use standard getaddrinfo + return getaddrinfo(node, service, hints, res); + } + +#ifdef _WIN64 + // Windows-specific implementation using GetAddrInfoEx with overlapped I/O + OVERLAPPED overlapped = {0}; + HANDLE event = CreateEventW(nullptr, TRUE, FALSE, nullptr); + if (!event) { return EAI_FAIL; } + + overlapped.hEvent = event; + + PADDRINFOEXW result_addrinfo = nullptr; + HANDLE cancel_handle = nullptr; + + ADDRINFOEXW hints_ex = {0}; + if (hints) { + hints_ex.ai_flags = hints->ai_flags; + hints_ex.ai_family = hints->ai_family; + hints_ex.ai_socktype = hints->ai_socktype; + hints_ex.ai_protocol = hints->ai_protocol; + } + + auto wnode = u8string_to_wstring(node); + auto wservice = u8string_to_wstring(service); + + auto ret = ::GetAddrInfoExW(wnode.data(), wservice.data(), NS_DNS, nullptr, + hints ? &hints_ex : nullptr, &result_addrinfo, + nullptr, &overlapped, nullptr, &cancel_handle); + + if (ret == WSA_IO_PENDING) { + auto wait_result = + ::WaitForSingleObject(event, static_cast(timeout_sec * 1000)); + if (wait_result == WAIT_TIMEOUT) { + if (cancel_handle) { ::GetAddrInfoExCancel(&cancel_handle); } + ::CloseHandle(event); + return EAI_AGAIN; + } + + DWORD bytes_returned; + if (!::GetOverlappedResult((HANDLE)INVALID_SOCKET, &overlapped, + &bytes_returned, FALSE)) { + ::CloseHandle(event); + return ::WSAGetLastError(); + } + } + + ::CloseHandle(event); + + if (ret == NO_ERROR || ret == WSA_IO_PENDING) { + *res = reinterpret_cast(result_addrinfo); + return 0; + } + + return ret; +#elif defined(TARGET_OS_OSX) + // macOS implementation using CFHost API for asynchronous DNS resolution + CFStringRef hostname_ref = CFStringCreateWithCString( + kCFAllocatorDefault, node, kCFStringEncodingUTF8); + if (!hostname_ref) { return EAI_MEMORY; } + + CFHostRef host_ref = CFHostCreateWithName(kCFAllocatorDefault, hostname_ref); + CFRelease(hostname_ref); + if (!host_ref) { return EAI_MEMORY; } + + // Set up context for callback + struct CFHostContext { + bool completed = false; + bool success = false; + CFArrayRef addresses = nullptr; + std::mutex mutex; + std::condition_variable cv; + } context; + + CFHostClientContext client_context; + memset(&client_context, 0, sizeof(client_context)); + client_context.info = &context; + + // Set callback + auto callback = [](CFHostRef theHost, CFHostInfoType /*typeInfo*/, + const CFStreamError *error, void *info) { + auto ctx = static_cast(info); + std::lock_guard lock(ctx->mutex); + + if (error && error->error != 0) { + ctx->success = false; + } else { + Boolean hasBeenResolved; + ctx->addresses = CFHostGetAddressing(theHost, &hasBeenResolved); + if (ctx->addresses && hasBeenResolved) { + CFRetain(ctx->addresses); + ctx->success = true; + } else { + ctx->success = false; + } + } + ctx->completed = true; + ctx->cv.notify_one(); + }; + + if (!CFHostSetClient(host_ref, callback, &client_context)) { + CFRelease(host_ref); + return EAI_SYSTEM; + } + + // Schedule on run loop + CFRunLoopRef run_loop = CFRunLoopGetCurrent(); + CFHostScheduleWithRunLoop(host_ref, run_loop, kCFRunLoopDefaultMode); + + // Start resolution + CFStreamError stream_error; + if (!CFHostStartInfoResolution(host_ref, kCFHostAddresses, &stream_error)) { + CFHostUnscheduleFromRunLoop(host_ref, run_loop, kCFRunLoopDefaultMode); + CFRelease(host_ref); + return EAI_FAIL; + } + + // Wait for completion with timeout + auto timeout_time = + std::chrono::steady_clock::now() + std::chrono::seconds(timeout_sec); + bool timed_out = false; + + { + std::unique_lock lock(context.mutex); + + while (!context.completed) { + auto now = std::chrono::steady_clock::now(); + if (now >= timeout_time) { + timed_out = true; + break; + } + + // Run the runloop for a short time + lock.unlock(); + CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, true); + lock.lock(); + } + } + + // Clean up + CFHostUnscheduleFromRunLoop(host_ref, run_loop, kCFRunLoopDefaultMode); + CFHostSetClient(host_ref, nullptr, nullptr); + + if (timed_out || !context.completed) { + CFHostCancelInfoResolution(host_ref, kCFHostAddresses); + CFRelease(host_ref); + return EAI_AGAIN; + } + + if (!context.success || !context.addresses) { + CFRelease(host_ref); + return EAI_NODATA; + } + + // Convert CFArray to addrinfo + CFIndex count = CFArrayGetCount(context.addresses); + if (count == 0) { + CFRelease(context.addresses); + CFRelease(host_ref); + return EAI_NODATA; + } + + struct addrinfo *result_addrinfo = nullptr; + struct addrinfo **current = &result_addrinfo; + + for (CFIndex i = 0; i < count; i++) { + CFDataRef addr_data = + static_cast(CFArrayGetValueAtIndex(context.addresses, i)); + if (!addr_data) continue; + + const struct sockaddr *sockaddr_ptr = + reinterpret_cast(CFDataGetBytePtr(addr_data)); + socklen_t sockaddr_len = static_cast(CFDataGetLength(addr_data)); + + // Allocate addrinfo structure + *current = static_cast(malloc(sizeof(struct addrinfo))); + if (!*current) { + freeaddrinfo(result_addrinfo); + CFRelease(context.addresses); + CFRelease(host_ref); + return EAI_MEMORY; + } + + memset(*current, 0, sizeof(struct addrinfo)); + + // Set up addrinfo fields + (*current)->ai_family = sockaddr_ptr->sa_family; + (*current)->ai_socktype = hints ? hints->ai_socktype : SOCK_STREAM; + (*current)->ai_protocol = hints ? hints->ai_protocol : IPPROTO_TCP; + (*current)->ai_addrlen = sockaddr_len; + + // Copy sockaddr + (*current)->ai_addr = static_cast(malloc(sockaddr_len)); + if (!(*current)->ai_addr) { + freeaddrinfo(result_addrinfo); + CFRelease(context.addresses); + CFRelease(host_ref); + return EAI_MEMORY; + } + memcpy((*current)->ai_addr, sockaddr_ptr, sockaddr_len); + + // Set port if service is specified + if (service && strlen(service) > 0) { + int port = atoi(service); + if (port > 0) { + if (sockaddr_ptr->sa_family == AF_INET) { + reinterpret_cast((*current)->ai_addr) + ->sin_port = htons(static_cast(port)); + } else if (sockaddr_ptr->sa_family == AF_INET6) { + reinterpret_cast((*current)->ai_addr) + ->sin6_port = htons(static_cast(port)); + } + } + } + + current = &((*current)->ai_next); + } + + CFRelease(context.addresses); + CFRelease(host_ref); + + *res = result_addrinfo; + return 0; +#elif defined(_GNU_SOURCE) && defined(__GLIBC__) && \ + (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 2)) + // Linux implementation using getaddrinfo_a for asynchronous DNS resolution + struct gaicb request; + struct gaicb *requests[1] = {&request}; + struct sigevent sevp; + struct timespec timeout; + + // Initialize the request structure + memset(&request, 0, sizeof(request)); + request.ar_name = node; + request.ar_service = service; + request.ar_request = hints; + + // Set up timeout + timeout.tv_sec = timeout_sec; + timeout.tv_nsec = 0; + + // Initialize sigevent structure (not used, but required) + memset(&sevp, 0, sizeof(sevp)); + sevp.sigev_notify = SIGEV_NONE; + + // Start asynchronous resolution + int start_result = getaddrinfo_a(GAI_NOWAIT, requests, 1, &sevp); + if (start_result != 0) { return start_result; } + + // Wait for completion with timeout + int wait_result = + gai_suspend((const struct gaicb *const *)requests, 1, &timeout); + + if (wait_result == 0) { + // Completed successfully, get the result + int gai_result = gai_error(&request); + if (gai_result == 0) { + *res = request.ar_result; + return 0; + } else { + // Clean up on error + if (request.ar_result) { freeaddrinfo(request.ar_result); } + return gai_result; + } + } else if (wait_result == EAI_AGAIN) { + // Timeout occurred, cancel the request + gai_cancel(&request); + return EAI_AGAIN; + } else { + // Other error occurred + gai_cancel(&request); + return wait_result; + } +#else + // Fallback implementation using thread-based timeout for other Unix systems + std::mutex result_mutex; + std::condition_variable result_cv; + auto completed = false; + auto result = EAI_SYSTEM; + struct addrinfo *result_addrinfo = nullptr; + + std::thread resolve_thread([&]() { + auto thread_result = getaddrinfo(node, service, hints, &result_addrinfo); + + std::lock_guard lock(result_mutex); + result = thread_result; + completed = true; + result_cv.notify_one(); + }); + + // Wait for completion or timeout + std::unique_lock lock(result_mutex); + auto finished = result_cv.wait_for(lock, std::chrono::seconds(timeout_sec), + [&] { return completed; }); + + if (finished) { + // Operation completed within timeout + resolve_thread.join(); + *res = result_addrinfo; + return result; + } else { + // Timeout occurred + resolve_thread.detach(); // Let the thread finish in background + return EAI_AGAIN; // Return timeout error + } +#endif +#else + (void)(timeout_sec); // Unused parameter for non-blocking getaddrinfo + return getaddrinfo(node, service, hints, res); +#endif +} + +template +socket_t create_socket(const std::string &host, const std::string &ip, int port, + int address_family, int socket_flags, bool tcp_nodelay, + bool ipv6_v6only, SocketOptions socket_options, + BindOrConnect bind_or_connect, time_t timeout_sec = 0) { + // Get address info + const char *node = nullptr; + struct addrinfo hints; + struct addrinfo *result; + + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_socktype = SOCK_STREAM; + hints.ai_protocol = IPPROTO_IP; + + if (!ip.empty()) { + node = ip.c_str(); + // Ask getaddrinfo to convert IP in c-string to address + hints.ai_family = AF_UNSPEC; + hints.ai_flags = AI_NUMERICHOST; + } else { + if (!host.empty()) { node = host.c_str(); } + hints.ai_family = address_family; + hints.ai_flags = socket_flags; + } + +#if !defined(_WIN64) || defined(CPPHTTPLIB_HAVE_AFUNIX_H) + if (hints.ai_family == AF_UNIX) { + const auto addrlen = host.length(); + if (addrlen > sizeof(sockaddr_un::sun_path)) { return INVALID_SOCKET; } + +#ifdef SOCK_CLOEXEC + auto sock = socket(hints.ai_family, hints.ai_socktype | SOCK_CLOEXEC, + hints.ai_protocol); +#else + auto sock = socket(hints.ai_family, hints.ai_socktype, hints.ai_protocol); +#endif + + if (sock != INVALID_SOCKET) { + sockaddr_un addr{}; + addr.sun_family = AF_UNIX; + + auto unescaped_host = unescape_abstract_namespace_unix_domain(host); + std::copy(unescaped_host.begin(), unescaped_host.end(), addr.sun_path); + + hints.ai_addr = reinterpret_cast(&addr); + hints.ai_addrlen = static_cast( + sizeof(addr) - sizeof(addr.sun_path) + addrlen); + +#ifndef SOCK_CLOEXEC +#ifndef _WIN64 + fcntl(sock, F_SETFD, FD_CLOEXEC); +#endif +#endif + + if (socket_options) { socket_options(sock); } + +#ifdef _WIN64 + // Setting SO_REUSEADDR seems not to work well with AF_UNIX on windows, so + // remove the option. + detail::set_socket_opt(sock, SOL_SOCKET, SO_REUSEADDR, 0); +#endif + + bool dummy; + if (!bind_or_connect(sock, hints, dummy)) { + close_socket(sock); + sock = INVALID_SOCKET; + } + } + return sock; + } +#endif + + auto service = std::to_string(port); + + if (getaddrinfo_with_timeout(node, service.c_str(), &hints, &result, + timeout_sec)) { +#if defined __linux__ && !defined __ANDROID__ + res_init(); +#endif + return INVALID_SOCKET; + } + auto se = detail::scope_exit([&] { freeaddrinfo(result); }); + + for (auto rp = result; rp; rp = rp->ai_next) { + // Create a socket +#ifdef _WIN64 + auto sock = + WSASocketW(rp->ai_family, rp->ai_socktype, rp->ai_protocol, nullptr, 0, + WSA_FLAG_NO_HANDLE_INHERIT | WSA_FLAG_OVERLAPPED); + /** + * Since the WSA_FLAG_NO_HANDLE_INHERIT is only supported on Windows 7 SP1 + * and above the socket creation fails on older Windows Systems. + * + * Let's try to create a socket the old way in this case. + * + * Reference: + * https://docs.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-wsasocketa + * + * WSA_FLAG_NO_HANDLE_INHERIT: + * This flag is supported on Windows 7 with SP1, Windows Server 2008 R2 with + * SP1, and later + * + */ + if (sock == INVALID_SOCKET) { + sock = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); + } +#else + +#ifdef SOCK_CLOEXEC + auto sock = + socket(rp->ai_family, rp->ai_socktype | SOCK_CLOEXEC, rp->ai_protocol); +#else + auto sock = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); +#endif + +#endif + if (sock == INVALID_SOCKET) { continue; } + +#if !defined _WIN64 && !defined SOCK_CLOEXEC + if (fcntl(sock, F_SETFD, FD_CLOEXEC) == -1) { + close_socket(sock); + continue; + } +#endif + + if (tcp_nodelay) { set_socket_opt(sock, IPPROTO_TCP, TCP_NODELAY, 1); } + + if (rp->ai_family == AF_INET6) { + set_socket_opt(sock, IPPROTO_IPV6, IPV6_V6ONLY, ipv6_v6only ? 1 : 0); + } + + if (socket_options) { socket_options(sock); } + + // bind or connect + auto quit = false; + if (bind_or_connect(sock, *rp, quit)) { return sock; } + + close_socket(sock); + + if (quit) { break; } + } + + return INVALID_SOCKET; +} + +inline void set_nonblocking(socket_t sock, bool nonblocking) { +#ifdef _WIN64 + auto flags = nonblocking ? 1UL : 0UL; + ioctlsocket(sock, FIONBIO, &flags); +#else + auto flags = fcntl(sock, F_GETFL, 0); + fcntl(sock, F_SETFL, + nonblocking ? (flags | O_NONBLOCK) : (flags & (~O_NONBLOCK))); +#endif +} + +inline bool is_connection_error() { +#ifdef _WIN64 + return WSAGetLastError() != WSAEWOULDBLOCK; +#else + return errno != EINPROGRESS; +#endif +} + +inline bool bind_ip_address(socket_t sock, const std::string &host) { + struct addrinfo hints; + struct addrinfo *result; + + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_protocol = 0; + + if (getaddrinfo_with_timeout(host.c_str(), "0", &hints, &result, 0)) { + return false; + } + + auto se = detail::scope_exit([&] { freeaddrinfo(result); }); + + auto ret = false; + for (auto rp = result; rp; rp = rp->ai_next) { + const auto &ai = *rp; + if (!::bind(sock, ai.ai_addr, static_cast(ai.ai_addrlen))) { + ret = true; + break; + } + } + + return ret; +} + +#if !defined _WIN64 && !defined ANDROID && !defined _AIX && !defined __MVS__ +#define USE_IF2IP +#endif + +#ifdef USE_IF2IP +inline std::string if2ip(int address_family, const std::string &ifn) { + struct ifaddrs *ifap; + getifaddrs(&ifap); + auto se = detail::scope_exit([&] { freeifaddrs(ifap); }); + + std::string addr_candidate; + for (auto ifa = ifap; ifa; ifa = ifa->ifa_next) { + if (ifa->ifa_addr && ifn == ifa->ifa_name && + (AF_UNSPEC == address_family || + ifa->ifa_addr->sa_family == address_family)) { + if (ifa->ifa_addr->sa_family == AF_INET) { + auto sa = reinterpret_cast(ifa->ifa_addr); + char buf[INET_ADDRSTRLEN]; + if (inet_ntop(AF_INET, &sa->sin_addr, buf, INET_ADDRSTRLEN)) { + return std::string(buf, INET_ADDRSTRLEN); + } + } else if (ifa->ifa_addr->sa_family == AF_INET6) { + auto sa = reinterpret_cast(ifa->ifa_addr); + if (!IN6_IS_ADDR_LINKLOCAL(&sa->sin6_addr)) { + char buf[INET6_ADDRSTRLEN] = {}; + if (inet_ntop(AF_INET6, &sa->sin6_addr, buf, INET6_ADDRSTRLEN)) { + // equivalent to mac's IN6_IS_ADDR_UNIQUE_LOCAL + auto s6_addr_head = sa->sin6_addr.s6_addr[0]; + if (s6_addr_head == 0xfc || s6_addr_head == 0xfd) { + addr_candidate = std::string(buf, INET6_ADDRSTRLEN); + } else { + return std::string(buf, INET6_ADDRSTRLEN); + } + } + } + } + } + } + return addr_candidate; +} +#endif + +inline socket_t create_client_socket( + const std::string &host, const std::string &ip, int port, + int address_family, bool tcp_nodelay, bool ipv6_v6only, + SocketOptions socket_options, time_t connection_timeout_sec, + time_t connection_timeout_usec, time_t read_timeout_sec, + time_t read_timeout_usec, time_t write_timeout_sec, + time_t write_timeout_usec, const std::string &intf, Error &error) { + auto sock = create_socket( + host, ip, port, address_family, 0, tcp_nodelay, ipv6_v6only, + std::move(socket_options), + [&](socket_t sock2, struct addrinfo &ai, bool &quit) -> bool { + if (!intf.empty()) { +#ifdef USE_IF2IP + auto ip_from_if = if2ip(address_family, intf); + if (ip_from_if.empty()) { ip_from_if = intf; } + if (!bind_ip_address(sock2, ip_from_if)) { + error = Error::BindIPAddress; + return false; + } +#endif + } + + set_nonblocking(sock2, true); + + auto ret = + ::connect(sock2, ai.ai_addr, static_cast(ai.ai_addrlen)); + + if (ret < 0) { + if (is_connection_error()) { + error = Error::Connection; + return false; + } + error = wait_until_socket_is_ready(sock2, connection_timeout_sec, + connection_timeout_usec); + if (error != Error::Success) { + if (error == Error::ConnectionTimeout) { quit = true; } + return false; + } + } + + set_nonblocking(sock2, false); + set_socket_opt_time(sock2, SOL_SOCKET, SO_RCVTIMEO, read_timeout_sec, + read_timeout_usec); + set_socket_opt_time(sock2, SOL_SOCKET, SO_SNDTIMEO, write_timeout_sec, + write_timeout_usec); + + error = Error::Success; + return true; + }, + connection_timeout_sec); // Pass DNS timeout + + if (sock != INVALID_SOCKET) { + error = Error::Success; + } else { + if (error == Error::Success) { error = Error::Connection; } + } + + return sock; +} + +inline bool get_ip_and_port(const struct sockaddr_storage &addr, + socklen_t addr_len, std::string &ip, int &port) { + if (addr.ss_family == AF_INET) { + port = ntohs(reinterpret_cast(&addr)->sin_port); + } else if (addr.ss_family == AF_INET6) { + port = + ntohs(reinterpret_cast(&addr)->sin6_port); + } else { + return false; + } + + std::array ipstr{}; + if (getnameinfo(reinterpret_cast(&addr), addr_len, + ipstr.data(), static_cast(ipstr.size()), nullptr, + 0, NI_NUMERICHOST)) { + return false; + } + + ip = ipstr.data(); + return true; +} + +inline void get_local_ip_and_port(socket_t sock, std::string &ip, int &port) { + struct sockaddr_storage addr; + socklen_t addr_len = sizeof(addr); + if (!getsockname(sock, reinterpret_cast(&addr), + &addr_len)) { + get_ip_and_port(addr, addr_len, ip, port); + } +} + +inline void get_remote_ip_and_port(socket_t sock, std::string &ip, int &port) { + struct sockaddr_storage addr; + socklen_t addr_len = sizeof(addr); + + if (!getpeername(sock, reinterpret_cast(&addr), + &addr_len)) { +#ifndef _WIN64 + if (addr.ss_family == AF_UNIX) { +#if defined(__linux__) + struct ucred ucred; + socklen_t len = sizeof(ucred); + if (getsockopt(sock, SOL_SOCKET, SO_PEERCRED, &ucred, &len) == 0) { + port = ucred.pid; + } +#elif defined(SOL_LOCAL) && defined(SO_PEERPID) + pid_t pid; + socklen_t len = sizeof(pid); + if (getsockopt(sock, SOL_LOCAL, SO_PEERPID, &pid, &len) == 0) { + port = pid; + } +#endif + return; + } +#endif + get_ip_and_port(addr, addr_len, ip, port); + } +} + +inline constexpr unsigned int str2tag_core(const char *s, size_t l, + unsigned int h) { + return (l == 0) + ? h + : str2tag_core( + s + 1, l - 1, + // Unsets the 6 high bits of h, therefore no overflow happens + (((std::numeric_limits::max)() >> 6) & + h * 33) ^ + static_cast(*s)); +} + +inline unsigned int str2tag(const std::string &s) { + return str2tag_core(s.data(), s.size(), 0); +} + +namespace udl { + +inline constexpr unsigned int operator""_t(const char *s, size_t l) { + return str2tag_core(s, l, 0); +} + +} // namespace udl + +inline std::string +find_content_type(const std::string &path, + const std::map &user_data, + const std::string &default_content_type) { + auto ext = file_extension(path); + + auto it = user_data.find(ext); + if (it != user_data.end()) { return it->second; } + + using udl::operator""_t; + + switch (str2tag(ext)) { + default: return default_content_type; + + case "css"_t: return "text/css"; + case "csv"_t: return "text/csv"; + case "htm"_t: + case "html"_t: return "text/html"; + case "js"_t: + case "mjs"_t: return "text/javascript"; + case "txt"_t: return "text/plain"; + case "vtt"_t: return "text/vtt"; + + case "apng"_t: return "image/apng"; + case "avif"_t: return "image/avif"; + case "bmp"_t: return "image/bmp"; + case "gif"_t: return "image/gif"; + case "png"_t: return "image/png"; + case "svg"_t: return "image/svg+xml"; + case "webp"_t: return "image/webp"; + case "ico"_t: return "image/x-icon"; + case "tif"_t: return "image/tiff"; + case "tiff"_t: return "image/tiff"; + case "jpg"_t: + case "jpeg"_t: return "image/jpeg"; + + case "mp4"_t: return "video/mp4"; + case "mpeg"_t: return "video/mpeg"; + case "webm"_t: return "video/webm"; + + case "mp3"_t: return "audio/mp3"; + case "mpga"_t: return "audio/mpeg"; + case "weba"_t: return "audio/webm"; + case "wav"_t: return "audio/wave"; + + case "otf"_t: return "font/otf"; + case "ttf"_t: return "font/ttf"; + case "woff"_t: return "font/woff"; + case "woff2"_t: return "font/woff2"; + + case "7z"_t: return "application/x-7z-compressed"; + case "atom"_t: return "application/atom+xml"; + case "pdf"_t: return "application/pdf"; + case "json"_t: return "application/json"; + case "rss"_t: return "application/rss+xml"; + case "tar"_t: return "application/x-tar"; + case "xht"_t: + case "xhtml"_t: return "application/xhtml+xml"; + case "xslt"_t: return "application/xslt+xml"; + case "xml"_t: return "application/xml"; + case "gz"_t: return "application/gzip"; + case "zip"_t: return "application/zip"; + case "wasm"_t: return "application/wasm"; + } +} + +inline bool can_compress_content_type(const std::string &content_type) { + using udl::operator""_t; + + auto tag = str2tag(content_type); + + switch (tag) { + case "image/svg+xml"_t: + case "application/javascript"_t: + case "application/json"_t: + case "application/xml"_t: + case "application/protobuf"_t: + case "application/xhtml+xml"_t: return true; + + case "text/event-stream"_t: return false; + + default: return !content_type.rfind("text/", 0); + } +} + +inline EncodingType encoding_type(const Request &req, const Response &res) { + auto ret = + detail::can_compress_content_type(res.get_header_value("Content-Type")); + if (!ret) { return EncodingType::None; } + + const auto &s = req.get_header_value("Accept-Encoding"); + (void)(s); + +#ifdef CPPHTTPLIB_BROTLI_SUPPORT + // TODO: 'Accept-Encoding' has br, not br;q=0 + ret = s.find("br") != std::string::npos; + if (ret) { return EncodingType::Brotli; } +#endif + +#ifdef CPPHTTPLIB_ZLIB_SUPPORT + // TODO: 'Accept-Encoding' has gzip, not gzip;q=0 + ret = s.find("gzip") != std::string::npos; + if (ret) { return EncodingType::Gzip; } +#endif + +#ifdef CPPHTTPLIB_ZSTD_SUPPORT + // TODO: 'Accept-Encoding' has zstd, not zstd;q=0 + ret = s.find("zstd") != std::string::npos; + if (ret) { return EncodingType::Zstd; } +#endif + + return EncodingType::None; +} + +inline bool nocompressor::compress(const char *data, size_t data_length, + bool /*last*/, Callback callback) { + if (!data_length) { return true; } + return callback(data, data_length); +} + +#ifdef CPPHTTPLIB_ZLIB_SUPPORT +inline gzip_compressor::gzip_compressor() { + std::memset(&strm_, 0, sizeof(strm_)); + strm_.zalloc = Z_NULL; + strm_.zfree = Z_NULL; + strm_.opaque = Z_NULL; + + is_valid_ = deflateInit2(&strm_, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 31, 8, + Z_DEFAULT_STRATEGY) == Z_OK; +} + +inline gzip_compressor::~gzip_compressor() { deflateEnd(&strm_); } + +inline bool gzip_compressor::compress(const char *data, size_t data_length, + bool last, Callback callback) { + assert(is_valid_); + + do { + constexpr size_t max_avail_in = + (std::numeric_limits::max)(); + + strm_.avail_in = static_cast( + (std::min)(data_length, max_avail_in)); + strm_.next_in = const_cast(reinterpret_cast(data)); + + data_length -= strm_.avail_in; + data += strm_.avail_in; + + auto flush = (last && data_length == 0) ? Z_FINISH : Z_NO_FLUSH; + auto ret = Z_OK; + + std::array buff{}; + do { + strm_.avail_out = static_cast(buff.size()); + strm_.next_out = reinterpret_cast(buff.data()); + + ret = deflate(&strm_, flush); + if (ret == Z_STREAM_ERROR) { return false; } + + if (!callback(buff.data(), buff.size() - strm_.avail_out)) { + return false; + } + } while (strm_.avail_out == 0); + + assert((flush == Z_FINISH && ret == Z_STREAM_END) || + (flush == Z_NO_FLUSH && ret == Z_OK)); + assert(strm_.avail_in == 0); + } while (data_length > 0); + + return true; +} + +inline gzip_decompressor::gzip_decompressor() { + std::memset(&strm_, 0, sizeof(strm_)); + strm_.zalloc = Z_NULL; + strm_.zfree = Z_NULL; + strm_.opaque = Z_NULL; + + // 15 is the value of wbits, which should be at the maximum possible value + // to ensure that any gzip stream can be decoded. The offset of 32 specifies + // that the stream type should be automatically detected either gzip or + // deflate. + is_valid_ = inflateInit2(&strm_, 32 + 15) == Z_OK; +} + +inline gzip_decompressor::~gzip_decompressor() { inflateEnd(&strm_); } + +inline bool gzip_decompressor::is_valid() const { return is_valid_; } + +inline bool gzip_decompressor::decompress(const char *data, size_t data_length, + Callback callback) { + assert(is_valid_); + + auto ret = Z_OK; + + do { + constexpr size_t max_avail_in = + (std::numeric_limits::max)(); + + strm_.avail_in = static_cast( + (std::min)(data_length, max_avail_in)); + strm_.next_in = const_cast(reinterpret_cast(data)); + + data_length -= strm_.avail_in; + data += strm_.avail_in; + + std::array buff{}; + while (strm_.avail_in > 0 && ret == Z_OK) { + strm_.avail_out = static_cast(buff.size()); + strm_.next_out = reinterpret_cast(buff.data()); + + ret = inflate(&strm_, Z_NO_FLUSH); + + assert(ret != Z_STREAM_ERROR); + switch (ret) { + case Z_NEED_DICT: + case Z_DATA_ERROR: + case Z_MEM_ERROR: inflateEnd(&strm_); return false; + } + + if (!callback(buff.data(), buff.size() - strm_.avail_out)) { + return false; + } + } + + if (ret != Z_OK && ret != Z_STREAM_END) { return false; } + + } while (data_length > 0); + + return true; +} +#endif + +#ifdef CPPHTTPLIB_BROTLI_SUPPORT +inline brotli_compressor::brotli_compressor() { + state_ = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr); +} + +inline brotli_compressor::~brotli_compressor() { + BrotliEncoderDestroyInstance(state_); +} + +inline bool brotli_compressor::compress(const char *data, size_t data_length, + bool last, Callback callback) { + std::array buff{}; + + auto operation = last ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_PROCESS; + auto available_in = data_length; + auto next_in = reinterpret_cast(data); + + for (;;) { + if (last) { + if (BrotliEncoderIsFinished(state_)) { break; } + } else { + if (!available_in) { break; } + } + + auto available_out = buff.size(); + auto next_out = buff.data(); + + if (!BrotliEncoderCompressStream(state_, operation, &available_in, &next_in, + &available_out, &next_out, nullptr)) { + return false; + } + + auto output_bytes = buff.size() - available_out; + if (output_bytes) { + callback(reinterpret_cast(buff.data()), output_bytes); + } + } + + return true; +} + +inline brotli_decompressor::brotli_decompressor() { + decoder_s = BrotliDecoderCreateInstance(0, 0, 0); + decoder_r = decoder_s ? BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT + : BROTLI_DECODER_RESULT_ERROR; +} + +inline brotli_decompressor::~brotli_decompressor() { + if (decoder_s) { BrotliDecoderDestroyInstance(decoder_s); } +} + +inline bool brotli_decompressor::is_valid() const { return decoder_s; } + +inline bool brotli_decompressor::decompress(const char *data, + size_t data_length, + Callback callback) { + if (decoder_r == BROTLI_DECODER_RESULT_SUCCESS || + decoder_r == BROTLI_DECODER_RESULT_ERROR) { + return 0; + } + + auto next_in = reinterpret_cast(data); + size_t avail_in = data_length; + size_t total_out; + + decoder_r = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT; + + std::array buff{}; + while (decoder_r == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) { + char *next_out = buff.data(); + size_t avail_out = buff.size(); + + decoder_r = BrotliDecoderDecompressStream( + decoder_s, &avail_in, &next_in, &avail_out, + reinterpret_cast(&next_out), &total_out); + + if (decoder_r == BROTLI_DECODER_RESULT_ERROR) { return false; } + + if (!callback(buff.data(), buff.size() - avail_out)) { return false; } + } + + return decoder_r == BROTLI_DECODER_RESULT_SUCCESS || + decoder_r == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT; +} +#endif + +#ifdef CPPHTTPLIB_ZSTD_SUPPORT +inline zstd_compressor::zstd_compressor() { + ctx_ = ZSTD_createCCtx(); + ZSTD_CCtx_setParameter(ctx_, ZSTD_c_compressionLevel, ZSTD_fast); +} + +inline zstd_compressor::~zstd_compressor() { ZSTD_freeCCtx(ctx_); } + +inline bool zstd_compressor::compress(const char *data, size_t data_length, + bool last, Callback callback) { + std::array buff{}; + + ZSTD_EndDirective mode = last ? ZSTD_e_end : ZSTD_e_continue; + ZSTD_inBuffer input = {data, data_length, 0}; + + bool finished; + do { + ZSTD_outBuffer output = {buff.data(), CPPHTTPLIB_COMPRESSION_BUFSIZ, 0}; + size_t const remaining = ZSTD_compressStream2(ctx_, &output, &input, mode); + + if (ZSTD_isError(remaining)) { return false; } + + if (!callback(buff.data(), output.pos)) { return false; } + + finished = last ? (remaining == 0) : (input.pos == input.size); + + } while (!finished); + + return true; +} + +inline zstd_decompressor::zstd_decompressor() { ctx_ = ZSTD_createDCtx(); } + +inline zstd_decompressor::~zstd_decompressor() { ZSTD_freeDCtx(ctx_); } + +inline bool zstd_decompressor::is_valid() const { return ctx_ != nullptr; } + +inline bool zstd_decompressor::decompress(const char *data, size_t data_length, + Callback callback) { + std::array buff{}; + ZSTD_inBuffer input = {data, data_length, 0}; + + while (input.pos < input.size) { + ZSTD_outBuffer output = {buff.data(), CPPHTTPLIB_COMPRESSION_BUFSIZ, 0}; + size_t const remaining = ZSTD_decompressStream(ctx_, &output, &input); + + if (ZSTD_isError(remaining)) { return false; } + + if (!callback(buff.data(), output.pos)) { return false; } + } + + return true; +} +#endif + +inline bool has_header(const Headers &headers, const std::string &key) { + return headers.find(key) != headers.end(); +} + +inline const char *get_header_value(const Headers &headers, + const std::string &key, const char *def, + size_t id) { + auto rng = headers.equal_range(key); + auto it = rng.first; + std::advance(it, static_cast(id)); + if (it != rng.second) { return it->second.c_str(); } + return def; +} + +template +inline bool parse_header(const char *beg, const char *end, T fn) { + // Skip trailing spaces and tabs. + while (beg < end && is_space_or_tab(end[-1])) { + end--; + } + + auto p = beg; + while (p < end && *p != ':') { + p++; + } + + auto name = std::string(beg, p); + if (!detail::fields::is_field_name(name)) { return false; } + + if (p == end) { return false; } + + auto key_end = p; + + if (*p++ != ':') { return false; } + + while (p < end && is_space_or_tab(*p)) { + p++; + } + + if (p <= end) { + auto key_len = key_end - beg; + if (!key_len) { return false; } + + auto key = std::string(beg, key_end); + auto val = std::string(p, end); + + if (!detail::fields::is_field_value(val)) { return false; } + + if (case_ignore::equal(key, "Location") || + case_ignore::equal(key, "Referer")) { + fn(key, val); + } else { + fn(key, decode_path_component(val)); + } + + return true; + } + + return false; +} + +inline bool read_headers(Stream &strm, Headers &headers) { + const auto bufsiz = 2048; + char buf[bufsiz]; + stream_line_reader line_reader(strm, buf, bufsiz); + + size_t header_count = 0; + + for (;;) { + if (!line_reader.getline()) { return false; } + + // Check if the line ends with CRLF. + auto line_terminator_len = 2; + if (line_reader.end_with_crlf()) { + // Blank line indicates end of headers. + if (line_reader.size() == 2) { break; } + } else { +#ifdef CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR + // Blank line indicates end of headers. + if (line_reader.size() == 1) { break; } + line_terminator_len = 1; +#else + continue; // Skip invalid line. +#endif + } + + if (line_reader.size() > CPPHTTPLIB_HEADER_MAX_LENGTH) { return false; } + + // Check header count limit + if (header_count >= CPPHTTPLIB_HEADER_MAX_COUNT) { return false; } + + // Exclude line terminator + auto end = line_reader.ptr() + line_reader.size() - line_terminator_len; + + if (!parse_header(line_reader.ptr(), end, + [&](const std::string &key, const std::string &val) { + headers.emplace(key, val); + })) { + return false; + } + + header_count++; + } + + return true; +} + +inline bool read_content_with_length(Stream &strm, size_t len, + DownloadProgress progress, + ContentReceiverWithProgress out) { + char buf[CPPHTTPLIB_RECV_BUFSIZ]; + + size_t r = 0; + while (r < len) { + auto read_len = static_cast(len - r); + auto n = strm.read(buf, (std::min)(read_len, CPPHTTPLIB_RECV_BUFSIZ)); + if (n <= 0) { return false; } + + if (!out(buf, static_cast(n), r, len)) { return false; } + r += static_cast(n); + + if (progress) { + if (!progress(r, len)) { return false; } + } + } + + return true; +} + +inline void skip_content_with_length(Stream &strm, size_t len) { + char buf[CPPHTTPLIB_RECV_BUFSIZ]; + size_t r = 0; + while (r < len) { + auto read_len = static_cast(len - r); + auto n = strm.read(buf, (std::min)(read_len, CPPHTTPLIB_RECV_BUFSIZ)); + if (n <= 0) { return; } + r += static_cast(n); + } +} + +enum class ReadContentResult { + Success, // Successfully read the content + PayloadTooLarge, // The content exceeds the specified payload limit + Error // An error occurred while reading the content +}; + +inline ReadContentResult +read_content_without_length(Stream &strm, size_t payload_max_length, + ContentReceiverWithProgress out) { + char buf[CPPHTTPLIB_RECV_BUFSIZ]; + size_t r = 0; + for (;;) { + auto n = strm.read(buf, CPPHTTPLIB_RECV_BUFSIZ); + if (n == 0) { return ReadContentResult::Success; } + if (n < 0) { return ReadContentResult::Error; } + + // Check if adding this data would exceed the payload limit + if (r > payload_max_length || + payload_max_length - r < static_cast(n)) { + return ReadContentResult::PayloadTooLarge; + } + + if (!out(buf, static_cast(n), r, 0)) { + return ReadContentResult::Error; + } + r += static_cast(n); + } + + return ReadContentResult::Success; +} + +template +inline ReadContentResult read_content_chunked(Stream &strm, T &x, + size_t payload_max_length, + ContentReceiverWithProgress out) { + const auto bufsiz = 16; + char buf[bufsiz]; + + stream_line_reader line_reader(strm, buf, bufsiz); + + if (!line_reader.getline()) { return ReadContentResult::Error; } + + unsigned long chunk_len; + size_t total_len = 0; + while (true) { + char *end_ptr; + + chunk_len = std::strtoul(line_reader.ptr(), &end_ptr, 16); + + if (end_ptr == line_reader.ptr()) { return ReadContentResult::Error; } + if (chunk_len == ULONG_MAX) { return ReadContentResult::Error; } + + if (chunk_len == 0) { break; } + + // Check if adding this chunk would exceed the payload limit + if (total_len > payload_max_length || + payload_max_length - total_len < chunk_len) { + return ReadContentResult::PayloadTooLarge; + } + + total_len += chunk_len; + + if (!read_content_with_length(strm, chunk_len, nullptr, out)) { + return ReadContentResult::Error; + } + + if (!line_reader.getline()) { return ReadContentResult::Error; } + + if (strcmp(line_reader.ptr(), "\r\n") != 0) { + return ReadContentResult::Error; + } + + if (!line_reader.getline()) { return ReadContentResult::Error; } + } + + assert(chunk_len == 0); + + // NOTE: In RFC 9112, '7.1 Chunked Transfer Coding' mentions "The chunked + // transfer coding is complete when a chunk with a chunk-size of zero is + // received, possibly followed by a trailer section, and finally terminated by + // an empty line". https://www.rfc-editor.org/rfc/rfc9112.html#section-7.1 + // + // In '7.1.3. Decoding Chunked', however, the pseudo-code in the section + // does't care for the existence of the final CRLF. In other words, it seems + // to be ok whether the final CRLF exists or not in the chunked data. + // https://www.rfc-editor.org/rfc/rfc9112.html#section-7.1.3 + // + // According to the reference code in RFC 9112, cpp-httplib now allows + // chunked transfer coding data without the final CRLF. + if (!line_reader.getline()) { return ReadContentResult::Success; } + + // RFC 7230 Section 4.1.2 - Headers prohibited in trailers + thread_local case_ignore::unordered_set prohibited_trailers = { + // Message framing + "transfer-encoding", "content-length", + + // Routing + "host", + + // Authentication + "authorization", "www-authenticate", "proxy-authenticate", + "proxy-authorization", "cookie", "set-cookie", + + // Request modifiers + "cache-control", "expect", "max-forwards", "pragma", "range", "te", + + // Response control + "age", "expires", "date", "location", "retry-after", "vary", "warning", + + // Payload processing + "content-encoding", "content-type", "content-range", "trailer"}; + + // Parse declared trailer headers once for performance + case_ignore::unordered_set declared_trailers; + if (has_header(x.headers, "Trailer")) { + auto trailer_header = get_header_value(x.headers, "Trailer", "", 0); + auto len = std::strlen(trailer_header); + + split(trailer_header, trailer_header + len, ',', + [&](const char *b, const char *e) { + std::string key(b, e); + if (prohibited_trailers.find(key) == prohibited_trailers.end()) { + declared_trailers.insert(key); + } + }); + } + + size_t trailer_header_count = 0; + while (strcmp(line_reader.ptr(), "\r\n") != 0) { + if (line_reader.size() > CPPHTTPLIB_HEADER_MAX_LENGTH) { + return ReadContentResult::Error; + } + + // Check trailer header count limit + if (trailer_header_count >= CPPHTTPLIB_HEADER_MAX_COUNT) { + return ReadContentResult::Error; + } + + // Exclude line terminator + constexpr auto line_terminator_len = 2; + auto end = line_reader.ptr() + line_reader.size() - line_terminator_len; + + parse_header(line_reader.ptr(), end, + [&](const std::string &key, const std::string &val) { + if (declared_trailers.find(key) != declared_trailers.end()) { + x.trailers.emplace(key, val); + trailer_header_count++; + } + }); + + if (!line_reader.getline()) { return ReadContentResult::Error; } + } + + return ReadContentResult::Success; +} + +inline bool is_chunked_transfer_encoding(const Headers &headers) { + return case_ignore::equal( + get_header_value(headers, "Transfer-Encoding", "", 0), "chunked"); +} + +template +bool prepare_content_receiver(T &x, int &status, + ContentReceiverWithProgress receiver, + bool decompress, U callback) { + if (decompress) { + std::string encoding = x.get_header_value("Content-Encoding"); + std::unique_ptr decompressor; + + if (encoding == "gzip" || encoding == "deflate") { +#ifdef CPPHTTPLIB_ZLIB_SUPPORT + decompressor = detail::make_unique(); +#else + status = StatusCode::UnsupportedMediaType_415; + return false; +#endif + } else if (encoding.find("br") != std::string::npos) { +#ifdef CPPHTTPLIB_BROTLI_SUPPORT + decompressor = detail::make_unique(); +#else + status = StatusCode::UnsupportedMediaType_415; + return false; +#endif + } else if (encoding == "zstd") { +#ifdef CPPHTTPLIB_ZSTD_SUPPORT + decompressor = detail::make_unique(); +#else + status = StatusCode::UnsupportedMediaType_415; + return false; +#endif + } + + if (decompressor) { + if (decompressor->is_valid()) { + ContentReceiverWithProgress out = [&](const char *buf, size_t n, + size_t off, size_t len) { + return decompressor->decompress(buf, n, + [&](const char *buf2, size_t n2) { + return receiver(buf2, n2, off, len); + }); + }; + return callback(std::move(out)); + } else { + status = StatusCode::InternalServerError_500; + return false; + } + } + } + + ContentReceiverWithProgress out = [&](const char *buf, size_t n, size_t off, + size_t len) { + return receiver(buf, n, off, len); + }; + return callback(std::move(out)); +} + +template +bool read_content(Stream &strm, T &x, size_t payload_max_length, int &status, + DownloadProgress progress, + ContentReceiverWithProgress receiver, bool decompress) { + return prepare_content_receiver( + x, status, std::move(receiver), decompress, + [&](const ContentReceiverWithProgress &out) { + auto ret = true; + auto exceed_payload_max_length = false; + + if (is_chunked_transfer_encoding(x.headers)) { + auto result = read_content_chunked(strm, x, payload_max_length, out); + if (result == ReadContentResult::Success) { + ret = true; + } else if (result == ReadContentResult::PayloadTooLarge) { + exceed_payload_max_length = true; + ret = false; + } else { + ret = false; + } + } else if (!has_header(x.headers, "Content-Length")) { + auto result = + read_content_without_length(strm, payload_max_length, out); + if (result == ReadContentResult::Success) { + ret = true; + } else if (result == ReadContentResult::PayloadTooLarge) { + exceed_payload_max_length = true; + ret = false; + } else { + ret = false; + } + } else { + auto is_invalid_value = false; + auto len = get_header_value_u64(x.headers, "Content-Length", + (std::numeric_limits::max)(), + 0, is_invalid_value); + + if (is_invalid_value) { + ret = false; + } else if (len > payload_max_length) { + exceed_payload_max_length = true; + skip_content_with_length(strm, len); + ret = false; + } else if (len > 0) { + ret = read_content_with_length(strm, len, std::move(progress), out); + } + } + + if (!ret) { + status = exceed_payload_max_length ? StatusCode::PayloadTooLarge_413 + : StatusCode::BadRequest_400; + } + return ret; + }); +} + +inline ssize_t write_request_line(Stream &strm, const std::string &method, + const std::string &path) { + std::string s = method; + s += " "; + s += path; + s += " HTTP/1.1\r\n"; + return strm.write(s.data(), s.size()); +} + +inline ssize_t write_response_line(Stream &strm, int status) { + std::string s = "HTTP/1.1 "; + s += std::to_string(status); + s += " "; + s += httplib::status_message(status); + s += "\r\n"; + return strm.write(s.data(), s.size()); +} + +inline ssize_t write_headers(Stream &strm, const Headers &headers) { + ssize_t write_len = 0; + for (const auto &x : headers) { + std::string s; + s = x.first; + s += ": "; + s += x.second; + s += "\r\n"; + + auto len = strm.write(s.data(), s.size()); + if (len < 0) { return len; } + write_len += len; + } + auto len = strm.write("\r\n"); + if (len < 0) { return len; } + write_len += len; + return write_len; +} + +inline bool write_data(Stream &strm, const char *d, size_t l) { + size_t offset = 0; + while (offset < l) { + auto length = strm.write(d + offset, l - offset); + if (length < 0) { return false; } + offset += static_cast(length); + } + return true; +} + +template +inline bool write_content_with_progress(Stream &strm, + const ContentProvider &content_provider, + size_t offset, size_t length, + T is_shutting_down, + const UploadProgress &upload_progress, + Error &error) { + size_t end_offset = offset + length; + size_t start_offset = offset; + auto ok = true; + DataSink data_sink; + + data_sink.write = [&](const char *d, size_t l) -> bool { + if (ok) { + if (write_data(strm, d, l)) { + offset += l; + + if (upload_progress && length > 0) { + size_t current_written = offset - start_offset; + if (!upload_progress(current_written, length)) { + ok = false; + return false; + } + } + } else { + ok = false; + } + } + return ok; + }; + + data_sink.is_writable = [&]() -> bool { return strm.wait_writable(); }; + + while (offset < end_offset && !is_shutting_down()) { + if (!strm.wait_writable()) { + error = Error::Write; + return false; + } else if (!content_provider(offset, end_offset - offset, data_sink)) { + error = Error::Canceled; + return false; + } else if (!ok) { + error = Error::Write; + return false; + } + } + + error = Error::Success; + return true; +} + +template +inline bool write_content(Stream &strm, const ContentProvider &content_provider, + size_t offset, size_t length, T is_shutting_down, + Error &error) { + return write_content_with_progress(strm, content_provider, offset, length, + is_shutting_down, nullptr, error); +} + +template +inline bool write_content(Stream &strm, const ContentProvider &content_provider, + size_t offset, size_t length, + const T &is_shutting_down) { + auto error = Error::Success; + return write_content(strm, content_provider, offset, length, is_shutting_down, + error); +} + +template +inline bool +write_content_without_length(Stream &strm, + const ContentProvider &content_provider, + const T &is_shutting_down) { + size_t offset = 0; + auto data_available = true; + auto ok = true; + DataSink data_sink; + + data_sink.write = [&](const char *d, size_t l) -> bool { + if (ok) { + offset += l; + if (!write_data(strm, d, l)) { ok = false; } + } + return ok; + }; + + data_sink.is_writable = [&]() -> bool { return strm.wait_writable(); }; + + data_sink.done = [&](void) { data_available = false; }; + + while (data_available && !is_shutting_down()) { + if (!strm.wait_writable()) { + return false; + } else if (!content_provider(offset, 0, data_sink)) { + return false; + } else if (!ok) { + return false; + } + } + return true; +} + +template +inline bool +write_content_chunked(Stream &strm, const ContentProvider &content_provider, + const T &is_shutting_down, U &compressor, Error &error) { + size_t offset = 0; + auto data_available = true; + auto ok = true; + DataSink data_sink; + + data_sink.write = [&](const char *d, size_t l) -> bool { + if (ok) { + data_available = l > 0; + offset += l; + + std::string payload; + if (compressor.compress(d, l, false, + [&](const char *data, size_t data_len) { + payload.append(data, data_len); + return true; + })) { + if (!payload.empty()) { + // Emit chunked response header and footer for each chunk + auto chunk = + from_i_to_hex(payload.size()) + "\r\n" + payload + "\r\n"; + if (!write_data(strm, chunk.data(), chunk.size())) { ok = false; } + } + } else { + ok = false; + } + } + return ok; + }; + + data_sink.is_writable = [&]() -> bool { return strm.wait_writable(); }; + + auto done_with_trailer = [&](const Headers *trailer) { + if (!ok) { return; } + + data_available = false; + + std::string payload; + if (!compressor.compress(nullptr, 0, true, + [&](const char *data, size_t data_len) { + payload.append(data, data_len); + return true; + })) { + ok = false; + return; + } + + if (!payload.empty()) { + // Emit chunked response header and footer for each chunk + auto chunk = from_i_to_hex(payload.size()) + "\r\n" + payload + "\r\n"; + if (!write_data(strm, chunk.data(), chunk.size())) { + ok = false; + return; + } + } + + constexpr const char done_marker[] = "0\r\n"; + if (!write_data(strm, done_marker, str_len(done_marker))) { ok = false; } + + // Trailer + if (trailer) { + for (const auto &kv : *trailer) { + std::string field_line = kv.first + ": " + kv.second + "\r\n"; + if (!write_data(strm, field_line.data(), field_line.size())) { + ok = false; + } + } + } + + constexpr const char crlf[] = "\r\n"; + if (!write_data(strm, crlf, str_len(crlf))) { ok = false; } + }; + + data_sink.done = [&](void) { done_with_trailer(nullptr); }; + + data_sink.done_with_trailer = [&](const Headers &trailer) { + done_with_trailer(&trailer); + }; + + while (data_available && !is_shutting_down()) { + if (!strm.wait_writable()) { + error = Error::Write; + return false; + } else if (!content_provider(offset, 0, data_sink)) { + error = Error::Canceled; + return false; + } else if (!ok) { + error = Error::Write; + return false; + } + } + + error = Error::Success; + return true; +} + +template +inline bool write_content_chunked(Stream &strm, + const ContentProvider &content_provider, + const T &is_shutting_down, U &compressor) { + auto error = Error::Success; + return write_content_chunked(strm, content_provider, is_shutting_down, + compressor, error); +} + +template +inline bool redirect(T &cli, Request &req, Response &res, + const std::string &path, const std::string &location, + Error &error) { + Request new_req = req; + new_req.path = path; + new_req.redirect_count_ -= 1; + + if (res.status == StatusCode::SeeOther_303 && + (req.method != "GET" && req.method != "HEAD")) { + new_req.method = "GET"; + new_req.body.clear(); + new_req.headers.clear(); + } + + Response new_res; + + auto ret = cli.send(new_req, new_res, error); + if (ret) { + req = new_req; + res = new_res; + + if (res.location.empty()) { res.location = location; } + } + return ret; +} + +inline std::string params_to_query_str(const Params ¶ms) { + std::string query; + + for (auto it = params.begin(); it != params.end(); ++it) { + if (it != params.begin()) { query += "&"; } + query += encode_query_component(it->first); + query += "="; + query += encode_query_component(it->second); + } + return query; +} + +inline void parse_query_text(const char *data, std::size_t size, + Params ¶ms) { + std::set cache; + split(data, data + size, '&', [&](const char *b, const char *e) { + std::string kv(b, e); + if (cache.find(kv) != cache.end()) { return; } + cache.insert(std::move(kv)); + + std::string key; + std::string val; + divide(b, static_cast(e - b), '=', + [&](const char *lhs_data, std::size_t lhs_size, const char *rhs_data, + std::size_t rhs_size) { + key.assign(lhs_data, lhs_size); + val.assign(rhs_data, rhs_size); + }); + + if (!key.empty()) { + params.emplace(decode_query_component(key), decode_query_component(val)); + } + }); +} + +inline void parse_query_text(const std::string &s, Params ¶ms) { + parse_query_text(s.data(), s.size(), params); +} + +inline bool parse_multipart_boundary(const std::string &content_type, + std::string &boundary) { + auto boundary_keyword = "boundary="; + auto pos = content_type.find(boundary_keyword); + if (pos == std::string::npos) { return false; } + auto end = content_type.find(';', pos); + auto beg = pos + strlen(boundary_keyword); + boundary = trim_double_quotes_copy(content_type.substr(beg, end - beg)); + return !boundary.empty(); +} + +inline void parse_disposition_params(const std::string &s, Params ¶ms) { + std::set cache; + split(s.data(), s.data() + s.size(), ';', [&](const char *b, const char *e) { + std::string kv(b, e); + if (cache.find(kv) != cache.end()) { return; } + cache.insert(kv); + + std::string key; + std::string val; + split(b, e, '=', [&](const char *b2, const char *e2) { + if (key.empty()) { + key.assign(b2, e2); + } else { + val.assign(b2, e2); + } + }); + + if (!key.empty()) { + params.emplace(trim_double_quotes_copy((key)), + trim_double_quotes_copy((val))); + } + }); +} + +#ifdef CPPHTTPLIB_NO_EXCEPTIONS +inline bool parse_range_header(const std::string &s, Ranges &ranges) { +#else +inline bool parse_range_header(const std::string &s, Ranges &ranges) try { +#endif + auto is_valid = [](const std::string &str) { + return std::all_of(str.cbegin(), str.cend(), + [](unsigned char c) { return std::isdigit(c); }); + }; + + if (s.size() > 7 && s.compare(0, 6, "bytes=") == 0) { + const auto pos = static_cast(6); + const auto len = static_cast(s.size() - 6); + auto all_valid_ranges = true; + split(&s[pos], &s[pos + len], ',', [&](const char *b, const char *e) { + if (!all_valid_ranges) { return; } + + const auto it = std::find(b, e, '-'); + if (it == e) { + all_valid_ranges = false; + return; + } + + const auto lhs = std::string(b, it); + const auto rhs = std::string(it + 1, e); + if (!is_valid(lhs) || !is_valid(rhs)) { + all_valid_ranges = false; + return; + } + + const auto first = + static_cast(lhs.empty() ? -1 : std::stoll(lhs)); + const auto last = + static_cast(rhs.empty() ? -1 : std::stoll(rhs)); + if ((first == -1 && last == -1) || + (first != -1 && last != -1 && first > last)) { + all_valid_ranges = false; + return; + } + + ranges.emplace_back(first, last); + }); + return all_valid_ranges && !ranges.empty(); + } + return false; +#ifdef CPPHTTPLIB_NO_EXCEPTIONS +} +#else +} catch (...) { return false; } +#endif + +inline bool parse_accept_header(const std::string &s, + std::vector &content_types) { + content_types.clear(); + + // Empty string is considered valid (no preference) + if (s.empty()) { return true; } + + // Check for invalid patterns: leading/trailing commas or consecutive commas + if (s.front() == ',' || s.back() == ',' || + s.find(",,") != std::string::npos) { + return false; + } + + struct AcceptEntry { + std::string media_type; + double quality; + int order; // Original order in header + }; + + std::vector entries; + int order = 0; + bool has_invalid_entry = false; + + // Split by comma and parse each entry + split(s.data(), s.data() + s.size(), ',', [&](const char *b, const char *e) { + std::string entry(b, e); + entry = trim_copy(entry); + + if (entry.empty()) { + has_invalid_entry = true; + return; + } + + AcceptEntry accept_entry; + accept_entry.quality = 1.0; // Default quality + accept_entry.order = order++; + + // Find q= parameter + auto q_pos = entry.find(";q="); + if (q_pos == std::string::npos) { q_pos = entry.find("; q="); } + + if (q_pos != std::string::npos) { + // Extract media type (before q parameter) + accept_entry.media_type = trim_copy(entry.substr(0, q_pos)); + + // Extract quality value + auto q_start = entry.find('=', q_pos) + 1; + auto q_end = entry.find(';', q_start); + if (q_end == std::string::npos) { q_end = entry.length(); } + + std::string quality_str = + trim_copy(entry.substr(q_start, q_end - q_start)); + if (quality_str.empty()) { + has_invalid_entry = true; + return; + } + +#ifdef CPPHTTPLIB_NO_EXCEPTIONS + { + std::istringstream iss(quality_str); + iss >> accept_entry.quality; + + // Check if conversion was successful and entire string was consumed + if (iss.fail() || !iss.eof()) { + has_invalid_entry = true; + return; + } + } +#else + try { + accept_entry.quality = std::stod(quality_str); + } catch (...) { + has_invalid_entry = true; + return; + } +#endif + // Check if quality is in valid range [0.0, 1.0] + if (accept_entry.quality < 0.0 || accept_entry.quality > 1.0) { + has_invalid_entry = true; + return; + } + } else { + // No quality parameter, use entire entry as media type + accept_entry.media_type = entry; + } + + // Remove additional parameters from media type + auto param_pos = accept_entry.media_type.find(';'); + if (param_pos != std::string::npos) { + accept_entry.media_type = + trim_copy(accept_entry.media_type.substr(0, param_pos)); + } + + // Basic validation of media type format + if (accept_entry.media_type.empty()) { + has_invalid_entry = true; + return; + } + + // Check for basic media type format (should contain '/' or be '*') + if (accept_entry.media_type != "*" && + accept_entry.media_type.find('/') == std::string::npos) { + has_invalid_entry = true; + return; + } + + entries.push_back(accept_entry); + }); + + // Return false if any invalid entry was found + if (has_invalid_entry) { return false; } + + // Sort by quality (descending), then by original order (ascending) + std::sort(entries.begin(), entries.end(), + [](const AcceptEntry &a, const AcceptEntry &b) { + if (a.quality != b.quality) { + return a.quality > b.quality; // Higher quality first + } + return a.order < b.order; // Earlier order first for same quality + }); + + // Extract sorted media types + content_types.reserve(entries.size()); + for (const auto &entry : entries) { + content_types.push_back(entry.media_type); + } + + return true; +} + +class FormDataParser { +public: + FormDataParser() = default; + + void set_boundary(std::string &&boundary) { + boundary_ = boundary; + dash_boundary_crlf_ = dash_ + boundary_ + crlf_; + crlf_dash_boundary_ = crlf_ + dash_ + boundary_; + } + + bool is_valid() const { return is_valid_; } + + bool parse(const char *buf, size_t n, const FormDataHeader &header_callback, + const ContentReceiver &content_callback) { + + buf_append(buf, n); + + while (buf_size() > 0) { + switch (state_) { + case 0: { // Initial boundary + auto pos = buf_find(dash_boundary_crlf_); + if (pos == buf_size()) { return true; } + buf_erase(pos + dash_boundary_crlf_.size()); + state_ = 1; + break; + } + case 1: { // New entry + clear_file_info(); + state_ = 2; + break; + } + case 2: { // Headers + auto pos = buf_find(crlf_); + if (pos > CPPHTTPLIB_HEADER_MAX_LENGTH) { return false; } + while (pos < buf_size()) { + // Empty line + if (pos == 0) { + if (!header_callback(file_)) { + is_valid_ = false; + return false; + } + buf_erase(crlf_.size()); + state_ = 3; + break; + } + + const auto header = buf_head(pos); + + if (!parse_header(header.data(), header.data() + header.size(), + [&](const std::string &, const std::string &) {})) { + is_valid_ = false; + return false; + } + + // Parse and emplace space trimmed headers into a map + if (!parse_header( + header.data(), header.data() + header.size(), + [&](const std::string &key, const std::string &val) { + file_.headers.emplace(key, val); + })) { + is_valid_ = false; + return false; + } + + constexpr const char header_content_type[] = "Content-Type:"; + + if (start_with_case_ignore(header, header_content_type)) { + file_.content_type = + trim_copy(header.substr(str_len(header_content_type))); + } else { + thread_local const std::regex re_content_disposition( + R"~(^Content-Disposition:\s*form-data;\s*(.*)$)~", + std::regex_constants::icase); + + std::smatch m; + if (std::regex_match(header, m, re_content_disposition)) { + Params params; + parse_disposition_params(m[1], params); + + auto it = params.find("name"); + if (it != params.end()) { + file_.name = it->second; + } else { + is_valid_ = false; + return false; + } + + it = params.find("filename"); + if (it != params.end()) { file_.filename = it->second; } + + it = params.find("filename*"); + if (it != params.end()) { + // Only allow UTF-8 encoding... + thread_local const std::regex re_rfc5987_encoding( + R"~(^UTF-8''(.+?)$)~", std::regex_constants::icase); + + std::smatch m2; + if (std::regex_match(it->second, m2, re_rfc5987_encoding)) { + file_.filename = decode_path_component(m2[1]); // override... + } else { + is_valid_ = false; + return false; + } + } + } + } + buf_erase(pos + crlf_.size()); + pos = buf_find(crlf_); + } + if (state_ != 3) { return true; } + break; + } + case 3: { // Body + if (crlf_dash_boundary_.size() > buf_size()) { return true; } + auto pos = buf_find(crlf_dash_boundary_); + if (pos < buf_size()) { + if (!content_callback(buf_data(), pos)) { + is_valid_ = false; + return false; + } + buf_erase(pos + crlf_dash_boundary_.size()); + state_ = 4; + } else { + auto len = buf_size() - crlf_dash_boundary_.size(); + if (len > 0) { + if (!content_callback(buf_data(), len)) { + is_valid_ = false; + return false; + } + buf_erase(len); + } + return true; + } + break; + } + case 4: { // Boundary + if (crlf_.size() > buf_size()) { return true; } + if (buf_start_with(crlf_)) { + buf_erase(crlf_.size()); + state_ = 1; + } else { + if (dash_.size() > buf_size()) { return true; } + if (buf_start_with(dash_)) { + buf_erase(dash_.size()); + is_valid_ = true; + buf_erase(buf_size()); // Remove epilogue + } else { + return true; + } + } + break; + } + } + } + + return true; + } + +private: + void clear_file_info() { + file_.name.clear(); + file_.filename.clear(); + file_.content_type.clear(); + file_.headers.clear(); + } + + bool start_with_case_ignore(const std::string &a, const char *b) const { + const auto b_len = strlen(b); + if (a.size() < b_len) { return false; } + for (size_t i = 0; i < b_len; i++) { + if (case_ignore::to_lower(a[i]) != case_ignore::to_lower(b[i])) { + return false; + } + } + return true; + } + + const std::string dash_ = "--"; + const std::string crlf_ = "\r\n"; + std::string boundary_; + std::string dash_boundary_crlf_; + std::string crlf_dash_boundary_; + + size_t state_ = 0; + bool is_valid_ = false; + FormData file_; + + // Buffer + bool start_with(const std::string &a, size_t spos, size_t epos, + const std::string &b) const { + if (epos - spos < b.size()) { return false; } + for (size_t i = 0; i < b.size(); i++) { + if (a[i + spos] != b[i]) { return false; } + } + return true; + } + + size_t buf_size() const { return buf_epos_ - buf_spos_; } + + const char *buf_data() const { return &buf_[buf_spos_]; } + + std::string buf_head(size_t l) const { return buf_.substr(buf_spos_, l); } + + bool buf_start_with(const std::string &s) const { + return start_with(buf_, buf_spos_, buf_epos_, s); + } + + size_t buf_find(const std::string &s) const { + auto c = s.front(); + + size_t off = buf_spos_; + while (off < buf_epos_) { + auto pos = off; + while (true) { + if (pos == buf_epos_) { return buf_size(); } + if (buf_[pos] == c) { break; } + pos++; + } + + auto remaining_size = buf_epos_ - pos; + if (s.size() > remaining_size) { return buf_size(); } + + if (start_with(buf_, pos, buf_epos_, s)) { return pos - buf_spos_; } + + off = pos + 1; + } + + return buf_size(); + } + + void buf_append(const char *data, size_t n) { + auto remaining_size = buf_size(); + if (remaining_size > 0 && buf_spos_ > 0) { + for (size_t i = 0; i < remaining_size; i++) { + buf_[i] = buf_[buf_spos_ + i]; + } + } + buf_spos_ = 0; + buf_epos_ = remaining_size; + + if (remaining_size + n > buf_.size()) { buf_.resize(remaining_size + n); } + + for (size_t i = 0; i < n; i++) { + buf_[buf_epos_ + i] = data[i]; + } + buf_epos_ += n; + } + + void buf_erase(size_t size) { buf_spos_ += size; } + + std::string buf_; + size_t buf_spos_ = 0; + size_t buf_epos_ = 0; +}; + +inline std::string random_string(size_t length) { + constexpr const char data[] = + "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + + thread_local auto engine([]() { + // std::random_device might actually be deterministic on some + // platforms, but due to lack of support in the c++ standard library, + // doing better requires either some ugly hacks or breaking portability. + std::random_device seed_gen; + // Request 128 bits of entropy for initialization + std::seed_seq seed_sequence{seed_gen(), seed_gen(), seed_gen(), seed_gen()}; + return std::mt19937(seed_sequence); + }()); + + std::string result; + for (size_t i = 0; i < length; i++) { + result += data[engine() % (sizeof(data) - 1)]; + } + return result; +} + +inline std::string make_multipart_data_boundary() { + return "--cpp-httplib-multipart-data-" + detail::random_string(16); +} + +inline bool is_multipart_boundary_chars_valid(const std::string &boundary) { + auto valid = true; + for (size_t i = 0; i < boundary.size(); i++) { + auto c = boundary[i]; + if (!std::isalnum(c) && c != '-' && c != '_') { + valid = false; + break; + } + } + return valid; +} + +template +inline std::string +serialize_multipart_formdata_item_begin(const T &item, + const std::string &boundary) { + std::string body = "--" + boundary + "\r\n"; + body += "Content-Disposition: form-data; name=\"" + item.name + "\""; + if (!item.filename.empty()) { + body += "; filename=\"" + item.filename + "\""; + } + body += "\r\n"; + if (!item.content_type.empty()) { + body += "Content-Type: " + item.content_type + "\r\n"; + } + body += "\r\n"; + + return body; +} + +inline std::string serialize_multipart_formdata_item_end() { return "\r\n"; } + +inline std::string +serialize_multipart_formdata_finish(const std::string &boundary) { + return "--" + boundary + "--\r\n"; +} + +inline std::string +serialize_multipart_formdata_get_content_type(const std::string &boundary) { + return "multipart/form-data; boundary=" + boundary; +} + +inline std::string +serialize_multipart_formdata(const UploadFormDataItems &items, + const std::string &boundary, bool finish = true) { + std::string body; + + for (const auto &item : items) { + body += serialize_multipart_formdata_item_begin(item, boundary); + body += item.content + serialize_multipart_formdata_item_end(); + } + + if (finish) { body += serialize_multipart_formdata_finish(boundary); } + + return body; +} + +inline void coalesce_ranges(Ranges &ranges, size_t content_length) { + if (ranges.size() <= 1) return; + + // Sort ranges by start position + std::sort(ranges.begin(), ranges.end(), + [](const Range &a, const Range &b) { return a.first < b.first; }); + + Ranges coalesced; + coalesced.reserve(ranges.size()); + + for (auto &r : ranges) { + auto first_pos = r.first; + auto last_pos = r.second; + + // Handle special cases like in range_error + if (first_pos == -1 && last_pos == -1) { + first_pos = 0; + last_pos = static_cast(content_length); + } + + if (first_pos == -1) { + first_pos = static_cast(content_length) - last_pos; + last_pos = static_cast(content_length) - 1; + } + + if (last_pos == -1 || last_pos >= static_cast(content_length)) { + last_pos = static_cast(content_length) - 1; + } + + // Skip invalid ranges + if (!(0 <= first_pos && first_pos <= last_pos && + last_pos < static_cast(content_length))) { + continue; + } + + // Coalesce with previous range if overlapping or adjacent (but not + // identical) + if (!coalesced.empty()) { + auto &prev = coalesced.back(); + // Check if current range overlaps or is adjacent to previous range + // but don't coalesce identical ranges (allow duplicates) + if (first_pos <= prev.second + 1 && + !(first_pos == prev.first && last_pos == prev.second)) { + // Extend the previous range + prev.second = (std::max)(prev.second, last_pos); + continue; + } + } + + // Add new range + coalesced.emplace_back(first_pos, last_pos); + } + + ranges = std::move(coalesced); +} + +inline bool range_error(Request &req, Response &res) { + if (!req.ranges.empty() && 200 <= res.status && res.status < 300) { + ssize_t content_len = static_cast( + res.content_length_ ? res.content_length_ : res.body.size()); + + std::vector> processed_ranges; + size_t overwrapping_count = 0; + + // NOTE: The following Range check is based on '14.2. Range' in RFC 9110 + // 'HTTP Semantics' to avoid potential denial-of-service attacks. + // https://www.rfc-editor.org/rfc/rfc9110#section-14.2 + + // Too many ranges + if (req.ranges.size() > CPPHTTPLIB_RANGE_MAX_COUNT) { return true; } + + for (auto &r : req.ranges) { + auto &first_pos = r.first; + auto &last_pos = r.second; + + if (first_pos == -1 && last_pos == -1) { + first_pos = 0; + last_pos = content_len; + } + + if (first_pos == -1) { + first_pos = content_len - last_pos; + last_pos = content_len - 1; + } + + // NOTE: RFC-9110 '14.1.2. Byte Ranges': + // A client can limit the number of bytes requested without knowing the + // size of the selected representation. If the last-pos value is absent, + // or if the value is greater than or equal to the current length of the + // representation data, the byte range is interpreted as the remainder of + // the representation (i.e., the server replaces the value of last-pos + // with a value that is one less than the current length of the selected + // representation). + // https://www.rfc-editor.org/rfc/rfc9110.html#section-14.1.2-6 + if (last_pos == -1 || last_pos >= content_len) { + last_pos = content_len - 1; + } + + // Range must be within content length + if (!(0 <= first_pos && first_pos <= last_pos && + last_pos <= content_len - 1)) { + return true; + } + + // Request must not have more than two overlapping ranges + for (const auto &processed_range : processed_ranges) { + if (!(last_pos < processed_range.first || + first_pos > processed_range.second)) { + overwrapping_count++; + if (overwrapping_count > 2) { return true; } + break; // Only count once per range + } + } + + processed_ranges.emplace_back(first_pos, last_pos); + } + + // After validation, coalesce overlapping ranges as per RFC 9110 + coalesce_ranges(req.ranges, static_cast(content_len)); + } + + return false; +} + +inline std::pair +get_range_offset_and_length(Range r, size_t content_length) { + assert(r.first != -1 && r.second != -1); + assert(0 <= r.first && r.first < static_cast(content_length)); + assert(r.first <= r.second && + r.second < static_cast(content_length)); + (void)(content_length); + return std::make_pair(r.first, static_cast(r.second - r.first) + 1); +} + +inline std::string make_content_range_header_field( + const std::pair &offset_and_length, size_t content_length) { + auto st = offset_and_length.first; + auto ed = st + offset_and_length.second - 1; + + std::string field = "bytes "; + field += std::to_string(st); + field += "-"; + field += std::to_string(ed); + field += "/"; + field += std::to_string(content_length); + return field; +} + +template +bool process_multipart_ranges_data(const Request &req, + const std::string &boundary, + const std::string &content_type, + size_t content_length, SToken stoken, + CToken ctoken, Content content) { + for (size_t i = 0; i < req.ranges.size(); i++) { + ctoken("--"); + stoken(boundary); + ctoken("\r\n"); + if (!content_type.empty()) { + ctoken("Content-Type: "); + stoken(content_type); + ctoken("\r\n"); + } + + auto offset_and_length = + get_range_offset_and_length(req.ranges[i], content_length); + + ctoken("Content-Range: "); + stoken(make_content_range_header_field(offset_and_length, content_length)); + ctoken("\r\n"); + ctoken("\r\n"); + + if (!content(offset_and_length.first, offset_and_length.second)) { + return false; + } + ctoken("\r\n"); + } + + ctoken("--"); + stoken(boundary); + ctoken("--"); + + return true; +} + +inline void make_multipart_ranges_data(const Request &req, Response &res, + const std::string &boundary, + const std::string &content_type, + size_t content_length, + std::string &data) { + process_multipart_ranges_data( + req, boundary, content_type, content_length, + [&](const std::string &token) { data += token; }, + [&](const std::string &token) { data += token; }, + [&](size_t offset, size_t length) { + assert(offset + length <= content_length); + data += res.body.substr(offset, length); + return true; + }); +} + +inline size_t get_multipart_ranges_data_length(const Request &req, + const std::string &boundary, + const std::string &content_type, + size_t content_length) { + size_t data_length = 0; + + process_multipart_ranges_data( + req, boundary, content_type, content_length, + [&](const std::string &token) { data_length += token.size(); }, + [&](const std::string &token) { data_length += token.size(); }, + [&](size_t /*offset*/, size_t length) { + data_length += length; + return true; + }); + + return data_length; +} + +template +inline bool +write_multipart_ranges_data(Stream &strm, const Request &req, Response &res, + const std::string &boundary, + const std::string &content_type, + size_t content_length, const T &is_shutting_down) { + return process_multipart_ranges_data( + req, boundary, content_type, content_length, + [&](const std::string &token) { strm.write(token); }, + [&](const std::string &token) { strm.write(token); }, + [&](size_t offset, size_t length) { + return write_content(strm, res.content_provider_, offset, length, + is_shutting_down); + }); +} + +inline bool expect_content(const Request &req) { + if (req.method == "POST" || req.method == "PUT" || req.method == "PATCH" || + req.method == "DELETE") { + return true; + } + if (req.has_header("Content-Length") && + req.get_header_value_u64("Content-Length") > 0) { + return true; + } + if (is_chunked_transfer_encoding(req.headers)) { return true; } + return false; +} + +inline bool has_crlf(const std::string &s) { + auto p = s.c_str(); + while (*p) { + if (*p == '\r' || *p == '\n') { return true; } + p++; + } + return false; +} + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +inline std::string message_digest(const std::string &s, const EVP_MD *algo) { + auto context = std::unique_ptr( + EVP_MD_CTX_new(), EVP_MD_CTX_free); + + unsigned int hash_length = 0; + unsigned char hash[EVP_MAX_MD_SIZE]; + + EVP_DigestInit_ex(context.get(), algo, nullptr); + EVP_DigestUpdate(context.get(), s.c_str(), s.size()); + EVP_DigestFinal_ex(context.get(), hash, &hash_length); + + std::stringstream ss; + for (auto i = 0u; i < hash_length; ++i) { + ss << std::hex << std::setw(2) << std::setfill('0') + << static_cast(hash[i]); + } + + return ss.str(); +} + +inline std::string MD5(const std::string &s) { + return message_digest(s, EVP_md5()); +} + +inline std::string SHA_256(const std::string &s) { + return message_digest(s, EVP_sha256()); +} + +inline std::string SHA_512(const std::string &s) { + return message_digest(s, EVP_sha512()); +} + +inline std::pair make_digest_authentication_header( + const Request &req, const std::map &auth, + size_t cnonce_count, const std::string &cnonce, const std::string &username, + const std::string &password, bool is_proxy = false) { + std::string nc; + { + std::stringstream ss; + ss << std::setfill('0') << std::setw(8) << std::hex << cnonce_count; + nc = ss.str(); + } + + std::string qop; + if (auth.find("qop") != auth.end()) { + qop = auth.at("qop"); + if (qop.find("auth-int") != std::string::npos) { + qop = "auth-int"; + } else if (qop.find("auth") != std::string::npos) { + qop = "auth"; + } else { + qop.clear(); + } + } + + std::string algo = "MD5"; + if (auth.find("algorithm") != auth.end()) { algo = auth.at("algorithm"); } + + std::string response; + { + auto H = algo == "SHA-256" ? detail::SHA_256 + : algo == "SHA-512" ? detail::SHA_512 + : detail::MD5; + + auto A1 = username + ":" + auth.at("realm") + ":" + password; + + auto A2 = req.method + ":" + req.path; + if (qop == "auth-int") { A2 += ":" + H(req.body); } + + if (qop.empty()) { + response = H(H(A1) + ":" + auth.at("nonce") + ":" + H(A2)); + } else { + response = H(H(A1) + ":" + auth.at("nonce") + ":" + nc + ":" + cnonce + + ":" + qop + ":" + H(A2)); + } + } + + auto opaque = (auth.find("opaque") != auth.end()) ? auth.at("opaque") : ""; + + auto field = "Digest username=\"" + username + "\", realm=\"" + + auth.at("realm") + "\", nonce=\"" + auth.at("nonce") + + "\", uri=\"" + req.path + "\", algorithm=" + algo + + (qop.empty() ? ", response=\"" + : ", qop=" + qop + ", nc=" + nc + ", cnonce=\"" + + cnonce + "\", response=\"") + + response + "\"" + + (opaque.empty() ? "" : ", opaque=\"" + opaque + "\""); + + auto key = is_proxy ? "Proxy-Authorization" : "Authorization"; + return std::make_pair(key, field); +} + +inline bool is_ssl_peer_could_be_closed(SSL *ssl, socket_t sock) { + detail::set_nonblocking(sock, true); + auto se = detail::scope_exit([&]() { detail::set_nonblocking(sock, false); }); + + char buf[1]; + return !SSL_peek(ssl, buf, 1) && + SSL_get_error(ssl, 0) == SSL_ERROR_ZERO_RETURN; +} + +#ifdef _WIN64 +// NOTE: This code came up with the following stackoverflow post: +// https://stackoverflow.com/questions/9507184/can-openssl-on-windows-use-the-system-certificate-store +inline bool load_system_certs_on_windows(X509_STORE *store) { + auto hStore = CertOpenSystemStoreW((HCRYPTPROV_LEGACY)NULL, L"ROOT"); + if (!hStore) { return false; } + + auto result = false; + PCCERT_CONTEXT pContext = NULL; + while ((pContext = CertEnumCertificatesInStore(hStore, pContext)) != + nullptr) { + auto encoded_cert = + static_cast(pContext->pbCertEncoded); + + auto x509 = d2i_X509(NULL, &encoded_cert, pContext->cbCertEncoded); + if (x509) { + X509_STORE_add_cert(store, x509); + X509_free(x509); + result = true; + } + } + + CertFreeCertificateContext(pContext); + CertCloseStore(hStore, 0); + + return result; +} +#elif defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) && \ + defined(TARGET_OS_OSX) +template +using CFObjectPtr = + std::unique_ptr::type, void (*)(CFTypeRef)>; + +inline void cf_object_ptr_deleter(CFTypeRef obj) { + if (obj) { CFRelease(obj); } +} + +inline bool retrieve_certs_from_keychain(CFObjectPtr &certs) { + CFStringRef keys[] = {kSecClass, kSecMatchLimit, kSecReturnRef}; + CFTypeRef values[] = {kSecClassCertificate, kSecMatchLimitAll, + kCFBooleanTrue}; + + CFObjectPtr query( + CFDictionaryCreate(nullptr, reinterpret_cast(keys), values, + sizeof(keys) / sizeof(keys[0]), + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks), + cf_object_ptr_deleter); + + if (!query) { return false; } + + CFTypeRef security_items = nullptr; + if (SecItemCopyMatching(query.get(), &security_items) != errSecSuccess || + CFArrayGetTypeID() != CFGetTypeID(security_items)) { + return false; + } + + certs.reset(reinterpret_cast(security_items)); + return true; +} + +inline bool retrieve_root_certs_from_keychain(CFObjectPtr &certs) { + CFArrayRef root_security_items = nullptr; + if (SecTrustCopyAnchorCertificates(&root_security_items) != errSecSuccess) { + return false; + } + + certs.reset(root_security_items); + return true; +} + +inline bool add_certs_to_x509_store(CFArrayRef certs, X509_STORE *store) { + auto result = false; + for (auto i = 0; i < CFArrayGetCount(certs); ++i) { + const auto cert = reinterpret_cast( + CFArrayGetValueAtIndex(certs, i)); + + if (SecCertificateGetTypeID() != CFGetTypeID(cert)) { continue; } + + CFDataRef cert_data = nullptr; + if (SecItemExport(cert, kSecFormatX509Cert, 0, nullptr, &cert_data) != + errSecSuccess) { + continue; + } + + CFObjectPtr cert_data_ptr(cert_data, cf_object_ptr_deleter); + + auto encoded_cert = static_cast( + CFDataGetBytePtr(cert_data_ptr.get())); + + auto x509 = + d2i_X509(NULL, &encoded_cert, CFDataGetLength(cert_data_ptr.get())); + + if (x509) { + X509_STORE_add_cert(store, x509); + X509_free(x509); + result = true; + } + } + + return result; +} + +inline bool load_system_certs_on_macos(X509_STORE *store) { + auto result = false; + CFObjectPtr certs(nullptr, cf_object_ptr_deleter); + if (retrieve_certs_from_keychain(certs) && certs) { + result = add_certs_to_x509_store(certs.get(), store); + } + + if (retrieve_root_certs_from_keychain(certs) && certs) { + result = add_certs_to_x509_store(certs.get(), store) || result; + } + + return result; +} +#endif // _WIN64 +#endif // CPPHTTPLIB_OPENSSL_SUPPORT + +#ifdef _WIN64 +class WSInit { +public: + WSInit() { + WSADATA wsaData; + if (WSAStartup(0x0002, &wsaData) == 0) is_valid_ = true; + } + + ~WSInit() { + if (is_valid_) WSACleanup(); + } + + bool is_valid_ = false; +}; + +static WSInit wsinit_; +#endif + +inline bool parse_www_authenticate(const Response &res, + std::map &auth, + bool is_proxy) { + auto auth_key = is_proxy ? "Proxy-Authenticate" : "WWW-Authenticate"; + if (res.has_header(auth_key)) { + thread_local auto re = + std::regex(R"~((?:(?:,\s*)?(.+?)=(?:"(.*?)"|([^,]*))))~"); + auto s = res.get_header_value(auth_key); + auto pos = s.find(' '); + if (pos != std::string::npos) { + auto type = s.substr(0, pos); + if (type == "Basic") { + return false; + } else if (type == "Digest") { + s = s.substr(pos + 1); + auto beg = std::sregex_iterator(s.begin(), s.end(), re); + for (auto i = beg; i != std::sregex_iterator(); ++i) { + const auto &m = *i; + auto key = s.substr(static_cast(m.position(1)), + static_cast(m.length(1))); + auto val = m.length(2) > 0 + ? s.substr(static_cast(m.position(2)), + static_cast(m.length(2))) + : s.substr(static_cast(m.position(3)), + static_cast(m.length(3))); + auth[key] = val; + } + return true; + } + } + } + return false; +} + +class ContentProviderAdapter { +public: + explicit ContentProviderAdapter( + ContentProviderWithoutLength &&content_provider) + : content_provider_(content_provider) {} + + bool operator()(size_t offset, size_t, DataSink &sink) { + return content_provider_(offset, sink); + } + +private: + ContentProviderWithoutLength content_provider_; +}; + +} // namespace detail + +inline std::string hosted_at(const std::string &hostname) { + std::vector addrs; + hosted_at(hostname, addrs); + if (addrs.empty()) { return std::string(); } + return addrs[0]; +} + +inline void hosted_at(const std::string &hostname, + std::vector &addrs) { + struct addrinfo hints; + struct addrinfo *result; + + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_protocol = 0; + + if (detail::getaddrinfo_with_timeout(hostname.c_str(), nullptr, &hints, + &result, 0)) { +#if defined __linux__ && !defined __ANDROID__ + res_init(); +#endif + return; + } + auto se = detail::scope_exit([&] { freeaddrinfo(result); }); + + for (auto rp = result; rp; rp = rp->ai_next) { + const auto &addr = + *reinterpret_cast(rp->ai_addr); + std::string ip; + auto dummy = -1; + if (detail::get_ip_and_port(addr, sizeof(struct sockaddr_storage), ip, + dummy)) { + addrs.push_back(ip); + } + } +} + +inline std::string encode_uri_component(const std::string &value) { + std::ostringstream escaped; + escaped.fill('0'); + escaped << std::hex; + + for (auto c : value) { + if (std::isalnum(static_cast(c)) || c == '-' || c == '_' || + c == '.' || c == '!' || c == '~' || c == '*' || c == '\'' || c == '(' || + c == ')') { + escaped << c; + } else { + escaped << std::uppercase; + escaped << '%' << std::setw(2) + << static_cast(static_cast(c)); + escaped << std::nouppercase; + } + } + + return escaped.str(); +} + +inline std::string encode_uri(const std::string &value) { + std::ostringstream escaped; + escaped.fill('0'); + escaped << std::hex; + + for (auto c : value) { + if (std::isalnum(static_cast(c)) || c == '-' || c == '_' || + c == '.' || c == '!' || c == '~' || c == '*' || c == '\'' || c == '(' || + c == ')' || c == ';' || c == '/' || c == '?' || c == ':' || c == '@' || + c == '&' || c == '=' || c == '+' || c == '$' || c == ',' || c == '#') { + escaped << c; + } else { + escaped << std::uppercase; + escaped << '%' << std::setw(2) + << static_cast(static_cast(c)); + escaped << std::nouppercase; + } + } + + return escaped.str(); +} + +inline std::string decode_uri_component(const std::string &value) { + std::string result; + + for (size_t i = 0; i < value.size(); i++) { + if (value[i] == '%' && i + 2 < value.size()) { + auto val = 0; + if (detail::from_hex_to_i(value, i + 1, 2, val)) { + result += static_cast(val); + i += 2; + } else { + result += value[i]; + } + } else { + result += value[i]; + } + } + + return result; +} + +inline std::string decode_uri(const std::string &value) { + std::string result; + + for (size_t i = 0; i < value.size(); i++) { + if (value[i] == '%' && i + 2 < value.size()) { + auto val = 0; + if (detail::from_hex_to_i(value, i + 1, 2, val)) { + result += static_cast(val); + i += 2; + } else { + result += value[i]; + } + } else { + result += value[i]; + } + } + + return result; +} + +inline std::string encode_path_component(const std::string &component) { + std::string result; + result.reserve(component.size() * 3); + + for (size_t i = 0; i < component.size(); i++) { + auto c = static_cast(component[i]); + + // Unreserved characters per RFC 3986: ALPHA / DIGIT / "-" / "." / "_" / "~" + if (std::isalnum(c) || c == '-' || c == '.' || c == '_' || c == '~') { + result += static_cast(c); + } + // Path-safe sub-delimiters: "!" / "$" / "&" / "'" / "(" / ")" / "*" / "+" / + // "," / ";" / "=" + else if (c == '!' || c == '$' || c == '&' || c == '\'' || c == '(' || + c == ')' || c == '*' || c == '+' || c == ',' || c == ';' || + c == '=') { + result += static_cast(c); + } + // Colon is allowed in path segments except first segment + else if (c == ':') { + result += static_cast(c); + } + // @ is allowed in path + else if (c == '@') { + result += static_cast(c); + } else { + result += '%'; + char hex[3]; + snprintf(hex, sizeof(hex), "%02X", c); + result.append(hex, 2); + } + } + return result; +} + +inline std::string decode_path_component(const std::string &component) { + std::string result; + result.reserve(component.size()); + + for (size_t i = 0; i < component.size(); i++) { + if (component[i] == '%' && i + 1 < component.size()) { + if (component[i + 1] == 'u') { + // Unicode %uXXXX encoding + auto val = 0; + if (detail::from_hex_to_i(component, i + 2, 4, val)) { + // 4 digits Unicode codes + char buff[4]; + size_t len = detail::to_utf8(val, buff); + if (len > 0) { result.append(buff, len); } + i += 5; // 'u0000' + } else { + result += component[i]; + } + } else { + // Standard %XX encoding + auto val = 0; + if (detail::from_hex_to_i(component, i + 1, 2, val)) { + // 2 digits hex codes + result += static_cast(val); + i += 2; // 'XX' + } else { + result += component[i]; + } + } + } else { + result += component[i]; + } + } + return result; +} + +inline std::string encode_query_component(const std::string &component, + bool space_as_plus) { + std::string result; + result.reserve(component.size() * 3); + + for (size_t i = 0; i < component.size(); i++) { + auto c = static_cast(component[i]); + + // Unreserved characters per RFC 3986 + if (std::isalnum(c) || c == '-' || c == '.' || c == '_' || c == '~') { + result += static_cast(c); + } + // Space handling + else if (c == ' ') { + if (space_as_plus) { + result += '+'; + } else { + result += "%20"; + } + } + // Plus sign handling + else if (c == '+') { + if (space_as_plus) { + result += "%2B"; + } else { + result += static_cast(c); + } + } + // Query-safe sub-delimiters (excluding & and = which are query delimiters) + else if (c == '!' || c == '$' || c == '\'' || c == '(' || c == ')' || + c == '*' || c == ',' || c == ';') { + result += static_cast(c); + } + // Colon and @ are allowed in query + else if (c == ':' || c == '@') { + result += static_cast(c); + } + // Forward slash is allowed in query values + else if (c == '/') { + result += static_cast(c); + } + // Question mark is allowed in query values (after first ?) + else if (c == '?') { + result += static_cast(c); + } else { + result += '%'; + char hex[3]; + snprintf(hex, sizeof(hex), "%02X", c); + result.append(hex, 2); + } + } + return result; +} + +inline std::string decode_query_component(const std::string &component, + bool plus_as_space) { + std::string result; + result.reserve(component.size()); + + for (size_t i = 0; i < component.size(); i++) { + if (component[i] == '%' && i + 2 < component.size()) { + std::string hex = component.substr(i + 1, 2); + char *end; + unsigned long value = std::strtoul(hex.c_str(), &end, 16); + if (end == hex.c_str() + 2) { + result += static_cast(value); + i += 2; + } else { + result += component[i]; + } + } else if (component[i] == '+' && plus_as_space) { + result += ' '; // + becomes space in form-urlencoded + } else { + result += component[i]; + } + } + return result; +} + +inline std::string append_query_params(const std::string &path, + const Params ¶ms) { + std::string path_with_query = path; + thread_local const std::regex re("[^?]+\\?.*"); + auto delm = std::regex_match(path, re) ? '&' : '?'; + path_with_query += delm + detail::params_to_query_str(params); + return path_with_query; +} + +// Header utilities +inline std::pair +make_range_header(const Ranges &ranges) { + std::string field = "bytes="; + auto i = 0; + for (const auto &r : ranges) { + if (i != 0) { field += ", "; } + if (r.first != -1) { field += std::to_string(r.first); } + field += '-'; + if (r.second != -1) { field += std::to_string(r.second); } + i++; + } + return std::make_pair("Range", std::move(field)); +} + +inline std::pair +make_basic_authentication_header(const std::string &username, + const std::string &password, bool is_proxy) { + auto field = "Basic " + detail::base64_encode(username + ":" + password); + auto key = is_proxy ? "Proxy-Authorization" : "Authorization"; + return std::make_pair(key, std::move(field)); +} + +inline std::pair +make_bearer_token_authentication_header(const std::string &token, + bool is_proxy = false) { + auto field = "Bearer " + token; + auto key = is_proxy ? "Proxy-Authorization" : "Authorization"; + return std::make_pair(key, std::move(field)); +} + +// Request implementation +inline bool Request::has_header(const std::string &key) const { + return detail::has_header(headers, key); +} + +inline std::string Request::get_header_value(const std::string &key, + const char *def, size_t id) const { + return detail::get_header_value(headers, key, def, id); +} + +inline size_t Request::get_header_value_count(const std::string &key) const { + auto r = headers.equal_range(key); + return static_cast(std::distance(r.first, r.second)); +} + +inline void Request::set_header(const std::string &key, + const std::string &val) { + if (detail::fields::is_field_name(key) && + detail::fields::is_field_value(val)) { + headers.emplace(key, val); + } +} + +inline bool Request::has_trailer(const std::string &key) const { + return trailers.find(key) != trailers.end(); +} + +inline std::string Request::get_trailer_value(const std::string &key, + size_t id) const { + auto rng = trailers.equal_range(key); + auto it = rng.first; + std::advance(it, static_cast(id)); + if (it != rng.second) { return it->second; } + return std::string(); +} + +inline size_t Request::get_trailer_value_count(const std::string &key) const { + auto r = trailers.equal_range(key); + return static_cast(std::distance(r.first, r.second)); +} + +inline bool Request::has_param(const std::string &key) const { + return params.find(key) != params.end(); +} + +inline std::string Request::get_param_value(const std::string &key, + size_t id) const { + auto rng = params.equal_range(key); + auto it = rng.first; + std::advance(it, static_cast(id)); + if (it != rng.second) { return it->second; } + return std::string(); +} + +inline size_t Request::get_param_value_count(const std::string &key) const { + auto r = params.equal_range(key); + return static_cast(std::distance(r.first, r.second)); +} + +inline bool Request::is_multipart_form_data() const { + const auto &content_type = get_header_value("Content-Type"); + return !content_type.rfind("multipart/form-data", 0); +} + +// Multipart FormData implementation +inline std::string MultipartFormData::get_field(const std::string &key, + size_t id) const { + auto rng = fields.equal_range(key); + auto it = rng.first; + std::advance(it, static_cast(id)); + if (it != rng.second) { return it->second.content; } + return std::string(); +} + +inline std::vector +MultipartFormData::get_fields(const std::string &key) const { + std::vector values; + auto rng = fields.equal_range(key); + for (auto it = rng.first; it != rng.second; it++) { + values.push_back(it->second.content); + } + return values; +} + +inline bool MultipartFormData::has_field(const std::string &key) const { + return fields.find(key) != fields.end(); +} + +inline size_t MultipartFormData::get_field_count(const std::string &key) const { + auto r = fields.equal_range(key); + return static_cast(std::distance(r.first, r.second)); +} + +inline FormData MultipartFormData::get_file(const std::string &key, + size_t id) const { + auto rng = files.equal_range(key); + auto it = rng.first; + std::advance(it, static_cast(id)); + if (it != rng.second) { return it->second; } + return FormData(); +} + +inline std::vector +MultipartFormData::get_files(const std::string &key) const { + std::vector values; + auto rng = files.equal_range(key); + for (auto it = rng.first; it != rng.second; it++) { + values.push_back(it->second); + } + return values; +} + +inline bool MultipartFormData::has_file(const std::string &key) const { + return files.find(key) != files.end(); +} + +inline size_t MultipartFormData::get_file_count(const std::string &key) const { + auto r = files.equal_range(key); + return static_cast(std::distance(r.first, r.second)); +} + +// Response implementation +inline bool Response::has_header(const std::string &key) const { + return headers.find(key) != headers.end(); +} + +inline std::string Response::get_header_value(const std::string &key, + const char *def, + size_t id) const { + return detail::get_header_value(headers, key, def, id); +} + +inline size_t Response::get_header_value_count(const std::string &key) const { + auto r = headers.equal_range(key); + return static_cast(std::distance(r.first, r.second)); +} + +inline void Response::set_header(const std::string &key, + const std::string &val) { + if (detail::fields::is_field_name(key) && + detail::fields::is_field_value(val)) { + headers.emplace(key, val); + } +} +inline bool Response::has_trailer(const std::string &key) const { + return trailers.find(key) != trailers.end(); +} + +inline std::string Response::get_trailer_value(const std::string &key, + size_t id) const { + auto rng = trailers.equal_range(key); + auto it = rng.first; + std::advance(it, static_cast(id)); + if (it != rng.second) { return it->second; } + return std::string(); +} + +inline size_t Response::get_trailer_value_count(const std::string &key) const { + auto r = trailers.equal_range(key); + return static_cast(std::distance(r.first, r.second)); +} + +inline void Response::set_redirect(const std::string &url, int stat) { + if (detail::fields::is_field_value(url)) { + set_header("Location", url); + if (300 <= stat && stat < 400) { + this->status = stat; + } else { + this->status = StatusCode::Found_302; + } + } +} + +inline void Response::set_content(const char *s, size_t n, + const std::string &content_type) { + body.assign(s, n); + + auto rng = headers.equal_range("Content-Type"); + headers.erase(rng.first, rng.second); + set_header("Content-Type", content_type); +} + +inline void Response::set_content(const std::string &s, + const std::string &content_type) { + set_content(s.data(), s.size(), content_type); +} + +inline void Response::set_content(std::string &&s, + const std::string &content_type) { + body = std::move(s); + + auto rng = headers.equal_range("Content-Type"); + headers.erase(rng.first, rng.second); + set_header("Content-Type", content_type); +} + +inline void Response::set_content_provider( + size_t in_length, const std::string &content_type, ContentProvider provider, + ContentProviderResourceReleaser resource_releaser) { + set_header("Content-Type", content_type); + content_length_ = in_length; + if (in_length > 0) { content_provider_ = std::move(provider); } + content_provider_resource_releaser_ = std::move(resource_releaser); + is_chunked_content_provider_ = false; +} + +inline void Response::set_content_provider( + const std::string &content_type, ContentProviderWithoutLength provider, + ContentProviderResourceReleaser resource_releaser) { + set_header("Content-Type", content_type); + content_length_ = 0; + content_provider_ = detail::ContentProviderAdapter(std::move(provider)); + content_provider_resource_releaser_ = std::move(resource_releaser); + is_chunked_content_provider_ = false; +} + +inline void Response::set_chunked_content_provider( + const std::string &content_type, ContentProviderWithoutLength provider, + ContentProviderResourceReleaser resource_releaser) { + set_header("Content-Type", content_type); + content_length_ = 0; + content_provider_ = detail::ContentProviderAdapter(std::move(provider)); + content_provider_resource_releaser_ = std::move(resource_releaser); + is_chunked_content_provider_ = true; +} + +inline void Response::set_file_content(const std::string &path, + const std::string &content_type) { + file_content_path_ = path; + file_content_content_type_ = content_type; +} + +inline void Response::set_file_content(const std::string &path) { + file_content_path_ = path; +} + +// Result implementation +inline bool Result::has_request_header(const std::string &key) const { + return request_headers_.find(key) != request_headers_.end(); +} + +inline std::string Result::get_request_header_value(const std::string &key, + const char *def, + size_t id) const { + return detail::get_header_value(request_headers_, key, def, id); +} + +inline size_t +Result::get_request_header_value_count(const std::string &key) const { + auto r = request_headers_.equal_range(key); + return static_cast(std::distance(r.first, r.second)); +} + +// Stream implementation +inline ssize_t Stream::write(const char *ptr) { + return write(ptr, strlen(ptr)); +} + +inline ssize_t Stream::write(const std::string &s) { + return write(s.data(), s.size()); +} + +namespace detail { + +inline void calc_actual_timeout(time_t max_timeout_msec, time_t duration_msec, + time_t timeout_sec, time_t timeout_usec, + time_t &actual_timeout_sec, + time_t &actual_timeout_usec) { + auto timeout_msec = (timeout_sec * 1000) + (timeout_usec / 1000); + + auto actual_timeout_msec = + (std::min)(max_timeout_msec - duration_msec, timeout_msec); + + if (actual_timeout_msec < 0) { actual_timeout_msec = 0; } + + actual_timeout_sec = actual_timeout_msec / 1000; + actual_timeout_usec = (actual_timeout_msec % 1000) * 1000; +} + +// Socket stream implementation +inline SocketStream::SocketStream( + socket_t sock, time_t read_timeout_sec, time_t read_timeout_usec, + time_t write_timeout_sec, time_t write_timeout_usec, + time_t max_timeout_msec, + std::chrono::time_point start_time) + : sock_(sock), read_timeout_sec_(read_timeout_sec), + read_timeout_usec_(read_timeout_usec), + write_timeout_sec_(write_timeout_sec), + write_timeout_usec_(write_timeout_usec), + max_timeout_msec_(max_timeout_msec), start_time_(start_time), + read_buff_(read_buff_size_, 0) {} + +inline SocketStream::~SocketStream() = default; + +inline bool SocketStream::is_readable() const { + return read_buff_off_ < read_buff_content_size_; +} + +inline bool SocketStream::wait_readable() const { + if (max_timeout_msec_ <= 0) { + return select_read(sock_, read_timeout_sec_, read_timeout_usec_) > 0; + } + + time_t read_timeout_sec; + time_t read_timeout_usec; + calc_actual_timeout(max_timeout_msec_, duration(), read_timeout_sec_, + read_timeout_usec_, read_timeout_sec, read_timeout_usec); + + return select_read(sock_, read_timeout_sec, read_timeout_usec) > 0; +} + +inline bool SocketStream::wait_writable() const { + return select_write(sock_, write_timeout_sec_, write_timeout_usec_) > 0 && + is_socket_alive(sock_); +} + +inline ssize_t SocketStream::read(char *ptr, size_t size) { +#ifdef _WIN64 + size = + (std::min)(size, static_cast((std::numeric_limits::max)())); +#else + size = (std::min)(size, + static_cast((std::numeric_limits::max)())); +#endif + + if (read_buff_off_ < read_buff_content_size_) { + auto remaining_size = read_buff_content_size_ - read_buff_off_; + if (size <= remaining_size) { + memcpy(ptr, read_buff_.data() + read_buff_off_, size); + read_buff_off_ += size; + return static_cast(size); + } else { + memcpy(ptr, read_buff_.data() + read_buff_off_, remaining_size); + read_buff_off_ += remaining_size; + return static_cast(remaining_size); + } + } + + if (!wait_readable()) { return -1; } + + read_buff_off_ = 0; + read_buff_content_size_ = 0; + + if (size < read_buff_size_) { + auto n = read_socket(sock_, read_buff_.data(), read_buff_size_, + CPPHTTPLIB_RECV_FLAGS); + if (n <= 0) { + return n; + } else if (n <= static_cast(size)) { + memcpy(ptr, read_buff_.data(), static_cast(n)); + return n; + } else { + memcpy(ptr, read_buff_.data(), size); + read_buff_off_ = size; + read_buff_content_size_ = static_cast(n); + return static_cast(size); + } + } else { + return read_socket(sock_, ptr, size, CPPHTTPLIB_RECV_FLAGS); + } +} + +inline ssize_t SocketStream::write(const char *ptr, size_t size) { + if (!wait_writable()) { return -1; } + +#if defined(_WIN64) && !defined(_WIN64) + size = + (std::min)(size, static_cast((std::numeric_limits::max)())); +#endif + + return send_socket(sock_, ptr, size, CPPHTTPLIB_SEND_FLAGS); +} + +inline void SocketStream::get_remote_ip_and_port(std::string &ip, + int &port) const { + return detail::get_remote_ip_and_port(sock_, ip, port); +} + +inline void SocketStream::get_local_ip_and_port(std::string &ip, + int &port) const { + return detail::get_local_ip_and_port(sock_, ip, port); +} + +inline socket_t SocketStream::socket() const { return sock_; } + +inline time_t SocketStream::duration() const { + return std::chrono::duration_cast( + std::chrono::steady_clock::now() - start_time_) + .count(); +} + +// Buffer stream implementation +inline bool BufferStream::is_readable() const { return true; } + +inline bool BufferStream::wait_readable() const { return true; } + +inline bool BufferStream::wait_writable() const { return true; } + +inline ssize_t BufferStream::read(char *ptr, size_t size) { +#if defined(_MSC_VER) && _MSC_VER < 1910 + auto len_read = buffer._Copy_s(ptr, size, size, position); +#else + auto len_read = buffer.copy(ptr, size, position); +#endif + position += static_cast(len_read); + return static_cast(len_read); +} + +inline ssize_t BufferStream::write(const char *ptr, size_t size) { + buffer.append(ptr, size); + return static_cast(size); +} + +inline void BufferStream::get_remote_ip_and_port(std::string & /*ip*/, + int & /*port*/) const {} + +inline void BufferStream::get_local_ip_and_port(std::string & /*ip*/, + int & /*port*/) const {} + +inline socket_t BufferStream::socket() const { return 0; } + +inline time_t BufferStream::duration() const { return 0; } + +inline const std::string &BufferStream::get_buffer() const { return buffer; } + +inline PathParamsMatcher::PathParamsMatcher(const std::string &pattern) + : MatcherBase(pattern) { + constexpr const char marker[] = "/:"; + + // One past the last ending position of a path param substring + std::size_t last_param_end = 0; + +#ifndef CPPHTTPLIB_NO_EXCEPTIONS + // Needed to ensure that parameter names are unique during matcher + // construction + // If exceptions are disabled, only last duplicate path + // parameter will be set + std::unordered_set param_name_set; +#endif + + while (true) { + const auto marker_pos = pattern.find( + marker, last_param_end == 0 ? last_param_end : last_param_end - 1); + if (marker_pos == std::string::npos) { break; } + + static_fragments_.push_back( + pattern.substr(last_param_end, marker_pos - last_param_end + 1)); + + const auto param_name_start = marker_pos + str_len(marker); + + auto sep_pos = pattern.find(separator, param_name_start); + if (sep_pos == std::string::npos) { sep_pos = pattern.length(); } + + auto param_name = + pattern.substr(param_name_start, sep_pos - param_name_start); + +#ifndef CPPHTTPLIB_NO_EXCEPTIONS + if (param_name_set.find(param_name) != param_name_set.cend()) { + std::string msg = "Encountered path parameter '" + param_name + + "' multiple times in route pattern '" + pattern + "'."; + throw std::invalid_argument(msg); + } +#endif + + param_names_.push_back(std::move(param_name)); + + last_param_end = sep_pos + 1; + } + + if (last_param_end < pattern.length()) { + static_fragments_.push_back(pattern.substr(last_param_end)); + } +} + +inline bool PathParamsMatcher::match(Request &request) const { + request.matches = std::smatch(); + request.path_params.clear(); + request.path_params.reserve(param_names_.size()); + + // One past the position at which the path matched the pattern last time + std::size_t starting_pos = 0; + for (size_t i = 0; i < static_fragments_.size(); ++i) { + const auto &fragment = static_fragments_[i]; + + if (starting_pos + fragment.length() > request.path.length()) { + return false; + } + + // Avoid unnecessary allocation by using strncmp instead of substr + + // comparison + if (std::strncmp(request.path.c_str() + starting_pos, fragment.c_str(), + fragment.length()) != 0) { + return false; + } + + starting_pos += fragment.length(); + + // Should only happen when we have a static fragment after a param + // Example: '/users/:id/subscriptions' + // The 'subscriptions' fragment here does not have a corresponding param + if (i >= param_names_.size()) { continue; } + + auto sep_pos = request.path.find(separator, starting_pos); + if (sep_pos == std::string::npos) { sep_pos = request.path.length(); } + + const auto ¶m_name = param_names_[i]; + + request.path_params.emplace( + param_name, request.path.substr(starting_pos, sep_pos - starting_pos)); + + // Mark everything up to '/' as matched + starting_pos = sep_pos + 1; + } + // Returns false if the path is longer than the pattern + return starting_pos >= request.path.length(); +} + +inline bool RegexMatcher::match(Request &request) const { + request.path_params.clear(); + return std::regex_match(request.path, request.matches, regex_); +} + +} // namespace detail + +// HTTP server implementation +inline Server::Server() + : new_task_queue( + [] { return new ThreadPool(CPPHTTPLIB_THREAD_POOL_COUNT); }) { +#ifndef _WIN64 + signal(SIGPIPE, SIG_IGN); +#endif +} + +inline Server::~Server() = default; + +inline std::unique_ptr +Server::make_matcher(const std::string &pattern) { + if (pattern.find("/:") != std::string::npos) { + return detail::make_unique(pattern); + } else { + return detail::make_unique(pattern); + } +} + +inline Server &Server::Get(const std::string &pattern, Handler handler) { + get_handlers_.emplace_back(make_matcher(pattern), std::move(handler)); + return *this; +} + +inline Server &Server::Post(const std::string &pattern, Handler handler) { + post_handlers_.emplace_back(make_matcher(pattern), std::move(handler)); + return *this; +} + +inline Server &Server::Post(const std::string &pattern, + HandlerWithContentReader handler) { + post_handlers_for_content_reader_.emplace_back(make_matcher(pattern), + std::move(handler)); + return *this; +} + +inline Server &Server::Put(const std::string &pattern, Handler handler) { + put_handlers_.emplace_back(make_matcher(pattern), std::move(handler)); + return *this; +} + +inline Server &Server::Put(const std::string &pattern, + HandlerWithContentReader handler) { + put_handlers_for_content_reader_.emplace_back(make_matcher(pattern), + std::move(handler)); + return *this; +} + +inline Server &Server::Patch(const std::string &pattern, Handler handler) { + patch_handlers_.emplace_back(make_matcher(pattern), std::move(handler)); + return *this; +} + +inline Server &Server::Patch(const std::string &pattern, + HandlerWithContentReader handler) { + patch_handlers_for_content_reader_.emplace_back(make_matcher(pattern), + std::move(handler)); + return *this; +} + +inline Server &Server::Delete(const std::string &pattern, Handler handler) { + delete_handlers_.emplace_back(make_matcher(pattern), std::move(handler)); + return *this; +} + +inline Server &Server::Delete(const std::string &pattern, + HandlerWithContentReader handler) { + delete_handlers_for_content_reader_.emplace_back(make_matcher(pattern), + std::move(handler)); + return *this; +} + +inline Server &Server::Options(const std::string &pattern, Handler handler) { + options_handlers_.emplace_back(make_matcher(pattern), std::move(handler)); + return *this; +} + +inline bool Server::set_base_dir(const std::string &dir, + const std::string &mount_point) { + return set_mount_point(mount_point, dir); +} + +inline bool Server::set_mount_point(const std::string &mount_point, + const std::string &dir, Headers headers) { + detail::FileStat stat(dir); + if (stat.is_dir()) { + std::string mnt = !mount_point.empty() ? mount_point : "/"; + if (!mnt.empty() && mnt[0] == '/') { + base_dirs_.push_back({mnt, dir, std::move(headers)}); + return true; + } + } + return false; +} + +inline bool Server::remove_mount_point(const std::string &mount_point) { + for (auto it = base_dirs_.begin(); it != base_dirs_.end(); ++it) { + if (it->mount_point == mount_point) { + base_dirs_.erase(it); + return true; + } + } + return false; +} + +inline Server & +Server::set_file_extension_and_mimetype_mapping(const std::string &ext, + const std::string &mime) { + file_extension_and_mimetype_map_[ext] = mime; + return *this; +} + +inline Server &Server::set_default_file_mimetype(const std::string &mime) { + default_file_mimetype_ = mime; + return *this; +} + +inline Server &Server::set_file_request_handler(Handler handler) { + file_request_handler_ = std::move(handler); + return *this; +} + +inline Server &Server::set_error_handler_core(HandlerWithResponse handler, + std::true_type) { + error_handler_ = std::move(handler); + return *this; +} + +inline Server &Server::set_error_handler_core(Handler handler, + std::false_type) { + error_handler_ = [handler](const Request &req, Response &res) { + handler(req, res); + return HandlerResponse::Handled; + }; + return *this; +} + +inline Server &Server::set_exception_handler(ExceptionHandler handler) { + exception_handler_ = std::move(handler); + return *this; +} + +inline Server &Server::set_pre_routing_handler(HandlerWithResponse handler) { + pre_routing_handler_ = std::move(handler); + return *this; +} + +inline Server &Server::set_post_routing_handler(Handler handler) { + post_routing_handler_ = std::move(handler); + return *this; +} + +inline Server &Server::set_pre_request_handler(HandlerWithResponse handler) { + pre_request_handler_ = std::move(handler); + return *this; +} + +inline Server &Server::set_logger(Logger logger) { + logger_ = std::move(logger); + return *this; +} + +inline Server &Server::set_error_logger(ErrorLogger error_logger) { + error_logger_ = std::move(error_logger); + return *this; +} + +inline Server &Server::set_pre_compression_logger(Logger logger) { + pre_compression_logger_ = std::move(logger); + return *this; +} + +inline Server & +Server::set_expect_100_continue_handler(Expect100ContinueHandler handler) { + expect_100_continue_handler_ = std::move(handler); + return *this; +} + +inline Server &Server::set_address_family(int family) { + address_family_ = family; + return *this; +} + +inline Server &Server::set_tcp_nodelay(bool on) { + tcp_nodelay_ = on; + return *this; +} + +inline Server &Server::set_ipv6_v6only(bool on) { + ipv6_v6only_ = on; + return *this; +} + +inline Server &Server::set_socket_options(SocketOptions socket_options) { + socket_options_ = std::move(socket_options); + return *this; +} + +inline Server &Server::set_default_headers(Headers headers) { + default_headers_ = std::move(headers); + return *this; +} + +inline Server &Server::set_header_writer( + std::function const &writer) { + header_writer_ = writer; + return *this; +} + +inline Server &Server::set_keep_alive_max_count(size_t count) { + keep_alive_max_count_ = count; + return *this; +} + +inline Server &Server::set_keep_alive_timeout(time_t sec) { + keep_alive_timeout_sec_ = sec; + return *this; +} + +inline Server &Server::set_read_timeout(time_t sec, time_t usec) { + read_timeout_sec_ = sec; + read_timeout_usec_ = usec; + return *this; +} + +inline Server &Server::set_write_timeout(time_t sec, time_t usec) { + write_timeout_sec_ = sec; + write_timeout_usec_ = usec; + return *this; +} + +inline Server &Server::set_idle_interval(time_t sec, time_t usec) { + idle_interval_sec_ = sec; + idle_interval_usec_ = usec; + return *this; +} + +inline Server &Server::set_payload_max_length(size_t length) { + payload_max_length_ = length; + return *this; +} + +inline bool Server::bind_to_port(const std::string &host, int port, + int socket_flags) { + auto ret = bind_internal(host, port, socket_flags); + if (ret == -1) { is_decommissioned = true; } + return ret >= 0; +} +inline int Server::bind_to_any_port(const std::string &host, int socket_flags) { + auto ret = bind_internal(host, 0, socket_flags); + if (ret == -1) { is_decommissioned = true; } + return ret; +} + +inline bool Server::listen_after_bind() { return listen_internal(); } + +inline bool Server::listen(const std::string &host, int port, + int socket_flags) { + return bind_to_port(host, port, socket_flags) && listen_internal(); +} + +inline bool Server::is_running() const { return is_running_; } + +inline void Server::wait_until_ready() const { + while (!is_running_ && !is_decommissioned) { + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + } +} + +inline void Server::stop() { + if (is_running_) { + assert(svr_sock_ != INVALID_SOCKET); + std::atomic sock(svr_sock_.exchange(INVALID_SOCKET)); + detail::shutdown_socket(sock); + detail::close_socket(sock); + } + is_decommissioned = false; +} + +inline void Server::decommission() { is_decommissioned = true; } + +inline bool Server::parse_request_line(const char *s, Request &req) const { + auto len = strlen(s); + if (len < 2 || s[len - 2] != '\r' || s[len - 1] != '\n') { return false; } + len -= 2; + + { + size_t count = 0; + + detail::split(s, s + len, ' ', [&](const char *b, const char *e) { + switch (count) { + case 0: req.method = std::string(b, e); break; + case 1: req.target = std::string(b, e); break; + case 2: req.version = std::string(b, e); break; + default: break; + } + count++; + }); + + if (count != 3) { return false; } + } + + thread_local const std::set methods{ + "GET", "HEAD", "POST", "PUT", "DELETE", + "CONNECT", "OPTIONS", "TRACE", "PATCH", "PRI"}; + + if (methods.find(req.method) == methods.end()) { + output_error_log(Error::InvalidHTTPMethod, &req); + return false; + } + + if (req.version != "HTTP/1.1" && req.version != "HTTP/1.0") { + output_error_log(Error::InvalidHTTPVersion, &req); + return false; + } + + { + // Skip URL fragment + for (size_t i = 0; i < req.target.size(); i++) { + if (req.target[i] == '#') { + req.target.erase(i); + break; + } + } + + detail::divide(req.target, '?', + [&](const char *lhs_data, std::size_t lhs_size, + const char *rhs_data, std::size_t rhs_size) { + req.path = + decode_path_component(std::string(lhs_data, lhs_size)); + detail::parse_query_text(rhs_data, rhs_size, req.params); + }); + } + + return true; +} + +inline bool Server::write_response(Stream &strm, bool close_connection, + Request &req, Response &res) { + // NOTE: `req.ranges` should be empty, otherwise it will be applied + // incorrectly to the error content. + req.ranges.clear(); + return write_response_core(strm, close_connection, req, res, false); +} + +inline bool Server::write_response_with_content(Stream &strm, + bool close_connection, + const Request &req, + Response &res) { + return write_response_core(strm, close_connection, req, res, true); +} + +inline bool Server::write_response_core(Stream &strm, bool close_connection, + const Request &req, Response &res, + bool need_apply_ranges) { + assert(res.status != -1); + + if (400 <= res.status && error_handler_ && + error_handler_(req, res) == HandlerResponse::Handled) { + need_apply_ranges = true; + } + + std::string content_type; + std::string boundary; + if (need_apply_ranges) { apply_ranges(req, res, content_type, boundary); } + + // Prepare additional headers + if (close_connection || req.get_header_value("Connection") == "close") { + res.set_header("Connection", "close"); + } else { + std::string s = "timeout="; + s += std::to_string(keep_alive_timeout_sec_); + s += ", max="; + s += std::to_string(keep_alive_max_count_); + res.set_header("Keep-Alive", s); + } + + if ((!res.body.empty() || res.content_length_ > 0 || res.content_provider_) && + !res.has_header("Content-Type")) { + res.set_header("Content-Type", "text/plain"); + } + + if (res.body.empty() && !res.content_length_ && !res.content_provider_ && + !res.has_header("Content-Length")) { + res.set_header("Content-Length", "0"); + } + + if (req.method == "HEAD" && !res.has_header("Accept-Ranges")) { + res.set_header("Accept-Ranges", "bytes"); + } + + if (post_routing_handler_) { post_routing_handler_(req, res); } + + // Response line and headers + { + detail::BufferStream bstrm; + if (!detail::write_response_line(bstrm, res.status)) { return false; } + if (!header_writer_(bstrm, res.headers)) { return false; } + + // Flush buffer + auto &data = bstrm.get_buffer(); + detail::write_data(strm, data.data(), data.size()); + } + + // Body + auto ret = true; + if (req.method != "HEAD") { + if (!res.body.empty()) { + if (!detail::write_data(strm, res.body.data(), res.body.size())) { + ret = false; + } + } else if (res.content_provider_) { + if (write_content_with_provider(strm, req, res, boundary, content_type)) { + res.content_provider_success_ = true; + } else { + ret = false; + } + } + } + + // Log + output_log(req, res); + + return ret; +} + +inline bool +Server::write_content_with_provider(Stream &strm, const Request &req, + Response &res, const std::string &boundary, + const std::string &content_type) { + auto is_shutting_down = [this]() { + return this->svr_sock_ == INVALID_SOCKET; + }; + + if (res.content_length_ > 0) { + if (req.ranges.empty()) { + return detail::write_content(strm, res.content_provider_, 0, + res.content_length_, is_shutting_down); + } else if (req.ranges.size() == 1) { + auto offset_and_length = detail::get_range_offset_and_length( + req.ranges[0], res.content_length_); + + return detail::write_content(strm, res.content_provider_, + offset_and_length.first, + offset_and_length.second, is_shutting_down); + } else { + return detail::write_multipart_ranges_data( + strm, req, res, boundary, content_type, res.content_length_, + is_shutting_down); + } + } else { + if (res.is_chunked_content_provider_) { + auto type = detail::encoding_type(req, res); + + std::unique_ptr compressor; + if (type == detail::EncodingType::Gzip) { +#ifdef CPPHTTPLIB_ZLIB_SUPPORT + compressor = detail::make_unique(); +#endif + } else if (type == detail::EncodingType::Brotli) { +#ifdef CPPHTTPLIB_BROTLI_SUPPORT + compressor = detail::make_unique(); +#endif + } else if (type == detail::EncodingType::Zstd) { +#ifdef CPPHTTPLIB_ZSTD_SUPPORT + compressor = detail::make_unique(); +#endif + } else { + compressor = detail::make_unique(); + } + assert(compressor != nullptr); + + return detail::write_content_chunked(strm, res.content_provider_, + is_shutting_down, *compressor); + } else { + return detail::write_content_without_length(strm, res.content_provider_, + is_shutting_down); + } + } +} + +inline bool Server::read_content(Stream &strm, Request &req, Response &res) { + FormFields::iterator cur_field; + FormFiles::iterator cur_file; + auto is_text_field = false; + size_t count = 0; + if (read_content_core( + strm, req, res, + // Regular + [&](const char *buf, size_t n) { + if (req.body.size() + n > req.body.max_size()) { return false; } + req.body.append(buf, n); + return true; + }, + // Multipart FormData + [&](const FormData &file) { + if (count++ == CPPHTTPLIB_MULTIPART_FORM_DATA_FILE_MAX_COUNT) { + output_error_log(Error::TooManyFormDataFiles, &req); + return false; + } + + if (file.filename.empty()) { + cur_field = req.form.fields.emplace( + file.name, FormField{file.name, file.content, file.headers}); + is_text_field = true; + } else { + cur_file = req.form.files.emplace(file.name, file); + is_text_field = false; + } + return true; + }, + [&](const char *buf, size_t n) { + if (is_text_field) { + auto &content = cur_field->second.content; + if (content.size() + n > content.max_size()) { return false; } + content.append(buf, n); + } else { + auto &content = cur_file->second.content; + if (content.size() + n > content.max_size()) { return false; } + content.append(buf, n); + } + return true; + })) { + const auto &content_type = req.get_header_value("Content-Type"); + if (!content_type.find("application/x-www-form-urlencoded")) { + if (req.body.size() > CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH) { + res.status = StatusCode::PayloadTooLarge_413; // NOTE: should be 414? + output_error_log(Error::ExceedMaxPayloadSize, &req); + return false; + } + detail::parse_query_text(req.body, req.params); + } + return true; + } + return false; +} + +inline bool Server::read_content_with_content_receiver( + Stream &strm, Request &req, Response &res, ContentReceiver receiver, + FormDataHeader multipart_header, ContentReceiver multipart_receiver) { + return read_content_core(strm, req, res, std::move(receiver), + std::move(multipart_header), + std::move(multipart_receiver)); +} + +inline bool Server::read_content_core( + Stream &strm, Request &req, Response &res, ContentReceiver receiver, + FormDataHeader multipart_header, ContentReceiver multipart_receiver) const { + detail::FormDataParser multipart_form_data_parser; + ContentReceiverWithProgress out; + + if (req.is_multipart_form_data()) { + const auto &content_type = req.get_header_value("Content-Type"); + std::string boundary; + if (!detail::parse_multipart_boundary(content_type, boundary)) { + res.status = StatusCode::BadRequest_400; + output_error_log(Error::MultipartParsing, &req); + return false; + } + + multipart_form_data_parser.set_boundary(std::move(boundary)); + out = [&](const char *buf, size_t n, size_t /*off*/, size_t /*len*/) { + return multipart_form_data_parser.parse(buf, n, multipart_header, + multipart_receiver); + }; + } else { + out = [receiver](const char *buf, size_t n, size_t /*off*/, + size_t /*len*/) { return receiver(buf, n); }; + } + + if (req.method == "DELETE" && !req.has_header("Content-Length")) { + return true; + } + + if (!detail::read_content(strm, req, payload_max_length_, res.status, nullptr, + out, true)) { + return false; + } + + if (req.is_multipart_form_data()) { + if (!multipart_form_data_parser.is_valid()) { + res.status = StatusCode::BadRequest_400; + output_error_log(Error::MultipartParsing, &req); + return false; + } + } + + return true; +} + +inline bool Server::handle_file_request(const Request &req, Response &res) { + for (const auto &entry : base_dirs_) { + // Prefix match + if (!req.path.compare(0, entry.mount_point.size(), entry.mount_point)) { + std::string sub_path = "/" + req.path.substr(entry.mount_point.size()); + if (detail::is_valid_path(sub_path)) { + auto path = entry.base_dir + sub_path; + if (path.back() == '/') { path += "index.html"; } + + detail::FileStat stat(path); + + if (stat.is_dir()) { + res.set_redirect(sub_path + "/", StatusCode::MovedPermanently_301); + return true; + } + + if (stat.is_file()) { + for (const auto &kv : entry.headers) { + res.set_header(kv.first, kv.second); + } + + auto mm = std::make_shared(path.c_str()); + if (!mm->is_open()) { + output_error_log(Error::OpenFile, &req); + return false; + } + + res.set_content_provider( + mm->size(), + detail::find_content_type(path, file_extension_and_mimetype_map_, + default_file_mimetype_), + [mm](size_t offset, size_t length, DataSink &sink) -> bool { + sink.write(mm->data() + offset, length); + return true; + }); + + if (req.method != "HEAD" && file_request_handler_) { + file_request_handler_(req, res); + } + + return true; + } else { + output_error_log(Error::OpenFile, &req); + } + } + } + } + return false; +} + +inline socket_t +Server::create_server_socket(const std::string &host, int port, + int socket_flags, + SocketOptions socket_options) const { + return detail::create_socket( + host, std::string(), port, address_family_, socket_flags, tcp_nodelay_, + ipv6_v6only_, std::move(socket_options), + [&](socket_t sock, struct addrinfo &ai, bool & /*quit*/) -> bool { + if (::bind(sock, ai.ai_addr, static_cast(ai.ai_addrlen))) { + output_error_log(Error::BindIPAddress, nullptr); + return false; + } + if (::listen(sock, CPPHTTPLIB_LISTEN_BACKLOG)) { + output_error_log(Error::Listen, nullptr); + return false; + } + return true; + }); +} + +inline int Server::bind_internal(const std::string &host, int port, + int socket_flags) { + if (is_decommissioned) { return -1; } + + if (!is_valid()) { return -1; } + + svr_sock_ = create_server_socket(host, port, socket_flags, socket_options_); + if (svr_sock_ == INVALID_SOCKET) { return -1; } + + if (port == 0) { + struct sockaddr_storage addr; + socklen_t addr_len = sizeof(addr); + if (getsockname(svr_sock_, reinterpret_cast(&addr), + &addr_len) == -1) { + output_error_log(Error::GetSockName, nullptr); + return -1; + } + if (addr.ss_family == AF_INET) { + return ntohs(reinterpret_cast(&addr)->sin_port); + } else if (addr.ss_family == AF_INET6) { + return ntohs(reinterpret_cast(&addr)->sin6_port); + } else { + output_error_log(Error::UnsupportedAddressFamily, nullptr); + return -1; + } + } else { + return port; + } +} + +inline bool Server::listen_internal() { + if (is_decommissioned) { return false; } + + auto ret = true; + is_running_ = true; + auto se = detail::scope_exit([&]() { is_running_ = false; }); + + { + std::unique_ptr task_queue(new_task_queue()); + + while (svr_sock_ != INVALID_SOCKET) { +#ifndef _WIN64 + if (idle_interval_sec_ > 0 || idle_interval_usec_ > 0) { +#endif + auto val = detail::select_read(svr_sock_, idle_interval_sec_, + idle_interval_usec_); + if (val == 0) { // Timeout + task_queue->on_idle(); + continue; + } +#ifndef _WIN64 + } +#endif + +#if defined _WIN64 + // sockets connected via WASAccept inherit flags NO_HANDLE_INHERIT, + // OVERLAPPED + socket_t sock = WSAAccept(svr_sock_, nullptr, nullptr, nullptr, 0); +#elif defined SOCK_CLOEXEC + socket_t sock = accept4(svr_sock_, nullptr, nullptr, SOCK_CLOEXEC); +#else + socket_t sock = accept(svr_sock_, nullptr, nullptr); +#endif + + if (sock == INVALID_SOCKET) { + if (errno == EMFILE) { + // The per-process limit of open file descriptors has been reached. + // Try to accept new connections after a short sleep. + std::this_thread::sleep_for(std::chrono::microseconds{1}); + continue; + } else if (errno == EINTR || errno == EAGAIN) { + continue; + } + if (svr_sock_ != INVALID_SOCKET) { + detail::close_socket(svr_sock_); + ret = false; + output_error_log(Error::Connection, nullptr); + } else { + ; // The server socket was closed by user. + } + break; + } + + detail::set_socket_opt_time(sock, SOL_SOCKET, SO_RCVTIMEO, + read_timeout_sec_, read_timeout_usec_); + detail::set_socket_opt_time(sock, SOL_SOCKET, SO_SNDTIMEO, + write_timeout_sec_, write_timeout_usec_); + + if (!task_queue->enqueue( + [this, sock]() { process_and_close_socket(sock); })) { + output_error_log(Error::ResourceExhaustion, nullptr); + detail::shutdown_socket(sock); + detail::close_socket(sock); + } + } + + task_queue->shutdown(); + } + + is_decommissioned = !ret; + return ret; +} + +inline bool Server::routing(Request &req, Response &res, Stream &strm) { + if (pre_routing_handler_ && + pre_routing_handler_(req, res) == HandlerResponse::Handled) { + return true; + } + + // File handler + if ((req.method == "GET" || req.method == "HEAD") && + handle_file_request(req, res)) { + return true; + } + + if (detail::expect_content(req)) { + // Content reader handler + { + ContentReader reader( + [&](ContentReceiver receiver) { + auto result = read_content_with_content_receiver( + strm, req, res, std::move(receiver), nullptr, nullptr); + if (!result) { output_error_log(Error::Read, &req); } + return result; + }, + [&](FormDataHeader header, ContentReceiver receiver) { + auto result = read_content_with_content_receiver( + strm, req, res, nullptr, std::move(header), + std::move(receiver)); + if (!result) { output_error_log(Error::Read, &req); } + return result; + }); + + if (req.method == "POST") { + if (dispatch_request_for_content_reader( + req, res, std::move(reader), + post_handlers_for_content_reader_)) { + return true; + } + } else if (req.method == "PUT") { + if (dispatch_request_for_content_reader( + req, res, std::move(reader), + put_handlers_for_content_reader_)) { + return true; + } + } else if (req.method == "PATCH") { + if (dispatch_request_for_content_reader( + req, res, std::move(reader), + patch_handlers_for_content_reader_)) { + return true; + } + } else if (req.method == "DELETE") { + if (dispatch_request_for_content_reader( + req, res, std::move(reader), + delete_handlers_for_content_reader_)) { + return true; + } + } + } + + // Read content into `req.body` + if (!read_content(strm, req, res)) { + output_error_log(Error::Read, &req); + return false; + } + } + + // Regular handler + if (req.method == "GET" || req.method == "HEAD") { + return dispatch_request(req, res, get_handlers_); + } else if (req.method == "POST") { + return dispatch_request(req, res, post_handlers_); + } else if (req.method == "PUT") { + return dispatch_request(req, res, put_handlers_); + } else if (req.method == "DELETE") { + return dispatch_request(req, res, delete_handlers_); + } else if (req.method == "OPTIONS") { + return dispatch_request(req, res, options_handlers_); + } else if (req.method == "PATCH") { + return dispatch_request(req, res, patch_handlers_); + } + + res.status = StatusCode::BadRequest_400; + return false; +} + +inline bool Server::dispatch_request(Request &req, Response &res, + const Handlers &handlers) const { + for (const auto &x : handlers) { + const auto &matcher = x.first; + const auto &handler = x.second; + + if (matcher->match(req)) { + req.matched_route = matcher->pattern(); + if (!pre_request_handler_ || + pre_request_handler_(req, res) != HandlerResponse::Handled) { + handler(req, res); + } + return true; + } + } + return false; +} + +inline void Server::apply_ranges(const Request &req, Response &res, + std::string &content_type, + std::string &boundary) const { + if (req.ranges.size() > 1 && res.status == StatusCode::PartialContent_206) { + auto it = res.headers.find("Content-Type"); + if (it != res.headers.end()) { + content_type = it->second; + res.headers.erase(it); + } + + boundary = detail::make_multipart_data_boundary(); + + res.set_header("Content-Type", + "multipart/byteranges; boundary=" + boundary); + } + + auto type = detail::encoding_type(req, res); + + if (res.body.empty()) { + if (res.content_length_ > 0) { + size_t length = 0; + if (req.ranges.empty() || res.status != StatusCode::PartialContent_206) { + length = res.content_length_; + } else if (req.ranges.size() == 1) { + auto offset_and_length = detail::get_range_offset_and_length( + req.ranges[0], res.content_length_); + + length = offset_and_length.second; + + auto content_range = detail::make_content_range_header_field( + offset_and_length, res.content_length_); + res.set_header("Content-Range", content_range); + } else { + length = detail::get_multipart_ranges_data_length( + req, boundary, content_type, res.content_length_); + } + res.set_header("Content-Length", std::to_string(length)); + } else { + if (res.content_provider_) { + if (res.is_chunked_content_provider_) { + res.set_header("Transfer-Encoding", "chunked"); + if (type == detail::EncodingType::Gzip) { + res.set_header("Content-Encoding", "gzip"); + } else if (type == detail::EncodingType::Brotli) { + res.set_header("Content-Encoding", "br"); + } else if (type == detail::EncodingType::Zstd) { + res.set_header("Content-Encoding", "zstd"); + } + } + } + } + } else { + if (req.ranges.empty() || res.status != StatusCode::PartialContent_206) { + ; + } else if (req.ranges.size() == 1) { + auto offset_and_length = + detail::get_range_offset_and_length(req.ranges[0], res.body.size()); + auto offset = offset_and_length.first; + auto length = offset_and_length.second; + + auto content_range = detail::make_content_range_header_field( + offset_and_length, res.body.size()); + res.set_header("Content-Range", content_range); + + assert(offset + length <= res.body.size()); + res.body = res.body.substr(offset, length); + } else { + std::string data; + detail::make_multipart_ranges_data(req, res, boundary, content_type, + res.body.size(), data); + res.body.swap(data); + } + + if (type != detail::EncodingType::None) { + output_pre_compression_log(req, res); + + std::unique_ptr compressor; + std::string content_encoding; + + if (type == detail::EncodingType::Gzip) { +#ifdef CPPHTTPLIB_ZLIB_SUPPORT + compressor = detail::make_unique(); + content_encoding = "gzip"; +#endif + } else if (type == detail::EncodingType::Brotli) { +#ifdef CPPHTTPLIB_BROTLI_SUPPORT + compressor = detail::make_unique(); + content_encoding = "br"; +#endif + } else if (type == detail::EncodingType::Zstd) { +#ifdef CPPHTTPLIB_ZSTD_SUPPORT + compressor = detail::make_unique(); + content_encoding = "zstd"; +#endif + } + + if (compressor) { + std::string compressed; + if (compressor->compress(res.body.data(), res.body.size(), true, + [&](const char *data, size_t data_len) { + compressed.append(data, data_len); + return true; + })) { + res.body.swap(compressed); + res.set_header("Content-Encoding", content_encoding); + } + } + } + + auto length = std::to_string(res.body.size()); + res.set_header("Content-Length", length); + } +} + +inline bool Server::dispatch_request_for_content_reader( + Request &req, Response &res, ContentReader content_reader, + const HandlersForContentReader &handlers) const { + for (const auto &x : handlers) { + const auto &matcher = x.first; + const auto &handler = x.second; + + if (matcher->match(req)) { + req.matched_route = matcher->pattern(); + if (!pre_request_handler_ || + pre_request_handler_(req, res) != HandlerResponse::Handled) { + handler(req, res, content_reader); + } + return true; + } + } + return false; +} + +inline bool +Server::process_request(Stream &strm, const std::string &remote_addr, + int remote_port, const std::string &local_addr, + int local_port, bool close_connection, + bool &connection_closed, + const std::function &setup_request) { + std::array buf{}; + + detail::stream_line_reader line_reader(strm, buf.data(), buf.size()); + + // Connection has been closed on client + if (!line_reader.getline()) { return false; } + + Request req; + + Response res; + res.version = "HTTP/1.1"; + res.headers = default_headers_; + +#ifdef __APPLE__ + // Socket file descriptor exceeded FD_SETSIZE... + if (strm.socket() >= FD_SETSIZE) { + Headers dummy; + detail::read_headers(strm, dummy); + res.status = StatusCode::InternalServerError_500; + output_error_log(Error::ExceedMaxSocketDescriptorCount, &req); + return write_response(strm, close_connection, req, res); + } +#endif + + // Request line and headers + if (!parse_request_line(line_reader.ptr(), req)) { + res.status = StatusCode::BadRequest_400; + output_error_log(Error::InvalidRequestLine, &req); + return write_response(strm, close_connection, req, res); + } + + // Request headers + if (!detail::read_headers(strm, req.headers)) { + res.status = StatusCode::BadRequest_400; + output_error_log(Error::InvalidHeaders, &req); + return write_response(strm, close_connection, req, res); + } + + // Check if the request URI doesn't exceed the limit + if (req.target.size() > CPPHTTPLIB_REQUEST_URI_MAX_LENGTH) { + Headers dummy; + detail::read_headers(strm, dummy); + res.status = StatusCode::UriTooLong_414; + output_error_log(Error::ExceedUriMaxLength, &req); + return write_response(strm, close_connection, req, res); + } + + if (req.get_header_value("Connection") == "close") { + connection_closed = true; + } + + if (req.version == "HTTP/1.0" && + req.get_header_value("Connection") != "Keep-Alive") { + connection_closed = true; + } + + req.remote_addr = remote_addr; + req.remote_port = remote_port; + req.set_header("REMOTE_ADDR", req.remote_addr); + req.set_header("REMOTE_PORT", std::to_string(req.remote_port)); + + req.local_addr = local_addr; + req.local_port = local_port; + req.set_header("LOCAL_ADDR", req.local_addr); + req.set_header("LOCAL_PORT", std::to_string(req.local_port)); + + if (req.has_header("Accept")) { + const auto &accept_header = req.get_header_value("Accept"); + if (!detail::parse_accept_header(accept_header, req.accept_content_types)) { + res.status = StatusCode::BadRequest_400; + output_error_log(Error::HTTPParsing, &req); + return write_response(strm, close_connection, req, res); + } + } + + if (req.has_header("Range")) { + const auto &range_header_value = req.get_header_value("Range"); + if (!detail::parse_range_header(range_header_value, req.ranges)) { + res.status = StatusCode::RangeNotSatisfiable_416; + output_error_log(Error::InvalidRangeHeader, &req); + return write_response(strm, close_connection, req, res); + } + } + + if (setup_request) { setup_request(req); } + + if (req.get_header_value("Expect") == "100-continue") { + int status = StatusCode::Continue_100; + if (expect_100_continue_handler_) { + status = expect_100_continue_handler_(req, res); + } + switch (status) { + case StatusCode::Continue_100: + case StatusCode::ExpectationFailed_417: + detail::write_response_line(strm, status); + strm.write("\r\n"); + break; + default: + connection_closed = true; + return write_response(strm, true, req, res); + } + } + + // Setup `is_connection_closed` method + auto sock = strm.socket(); + req.is_connection_closed = [sock]() { + return !detail::is_socket_alive(sock); + }; + + // Routing + auto routed = false; +#ifdef CPPHTTPLIB_NO_EXCEPTIONS + routed = routing(req, res, strm); +#else + try { + routed = routing(req, res, strm); + } catch (std::exception &e) { + if (exception_handler_) { + auto ep = std::current_exception(); + exception_handler_(req, res, ep); + routed = true; + } else { + res.status = StatusCode::InternalServerError_500; + std::string val; + auto s = e.what(); + for (size_t i = 0; s[i]; i++) { + switch (s[i]) { + case '\r': val += "\\r"; break; + case '\n': val += "\\n"; break; + default: val += s[i]; break; + } + } + res.set_header("EXCEPTION_WHAT", val); + } + } catch (...) { + if (exception_handler_) { + auto ep = std::current_exception(); + exception_handler_(req, res, ep); + routed = true; + } else { + res.status = StatusCode::InternalServerError_500; + res.set_header("EXCEPTION_WHAT", "UNKNOWN"); + } + } +#endif + if (routed) { + if (res.status == -1) { + res.status = req.ranges.empty() ? StatusCode::OK_200 + : StatusCode::PartialContent_206; + } + + // Serve file content by using a content provider + if (!res.file_content_path_.empty()) { + const auto &path = res.file_content_path_; + auto mm = std::make_shared(path.c_str()); + if (!mm->is_open()) { + res.body.clear(); + res.content_length_ = 0; + res.content_provider_ = nullptr; + res.status = StatusCode::NotFound_404; + output_error_log(Error::OpenFile, &req); + return write_response(strm, close_connection, req, res); + } + + auto content_type = res.file_content_content_type_; + if (content_type.empty()) { + content_type = detail::find_content_type( + path, file_extension_and_mimetype_map_, default_file_mimetype_); + } + + res.set_content_provider( + mm->size(), content_type, + [mm](size_t offset, size_t length, DataSink &sink) -> bool { + sink.write(mm->data() + offset, length); + return true; + }); + } + + if (detail::range_error(req, res)) { + res.body.clear(); + res.content_length_ = 0; + res.content_provider_ = nullptr; + res.status = StatusCode::RangeNotSatisfiable_416; + return write_response(strm, close_connection, req, res); + } + + return write_response_with_content(strm, close_connection, req, res); + } else { + if (res.status == -1) { res.status = StatusCode::NotFound_404; } + + return write_response(strm, close_connection, req, res); + } +} + +inline bool Server::is_valid() const { return true; } + +inline bool Server::process_and_close_socket(socket_t sock) { + std::string remote_addr; + int remote_port = 0; + detail::get_remote_ip_and_port(sock, remote_addr, remote_port); + + std::string local_addr; + int local_port = 0; + detail::get_local_ip_and_port(sock, local_addr, local_port); + + auto ret = detail::process_server_socket( + svr_sock_, sock, keep_alive_max_count_, keep_alive_timeout_sec_, + read_timeout_sec_, read_timeout_usec_, write_timeout_sec_, + write_timeout_usec_, + [&](Stream &strm, bool close_connection, bool &connection_closed) { + return process_request(strm, remote_addr, remote_port, local_addr, + local_port, close_connection, connection_closed, + nullptr); + }); + + detail::shutdown_socket(sock); + detail::close_socket(sock); + return ret; +} + +inline void Server::output_log(const Request &req, const Response &res) const { + if (logger_) { + std::lock_guard guard(logger_mutex_); + logger_(req, res); + } +} + +inline void Server::output_pre_compression_log(const Request &req, + const Response &res) const { + if (pre_compression_logger_) { + std::lock_guard guard(logger_mutex_); + pre_compression_logger_(req, res); + } +} + +inline void Server::output_error_log(const Error &err, + const Request *req) const { + if (error_logger_) { + std::lock_guard guard(logger_mutex_); + error_logger_(err, req); + } +} + +// HTTP client implementation +inline ClientImpl::ClientImpl(const std::string &host) + : ClientImpl(host, 80, std::string(), std::string()) {} + +inline ClientImpl::ClientImpl(const std::string &host, int port) + : ClientImpl(host, port, std::string(), std::string()) {} + +inline ClientImpl::ClientImpl(const std::string &host, int port, + const std::string &client_cert_path, + const std::string &client_key_path) + : host_(detail::escape_abstract_namespace_unix_domain(host)), port_(port), + host_and_port_(adjust_host_string(host_) + ":" + std::to_string(port)), + client_cert_path_(client_cert_path), client_key_path_(client_key_path) {} + +inline ClientImpl::~ClientImpl() { + // Wait until all the requests in flight are handled. + size_t retry_count = 10; + while (retry_count-- > 0) { + { + std::lock_guard guard(socket_mutex_); + if (socket_requests_in_flight_ == 0) { break; } + } + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + } + + std::lock_guard guard(socket_mutex_); + shutdown_socket(socket_); + close_socket(socket_); +} + +inline bool ClientImpl::is_valid() const { return true; } + +inline void ClientImpl::copy_settings(const ClientImpl &rhs) { + client_cert_path_ = rhs.client_cert_path_; + client_key_path_ = rhs.client_key_path_; + connection_timeout_sec_ = rhs.connection_timeout_sec_; + read_timeout_sec_ = rhs.read_timeout_sec_; + read_timeout_usec_ = rhs.read_timeout_usec_; + write_timeout_sec_ = rhs.write_timeout_sec_; + write_timeout_usec_ = rhs.write_timeout_usec_; + max_timeout_msec_ = rhs.max_timeout_msec_; + basic_auth_username_ = rhs.basic_auth_username_; + basic_auth_password_ = rhs.basic_auth_password_; + bearer_token_auth_token_ = rhs.bearer_token_auth_token_; +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + digest_auth_username_ = rhs.digest_auth_username_; + digest_auth_password_ = rhs.digest_auth_password_; +#endif + keep_alive_ = rhs.keep_alive_; + follow_location_ = rhs.follow_location_; + path_encode_ = rhs.path_encode_; + address_family_ = rhs.address_family_; + tcp_nodelay_ = rhs.tcp_nodelay_; + ipv6_v6only_ = rhs.ipv6_v6only_; + socket_options_ = rhs.socket_options_; + compress_ = rhs.compress_; + decompress_ = rhs.decompress_; + interface_ = rhs.interface_; + proxy_host_ = rhs.proxy_host_; + proxy_port_ = rhs.proxy_port_; + proxy_basic_auth_username_ = rhs.proxy_basic_auth_username_; + proxy_basic_auth_password_ = rhs.proxy_basic_auth_password_; + proxy_bearer_token_auth_token_ = rhs.proxy_bearer_token_auth_token_; +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + proxy_digest_auth_username_ = rhs.proxy_digest_auth_username_; + proxy_digest_auth_password_ = rhs.proxy_digest_auth_password_; +#endif +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + ca_cert_file_path_ = rhs.ca_cert_file_path_; + ca_cert_dir_path_ = rhs.ca_cert_dir_path_; + ca_cert_store_ = rhs.ca_cert_store_; +#endif +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + server_certificate_verification_ = rhs.server_certificate_verification_; + server_hostname_verification_ = rhs.server_hostname_verification_; + server_certificate_verifier_ = rhs.server_certificate_verifier_; +#endif + logger_ = rhs.logger_; + error_logger_ = rhs.error_logger_; +} + +inline socket_t ClientImpl::create_client_socket(Error &error) const { + if (!proxy_host_.empty() && proxy_port_ != -1) { + return detail::create_client_socket( + proxy_host_, std::string(), proxy_port_, address_family_, tcp_nodelay_, + ipv6_v6only_, socket_options_, connection_timeout_sec_, + connection_timeout_usec_, read_timeout_sec_, read_timeout_usec_, + write_timeout_sec_, write_timeout_usec_, interface_, error); + } + + // Check is custom IP specified for host_ + std::string ip; + auto it = addr_map_.find(host_); + if (it != addr_map_.end()) { ip = it->second; } + + return detail::create_client_socket( + host_, ip, port_, address_family_, tcp_nodelay_, ipv6_v6only_, + socket_options_, connection_timeout_sec_, connection_timeout_usec_, + read_timeout_sec_, read_timeout_usec_, write_timeout_sec_, + write_timeout_usec_, interface_, error); +} + +inline bool ClientImpl::create_and_connect_socket(Socket &socket, + Error &error) { + auto sock = create_client_socket(error); + if (sock == INVALID_SOCKET) { return false; } + socket.sock = sock; + return true; +} + +inline void ClientImpl::shutdown_ssl(Socket & /*socket*/, + bool /*shutdown_gracefully*/) { + // If there are any requests in flight from threads other than us, then it's + // a thread-unsafe race because individual ssl* objects are not thread-safe. + assert(socket_requests_in_flight_ == 0 || + socket_requests_are_from_thread_ == std::this_thread::get_id()); +} + +inline void ClientImpl::shutdown_socket(Socket &socket) const { + if (socket.sock == INVALID_SOCKET) { return; } + detail::shutdown_socket(socket.sock); +} + +inline void ClientImpl::close_socket(Socket &socket) { + // If there are requests in flight in another thread, usually closing + // the socket will be fine and they will simply receive an error when + // using the closed socket, but it is still a bug since rarely the OS + // may reassign the socket id to be used for a new socket, and then + // suddenly they will be operating on a live socket that is different + // than the one they intended! + assert(socket_requests_in_flight_ == 0 || + socket_requests_are_from_thread_ == std::this_thread::get_id()); + + // It is also a bug if this happens while SSL is still active +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + assert(socket.ssl == nullptr); +#endif + if (socket.sock == INVALID_SOCKET) { return; } + detail::close_socket(socket.sock); + socket.sock = INVALID_SOCKET; +} + +inline bool ClientImpl::read_response_line(Stream &strm, const Request &req, + Response &res) const { + std::array buf{}; + + detail::stream_line_reader line_reader(strm, buf.data(), buf.size()); + + if (!line_reader.getline()) { return false; } + +#ifdef CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR + thread_local const std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r?\n"); +#else + thread_local const std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r\n"); +#endif + + std::cmatch m; + if (!std::regex_match(line_reader.ptr(), m, re)) { + return req.method == "CONNECT"; + } + res.version = std::string(m[1]); + res.status = std::stoi(std::string(m[2])); + res.reason = std::string(m[3]); + + // Ignore '100 Continue' + while (res.status == StatusCode::Continue_100) { + if (!line_reader.getline()) { return false; } // CRLF + if (!line_reader.getline()) { return false; } // next response line + + if (!std::regex_match(line_reader.ptr(), m, re)) { return false; } + res.version = std::string(m[1]); + res.status = std::stoi(std::string(m[2])); + res.reason = std::string(m[3]); + } + + return true; +} + +inline bool ClientImpl::send(Request &req, Response &res, Error &error) { + std::lock_guard request_mutex_guard(request_mutex_); + auto ret = send_(req, res, error); + if (error == Error::SSLPeerCouldBeClosed_) { + assert(!ret); + ret = send_(req, res, error); + } + return ret; +} + +inline bool ClientImpl::send_(Request &req, Response &res, Error &error) { + { + std::lock_guard guard(socket_mutex_); + + // Set this to false immediately - if it ever gets set to true by the end of + // the request, we know another thread instructed us to close the socket. + socket_should_be_closed_when_request_is_done_ = false; + + auto is_alive = false; + if (socket_.is_open()) { + is_alive = detail::is_socket_alive(socket_.sock); + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + if (is_alive && is_ssl()) { + if (detail::is_ssl_peer_could_be_closed(socket_.ssl, socket_.sock)) { + is_alive = false; + } + } +#endif + + if (!is_alive) { + // Attempt to avoid sigpipe by shutting down non-gracefully if it seems + // like the other side has already closed the connection Also, there + // cannot be any requests in flight from other threads since we locked + // request_mutex_, so safe to close everything immediately + const bool shutdown_gracefully = false; + shutdown_ssl(socket_, shutdown_gracefully); + shutdown_socket(socket_); + close_socket(socket_); + } + } + + if (!is_alive) { + if (!create_and_connect_socket(socket_, error)) { + output_error_log(error, &req); + return false; + } + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + // TODO: refactoring + if (is_ssl()) { + auto &scli = static_cast(*this); + if (!proxy_host_.empty() && proxy_port_ != -1) { + auto success = false; + if (!scli.connect_with_proxy(socket_, req.start_time_, res, success, + error)) { + if (!success) { output_error_log(error, &req); } + return success; + } + } + + if (!scli.initialize_ssl(socket_, error)) { + output_error_log(error, &req); + return false; + } + } +#endif + } + + // Mark the current socket as being in use so that it cannot be closed by + // anyone else while this request is ongoing, even though we will be + // releasing the mutex. + if (socket_requests_in_flight_ > 1) { + assert(socket_requests_are_from_thread_ == std::this_thread::get_id()); + } + socket_requests_in_flight_ += 1; + socket_requests_are_from_thread_ = std::this_thread::get_id(); + } + + for (const auto &header : default_headers_) { + if (req.headers.find(header.first) == req.headers.end()) { + req.headers.insert(header); + } + } + + auto ret = false; + auto close_connection = !keep_alive_; + + auto se = detail::scope_exit([&]() { + // Briefly lock mutex in order to mark that a request is no longer ongoing + std::lock_guard guard(socket_mutex_); + socket_requests_in_flight_ -= 1; + if (socket_requests_in_flight_ <= 0) { + assert(socket_requests_in_flight_ == 0); + socket_requests_are_from_thread_ = std::thread::id(); + } + + if (socket_should_be_closed_when_request_is_done_ || close_connection || + !ret) { + shutdown_ssl(socket_, true); + shutdown_socket(socket_); + close_socket(socket_); + } + }); + + ret = process_socket(socket_, req.start_time_, [&](Stream &strm) { + return handle_request(strm, req, res, close_connection, error); + }); + + if (!ret) { + if (error == Error::Success) { + error = Error::Unknown; + output_error_log(error, &req); + } + } + + return ret; +} + +inline Result ClientImpl::send(const Request &req) { + auto req2 = req; + return send_(std::move(req2)); +} + +inline Result ClientImpl::send_(Request &&req) { + auto res = detail::make_unique(); + auto error = Error::Success; + auto ret = send(req, *res, error); +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + return Result{ret ? std::move(res) : nullptr, error, std::move(req.headers), + last_ssl_error_, last_openssl_error_}; +#else + return Result{ret ? std::move(res) : nullptr, error, std::move(req.headers)}; +#endif +} + +inline bool ClientImpl::handle_request(Stream &strm, Request &req, + Response &res, bool close_connection, + Error &error) { + if (req.path.empty()) { + error = Error::Connection; + output_error_log(error, &req); + return false; + } + + auto req_save = req; + + bool ret; + + if (!is_ssl() && !proxy_host_.empty() && proxy_port_ != -1) { + auto req2 = req; + req2.path = "http://" + host_and_port_ + req.path; + ret = process_request(strm, req2, res, close_connection, error); + req = req2; + req.path = req_save.path; + } else { + ret = process_request(strm, req, res, close_connection, error); + } + + if (!ret) { return false; } + + if (res.get_header_value("Connection") == "close" || + (res.version == "HTTP/1.0" && res.reason != "Connection established")) { + // TODO this requires a not-entirely-obvious chain of calls to be correct + // for this to be safe. + + // This is safe to call because handle_request is only called by send_ + // which locks the request mutex during the process. It would be a bug + // to call it from a different thread since it's a thread-safety issue + // to do these things to the socket if another thread is using the socket. + std::lock_guard guard(socket_mutex_); + shutdown_ssl(socket_, true); + shutdown_socket(socket_); + close_socket(socket_); + } + + if (300 < res.status && res.status < 400 && follow_location_) { + req = req_save; + ret = redirect(req, res, error); + } + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + if ((res.status == StatusCode::Unauthorized_401 || + res.status == StatusCode::ProxyAuthenticationRequired_407) && + req.authorization_count_ < 5) { + auto is_proxy = res.status == StatusCode::ProxyAuthenticationRequired_407; + const auto &username = + is_proxy ? proxy_digest_auth_username_ : digest_auth_username_; + const auto &password = + is_proxy ? proxy_digest_auth_password_ : digest_auth_password_; + + if (!username.empty() && !password.empty()) { + std::map auth; + if (detail::parse_www_authenticate(res, auth, is_proxy)) { + Request new_req = req; + new_req.authorization_count_ += 1; + new_req.headers.erase(is_proxy ? "Proxy-Authorization" + : "Authorization"); + new_req.headers.insert(detail::make_digest_authentication_header( + req, auth, new_req.authorization_count_, detail::random_string(10), + username, password, is_proxy)); + + Response new_res; + + ret = send(new_req, new_res, error); + if (ret) { res = new_res; } + } + } + } +#endif + + return ret; +} + +inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) { + if (req.redirect_count_ == 0) { + error = Error::ExceedRedirectCount; + output_error_log(error, &req); + return false; + } + + auto location = res.get_header_value("location"); + if (location.empty()) { return false; } + + thread_local const std::regex re( + R"((?:(https?):)?(?://(?:\[([a-fA-F\d:]+)\]|([^:/?#]+))(?::(\d+))?)?([^?#]*)(\?[^#]*)?(?:#.*)?)"); + + std::smatch m; + if (!std::regex_match(location, m, re)) { return false; } + + auto scheme = is_ssl() ? "https" : "http"; + + auto next_scheme = m[1].str(); + auto next_host = m[2].str(); + if (next_host.empty()) { next_host = m[3].str(); } + auto port_str = m[4].str(); + auto next_path = m[5].str(); + auto next_query = m[6].str(); + + auto next_port = port_; + if (!port_str.empty()) { + next_port = std::stoi(port_str); + } else if (!next_scheme.empty()) { + next_port = next_scheme == "https" ? 443 : 80; + } + + if (next_scheme.empty()) { next_scheme = scheme; } + if (next_host.empty()) { next_host = host_; } + if (next_path.empty()) { next_path = "/"; } + + auto path = decode_query_component(next_path, true) + next_query; + + // Same host redirect - use current client + if (next_scheme == scheme && next_host == host_ && next_port == port_) { + return detail::redirect(*this, req, res, path, location, error); + } + + // Cross-host/scheme redirect - create new client with robust setup + return create_redirect_client(next_scheme, next_host, next_port, req, res, + path, location, error); +} + +// New method for robust redirect client creation +inline bool ClientImpl::create_redirect_client( + const std::string &scheme, const std::string &host, int port, Request &req, + Response &res, const std::string &path, const std::string &location, + Error &error) { + // Determine if we need SSL + auto need_ssl = (scheme == "https"); + + // Clean up request headers that are host/client specific + // Remove headers that should not be carried over to new host + auto headers_to_remove = + std::vector{"Host", "Proxy-Authorization", "Authorization"}; + + for (const auto &header_name : headers_to_remove) { + auto it = req.headers.find(header_name); + while (it != req.headers.end()) { + it = req.headers.erase(it); + it = req.headers.find(header_name); + } + } + + // Create appropriate client type and handle redirect + if (need_ssl) { +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + // Create SSL client for HTTPS redirect + SSLClient redirect_client(host, port); + + // Setup basic client configuration first + setup_redirect_client(redirect_client); + + // SSL-specific configuration for proxy environments + if (!proxy_host_.empty() && proxy_port_ != -1) { + // Critical: Disable SSL verification for proxy environments + redirect_client.enable_server_certificate_verification(false); + redirect_client.enable_server_hostname_verification(false); + } else { + // For direct SSL connections, copy SSL verification settings + redirect_client.enable_server_certificate_verification( + server_certificate_verification_); + redirect_client.enable_server_hostname_verification( + server_hostname_verification_); + } + + // Handle CA certificate store and paths if available + if (ca_cert_store_) { redirect_client.set_ca_cert_store(ca_cert_store_); } + if (!ca_cert_file_path_.empty()) { + redirect_client.set_ca_cert_path(ca_cert_file_path_, ca_cert_dir_path_); + } + + // Client certificates are set through constructor for SSLClient + // NOTE: SSLClient constructor already takes client_cert_path and + // client_key_path so we need to create it properly if client certs are + // needed + + // Execute the redirect + return detail::redirect(redirect_client, req, res, path, location, error); +#else + // SSL not supported - set appropriate error + error = Error::SSLConnection; + output_error_log(error, &req); + return false; +#endif + } else { + // HTTP redirect + ClientImpl redirect_client(host, port); + + // Setup client with robust configuration + setup_redirect_client(redirect_client); + + // Execute the redirect + return detail::redirect(redirect_client, req, res, path, location, error); + } +} + +// New method for robust client setup (based on basic_manual_redirect.cpp logic) +template +inline void ClientImpl::setup_redirect_client(ClientType &client) { + // Copy basic settings first + client.set_connection_timeout(connection_timeout_sec_); + client.set_read_timeout(read_timeout_sec_, read_timeout_usec_); + client.set_write_timeout(write_timeout_sec_, write_timeout_usec_); + client.set_keep_alive(keep_alive_); + client.set_follow_location( + true); // Enable redirects to handle multi-step redirects + client.set_path_encode(path_encode_); + client.set_compress(compress_); + client.set_decompress(decompress_); + + // Copy authentication settings BEFORE proxy setup + if (!basic_auth_username_.empty()) { + client.set_basic_auth(basic_auth_username_, basic_auth_password_); + } + if (!bearer_token_auth_token_.empty()) { + client.set_bearer_token_auth(bearer_token_auth_token_); + } +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + if (!digest_auth_username_.empty()) { + client.set_digest_auth(digest_auth_username_, digest_auth_password_); + } +#endif + + // Setup proxy configuration (CRITICAL ORDER - proxy must be set + // before proxy auth) + if (!proxy_host_.empty() && proxy_port_ != -1) { + // First set proxy host and port + client.set_proxy(proxy_host_, proxy_port_); + + // Then set proxy authentication (order matters!) + if (!proxy_basic_auth_username_.empty()) { + client.set_proxy_basic_auth(proxy_basic_auth_username_, + proxy_basic_auth_password_); + } + if (!proxy_bearer_token_auth_token_.empty()) { + client.set_proxy_bearer_token_auth(proxy_bearer_token_auth_token_); + } +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + if (!proxy_digest_auth_username_.empty()) { + client.set_proxy_digest_auth(proxy_digest_auth_username_, + proxy_digest_auth_password_); + } +#endif + } + + // Copy network and socket settings + client.set_address_family(address_family_); + client.set_tcp_nodelay(tcp_nodelay_); + client.set_ipv6_v6only(ipv6_v6only_); + if (socket_options_) { client.set_socket_options(socket_options_); } + if (!interface_.empty()) { client.set_interface(interface_); } + + // Copy logging and headers + if (logger_) { client.set_logger(logger_); } + if (error_logger_) { client.set_error_logger(error_logger_); } + + // NOTE: DO NOT copy default_headers_ as they may contain stale Host headers + // Each new client should generate its own headers based on its target host +} + +inline bool ClientImpl::write_content_with_provider(Stream &strm, + const Request &req, + Error &error) const { + auto is_shutting_down = []() { return false; }; + + if (req.is_chunked_content_provider_) { + // TODO: Brotli support + std::unique_ptr compressor; +#ifdef CPPHTTPLIB_ZLIB_SUPPORT + if (compress_) { + compressor = detail::make_unique(); + } else +#endif + { + compressor = detail::make_unique(); + } + + return detail::write_content_chunked(strm, req.content_provider_, + is_shutting_down, *compressor, error); + } else { + return detail::write_content_with_progress( + strm, req.content_provider_, 0, req.content_length_, is_shutting_down, + req.upload_progress, error); + } +} + +inline bool ClientImpl::write_request(Stream &strm, Request &req, + bool close_connection, Error &error) { + // Prepare additional headers + if (close_connection) { + if (!req.has_header("Connection")) { + req.set_header("Connection", "close"); + } + } + + if (!req.has_header("Host")) { + // For Unix socket connections, use "localhost" as Host header (similar to + // curl behavior) + if (address_family_ == AF_UNIX) { + req.set_header("Host", "localhost"); + } else if (is_ssl()) { + if (port_ == 443) { + req.set_header("Host", host_); + } else { + req.set_header("Host", host_and_port_); + } + } else { + if (port_ == 80) { + req.set_header("Host", host_); + } else { + req.set_header("Host", host_and_port_); + } + } + } + + if (!req.has_header("Accept")) { req.set_header("Accept", "*/*"); } + + if (!req.content_receiver) { + if (!req.has_header("Accept-Encoding")) { + std::string accept_encoding; +#ifdef CPPHTTPLIB_BROTLI_SUPPORT + accept_encoding = "br"; +#endif +#ifdef CPPHTTPLIB_ZLIB_SUPPORT + if (!accept_encoding.empty()) { accept_encoding += ", "; } + accept_encoding += "gzip, deflate"; +#endif +#ifdef CPPHTTPLIB_ZSTD_SUPPORT + if (!accept_encoding.empty()) { accept_encoding += ", "; } + accept_encoding += "zstd"; +#endif + req.set_header("Accept-Encoding", accept_encoding); + } + +#ifndef CPPHTTPLIB_NO_DEFAULT_USER_AGENT + if (!req.has_header("User-Agent")) { + auto agent = std::string("cpp-httplib/") + CPPHTTPLIB_VERSION; + req.set_header("User-Agent", agent); + } +#endif + }; + + if (req.body.empty()) { + if (req.content_provider_) { + if (!req.is_chunked_content_provider_) { + if (!req.has_header("Content-Length")) { + auto length = std::to_string(req.content_length_); + req.set_header("Content-Length", length); + } + } + } else { + if (req.method == "POST" || req.method == "PUT" || + req.method == "PATCH") { + req.set_header("Content-Length", "0"); + } + } + } else { + if (!req.has_header("Content-Type")) { + req.set_header("Content-Type", "text/plain"); + } + + if (!req.has_header("Content-Length")) { + auto length = std::to_string(req.body.size()); + req.set_header("Content-Length", length); + } + } + + if (!basic_auth_password_.empty() || !basic_auth_username_.empty()) { + if (!req.has_header("Authorization")) { + req.headers.insert(make_basic_authentication_header( + basic_auth_username_, basic_auth_password_, false)); + } + } + + if (!proxy_basic_auth_username_.empty() && + !proxy_basic_auth_password_.empty()) { + if (!req.has_header("Proxy-Authorization")) { + req.headers.insert(make_basic_authentication_header( + proxy_basic_auth_username_, proxy_basic_auth_password_, true)); + } + } + + if (!bearer_token_auth_token_.empty()) { + if (!req.has_header("Authorization")) { + req.headers.insert(make_bearer_token_authentication_header( + bearer_token_auth_token_, false)); + } + } + + if (!proxy_bearer_token_auth_token_.empty()) { + if (!req.has_header("Proxy-Authorization")) { + req.headers.insert(make_bearer_token_authentication_header( + proxy_bearer_token_auth_token_, true)); + } + } + + // Request line and headers + { + detail::BufferStream bstrm; + + // Extract path and query from req.path + std::string path_part, query_part; + auto query_pos = req.path.find('?'); + if (query_pos != std::string::npos) { + path_part = req.path.substr(0, query_pos); + query_part = req.path.substr(query_pos + 1); + } else { + path_part = req.path; + query_part = ""; + } + + // Encode path and query + auto path_with_query = + path_encode_ ? detail::encode_path(path_part) : path_part; + + detail::parse_query_text(query_part, req.params); + if (!req.params.empty()) { + path_with_query = append_query_params(path_with_query, req.params); + } + + // Write request line and headers + detail::write_request_line(bstrm, req.method, path_with_query); + header_writer_(bstrm, req.headers); + + // Flush buffer + auto &data = bstrm.get_buffer(); + if (!detail::write_data(strm, data.data(), data.size())) { + error = Error::Write; + output_error_log(error, &req); + return false; + } + } + + // Body + if (req.body.empty()) { + return write_content_with_provider(strm, req, error); + } + + if (req.upload_progress) { + auto body_size = req.body.size(); + size_t written = 0; + auto data = req.body.data(); + + while (written < body_size) { + size_t to_write = (std::min)(CPPHTTPLIB_SEND_BUFSIZ, body_size - written); + if (!detail::write_data(strm, data + written, to_write)) { + error = Error::Write; + output_error_log(error, &req); + return false; + } + written += to_write; + + if (!req.upload_progress(written, body_size)) { + error = Error::Canceled; + output_error_log(error, &req); + return false; + } + } + } else { + if (!detail::write_data(strm, req.body.data(), req.body.size())) { + error = Error::Write; + output_error_log(error, &req); + return false; + } + } + + return true; +} + +inline std::unique_ptr ClientImpl::send_with_content_provider( + Request &req, const char *body, size_t content_length, + ContentProvider content_provider, + ContentProviderWithoutLength content_provider_without_length, + const std::string &content_type, Error &error) { + if (!content_type.empty()) { req.set_header("Content-Type", content_type); } + +#ifdef CPPHTTPLIB_ZLIB_SUPPORT + if (compress_) { req.set_header("Content-Encoding", "gzip"); } +#endif + +#ifdef CPPHTTPLIB_ZLIB_SUPPORT + if (compress_ && !content_provider_without_length) { + // TODO: Brotli support + detail::gzip_compressor compressor; + + if (content_provider) { + auto ok = true; + size_t offset = 0; + DataSink data_sink; + + data_sink.write = [&](const char *data, size_t data_len) -> bool { + if (ok) { + auto last = offset + data_len == content_length; + + auto ret = compressor.compress( + data, data_len, last, + [&](const char *compressed_data, size_t compressed_data_len) { + req.body.append(compressed_data, compressed_data_len); + return true; + }); + + if (ret) { + offset += data_len; + } else { + ok = false; + } + } + return ok; + }; + + while (ok && offset < content_length) { + if (!content_provider(offset, content_length - offset, data_sink)) { + error = Error::Canceled; + output_error_log(error, &req); + return nullptr; + } + } + } else { + if (!compressor.compress(body, content_length, true, + [&](const char *data, size_t data_len) { + req.body.append(data, data_len); + return true; + })) { + error = Error::Compression; + output_error_log(error, &req); + return nullptr; + } + } + } else +#endif + { + if (content_provider) { + req.content_length_ = content_length; + req.content_provider_ = std::move(content_provider); + req.is_chunked_content_provider_ = false; + } else if (content_provider_without_length) { + req.content_length_ = 0; + req.content_provider_ = detail::ContentProviderAdapter( + std::move(content_provider_without_length)); + req.is_chunked_content_provider_ = true; + req.set_header("Transfer-Encoding", "chunked"); + } else { + req.body.assign(body, content_length); + } + } + + auto res = detail::make_unique(); + return send(req, *res, error) ? std::move(res) : nullptr; +} + +inline Result ClientImpl::send_with_content_provider( + const std::string &method, const std::string &path, const Headers &headers, + const char *body, size_t content_length, ContentProvider content_provider, + ContentProviderWithoutLength content_provider_without_length, + const std::string &content_type, UploadProgress progress) { + Request req; + req.method = method; + req.headers = headers; + req.path = path; + req.upload_progress = std::move(progress); + if (max_timeout_msec_ > 0) { + req.start_time_ = std::chrono::steady_clock::now(); + } + + auto error = Error::Success; + + auto res = send_with_content_provider( + req, body, content_length, std::move(content_provider), + std::move(content_provider_without_length), content_type, error); + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + return Result{std::move(res), error, std::move(req.headers), last_ssl_error_, + last_openssl_error_}; +#else + return Result{std::move(res), error, std::move(req.headers)}; +#endif +} + +inline std::string +ClientImpl::adjust_host_string(const std::string &host) const { + if (host.find(':') != std::string::npos) { return "[" + host + "]"; } + return host; +} + +inline void ClientImpl::output_log(const Request &req, + const Response &res) const { + if (logger_) { + std::lock_guard guard(logger_mutex_); + logger_(req, res); + } +} + +inline void ClientImpl::output_error_log(const Error &err, + const Request *req) const { + if (error_logger_) { + std::lock_guard guard(logger_mutex_); + error_logger_(err, req); + } +} + +inline bool ClientImpl::process_request(Stream &strm, Request &req, + Response &res, bool close_connection, + Error &error) { + // Send request + if (!write_request(strm, req, close_connection, error)) { return false; } + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + if (is_ssl()) { + auto is_proxy_enabled = !proxy_host_.empty() && proxy_port_ != -1; + if (!is_proxy_enabled) { + if (detail::is_ssl_peer_could_be_closed(socket_.ssl, socket_.sock)) { + error = Error::SSLPeerCouldBeClosed_; + output_error_log(error, &req); + return false; + } + } + } +#endif + + // Receive response and headers + if (!read_response_line(strm, req, res) || + !detail::read_headers(strm, res.headers)) { + error = Error::Read; + output_error_log(error, &req); + return false; + } + + // Body + if ((res.status != StatusCode::NoContent_204) && req.method != "HEAD" && + req.method != "CONNECT") { + auto redirect = 300 < res.status && res.status < 400 && + res.status != StatusCode::NotModified_304 && + follow_location_; + + if (req.response_handler && !redirect) { + if (!req.response_handler(res)) { + error = Error::Canceled; + output_error_log(error, &req); + return false; + } + } + + auto out = + req.content_receiver + ? static_cast( + [&](const char *buf, size_t n, size_t off, size_t len) { + if (redirect) { return true; } + auto ret = req.content_receiver(buf, n, off, len); + if (!ret) { + error = Error::Canceled; + output_error_log(error, &req); + } + return ret; + }) + : static_cast( + [&](const char *buf, size_t n, size_t /*off*/, + size_t /*len*/) { + assert(res.body.size() + n <= res.body.max_size()); + res.body.append(buf, n); + return true; + }); + + auto progress = [&](size_t current, size_t total) { + if (!req.download_progress || redirect) { return true; } + auto ret = req.download_progress(current, total); + if (!ret) { + error = Error::Canceled; + output_error_log(error, &req); + } + return ret; + }; + + if (res.has_header("Content-Length")) { + if (!req.content_receiver) { + auto len = res.get_header_value_u64("Content-Length"); + if (len > res.body.max_size()) { + error = Error::Read; + output_error_log(error, &req); + return false; + } + res.body.reserve(static_cast(len)); + } + } + + if (res.status != StatusCode::NotModified_304) { + int dummy_status; + if (!detail::read_content(strm, res, (std::numeric_limits::max)(), + dummy_status, std::move(progress), + std::move(out), decompress_)) { + if (error != Error::Canceled) { error = Error::Read; } + output_error_log(error, &req); + return false; + } + } + } + + // Log + output_log(req, res); + + return true; +} + +inline ContentProviderWithoutLength ClientImpl::get_multipart_content_provider( + const std::string &boundary, const UploadFormDataItems &items, + const FormDataProviderItems &provider_items) const { + size_t cur_item = 0; + size_t cur_start = 0; + // cur_item and cur_start are copied to within the std::function and maintain + // state between successive calls + return [&, cur_item, cur_start](size_t offset, + DataSink &sink) mutable -> bool { + if (!offset && !items.empty()) { + sink.os << detail::serialize_multipart_formdata(items, boundary, false); + return true; + } else if (cur_item < provider_items.size()) { + if (!cur_start) { + const auto &begin = detail::serialize_multipart_formdata_item_begin( + provider_items[cur_item], boundary); + offset += begin.size(); + cur_start = offset; + sink.os << begin; + } + + DataSink cur_sink; + auto has_data = true; + cur_sink.write = sink.write; + cur_sink.done = [&]() { has_data = false; }; + + if (!provider_items[cur_item].provider(offset - cur_start, cur_sink)) { + return false; + } + + if (!has_data) { + sink.os << detail::serialize_multipart_formdata_item_end(); + cur_item++; + cur_start = 0; + } + return true; + } else { + sink.os << detail::serialize_multipart_formdata_finish(boundary); + sink.done(); + return true; + } + }; +} + +inline bool ClientImpl::process_socket( + const Socket &socket, + std::chrono::time_point start_time, + std::function callback) { + return detail::process_client_socket( + socket.sock, read_timeout_sec_, read_timeout_usec_, write_timeout_sec_, + write_timeout_usec_, max_timeout_msec_, start_time, std::move(callback)); +} + +inline bool ClientImpl::is_ssl() const { return false; } + +inline Result ClientImpl::Get(const std::string &path, + DownloadProgress progress) { + return Get(path, Headers(), std::move(progress)); +} + +inline Result ClientImpl::Get(const std::string &path, const Params ¶ms, + const Headers &headers, + DownloadProgress progress) { + if (params.empty()) { return Get(path, headers); } + + std::string path_with_query = append_query_params(path, params); + return Get(path_with_query, headers, std::move(progress)); +} + +inline Result ClientImpl::Get(const std::string &path, const Headers &headers, + DownloadProgress progress) { + Request req; + req.method = "GET"; + req.path = path; + req.headers = headers; + req.download_progress = std::move(progress); + if (max_timeout_msec_ > 0) { + req.start_time_ = std::chrono::steady_clock::now(); + } + + return send_(std::move(req)); +} + +inline Result ClientImpl::Get(const std::string &path, + ContentReceiver content_receiver, + DownloadProgress progress) { + return Get(path, Headers(), nullptr, std::move(content_receiver), + std::move(progress)); +} + +inline Result ClientImpl::Get(const std::string &path, const Headers &headers, + ContentReceiver content_receiver, + DownloadProgress progress) { + return Get(path, headers, nullptr, std::move(content_receiver), + std::move(progress)); +} + +inline Result ClientImpl::Get(const std::string &path, + ResponseHandler response_handler, + ContentReceiver content_receiver, + DownloadProgress progress) { + return Get(path, Headers(), std::move(response_handler), + std::move(content_receiver), std::move(progress)); +} + +inline Result ClientImpl::Get(const std::string &path, const Headers &headers, + ResponseHandler response_handler, + ContentReceiver content_receiver, + DownloadProgress progress) { + Request req; + req.method = "GET"; + req.path = path; + req.headers = headers; + req.response_handler = std::move(response_handler); + req.content_receiver = + [content_receiver](const char *data, size_t data_length, + size_t /*offset*/, size_t /*total_length*/) { + return content_receiver(data, data_length); + }; + req.download_progress = std::move(progress); + if (max_timeout_msec_ > 0) { + req.start_time_ = std::chrono::steady_clock::now(); + } + + return send_(std::move(req)); +} + +inline Result ClientImpl::Get(const std::string &path, const Params ¶ms, + const Headers &headers, + ContentReceiver content_receiver, + DownloadProgress progress) { + return Get(path, params, headers, nullptr, std::move(content_receiver), + std::move(progress)); +} + +inline Result ClientImpl::Get(const std::string &path, const Params ¶ms, + const Headers &headers, + ResponseHandler response_handler, + ContentReceiver content_receiver, + DownloadProgress progress) { + if (params.empty()) { + return Get(path, headers, std::move(response_handler), + std::move(content_receiver), std::move(progress)); + } + + std::string path_with_query = append_query_params(path, params); + return Get(path_with_query, headers, std::move(response_handler), + std::move(content_receiver), std::move(progress)); +} + +inline Result ClientImpl::Head(const std::string &path) { + return Head(path, Headers()); +} + +inline Result ClientImpl::Head(const std::string &path, + const Headers &headers) { + Request req; + req.method = "HEAD"; + req.headers = headers; + req.path = path; + if (max_timeout_msec_ > 0) { + req.start_time_ = std::chrono::steady_clock::now(); + } + + return send_(std::move(req)); +} + +inline Result ClientImpl::Post(const std::string &path) { + return Post(path, std::string(), std::string()); +} + +inline Result ClientImpl::Post(const std::string &path, + const Headers &headers) { + return Post(path, headers, nullptr, 0, std::string()); +} + +inline Result ClientImpl::Post(const std::string &path, const char *body, + size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return Post(path, Headers(), body, content_length, content_type, progress); +} + +inline Result ClientImpl::Post(const std::string &path, const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return Post(path, Headers(), body, content_type, progress); +} + +inline Result ClientImpl::Post(const std::string &path, const Params ¶ms) { + return Post(path, Headers(), params); +} + +inline Result ClientImpl::Post(const std::string &path, size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return Post(path, Headers(), content_length, std::move(content_provider), + content_type, progress); +} + +inline Result ClientImpl::Post(const std::string &path, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return Post(path, Headers(), std::move(content_provider), content_type, + progress); +} + +inline Result ClientImpl::Post(const std::string &path, const Headers &headers, + const Params ¶ms) { + auto query = detail::params_to_query_str(params); + return Post(path, headers, query, "application/x-www-form-urlencoded"); +} + +inline Result ClientImpl::Post(const std::string &path, + const UploadFormDataItems &items, + UploadProgress progress) { + return Post(path, Headers(), items, progress); +} + +inline Result ClientImpl::Post(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + UploadProgress progress) { + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + const auto &body = detail::serialize_multipart_formdata(items, boundary); + return Post(path, headers, body, content_type, progress); +} + +inline Result ClientImpl::Post(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const std::string &boundary, + UploadProgress progress) { + if (!detail::is_multipart_boundary_chars_valid(boundary)) { + return Result{nullptr, Error::UnsupportedMultipartBoundaryChars}; + } + + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + const auto &body = detail::serialize_multipart_formdata(items, boundary); + return Post(path, headers, body, content_type, progress); +} + +inline Result ClientImpl::Post(const std::string &path, const Headers &headers, + const char *body, size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider("POST", path, headers, body, content_length, + nullptr, nullptr, content_type, progress); +} + +inline Result ClientImpl::Post(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider("POST", path, headers, body.data(), + body.size(), nullptr, nullptr, content_type, + progress); +} + +inline Result ClientImpl::Post(const std::string &path, const Headers &headers, + size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider("POST", path, headers, nullptr, + content_length, std::move(content_provider), + nullptr, content_type, progress); +} + +inline Result ClientImpl::Post(const std::string &path, const Headers &headers, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider("POST", path, headers, nullptr, 0, nullptr, + std::move(content_provider), content_type, + progress); +} + +inline Result ClientImpl::Post(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const FormDataProviderItems &provider_items, + UploadProgress progress) { + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + return send_with_content_provider( + "POST", path, headers, nullptr, 0, nullptr, + get_multipart_content_provider(boundary, items, provider_items), + content_type, progress); +} + +inline Result ClientImpl::Post(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + ContentReceiver content_receiver, + DownloadProgress progress) { + Request req; + req.method = "POST"; + req.path = path; + req.headers = headers; + req.body = body; + req.content_receiver = + [content_receiver](const char *data, size_t data_length, + size_t /*offset*/, size_t /*total_length*/) { + return content_receiver(data, data_length); + }; + req.download_progress = std::move(progress); + + if (max_timeout_msec_ > 0) { + req.start_time_ = std::chrono::steady_clock::now(); + } + + if (!content_type.empty()) { req.set_header("Content-Type", content_type); } + + return send_(std::move(req)); +} + +inline Result ClientImpl::Put(const std::string &path) { + return Put(path, std::string(), std::string()); +} + +inline Result ClientImpl::Put(const std::string &path, const Headers &headers) { + return Put(path, headers, nullptr, 0, std::string()); +} + +inline Result ClientImpl::Put(const std::string &path, const char *body, + size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return Put(path, Headers(), body, content_length, content_type, progress); +} + +inline Result ClientImpl::Put(const std::string &path, const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return Put(path, Headers(), body, content_type, progress); +} + +inline Result ClientImpl::Put(const std::string &path, const Params ¶ms) { + return Put(path, Headers(), params); +} + +inline Result ClientImpl::Put(const std::string &path, size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return Put(path, Headers(), content_length, std::move(content_provider), + content_type, progress); +} + +inline Result ClientImpl::Put(const std::string &path, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return Put(path, Headers(), std::move(content_provider), content_type, + progress); +} + +inline Result ClientImpl::Put(const std::string &path, const Headers &headers, + const Params ¶ms) { + auto query = detail::params_to_query_str(params); + return Put(path, headers, query, "application/x-www-form-urlencoded"); +} + +inline Result ClientImpl::Put(const std::string &path, + const UploadFormDataItems &items, + UploadProgress progress) { + return Put(path, Headers(), items, progress); +} + +inline Result ClientImpl::Put(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + UploadProgress progress) { + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + const auto &body = detail::serialize_multipart_formdata(items, boundary); + return Put(path, headers, body, content_type, progress); +} + +inline Result ClientImpl::Put(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const std::string &boundary, + UploadProgress progress) { + if (!detail::is_multipart_boundary_chars_valid(boundary)) { + return Result{nullptr, Error::UnsupportedMultipartBoundaryChars}; + } + + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + const auto &body = detail::serialize_multipart_formdata(items, boundary); + return Put(path, headers, body, content_type, progress); +} + +inline Result ClientImpl::Put(const std::string &path, const Headers &headers, + const char *body, size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider("PUT", path, headers, body, content_length, + nullptr, nullptr, content_type, progress); +} + +inline Result ClientImpl::Put(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider("PUT", path, headers, body.data(), + body.size(), nullptr, nullptr, content_type, + progress); +} + +inline Result ClientImpl::Put(const std::string &path, const Headers &headers, + size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider("PUT", path, headers, nullptr, + content_length, std::move(content_provider), + nullptr, content_type, progress); +} + +inline Result ClientImpl::Put(const std::string &path, const Headers &headers, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider("PUT", path, headers, nullptr, 0, nullptr, + std::move(content_provider), content_type, + progress); +} + +inline Result ClientImpl::Put(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const FormDataProviderItems &provider_items, + UploadProgress progress) { + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + return send_with_content_provider( + "PUT", path, headers, nullptr, 0, nullptr, + get_multipart_content_provider(boundary, items, provider_items), + content_type, progress); +} + +inline Result ClientImpl::Put(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + ContentReceiver content_receiver, + DownloadProgress progress) { + Request req; + req.method = "PUT"; + req.path = path; + req.headers = headers; + req.body = body; + req.content_receiver = + [content_receiver](const char *data, size_t data_length, + size_t /*offset*/, size_t /*total_length*/) { + return content_receiver(data, data_length); + }; + req.download_progress = std::move(progress); + + if (max_timeout_msec_ > 0) { + req.start_time_ = std::chrono::steady_clock::now(); + } + + if (!content_type.empty()) { req.set_header("Content-Type", content_type); } + + return send_(std::move(req)); +} + +inline Result ClientImpl::Patch(const std::string &path) { + return Patch(path, std::string(), std::string()); +} + +inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, + UploadProgress progress) { + return Patch(path, headers, nullptr, 0, std::string(), progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const char *body, + size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return Patch(path, Headers(), body, content_length, content_type, progress); +} + +inline Result ClientImpl::Patch(const std::string &path, + const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return Patch(path, Headers(), body, content_type, progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const Params ¶ms) { + return Patch(path, Headers(), params); +} + +inline Result ClientImpl::Patch(const std::string &path, size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return Patch(path, Headers(), content_length, std::move(content_provider), + content_type, progress); +} + +inline Result ClientImpl::Patch(const std::string &path, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return Patch(path, Headers(), std::move(content_provider), content_type, + progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, + const Params ¶ms) { + auto query = detail::params_to_query_str(params); + return Patch(path, headers, query, "application/x-www-form-urlencoded"); +} + +inline Result ClientImpl::Patch(const std::string &path, + const UploadFormDataItems &items, + UploadProgress progress) { + return Patch(path, Headers(), items, progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + UploadProgress progress) { + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + const auto &body = detail::serialize_multipart_formdata(items, boundary); + return Patch(path, headers, body, content_type, progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const std::string &boundary, + UploadProgress progress) { + if (!detail::is_multipart_boundary_chars_valid(boundary)) { + return Result{nullptr, Error::UnsupportedMultipartBoundaryChars}; + } + + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + const auto &body = detail::serialize_multipart_formdata(items, boundary); + return Patch(path, headers, body, content_type, progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, + const char *body, size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider("PATCH", path, headers, body, + content_length, nullptr, nullptr, + content_type, progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider("PATCH", path, headers, body.data(), + body.size(), nullptr, nullptr, content_type, + progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, + size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider("PATCH", path, headers, nullptr, + content_length, std::move(content_provider), + nullptr, content_type, progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return send_with_content_provider("PATCH", path, headers, nullptr, 0, nullptr, + std::move(content_provider), content_type, + progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const FormDataProviderItems &provider_items, + UploadProgress progress) { + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + return send_with_content_provider( + "PATCH", path, headers, nullptr, 0, nullptr, + get_multipart_content_provider(boundary, items, provider_items), + content_type, progress); +} + +inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + ContentReceiver content_receiver, + DownloadProgress progress) { + Request req; + req.method = "PATCH"; + req.path = path; + req.headers = headers; + req.body = body; + req.content_receiver = + [content_receiver](const char *data, size_t data_length, + size_t /*offset*/, size_t /*total_length*/) { + return content_receiver(data, data_length); + }; + req.download_progress = std::move(progress); + + if (max_timeout_msec_ > 0) { + req.start_time_ = std::chrono::steady_clock::now(); + } + + if (!content_type.empty()) { req.set_header("Content-Type", content_type); } + + return send_(std::move(req)); +} + +inline Result ClientImpl::Delete(const std::string &path, + DownloadProgress progress) { + return Delete(path, Headers(), std::string(), std::string(), progress); +} + +inline Result ClientImpl::Delete(const std::string &path, + const Headers &headers, + DownloadProgress progress) { + return Delete(path, headers, std::string(), std::string(), progress); +} + +inline Result ClientImpl::Delete(const std::string &path, const char *body, + size_t content_length, + const std::string &content_type, + DownloadProgress progress) { + return Delete(path, Headers(), body, content_length, content_type, progress); +} + +inline Result ClientImpl::Delete(const std::string &path, + const std::string &body, + const std::string &content_type, + DownloadProgress progress) { + return Delete(path, Headers(), body.data(), body.size(), content_type, + progress); +} + +inline Result ClientImpl::Delete(const std::string &path, + const Headers &headers, + const std::string &body, + const std::string &content_type, + DownloadProgress progress) { + return Delete(path, headers, body.data(), body.size(), content_type, + progress); +} + +inline Result ClientImpl::Delete(const std::string &path, const Params ¶ms, + DownloadProgress progress) { + return Delete(path, Headers(), params, progress); +} + +inline Result ClientImpl::Delete(const std::string &path, + const Headers &headers, const Params ¶ms, + DownloadProgress progress) { + auto query = detail::params_to_query_str(params); + return Delete(path, headers, query, "application/x-www-form-urlencoded", + progress); +} + +inline Result ClientImpl::Delete(const std::string &path, + const Headers &headers, const char *body, + size_t content_length, + const std::string &content_type, + DownloadProgress progress) { + Request req; + req.method = "DELETE"; + req.headers = headers; + req.path = path; + req.download_progress = std::move(progress); + if (max_timeout_msec_ > 0) { + req.start_time_ = std::chrono::steady_clock::now(); + } + + if (!content_type.empty()) { req.set_header("Content-Type", content_type); } + req.body.assign(body, content_length); + + return send_(std::move(req)); +} + +inline Result ClientImpl::Options(const std::string &path) { + return Options(path, Headers()); +} + +inline Result ClientImpl::Options(const std::string &path, + const Headers &headers) { + Request req; + req.method = "OPTIONS"; + req.headers = headers; + req.path = path; + if (max_timeout_msec_ > 0) { + req.start_time_ = std::chrono::steady_clock::now(); + } + + return send_(std::move(req)); +} + +inline void ClientImpl::stop() { + std::lock_guard guard(socket_mutex_); + + // If there is anything ongoing right now, the ONLY thread-safe thing we can + // do is to shutdown_socket, so that threads using this socket suddenly + // discover they can't read/write any more and error out. Everything else + // (closing the socket, shutting ssl down) is unsafe because these actions are + // not thread-safe. + if (socket_requests_in_flight_ > 0) { + shutdown_socket(socket_); + + // Aside from that, we set a flag for the socket to be closed when we're + // done. + socket_should_be_closed_when_request_is_done_ = true; + return; + } + + // Otherwise, still holding the mutex, we can shut everything down ourselves + shutdown_ssl(socket_, true); + shutdown_socket(socket_); + close_socket(socket_); +} + +inline std::string ClientImpl::host() const { return host_; } + +inline int ClientImpl::port() const { return port_; } + +inline size_t ClientImpl::is_socket_open() const { + std::lock_guard guard(socket_mutex_); + return socket_.is_open(); +} + +inline socket_t ClientImpl::socket() const { return socket_.sock; } + +inline void ClientImpl::set_connection_timeout(time_t sec, time_t usec) { + connection_timeout_sec_ = sec; + connection_timeout_usec_ = usec; +} + +inline void ClientImpl::set_read_timeout(time_t sec, time_t usec) { + read_timeout_sec_ = sec; + read_timeout_usec_ = usec; +} + +inline void ClientImpl::set_write_timeout(time_t sec, time_t usec) { + write_timeout_sec_ = sec; + write_timeout_usec_ = usec; +} + +inline void ClientImpl::set_max_timeout(time_t msec) { + max_timeout_msec_ = msec; +} + +inline void ClientImpl::set_basic_auth(const std::string &username, + const std::string &password) { + basic_auth_username_ = username; + basic_auth_password_ = password; +} + +inline void ClientImpl::set_bearer_token_auth(const std::string &token) { + bearer_token_auth_token_ = token; +} + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +inline void ClientImpl::set_digest_auth(const std::string &username, + const std::string &password) { + digest_auth_username_ = username; + digest_auth_password_ = password; +} +#endif + +inline void ClientImpl::set_keep_alive(bool on) { keep_alive_ = on; } + +inline void ClientImpl::set_follow_location(bool on) { follow_location_ = on; } + +inline void ClientImpl::set_path_encode(bool on) { path_encode_ = on; } + +inline void +ClientImpl::set_hostname_addr_map(std::map addr_map) { + addr_map_ = std::move(addr_map); +} + +inline void ClientImpl::set_default_headers(Headers headers) { + default_headers_ = std::move(headers); +} + +inline void ClientImpl::set_header_writer( + std::function const &writer) { + header_writer_ = writer; +} + +inline void ClientImpl::set_address_family(int family) { + address_family_ = family; +} + +inline void ClientImpl::set_tcp_nodelay(bool on) { tcp_nodelay_ = on; } + +inline void ClientImpl::set_ipv6_v6only(bool on) { ipv6_v6only_ = on; } + +inline void ClientImpl::set_socket_options(SocketOptions socket_options) { + socket_options_ = std::move(socket_options); +} + +inline void ClientImpl::set_compress(bool on) { compress_ = on; } + +inline void ClientImpl::set_decompress(bool on) { decompress_ = on; } + +inline void ClientImpl::set_interface(const std::string &intf) { + interface_ = intf; +} + +inline void ClientImpl::set_proxy(const std::string &host, int port) { + proxy_host_ = host; + proxy_port_ = port; +} + +inline void ClientImpl::set_proxy_basic_auth(const std::string &username, + const std::string &password) { + proxy_basic_auth_username_ = username; + proxy_basic_auth_password_ = password; +} + +inline void ClientImpl::set_proxy_bearer_token_auth(const std::string &token) { + proxy_bearer_token_auth_token_ = token; +} + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +inline void ClientImpl::set_proxy_digest_auth(const std::string &username, + const std::string &password) { + proxy_digest_auth_username_ = username; + proxy_digest_auth_password_ = password; +} + +inline void ClientImpl::set_ca_cert_path(const std::string &ca_cert_file_path, + const std::string &ca_cert_dir_path) { + ca_cert_file_path_ = ca_cert_file_path; + ca_cert_dir_path_ = ca_cert_dir_path; +} + +inline void ClientImpl::set_ca_cert_store(X509_STORE *ca_cert_store) { + if (ca_cert_store && ca_cert_store != ca_cert_store_) { + ca_cert_store_ = ca_cert_store; + } +} + +inline X509_STORE *ClientImpl::create_ca_cert_store(const char *ca_cert, + std::size_t size) const { + auto mem = BIO_new_mem_buf(ca_cert, static_cast(size)); + auto se = detail::scope_exit([&] { BIO_free_all(mem); }); + if (!mem) { return nullptr; } + + auto inf = PEM_X509_INFO_read_bio(mem, nullptr, nullptr, nullptr); + if (!inf) { return nullptr; } + + auto cts = X509_STORE_new(); + if (cts) { + for (auto i = 0; i < static_cast(sk_X509_INFO_num(inf)); i++) { + auto itmp = sk_X509_INFO_value(inf, i); + if (!itmp) { continue; } + + if (itmp->x509) { X509_STORE_add_cert(cts, itmp->x509); } + if (itmp->crl) { X509_STORE_add_crl(cts, itmp->crl); } + } + } + + sk_X509_INFO_pop_free(inf, X509_INFO_free); + return cts; +} + +inline void ClientImpl::enable_server_certificate_verification(bool enabled) { + server_certificate_verification_ = enabled; +} + +inline void ClientImpl::enable_server_hostname_verification(bool enabled) { + server_hostname_verification_ = enabled; +} + +inline void ClientImpl::set_server_certificate_verifier( + std::function verifier) { + server_certificate_verifier_ = verifier; +} +#endif + +inline void ClientImpl::set_logger(Logger logger) { + logger_ = std::move(logger); +} + +inline void ClientImpl::set_error_logger(ErrorLogger error_logger) { + error_logger_ = std::move(error_logger); +} + +/* + * SSL Implementation + */ +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +namespace detail { + +template +inline SSL *ssl_new(socket_t sock, SSL_CTX *ctx, std::mutex &ctx_mutex, + U SSL_connect_or_accept, V setup) { + SSL *ssl = nullptr; + { + std::lock_guard guard(ctx_mutex); + ssl = SSL_new(ctx); + } + + if (ssl) { + set_nonblocking(sock, true); + auto bio = BIO_new_socket(static_cast(sock), BIO_NOCLOSE); + BIO_set_nbio(bio, 1); + SSL_set_bio(ssl, bio, bio); + + if (!setup(ssl) || SSL_connect_or_accept(ssl) != 1) { + SSL_shutdown(ssl); + { + std::lock_guard guard(ctx_mutex); + SSL_free(ssl); + } + set_nonblocking(sock, false); + return nullptr; + } + BIO_set_nbio(bio, 0); + set_nonblocking(sock, false); + } + + return ssl; +} + +inline void ssl_delete(std::mutex &ctx_mutex, SSL *ssl, socket_t sock, + bool shutdown_gracefully) { + // sometimes we may want to skip this to try to avoid SIGPIPE if we know + // the remote has closed the network connection + // Note that it is not always possible to avoid SIGPIPE, this is merely a + // best-efforts. + if (shutdown_gracefully) { + (void)(sock); + // SSL_shutdown() returns 0 on first call (indicating close_notify alert + // sent) and 1 on subsequent call (indicating close_notify alert received) + if (SSL_shutdown(ssl) == 0) { + // Expected to return 1, but even if it doesn't, we free ssl + SSL_shutdown(ssl); + } + } + + std::lock_guard guard(ctx_mutex); + SSL_free(ssl); +} + +template +bool ssl_connect_or_accept_nonblocking(socket_t sock, SSL *ssl, + U ssl_connect_or_accept, + time_t timeout_sec, time_t timeout_usec, + int *ssl_error) { + auto res = 0; + while ((res = ssl_connect_or_accept(ssl)) != 1) { + auto err = SSL_get_error(ssl, res); + switch (err) { + case SSL_ERROR_WANT_READ: + if (select_read(sock, timeout_sec, timeout_usec) > 0) { continue; } + break; + case SSL_ERROR_WANT_WRITE: + if (select_write(sock, timeout_sec, timeout_usec) > 0) { continue; } + break; + default: break; + } + if (ssl_error) { *ssl_error = err; } + return false; + } + return true; +} + +template +inline bool process_server_socket_ssl( + const std::atomic &svr_sock, SSL *ssl, socket_t sock, + size_t keep_alive_max_count, time_t keep_alive_timeout_sec, + time_t read_timeout_sec, time_t read_timeout_usec, time_t write_timeout_sec, + time_t write_timeout_usec, T callback) { + return process_server_socket_core( + svr_sock, sock, keep_alive_max_count, keep_alive_timeout_sec, + [&](bool close_connection, bool &connection_closed) { + SSLSocketStream strm(sock, ssl, read_timeout_sec, read_timeout_usec, + write_timeout_sec, write_timeout_usec); + return callback(strm, close_connection, connection_closed); + }); +} + +template +inline bool process_client_socket_ssl( + SSL *ssl, socket_t sock, time_t read_timeout_sec, time_t read_timeout_usec, + time_t write_timeout_sec, time_t write_timeout_usec, + time_t max_timeout_msec, + std::chrono::time_point start_time, T callback) { + SSLSocketStream strm(sock, ssl, read_timeout_sec, read_timeout_usec, + write_timeout_sec, write_timeout_usec, max_timeout_msec, + start_time); + return callback(strm); +} + +// SSL socket stream implementation +inline SSLSocketStream::SSLSocketStream( + socket_t sock, SSL *ssl, time_t read_timeout_sec, time_t read_timeout_usec, + time_t write_timeout_sec, time_t write_timeout_usec, + time_t max_timeout_msec, + std::chrono::time_point start_time) + : sock_(sock), ssl_(ssl), read_timeout_sec_(read_timeout_sec), + read_timeout_usec_(read_timeout_usec), + write_timeout_sec_(write_timeout_sec), + write_timeout_usec_(write_timeout_usec), + max_timeout_msec_(max_timeout_msec), start_time_(start_time) { + SSL_clear_mode(ssl, SSL_MODE_AUTO_RETRY); +} + +inline SSLSocketStream::~SSLSocketStream() = default; + +inline bool SSLSocketStream::is_readable() const { + return SSL_pending(ssl_) > 0; +} + +inline bool SSLSocketStream::wait_readable() const { + if (max_timeout_msec_ <= 0) { + return select_read(sock_, read_timeout_sec_, read_timeout_usec_) > 0; + } + + time_t read_timeout_sec; + time_t read_timeout_usec; + calc_actual_timeout(max_timeout_msec_, duration(), read_timeout_sec_, + read_timeout_usec_, read_timeout_sec, read_timeout_usec); + + return select_read(sock_, read_timeout_sec, read_timeout_usec) > 0; +} + +inline bool SSLSocketStream::wait_writable() const { + return select_write(sock_, write_timeout_sec_, write_timeout_usec_) > 0 && + is_socket_alive(sock_) && !is_ssl_peer_could_be_closed(ssl_, sock_); +} + +inline ssize_t SSLSocketStream::read(char *ptr, size_t size) { + if (SSL_pending(ssl_) > 0) { + return SSL_read(ssl_, ptr, static_cast(size)); + } else if (wait_readable()) { + auto ret = SSL_read(ssl_, ptr, static_cast(size)); + if (ret < 0) { + auto err = SSL_get_error(ssl_, ret); + auto n = 1000; +#ifdef _WIN64 + while (--n >= 0 && (err == SSL_ERROR_WANT_READ || + (err == SSL_ERROR_SYSCALL && + WSAGetLastError() == WSAETIMEDOUT))) { +#else + while (--n >= 0 && err == SSL_ERROR_WANT_READ) { +#endif + if (SSL_pending(ssl_) > 0) { + return SSL_read(ssl_, ptr, static_cast(size)); + } else if (wait_readable()) { + std::this_thread::sleep_for(std::chrono::microseconds{10}); + ret = SSL_read(ssl_, ptr, static_cast(size)); + if (ret >= 0) { return ret; } + err = SSL_get_error(ssl_, ret); + } else { + break; + } + } + assert(ret < 0); + } + return ret; + } else { + return -1; + } +} + +inline ssize_t SSLSocketStream::write(const char *ptr, size_t size) { + if (wait_writable()) { + auto handle_size = static_cast( + std::min(size, (std::numeric_limits::max)())); + + auto ret = SSL_write(ssl_, ptr, static_cast(handle_size)); + if (ret < 0) { + auto err = SSL_get_error(ssl_, ret); + auto n = 1000; +#ifdef _WIN64 + while (--n >= 0 && (err == SSL_ERROR_WANT_WRITE || + (err == SSL_ERROR_SYSCALL && + WSAGetLastError() == WSAETIMEDOUT))) { +#else + while (--n >= 0 && err == SSL_ERROR_WANT_WRITE) { +#endif + if (wait_writable()) { + std::this_thread::sleep_for(std::chrono::microseconds{10}); + ret = SSL_write(ssl_, ptr, static_cast(handle_size)); + if (ret >= 0) { return ret; } + err = SSL_get_error(ssl_, ret); + } else { + break; + } + } + assert(ret < 0); + } + return ret; + } + return -1; +} + +inline void SSLSocketStream::get_remote_ip_and_port(std::string &ip, + int &port) const { + detail::get_remote_ip_and_port(sock_, ip, port); +} + +inline void SSLSocketStream::get_local_ip_and_port(std::string &ip, + int &port) const { + detail::get_local_ip_and_port(sock_, ip, port); +} + +inline socket_t SSLSocketStream::socket() const { return sock_; } + +inline time_t SSLSocketStream::duration() const { + return std::chrono::duration_cast( + std::chrono::steady_clock::now() - start_time_) + .count(); +} + +} // namespace detail + +// SSL HTTP server implementation +inline SSLServer::SSLServer(const char *cert_path, const char *private_key_path, + const char *client_ca_cert_file_path, + const char *client_ca_cert_dir_path, + const char *private_key_password) { + ctx_ = SSL_CTX_new(TLS_server_method()); + + if (ctx_) { + SSL_CTX_set_options(ctx_, + SSL_OP_NO_COMPRESSION | + SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION); + + SSL_CTX_set_min_proto_version(ctx_, TLS1_2_VERSION); + + if (private_key_password != nullptr && (private_key_password[0] != '\0')) { + SSL_CTX_set_default_passwd_cb_userdata( + ctx_, + reinterpret_cast(const_cast(private_key_password))); + } + + if (SSL_CTX_use_certificate_chain_file(ctx_, cert_path) != 1 || + SSL_CTX_use_PrivateKey_file(ctx_, private_key_path, SSL_FILETYPE_PEM) != + 1 || + SSL_CTX_check_private_key(ctx_) != 1) { + last_ssl_error_ = static_cast(ERR_get_error()); + SSL_CTX_free(ctx_); + ctx_ = nullptr; + } else if (client_ca_cert_file_path || client_ca_cert_dir_path) { + SSL_CTX_load_verify_locations(ctx_, client_ca_cert_file_path, + client_ca_cert_dir_path); + + SSL_CTX_set_verify( + ctx_, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, nullptr); + } + } +} + +inline SSLServer::SSLServer(X509 *cert, EVP_PKEY *private_key, + X509_STORE *client_ca_cert_store) { + ctx_ = SSL_CTX_new(TLS_server_method()); + + if (ctx_) { + SSL_CTX_set_options(ctx_, + SSL_OP_NO_COMPRESSION | + SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION); + + SSL_CTX_set_min_proto_version(ctx_, TLS1_2_VERSION); + + if (SSL_CTX_use_certificate(ctx_, cert) != 1 || + SSL_CTX_use_PrivateKey(ctx_, private_key) != 1) { + SSL_CTX_free(ctx_); + ctx_ = nullptr; + } else if (client_ca_cert_store) { + SSL_CTX_set_cert_store(ctx_, client_ca_cert_store); + + SSL_CTX_set_verify( + ctx_, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, nullptr); + } + } +} + +inline SSLServer::SSLServer( + const std::function &setup_ssl_ctx_callback) { + ctx_ = SSL_CTX_new(TLS_method()); + if (ctx_) { + if (!setup_ssl_ctx_callback(*ctx_)) { + SSL_CTX_free(ctx_); + ctx_ = nullptr; + } + } +} + +inline SSLServer::~SSLServer() { + if (ctx_) { SSL_CTX_free(ctx_); } +} + +inline bool SSLServer::is_valid() const { return ctx_; } + +inline SSL_CTX *SSLServer::ssl_context() const { return ctx_; } + +inline void SSLServer::update_certs(X509 *cert, EVP_PKEY *private_key, + X509_STORE *client_ca_cert_store) { + + std::lock_guard guard(ctx_mutex_); + + SSL_CTX_use_certificate(ctx_, cert); + SSL_CTX_use_PrivateKey(ctx_, private_key); + + if (client_ca_cert_store != nullptr) { + SSL_CTX_set_cert_store(ctx_, client_ca_cert_store); + } +} + +inline bool SSLServer::process_and_close_socket(socket_t sock) { + auto ssl = detail::ssl_new( + sock, ctx_, ctx_mutex_, + [&](SSL *ssl2) { + return detail::ssl_connect_or_accept_nonblocking( + sock, ssl2, SSL_accept, read_timeout_sec_, read_timeout_usec_, + &last_ssl_error_); + }, + [](SSL * /*ssl2*/) { return true; }); + + auto ret = false; + if (ssl) { + std::string remote_addr; + int remote_port = 0; + detail::get_remote_ip_and_port(sock, remote_addr, remote_port); + + std::string local_addr; + int local_port = 0; + detail::get_local_ip_and_port(sock, local_addr, local_port); + + ret = detail::process_server_socket_ssl( + svr_sock_, ssl, sock, keep_alive_max_count_, keep_alive_timeout_sec_, + read_timeout_sec_, read_timeout_usec_, write_timeout_sec_, + write_timeout_usec_, + [&](Stream &strm, bool close_connection, bool &connection_closed) { + return process_request(strm, remote_addr, remote_port, local_addr, + local_port, close_connection, + connection_closed, + [&](Request &req) { req.ssl = ssl; }); + }); + + // Shutdown gracefully if the result seemed successful, non-gracefully if + // the connection appeared to be closed. + const bool shutdown_gracefully = ret; + detail::ssl_delete(ctx_mutex_, ssl, sock, shutdown_gracefully); + } + + detail::shutdown_socket(sock); + detail::close_socket(sock); + return ret; +} + +// SSL HTTP client implementation +inline SSLClient::SSLClient(const std::string &host) + : SSLClient(host, 443, std::string(), std::string()) {} + +inline SSLClient::SSLClient(const std::string &host, int port) + : SSLClient(host, port, std::string(), std::string()) {} + +inline SSLClient::SSLClient(const std::string &host, int port, + const std::string &client_cert_path, + const std::string &client_key_path, + const std::string &private_key_password) + : ClientImpl(host, port, client_cert_path, client_key_path) { + ctx_ = SSL_CTX_new(TLS_client_method()); + + SSL_CTX_set_min_proto_version(ctx_, TLS1_2_VERSION); + + detail::split(&host_[0], &host_[host_.size()], '.', + [&](const char *b, const char *e) { + host_components_.emplace_back(b, e); + }); + + if (!client_cert_path.empty() && !client_key_path.empty()) { + if (!private_key_password.empty()) { + SSL_CTX_set_default_passwd_cb_userdata( + ctx_, reinterpret_cast( + const_cast(private_key_password.c_str()))); + } + + if (SSL_CTX_use_certificate_file(ctx_, client_cert_path.c_str(), + SSL_FILETYPE_PEM) != 1 || + SSL_CTX_use_PrivateKey_file(ctx_, client_key_path.c_str(), + SSL_FILETYPE_PEM) != 1) { + last_openssl_error_ = ERR_get_error(); + SSL_CTX_free(ctx_); + ctx_ = nullptr; + } + } +} + +inline SSLClient::SSLClient(const std::string &host, int port, + X509 *client_cert, EVP_PKEY *client_key, + const std::string &private_key_password) + : ClientImpl(host, port) { + ctx_ = SSL_CTX_new(TLS_client_method()); + + detail::split(&host_[0], &host_[host_.size()], '.', + [&](const char *b, const char *e) { + host_components_.emplace_back(b, e); + }); + + if (client_cert != nullptr && client_key != nullptr) { + if (!private_key_password.empty()) { + SSL_CTX_set_default_passwd_cb_userdata( + ctx_, reinterpret_cast( + const_cast(private_key_password.c_str()))); + } + + if (SSL_CTX_use_certificate(ctx_, client_cert) != 1 || + SSL_CTX_use_PrivateKey(ctx_, client_key) != 1) { + last_openssl_error_ = ERR_get_error(); + SSL_CTX_free(ctx_); + ctx_ = nullptr; + } + } +} + +inline SSLClient::~SSLClient() { + if (ctx_) { SSL_CTX_free(ctx_); } + // Make sure to shut down SSL since shutdown_ssl will resolve to the + // base function rather than the derived function once we get to the + // base class destructor, and won't free the SSL (causing a leak). + shutdown_ssl_impl(socket_, true); +} + +inline bool SSLClient::is_valid() const { return ctx_; } + +inline void SSLClient::set_ca_cert_store(X509_STORE *ca_cert_store) { + if (ca_cert_store) { + if (ctx_) { + if (SSL_CTX_get_cert_store(ctx_) != ca_cert_store) { + // Free memory allocated for old cert and use new store `ca_cert_store` + SSL_CTX_set_cert_store(ctx_, ca_cert_store); + } + } else { + X509_STORE_free(ca_cert_store); + } + } +} + +inline void SSLClient::load_ca_cert_store(const char *ca_cert, + std::size_t size) { + set_ca_cert_store(ClientImpl::create_ca_cert_store(ca_cert, size)); +} + +inline long SSLClient::get_openssl_verify_result() const { + return verify_result_; +} + +inline SSL_CTX *SSLClient::ssl_context() const { return ctx_; } + +inline bool SSLClient::create_and_connect_socket(Socket &socket, Error &error) { + return is_valid() && ClientImpl::create_and_connect_socket(socket, error); +} + +// Assumes that socket_mutex_ is locked and that there are no requests in flight +inline bool SSLClient::connect_with_proxy( + Socket &socket, + std::chrono::time_point start_time, + Response &res, bool &success, Error &error) { + success = true; + Response proxy_res; + if (!detail::process_client_socket( + socket.sock, read_timeout_sec_, read_timeout_usec_, + write_timeout_sec_, write_timeout_usec_, max_timeout_msec_, + start_time, [&](Stream &strm) { + Request req2; + req2.method = "CONNECT"; + req2.path = host_and_port_; + if (max_timeout_msec_ > 0) { + req2.start_time_ = std::chrono::steady_clock::now(); + } + return process_request(strm, req2, proxy_res, false, error); + })) { + // Thread-safe to close everything because we are assuming there are no + // requests in flight + shutdown_ssl(socket, true); + shutdown_socket(socket); + close_socket(socket); + success = false; + return false; + } + + if (proxy_res.status == StatusCode::ProxyAuthenticationRequired_407) { + if (!proxy_digest_auth_username_.empty() && + !proxy_digest_auth_password_.empty()) { + std::map auth; + if (detail::parse_www_authenticate(proxy_res, auth, true)) { + // Close the current socket and create a new one for the authenticated + // request + shutdown_ssl(socket, true); + shutdown_socket(socket); + close_socket(socket); + + // Create a new socket for the authenticated CONNECT request + if (!create_and_connect_socket(socket, error)) { + success = false; + output_error_log(error, nullptr); + return false; + } + + proxy_res = Response(); + if (!detail::process_client_socket( + socket.sock, read_timeout_sec_, read_timeout_usec_, + write_timeout_sec_, write_timeout_usec_, max_timeout_msec_, + start_time, [&](Stream &strm) { + Request req3; + req3.method = "CONNECT"; + req3.path = host_and_port_; + req3.headers.insert(detail::make_digest_authentication_header( + req3, auth, 1, detail::random_string(10), + proxy_digest_auth_username_, proxy_digest_auth_password_, + true)); + if (max_timeout_msec_ > 0) { + req3.start_time_ = std::chrono::steady_clock::now(); + } + return process_request(strm, req3, proxy_res, false, error); + })) { + // Thread-safe to close everything because we are assuming there are + // no requests in flight + shutdown_ssl(socket, true); + shutdown_socket(socket); + close_socket(socket); + success = false; + return false; + } + } + } + } + + // If status code is not 200, proxy request is failed. + // Set error to ProxyConnection and return proxy response + // as the response of the request + if (proxy_res.status != StatusCode::OK_200) { + error = Error::ProxyConnection; + output_error_log(error, nullptr); + res = std::move(proxy_res); + // Thread-safe to close everything because we are assuming there are + // no requests in flight + shutdown_ssl(socket, true); + shutdown_socket(socket); + close_socket(socket); + return false; + } + + return true; +} + +inline bool SSLClient::load_certs() { + auto ret = true; + + std::call_once(initialize_cert_, [&]() { + std::lock_guard guard(ctx_mutex_); + if (!ca_cert_file_path_.empty()) { + if (!SSL_CTX_load_verify_locations(ctx_, ca_cert_file_path_.c_str(), + nullptr)) { + last_openssl_error_ = ERR_get_error(); + ret = false; + } + } else if (!ca_cert_dir_path_.empty()) { + if (!SSL_CTX_load_verify_locations(ctx_, nullptr, + ca_cert_dir_path_.c_str())) { + last_openssl_error_ = ERR_get_error(); + ret = false; + } + } else { + auto loaded = false; +#ifdef _WIN64 + loaded = + detail::load_system_certs_on_windows(SSL_CTX_get_cert_store(ctx_)); +#elif defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) && \ + defined(TARGET_OS_OSX) + loaded = detail::load_system_certs_on_macos(SSL_CTX_get_cert_store(ctx_)); +#endif // _WIN64 + if (!loaded) { SSL_CTX_set_default_verify_paths(ctx_); } + } + }); + + return ret; +} + +inline bool SSLClient::initialize_ssl(Socket &socket, Error &error) { + auto ssl = detail::ssl_new( + socket.sock, ctx_, ctx_mutex_, + [&](SSL *ssl2) { + if (server_certificate_verification_) { + if (!load_certs()) { + error = Error::SSLLoadingCerts; + output_error_log(error, nullptr); + return false; + } + SSL_set_verify(ssl2, SSL_VERIFY_NONE, nullptr); + } + + if (!detail::ssl_connect_or_accept_nonblocking( + socket.sock, ssl2, SSL_connect, connection_timeout_sec_, + connection_timeout_usec_, &last_ssl_error_)) { + error = Error::SSLConnection; + output_error_log(error, nullptr); + return false; + } + + if (server_certificate_verification_) { + auto verification_status = SSLVerifierResponse::NoDecisionMade; + + if (server_certificate_verifier_) { + verification_status = server_certificate_verifier_(ssl2); + } + + if (verification_status == SSLVerifierResponse::CertificateRejected) { + last_openssl_error_ = ERR_get_error(); + error = Error::SSLServerVerification; + output_error_log(error, nullptr); + return false; + } + + if (verification_status == SSLVerifierResponse::NoDecisionMade) { + verify_result_ = SSL_get_verify_result(ssl2); + + if (verify_result_ != X509_V_OK) { + last_openssl_error_ = static_cast(verify_result_); + error = Error::SSLServerVerification; + output_error_log(error, nullptr); + return false; + } + + auto server_cert = SSL_get1_peer_certificate(ssl2); + auto se = detail::scope_exit([&] { X509_free(server_cert); }); + + if (server_cert == nullptr) { + last_openssl_error_ = ERR_get_error(); + error = Error::SSLServerVerification; + output_error_log(error, nullptr); + return false; + } + + if (server_hostname_verification_) { + if (!verify_host(server_cert)) { + last_openssl_error_ = X509_V_ERR_HOSTNAME_MISMATCH; + error = Error::SSLServerHostnameVerification; + output_error_log(error, nullptr); + return false; + } + } + } + } + + return true; + }, + [&](SSL *ssl2) { +#if defined(OPENSSL_IS_BORINGSSL) + SSL_set_tlsext_host_name(ssl2, host_.c_str()); +#else + // NOTE: Direct call instead of using the OpenSSL macro to suppress + // -Wold-style-cast warning + SSL_ctrl(ssl2, SSL_CTRL_SET_TLSEXT_HOSTNAME, TLSEXT_NAMETYPE_host_name, + static_cast(const_cast(host_.c_str()))); +#endif + return true; + }); + + if (ssl) { + socket.ssl = ssl; + return true; + } + + shutdown_socket(socket); + close_socket(socket); + return false; +} + +inline void SSLClient::shutdown_ssl(Socket &socket, bool shutdown_gracefully) { + shutdown_ssl_impl(socket, shutdown_gracefully); +} + +inline void SSLClient::shutdown_ssl_impl(Socket &socket, + bool shutdown_gracefully) { + if (socket.sock == INVALID_SOCKET) { + assert(socket.ssl == nullptr); + return; + } + if (socket.ssl) { + detail::ssl_delete(ctx_mutex_, socket.ssl, socket.sock, + shutdown_gracefully); + socket.ssl = nullptr; + } + assert(socket.ssl == nullptr); +} + +inline bool SSLClient::process_socket( + const Socket &socket, + std::chrono::time_point start_time, + std::function callback) { + assert(socket.ssl); + return detail::process_client_socket_ssl( + socket.ssl, socket.sock, read_timeout_sec_, read_timeout_usec_, + write_timeout_sec_, write_timeout_usec_, max_timeout_msec_, start_time, + std::move(callback)); +} + +inline bool SSLClient::is_ssl() const { return true; } + +inline bool SSLClient::verify_host(X509 *server_cert) const { + /* Quote from RFC2818 section 3.1 "Server Identity" + + If a subjectAltName extension of type dNSName is present, that MUST + be used as the identity. Otherwise, the (most specific) Common Name + field in the Subject field of the certificate MUST be used. Although + the use of the Common Name is existing practice, it is deprecated and + Certification Authorities are encouraged to use the dNSName instead. + + Matching is performed using the matching rules specified by + [RFC2459]. If more than one identity of a given type is present in + the certificate (e.g., more than one dNSName name, a match in any one + of the set is considered acceptable.) Names may contain the wildcard + character * which is considered to match any single domain name + component or component fragment. E.g., *.a.com matches foo.a.com but + not bar.foo.a.com. f*.com matches foo.com but not bar.com. + + In some cases, the URI is specified as an IP address rather than a + hostname. In this case, the iPAddress subjectAltName must be present + in the certificate and must exactly match the IP in the URI. + + */ + return verify_host_with_subject_alt_name(server_cert) || + verify_host_with_common_name(server_cert); +} + +inline bool +SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const { + auto ret = false; + + auto type = GEN_DNS; + + struct in6_addr addr6 = {}; + struct in_addr addr = {}; + size_t addr_len = 0; + +#ifndef __MINGW32__ + if (inet_pton(AF_INET6, host_.c_str(), &addr6)) { + type = GEN_IPADD; + addr_len = sizeof(struct in6_addr); + } else if (inet_pton(AF_INET, host_.c_str(), &addr)) { + type = GEN_IPADD; + addr_len = sizeof(struct in_addr); + } +#endif + + auto alt_names = static_cast( + X509_get_ext_d2i(server_cert, NID_subject_alt_name, nullptr, nullptr)); + + if (alt_names) { + auto dsn_matched = false; + auto ip_matched = false; + + auto count = sk_GENERAL_NAME_num(alt_names); + + for (decltype(count) i = 0; i < count && !dsn_matched; i++) { + auto val = sk_GENERAL_NAME_value(alt_names, i); + if (val->type == type) { + auto name = + reinterpret_cast(ASN1_STRING_get0_data(val->d.ia5)); + auto name_len = static_cast(ASN1_STRING_length(val->d.ia5)); + + switch (type) { + case GEN_DNS: dsn_matched = check_host_name(name, name_len); break; + + case GEN_IPADD: + if (!memcmp(&addr6, name, addr_len) || + !memcmp(&addr, name, addr_len)) { + ip_matched = true; + } + break; + } + } + } + + if (dsn_matched || ip_matched) { ret = true; } + } + + GENERAL_NAMES_free(const_cast( + reinterpret_cast(alt_names))); + return ret; +} + +inline bool SSLClient::verify_host_with_common_name(X509 *server_cert) const { + const auto subject_name = X509_get_subject_name(server_cert); + + if (subject_name != nullptr) { + char name[BUFSIZ]; + auto name_len = X509_NAME_get_text_by_NID(subject_name, NID_commonName, + name, sizeof(name)); + + if (name_len != -1) { + return check_host_name(name, static_cast(name_len)); + } + } + + return false; +} + +inline bool SSLClient::check_host_name(const char *pattern, + size_t pattern_len) const { + if (host_.size() == pattern_len && host_ == pattern) { return true; } + + // Wildcard match + // https://bugs.launchpad.net/ubuntu/+source/firefox-3.0/+bug/376484 + std::vector pattern_components; + detail::split(&pattern[0], &pattern[pattern_len], '.', + [&](const char *b, const char *e) { + pattern_components.emplace_back(b, e); + }); + + if (host_components_.size() != pattern_components.size()) { return false; } + + auto itr = pattern_components.begin(); + for (const auto &h : host_components_) { + auto &p = *itr; + if (p != h && p != "*") { + auto partial_match = (p.size() > 0 && p[p.size() - 1] == '*' && + !p.compare(0, p.size() - 1, h)); + if (!partial_match) { return false; } + } + ++itr; + } + + return true; +} +#endif + +// Universal client implementation +inline Client::Client(const std::string &scheme_host_port) + : Client(scheme_host_port, std::string(), std::string()) {} + +inline Client::Client(const std::string &scheme_host_port, + const std::string &client_cert_path, + const std::string &client_key_path) { + const static std::regex re( + R"((?:([a-z]+):\/\/)?(?:\[([a-fA-F\d:]+)\]|([^:/?#]+))(?::(\d+))?)"); + + std::smatch m; + if (std::regex_match(scheme_host_port, m, re)) { + auto scheme = m[1].str(); + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + if (!scheme.empty() && (scheme != "http" && scheme != "https")) { +#else + if (!scheme.empty() && scheme != "http") { +#endif +#ifndef CPPHTTPLIB_NO_EXCEPTIONS + std::string msg = "'" + scheme + "' scheme is not supported."; + throw std::invalid_argument(msg); +#endif + return; + } + + auto is_ssl = scheme == "https"; + + auto host = m[2].str(); + if (host.empty()) { host = m[3].str(); } + + auto port_str = m[4].str(); + auto port = !port_str.empty() ? std::stoi(port_str) : (is_ssl ? 443 : 80); + + if (is_ssl) { +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + cli_ = detail::make_unique(host, port, client_cert_path, + client_key_path); + is_ssl_ = is_ssl; +#endif + } else { + cli_ = detail::make_unique(host, port, client_cert_path, + client_key_path); + } + } else { + // NOTE: Update TEST(UniversalClientImplTest, Ipv6LiteralAddress) + // if port param below changes. + cli_ = detail::make_unique(scheme_host_port, 80, + client_cert_path, client_key_path); + } +} // namespace detail + +inline Client::Client(const std::string &host, int port) + : cli_(detail::make_unique(host, port)) {} + +inline Client::Client(const std::string &host, int port, + const std::string &client_cert_path, + const std::string &client_key_path) + : cli_(detail::make_unique(host, port, client_cert_path, + client_key_path)) {} + +inline Client::~Client() = default; + +inline bool Client::is_valid() const { + return cli_ != nullptr && cli_->is_valid(); +} + +inline Result Client::Get(const std::string &path, DownloadProgress progress) { + return cli_->Get(path, std::move(progress)); +} +inline Result Client::Get(const std::string &path, const Headers &headers, + DownloadProgress progress) { + return cli_->Get(path, headers, std::move(progress)); +} +inline Result Client::Get(const std::string &path, + ContentReceiver content_receiver, + DownloadProgress progress) { + return cli_->Get(path, std::move(content_receiver), std::move(progress)); +} +inline Result Client::Get(const std::string &path, const Headers &headers, + ContentReceiver content_receiver, + DownloadProgress progress) { + return cli_->Get(path, headers, std::move(content_receiver), + std::move(progress)); +} +inline Result Client::Get(const std::string &path, + ResponseHandler response_handler, + ContentReceiver content_receiver, + DownloadProgress progress) { + return cli_->Get(path, std::move(response_handler), + std::move(content_receiver), std::move(progress)); +} +inline Result Client::Get(const std::string &path, const Headers &headers, + ResponseHandler response_handler, + ContentReceiver content_receiver, + DownloadProgress progress) { + return cli_->Get(path, headers, std::move(response_handler), + std::move(content_receiver), std::move(progress)); +} +inline Result Client::Get(const std::string &path, const Params ¶ms, + const Headers &headers, DownloadProgress progress) { + return cli_->Get(path, params, headers, std::move(progress)); +} +inline Result Client::Get(const std::string &path, const Params ¶ms, + const Headers &headers, + ContentReceiver content_receiver, + DownloadProgress progress) { + return cli_->Get(path, params, headers, std::move(content_receiver), + std::move(progress)); +} +inline Result Client::Get(const std::string &path, const Params ¶ms, + const Headers &headers, + ResponseHandler response_handler, + ContentReceiver content_receiver, + DownloadProgress progress) { + return cli_->Get(path, params, headers, std::move(response_handler), + std::move(content_receiver), std::move(progress)); +} + +inline Result Client::Head(const std::string &path) { return cli_->Head(path); } +inline Result Client::Head(const std::string &path, const Headers &headers) { + return cli_->Head(path, headers); +} + +inline Result Client::Post(const std::string &path) { return cli_->Post(path); } +inline Result Client::Post(const std::string &path, const Headers &headers) { + return cli_->Post(path, headers); +} +inline Result Client::Post(const std::string &path, const char *body, + size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return cli_->Post(path, body, content_length, content_type, progress); +} +inline Result Client::Post(const std::string &path, const Headers &headers, + const char *body, size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return cli_->Post(path, headers, body, content_length, content_type, + progress); +} +inline Result Client::Post(const std::string &path, const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return cli_->Post(path, body, content_type, progress); +} +inline Result Client::Post(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return cli_->Post(path, headers, body, content_type, progress); +} +inline Result Client::Post(const std::string &path, size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Post(path, content_length, std::move(content_provider), + content_type, progress); +} +inline Result Client::Post(const std::string &path, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Post(path, std::move(content_provider), content_type, progress); +} +inline Result Client::Post(const std::string &path, const Headers &headers, + size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Post(path, headers, content_length, std::move(content_provider), + content_type, progress); +} +inline Result Client::Post(const std::string &path, const Headers &headers, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Post(path, headers, std::move(content_provider), content_type, + progress); +} +inline Result Client::Post(const std::string &path, const Params ¶ms) { + return cli_->Post(path, params); +} +inline Result Client::Post(const std::string &path, const Headers &headers, + const Params ¶ms) { + return cli_->Post(path, headers, params); +} +inline Result Client::Post(const std::string &path, + const UploadFormDataItems &items, + UploadProgress progress) { + return cli_->Post(path, items, progress); +} +inline Result Client::Post(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + UploadProgress progress) { + return cli_->Post(path, headers, items, progress); +} +inline Result Client::Post(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const std::string &boundary, + UploadProgress progress) { + return cli_->Post(path, headers, items, boundary, progress); +} +inline Result Client::Post(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const FormDataProviderItems &provider_items, + UploadProgress progress) { + return cli_->Post(path, headers, items, provider_items, progress); +} +inline Result Client::Post(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + ContentReceiver content_receiver, + DownloadProgress progress) { + return cli_->Post(path, headers, body, content_type, content_receiver, + progress); +} + +inline Result Client::Put(const std::string &path) { return cli_->Put(path); } +inline Result Client::Put(const std::string &path, const Headers &headers) { + return cli_->Put(path, headers); +} +inline Result Client::Put(const std::string &path, const char *body, + size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return cli_->Put(path, body, content_length, content_type, progress); +} +inline Result Client::Put(const std::string &path, const Headers &headers, + const char *body, size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return cli_->Put(path, headers, body, content_length, content_type, progress); +} +inline Result Client::Put(const std::string &path, const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return cli_->Put(path, body, content_type, progress); +} +inline Result Client::Put(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return cli_->Put(path, headers, body, content_type, progress); +} +inline Result Client::Put(const std::string &path, size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Put(path, content_length, std::move(content_provider), + content_type, progress); +} +inline Result Client::Put(const std::string &path, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Put(path, std::move(content_provider), content_type, progress); +} +inline Result Client::Put(const std::string &path, const Headers &headers, + size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Put(path, headers, content_length, std::move(content_provider), + content_type, progress); +} +inline Result Client::Put(const std::string &path, const Headers &headers, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Put(path, headers, std::move(content_provider), content_type, + progress); +} +inline Result Client::Put(const std::string &path, const Params ¶ms) { + return cli_->Put(path, params); +} +inline Result Client::Put(const std::string &path, const Headers &headers, + const Params ¶ms) { + return cli_->Put(path, headers, params); +} +inline Result Client::Put(const std::string &path, + const UploadFormDataItems &items, + UploadProgress progress) { + return cli_->Put(path, items, progress); +} +inline Result Client::Put(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + UploadProgress progress) { + return cli_->Put(path, headers, items, progress); +} +inline Result Client::Put(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const std::string &boundary, + UploadProgress progress) { + return cli_->Put(path, headers, items, boundary, progress); +} +inline Result Client::Put(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const FormDataProviderItems &provider_items, + UploadProgress progress) { + return cli_->Put(path, headers, items, provider_items, progress); +} +inline Result Client::Put(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + ContentReceiver content_receiver, + DownloadProgress progress) { + return cli_->Put(path, headers, body, content_type, content_receiver, + progress); +} + +inline Result Client::Patch(const std::string &path) { + return cli_->Patch(path); +} +inline Result Client::Patch(const std::string &path, const Headers &headers) { + return cli_->Patch(path, headers); +} +inline Result Client::Patch(const std::string &path, const char *body, + size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return cli_->Patch(path, body, content_length, content_type, progress); +} +inline Result Client::Patch(const std::string &path, const Headers &headers, + const char *body, size_t content_length, + const std::string &content_type, + UploadProgress progress) { + return cli_->Patch(path, headers, body, content_length, content_type, + progress); +} +inline Result Client::Patch(const std::string &path, const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return cli_->Patch(path, body, content_type, progress); +} +inline Result Client::Patch(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + UploadProgress progress) { + return cli_->Patch(path, headers, body, content_type, progress); +} +inline Result Client::Patch(const std::string &path, size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Patch(path, content_length, std::move(content_provider), + content_type, progress); +} +inline Result Client::Patch(const std::string &path, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Patch(path, std::move(content_provider), content_type, progress); +} +inline Result Client::Patch(const std::string &path, const Headers &headers, + size_t content_length, + ContentProvider content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Patch(path, headers, content_length, std::move(content_provider), + content_type, progress); +} +inline Result Client::Patch(const std::string &path, const Headers &headers, + ContentProviderWithoutLength content_provider, + const std::string &content_type, + UploadProgress progress) { + return cli_->Patch(path, headers, std::move(content_provider), content_type, + progress); +} +inline Result Client::Patch(const std::string &path, const Params ¶ms) { + return cli_->Patch(path, params); +} +inline Result Client::Patch(const std::string &path, const Headers &headers, + const Params ¶ms) { + return cli_->Patch(path, headers, params); +} +inline Result Client::Patch(const std::string &path, + const UploadFormDataItems &items, + UploadProgress progress) { + return cli_->Patch(path, items, progress); +} +inline Result Client::Patch(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + UploadProgress progress) { + return cli_->Patch(path, headers, items, progress); +} +inline Result Client::Patch(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const std::string &boundary, + UploadProgress progress) { + return cli_->Patch(path, headers, items, boundary, progress); +} +inline Result Client::Patch(const std::string &path, const Headers &headers, + const UploadFormDataItems &items, + const FormDataProviderItems &provider_items, + UploadProgress progress) { + return cli_->Patch(path, headers, items, provider_items, progress); +} +inline Result Client::Patch(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + ContentReceiver content_receiver, + DownloadProgress progress) { + return cli_->Patch(path, headers, body, content_type, content_receiver, + progress); +} + +inline Result Client::Delete(const std::string &path, + DownloadProgress progress) { + return cli_->Delete(path, progress); +} +inline Result Client::Delete(const std::string &path, const Headers &headers, + DownloadProgress progress) { + return cli_->Delete(path, headers, progress); +} +inline Result Client::Delete(const std::string &path, const char *body, + size_t content_length, + const std::string &content_type, + DownloadProgress progress) { + return cli_->Delete(path, body, content_length, content_type, progress); +} +inline Result Client::Delete(const std::string &path, const Headers &headers, + const char *body, size_t content_length, + const std::string &content_type, + DownloadProgress progress) { + return cli_->Delete(path, headers, body, content_length, content_type, + progress); +} +inline Result Client::Delete(const std::string &path, const std::string &body, + const std::string &content_type, + DownloadProgress progress) { + return cli_->Delete(path, body, content_type, progress); +} +inline Result Client::Delete(const std::string &path, const Headers &headers, + const std::string &body, + const std::string &content_type, + DownloadProgress progress) { + return cli_->Delete(path, headers, body, content_type, progress); +} +inline Result Client::Delete(const std::string &path, const Params ¶ms, + DownloadProgress progress) { + return cli_->Delete(path, params, progress); +} +inline Result Client::Delete(const std::string &path, const Headers &headers, + const Params ¶ms, DownloadProgress progress) { + return cli_->Delete(path, headers, params, progress); +} + +inline Result Client::Options(const std::string &path) { + return cli_->Options(path); +} +inline Result Client::Options(const std::string &path, const Headers &headers) { + return cli_->Options(path, headers); +} + +inline bool Client::send(Request &req, Response &res, Error &error) { + return cli_->send(req, res, error); +} + +inline Result Client::send(const Request &req) { return cli_->send(req); } + +inline void Client::stop() { cli_->stop(); } + +inline std::string Client::host() const { return cli_->host(); } + +inline int Client::port() const { return cli_->port(); } + +inline size_t Client::is_socket_open() const { return cli_->is_socket_open(); } + +inline socket_t Client::socket() const { return cli_->socket(); } + +inline void +Client::set_hostname_addr_map(std::map addr_map) { + cli_->set_hostname_addr_map(std::move(addr_map)); +} + +inline void Client::set_default_headers(Headers headers) { + cli_->set_default_headers(std::move(headers)); +} + +inline void Client::set_header_writer( + std::function const &writer) { + cli_->set_header_writer(writer); +} + +inline void Client::set_address_family(int family) { + cli_->set_address_family(family); +} + +inline void Client::set_tcp_nodelay(bool on) { cli_->set_tcp_nodelay(on); } + +inline void Client::set_socket_options(SocketOptions socket_options) { + cli_->set_socket_options(std::move(socket_options)); +} + +inline void Client::set_connection_timeout(time_t sec, time_t usec) { + cli_->set_connection_timeout(sec, usec); +} + +inline void Client::set_read_timeout(time_t sec, time_t usec) { + cli_->set_read_timeout(sec, usec); +} + +inline void Client::set_write_timeout(time_t sec, time_t usec) { + cli_->set_write_timeout(sec, usec); +} + +inline void Client::set_basic_auth(const std::string &username, + const std::string &password) { + cli_->set_basic_auth(username, password); +} +inline void Client::set_bearer_token_auth(const std::string &token) { + cli_->set_bearer_token_auth(token); +} +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +inline void Client::set_digest_auth(const std::string &username, + const std::string &password) { + cli_->set_digest_auth(username, password); +} +#endif + +inline void Client::set_keep_alive(bool on) { cli_->set_keep_alive(on); } +inline void Client::set_follow_location(bool on) { + cli_->set_follow_location(on); +} + +inline void Client::set_path_encode(bool on) { cli_->set_path_encode(on); } + +[[deprecated("Use set_path_encode instead")]] +inline void Client::set_url_encode(bool on) { + cli_->set_path_encode(on); +} + +inline void Client::set_compress(bool on) { cli_->set_compress(on); } + +inline void Client::set_decompress(bool on) { cli_->set_decompress(on); } + +inline void Client::set_interface(const std::string &intf) { + cli_->set_interface(intf); +} + +inline void Client::set_proxy(const std::string &host, int port) { + cli_->set_proxy(host, port); +} +inline void Client::set_proxy_basic_auth(const std::string &username, + const std::string &password) { + cli_->set_proxy_basic_auth(username, password); +} +inline void Client::set_proxy_bearer_token_auth(const std::string &token) { + cli_->set_proxy_bearer_token_auth(token); +} +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +inline void Client::set_proxy_digest_auth(const std::string &username, + const std::string &password) { + cli_->set_proxy_digest_auth(username, password); +} +#endif + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +inline void Client::enable_server_certificate_verification(bool enabled) { + cli_->enable_server_certificate_verification(enabled); +} + +inline void Client::enable_server_hostname_verification(bool enabled) { + cli_->enable_server_hostname_verification(enabled); +} + +inline void Client::set_server_certificate_verifier( + std::function verifier) { + cli_->set_server_certificate_verifier(verifier); +} +#endif + +inline void Client::set_logger(Logger logger) { + cli_->set_logger(std::move(logger)); +} + +inline void Client::set_error_logger(ErrorLogger error_logger) { + cli_->set_error_logger(std::move(error_logger)); +} + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +inline void Client::set_ca_cert_path(const std::string &ca_cert_file_path, + const std::string &ca_cert_dir_path) { + cli_->set_ca_cert_path(ca_cert_file_path, ca_cert_dir_path); +} + +inline void Client::set_ca_cert_store(X509_STORE *ca_cert_store) { + if (is_ssl_) { + static_cast(*cli_).set_ca_cert_store(ca_cert_store); + } else { + cli_->set_ca_cert_store(ca_cert_store); + } +} + +inline void Client::load_ca_cert_store(const char *ca_cert, std::size_t size) { + set_ca_cert_store(cli_->create_ca_cert_store(ca_cert, size)); +} + +inline long Client::get_openssl_verify_result() const { + if (is_ssl_) { + return static_cast(*cli_).get_openssl_verify_result(); + } + return -1; // NOTE: -1 doesn't match any of X509_V_ERR_??? +} + +inline SSL_CTX *Client::ssl_context() const { + if (is_ssl_) { return static_cast(*cli_).ssl_context(); } + return nullptr; +} +#endif + +// ---------------------------------------------------------------------------- + +} // namespace httplib + +#endif // CPPHTTPLIB_HTTPLIB_H diff --git a/third_party/cpp_httplib/httplib_test.cpp b/third_party/cpp_httplib/httplib_test.cpp new file mode 100644 index 0000000..fcbfe41 --- /dev/null +++ b/third_party/cpp_httplib/httplib_test.cpp @@ -0,0 +1,47 @@ +#include +#include "httplib.h" + +int main() { + /** + * GET html from website: http://www.videopipe.cool + */ + httplib::Client get_cli("http://www.videopipe1.cool"); + if (auto res = get_cli.Get("/index.php/bloglist/")) { + if (res->status == httplib::StatusCode::OK_200) { + std::cout << res->body << std::endl; + } + else { + std::cout << res->status << std::endl; + } + } + else { + auto err = res.error(); + std::cout << "HTTP error: " << httplib::to_string(err) << std::endl; + } + + /** + * POST data with headers to LLM service based on Ollama. + */ + httplib::Client post_cli("http://192.168.77.219:11434"); + httplib::Headers headers = {{"myheader1", "1111"}, {"myheader2", "2222"}}; + auto payload = R"( + { + "model": "qwen2.5:7b", + "prompt": "为什么坐火车要给钱？", + "stream": false + } + )"; + if (auto res = post_cli.Post("/api/generate", headers, payload, "application/json")) { + if (res->status == httplib::StatusCode::OK_200) { + std::cout << res->body << std::endl; + } + else { + std::cout << res->body << std::endl; + std::cout << res->status << std::endl; + } + } + else { + auto err = res.error(); + std::cout << "HTTP error: " << httplib::to_string(err) << std::endl; + } +} \ No newline at end of file diff --git a/third_party/cpp_llmlib/CMakeLists.txt b/third_party/cpp_llmlib/CMakeLists.txt new file mode 100644 index 0000000..b2d41d0 --- /dev/null +++ b/third_party/cpp_llmlib/CMakeLists.txt @@ -0,0 +1,30 @@ +cmake_minimum_required(VERSION 3.10) +project(llmlib_test VERSION 1.0) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_BUILD_TYPE Debug) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fPIC -w -fdiagnostics-color=always -pthread") + +# OpenCV required +find_package(OpenCV REQUIRED) +message(STATUS "OpenCV library status:") +message(STATUS " version: ${OpenCV_VERSION}") +message(STATUS " libraries: ${OpenCV_LIBS}") +message(STATUS " include path: ${OpenCV_INCLUDE_DIRS}") +include_directories(${OpenCV_INCLUDE_DIRS}) + +find_package(OpenSSL REQUIRED) +message(STATUS "OpenSSL library status:") +message(STATUS " version: ${OPENSSL_VERSION}") +message(STATUS " libraries: ${OPENSSL_LIBRARIES}") +message(STATUS " include path: ${OPENSSL_INCLUDE_DIR}") +include_directories(${OPENSSL_INCLUDE_DIR}) + +add_executable(llmlib_ollama_test "llmlib_ollama_test.cpp") +target_link_libraries(llmlib_ollama_test ${OpenCV_LIBS} ${OPENSSL_LIBRARIES}) + +add_executable(llmlib_openai_test "llmlib_openai_test.cpp") +target_link_libraries(llmlib_openai_test ${OpenCV_LIBS} ${OPENSSL_LIBRARIES}) + +add_executable(llmlib_multi_chat_test "llmlib_multi_chat_test.cpp") +target_link_libraries(llmlib_multi_chat_test ${OpenCV_LIBS} ${OPENSSL_LIBRARIES}) \ No newline at end of file diff --git a/third_party/cpp_llmlib/README.md b/third_party/cpp_llmlib/README.md new file mode 100644 index 0000000..ead84d3 --- /dev/null +++ b/third_party/cpp_llmlib/README.md @@ -0,0 +1,13 @@ + + a lightweight LLM Client SDK using modern C++ Language, supports calling OpenAI-compatible online services — such as Alibaba Cloud, OpenAI, and other API providers — as well as locally hosted inference backends like Ollama, vLLM, and Hugging Face TGI. + +> NOTE: OpenSSL >= 3.0 required for llmlib.hpp. +``` +mkdir build +cd build +cmake .. +make -j8 + +./llmlib_ollama_test # test chat REST API for Ollama +./llmlib_openai_test # test chat REST API for OpenAI-compatible protocol(aliyun LLM API provider/vLLM/...) +``` \ No newline at end of file diff --git a/third_party/cpp_llmlib/llmlib.hpp b/third_party/cpp_llmlib/llmlib.hpp new file mode 100644 index 0000000..6618456 --- /dev/null +++ b/third_party/cpp_llmlib/llmlib.hpp @@ -0,0 +1,294 @@ +#define CPPHTTPLIB_OPENSSL_SUPPORT +#include +#include +#include +#include +#include "../cpp_httplib/httplib.h" +#include "../cpp_base64/base64.h" +#include "../nlohmann/json.hpp" +using json = nlohmann::json; + +namespace llmlib { + /** + * LLM backends: + * 1. OpenAI: OpenAI-compatible LLM service API providers + * 2. Ollama: LLM services hosted locally by Ollama framework + * + */ + enum class LLMBackendType { + OpenAI, + Ollama + }; + + + /** + * LLM Client: + * + * support chat with LLM with text & image as input. + */ + class LLMClient { + public: + LLMClient() {} + LLMClient(const std::string& api_base_url, + const std::string& api_key, + LLMBackendType backend_type = LLMBackendType::OpenAI): + __api_base_url(api_base_url), + __api_key(api_key), + __backend_type(backend_type) + {} + + /** + * set default headers for all requests. + */ + void set_default_header(const std::string& key, const std::string& value) { + __default_headers[key] = value; + } + + /** + * set connection timeout(seconds) for all requests. + */ + void set_connection_timeout(int connection_timeout) { + __connection_timeout = connection_timeout; + } + + /** + * simple chat interface. + * support single text prompt and multi images as input, history messages not Supported. + * + * @param model_name LLM to be used. + * @param prompt input text prompt. + * @param images input images. + * @param options LLM parameters: temperature, top_k, etc. + */ + std::string simple_chat(const std::string& model_name, + const std::string& prompt, + const std::vector& images, + const json& options) { + // construct json payload + json payload; + payload["model"] = model_name; + payload["stream"] = false; + + if (__backend_type == LLMBackendType::OpenAI) { + json message; + message["role"] = "user"; + json content = json::array(); + content.push_back({{"type", "text"}, {"text", prompt}}); + if (images.size() != 0) { + for(auto& image: images) { + auto base64_img = mat_to_base64(image); + content.push_back({{"type", "image_url"}, {"image_url", {{"url", "data:image/jpg;base64," + base64_img}}}}); + } + } + + message["content"] = content; + payload["messages"] = {message}; + if (!options.is_null()) { + payload.update(options); + } + } else if (__backend_type == LLMBackendType::Ollama) { + json message; + message["role"] = "user"; + message["content"] = prompt; + if (images.size() != 0) { + json imgs = json::array(); + for(auto& image: images) { + auto base64_img = mat_to_base64(image); + imgs.push_back(base64_img); + } + message["images"] = imgs; + } + payload["messages"] = {message}; + payload["options"] = options; + } + + // send request + auto res = request(payload.dump(4)); + auto res_json = json::parse(res); + + // std::cout << payload.dump(4) << std::endl; + // std::cout << res << std::endl; + + // try to get result extracted from response + try + { + if (__backend_type == LLMBackendType::OpenAI) { + return res_json["choices"][0]["message"]["content"]; + } else if (__backend_type == LLMBackendType::Ollama) { + return res_json["message"]["content"]; + } else { + return ""; + } + } + catch(const std::exception& e) + { + std::cerr << e.what() << '\n'; + } + return ""; + } + + /** + * chat interface. + * support messages(json array) as input, history messages Supported. + * + * @param model_name LLM to be used. + * @param messages a json array containing history messages. + * @param options LLM parameters: temperature, top_k, etc. + * + * NOTE: + * messages MUST be constructed according to appropriate protocols by caller outside of chat. + */ + std::string chat(const std::string& model_name, + const json& messages, + const json& options) { + // construct json payload + json payload; + payload["model"] = model_name; + payload["messages"] = messages; + payload["stream"] = false; + + if (__backend_type == LLMBackendType::OpenAI) { + if (!options.is_null()) { + payload.update(options); + } + } else if (__backend_type == LLMBackendType::Ollama) { + payload["options"] = options; + } + + // send request + auto res = request(payload.dump(4)); + auto res_json = json::parse(res); + + //std::cout << payload.dump(4) << std::endl; + //std::cout << res << std::endl; + + // try to get result extracted from response + try + { + if (__backend_type == LLMBackendType::OpenAI) { + return res_json["choices"][0]["message"]["content"]; + } else if (__backend_type == LLMBackendType::Ollama) { + return res_json["message"]["content"]; + } else { + return ""; + } + } + catch(const std::exception& e) + { + std::cerr << e.what() << '\n'; + } + return ""; + } + + /** + * convert base64 string to cv::Mat + */ + cv::Mat base64_to_mat(const std::string& base64_data) { + std::string decoded_data = base64_decode(base64_data); + std::vector data(decoded_data.begin(), decoded_data.end()); + cv::Mat img = cv::imdecode(data, cv::IMREAD_UNCHANGED); + return img; + } + + /** + * convert cv::Mat to base64 string + */ + std::string mat_to_base64(const cv::Mat& img, const std::string& ext = ".jpg") { + std::vector buf; + cv::imencode(ext, img, buf); + std::string encoded = base64_encode(buf.data(), buf.size()); + return encoded; + } + private: + std::string __api_base_url; + std::string __api_key; + LLMBackendType __backend_type = LLMBackendType::OpenAI; + std::map __default_headers; + int __connection_timeout = 1; + /**/ + std::string __openai_chat_completions_path = "/chat/completions"; + std::string __ollama_chat_completions_path = "/api/chat"; + + std::string request(const std::string& payload) { + auto parsed_base_url = split_base_url(__api_base_url); + httplib::Client cli(parsed_base_url.base_host.c_str()); + cli.set_connection_timeout(__connection_timeout, 0); + + httplib::Headers headers; + auto chat_path = __backend_type == LLMBackendType::OpenAI ? __openai_chat_completions_path : __ollama_chat_completions_path; + if (__backend_type == LLMBackendType::OpenAI) { + headers = { + {"Content-Type", "application/json"}, + {"Authorization", "Bearer " + __api_key} + }; + } else if (__backend_type == LLMBackendType::Ollama) { + headers = { + {"Content-Type", "application/json"} + }; + } else { + + } + for (const auto& kv : __default_headers) { + headers.emplace(kv.first, kv.second); + } + + auto final_path = parsed_base_url.base_path + chat_path; + auto res = cli.Post(final_path, headers, payload, "application/json"); + if (res) { + if (res->status == httplib::StatusCode::OK_200) { + return res->body; + } + else { + std::cout << "[llmlib] HTTP status code: " << res->status << std::endl; + return "{}"; + } + } else { + auto err = res.error(); + std::cout << "[llmlib] HTTP error: " << httplib::to_string(err) << std::endl; + return "{}"; + } + } + + struct ParsedBaseUrl { + std::string base_host; // protocols + host + port（if exists） + std::string base_path; // start with '/', not end with '/' + }; + + ParsedBaseUrl split_base_url(const std::string& url) { + ParsedBaseUrl result; + + std::regex re(R"(^(https?)://([^:/\s]+)(?::(\d+))?(/.*|$))", std::regex::icase); + std::smatch match; + + if (!std::regex_search(url, match, re)) { + throw std::invalid_argument("Invalid URL format: " + url); + } + + std::string protocol = match[1].str(); + std::string host = match[2].str(); + std::string port_str = match[3].str(); + std::string path = match[4].str(); + + int port = (protocol == "https") ? 443 : 80; + if (!port_str.empty()) { + port = std::stoi(port_str); + } + + if (port_str.empty()) { + result.base_host = protocol + "://" + host; + } else { + result.base_host = protocol + "://" + host + ":" + port_str; + } + + result.base_path = path.empty() ? "/" : path; + if (result.base_path.length() > 1 && result.base_path.back() == '/') { + result.base_path.pop_back(); + } + if (result.base_path == "/") { + result.base_path.clear(); + } + + return result; + } + }; +} \ No newline at end of file diff --git a/third_party/cpp_llmlib/llmlib_multi_chat_test.cpp b/third_party/cpp_llmlib/llmlib_multi_chat_test.cpp new file mode 100644 index 0000000..116db6b --- /dev/null +++ b/third_party/cpp_llmlib/llmlib_multi_chat_test.cpp @@ -0,0 +1,35 @@ +#include +#include "llmlib.hpp" +using namespace llmlib; + +int main() { + // create LLMClient + LLMClient llmcli("http://192.168.77.219:11434", "", LLMBackendType::Ollama); + + // define parameters + auto model_name = "qwen2.5:7b"; + json options = {{"temperature", 0.1}, {"top_k", 1}}; + json messages = {{{"role", "system"}, {"content", "你是一个非常专业的聊天助手，你的名字叫小王。"}}}; + + // chat loop runing on console + std::string input = ""; + std::cout << "##let's chat with LLM hosted by Ollama##" << std::endl; + std::cout << "YOU: "; + do + { + std::cin >> input; + if (input != "quit") { + // append user's input to history messages + messages.push_back({{"role", "user"}, {"content", input}}); + + // call chat() + auto output = llmcli.chat(model_name, messages, options); + std::cout << "-----------------------------------------" << std::endl << "LLM: " << output << std::endl; + std::cout << "-----------------------------------------" << std::endl << "YOU: "; + + // append assistant's output to history messages + messages.push_back({{"role", "assistant"}, {"content", output}}); + } + } while (input != "quit"); + std::cout << "##finish chat with LLM hosted by Ollama##" << std::endl; +} \ No newline at end of file diff --git a/third_party/cpp_llmlib/llmlib_ollama_test.cpp b/third_party/cpp_llmlib/llmlib_ollama_test.cpp new file mode 100644 index 0000000..c2454ef --- /dev/null +++ b/third_party/cpp_llmlib/llmlib_ollama_test.cpp @@ -0,0 +1,68 @@ + +#include +#include "llmlib.hpp" +using namespace llmlib; + +/** + * 1. install Ollama locally + * 2. start Ollama service and get api base url(no api key required for Ollama) + * +*/ +int main() { + LLMClient llmcli("http://192.168.77.219:11434", "", LLMBackendType::Ollama); + /** + * 1. test for simple chat + */ + auto res1 = llmcli.simple_chat("qwen2.5:7b", "坐飞机为什么要付机票钱？", {}, {{"temperature", 0.1}, {"top_k", 1}}); + std::cout << "------test for simple chat-----" << std::endl; + std::cout << res1 << std::endl; + std::cout << "-------------------------------" << std::endl; + + /** + * 2. test for simple chat with images + */ + auto image1 = cv::imread("/windows2/zhzhi/github/vp_data/test_images/vehicle/0.jpg"); + auto image2 = cv::imread("/windows2/zhzhi/github/vp_data/test_images/vehicle/27.jpg"); + auto res2 = llmcli.simple_chat("minicpm-v:8b", "描述这两幅图片的差异", {image1, image2}, {}); + std::cout << "-----test for simple chat with images-----" << std::endl; + std::cout << res2 << std::endl; + std::cout << "------------------------------------------" << std::endl; + + /** + * 3. test for chat + */ + auto messages = R"( + [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "你是谁？" + } + ] + )"_json; + auto options = R"( + { + "temperature": 0.4, + "top_k": 2 + } + )"_json; + auto res3 = llmcli.chat("qwen2.5:7b", messages, options); + std::cout << "---------test for chat---------" << std::endl; + std::cout << res3 << std::endl; + std::cout << "-------------------------------" << std::endl; + + /** + * 4. test for chat with images + */ + json messages4 = { + {{"role", "system"}, {"content", "You are a helpful assistant."}}, + {{"role", "user"}, {"content", "这是一张什么风格的图片？"}, {"images", {"iVBORw0KGgoAAAANSUhEUgAAAG0AAABmCAYAAADBPx+VAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAA3VSURBVHgB7Z27r0zdG8fX743i1bi1ikMoFMQloXRpKFFIqI7LH4BEQ+NWIkjQuSWCRIEoULk0gsK1kCBI0IhrQVT7tz/7zZo888yz1r7MnDl7z5xvsjkzs2fP3uu71nNfa7lkAsm7d++Sffv2JbNmzUqcc8m0adOSzZs3Z+/XES4ZckAWJEGWPiCxjsQNLWmQsWjRIpMseaxcuTKpG/7HP27I8P79e7dq1ars/yL4/v27S0ejqwv+cUOGEGGpKHR37tzJCEpHV9tnT58+dXXCJDdECBE2Ojrqjh071hpNECjx4cMHVycM1Uhbv359B2F79+51586daxN/+pyRkRFXKyRDAqxEp4yMlDDzXG1NPnnyJKkThoK0VFd1ELZu3TrzXKxKfW7dMBQ6bcuWLW2v0VlHjx41z717927ba22U9APcw7Nnz1oGEPeL3m3p2mTAYYnFmMOMXybPPXv2bNIPpFZr1NHn4HMw0KRBjg9NuRw95s8PEcz/6DZELQd/09C9QGq5RsmSRybqkwHGjh07OsJSsYYm3ijPpyHzoiacg35MLdDSIS/O1yM778jOTwYUkKNHWUzUWaOsylE00MyI0fcnOwIdjvtNdW/HZwNLGg+sR1kMepSNJXmIwxBZiG8tDTpEZzKg0GItNsosY8USkxDhD0Rinuiko2gfL/RbiD2LZAjU9zKQJj8RDR0vJBR1/Phx9+PHj9Z7REF4nTZkxzX4LCXHrV271qXkBAPGfP/atWvu/PnzHe4C97F48eIsRLZ9+3a3f/9+87dwP1JxaF7/3r17ba+5l4EcaVo0lj3SBq5kGTJSQmLWMjgYNei2GPT1MuMqGTDEFHzeQSP2wi/jGnkmPJ/nhccs44jvDAxpVcxnq0F6eT8h4ni/iIWpR5lPyA6ETkNXoSukvpJAD3AsXLiwpZs49+fPn5ke4j10TqYvegSfn0OnafC+Tv9ooA/JPkgQysqQNBzagXY55nO/oa1F7qvIPWkRL12WRpMWUvpVDYmxAPehxWSe8ZEXL20sadYIozfmNch4QJPAfeJgW3rNsnzphBKNJM2KKODo1rVOMRYik5ETy3ix4qWNI81qAAirizgMIc+yhTytx0JWZuNI03qsrgWlGtwjoS9XwgUhWGyhUaRZZQNNIEwCiXD16tXcAHUs79co0vSD8rrJCIW98pzvxpAWyyo3HYwqS0+H0BjStClcZJT5coMm6D2LOF8TolGJtK9fvyZpyiC5ePFi9nc/oJU4eiEP0jVoAnHa9wyJycITMP78+eMeP37sXrx44d6+fdt6f82aNdkx1pg9e3Zb5W+RSRE+n+VjksQWifvVaTKFhn5O8my63K8Qabdv33b379/PiAP//vuvW7BggZszZ072/+TJk91YgkafPn166zXB1rQHFvouAWHq9z3SEevSUerqCn2/dDCeta2jxYbr69evk4MHDyY7d+7MjhMnTiTPnz9Pfv/+nfQT2ggpO2dMF8cghuoM7Ygj5iWCqRlGFml0QC/ftGmTmzt3rmsaKDsgBSPh0/8yPeLLBihLkOKJc0jp8H8vUzcxIA1k6QJ/c78tWEyj5P3o4u9+jywNPdJi5rAH9x0KHcl4Hg570eQp3+vHXGyrmEeigzQsQsjavXt38ujRo44LQuDDhw+TW7duRS1HGgMxhNXHgflaNTOsHyKvHK5Ijo2jbFjJBQK9YwFd6RVMzfgRBmEfP37suBBm/p49e1qjEP2mwTViNRo0VJWH1deMXcNK08uUjVUu7s/zRaL+oLNxz1bpANco4npUgX4G2eFbpDFyQoQxojBCpEGSytmOH8qrH5Q9vuzD6ofQylkCUmh8DBAr+q8JCyVNtWQIidKQE9wNtLSQnS4jDSsxNHogzFuQBw4cyM61UKVsjfr3ooBkPSqqQHesUPWVtzi9/vQi1T+rJj7WiTz4Pt/l3LxUkr5P2VYZaZ4URpsE+st/dujQoaBBYokbrz/8TJNQYLSonrPS9kUaSkPeZyj1AWSj+d+VBoy1pIWVNed8P0Ll/ee5HdGRhrHhR5GGN0r4LGZBaj8oFDJitBTJzIZgFcmU0Y8ytWMZMzJOaXUSrUs5RxKnrxmbb5YXO9VGUhtpXldhEUogFr3IzIsvlpmdosVcGVGXFWp2oU9kLFL3dEkSz6NHEY1sjSRdIuDFWEhd8KxFqsRi1uM/nz9/zpxnwlESONdg6dKlbsaMGS4EHFHtjFIDHwKOo46l4TxSuxgDzi+rE2jg+BaFruOX4HXa0Nnf1lwAPufZeF8/r6zD97WK2qFnGjBxTw5qNGPxT+5T/r7/7RawFC3j4vTp09koCxkeHjqbHJqArmH5UrFKKksnxrK7FuRIs8STfBZv+luugXZ2pR/pP9Ois4z+TiMzUUkUjD0iEi1fzX8GmXyuxUBRcaUfykV0YZnlJGKQpOiGB76x5GeWkWWJc3mOrK6S7xdND+W5N6XyaRgtWJFe13GkaZnKOsYqGdOVVVbGupsyA/l7emTLHi7vwTdirNEt0qxnzAvBFcnQF16xh/TMpUuXHDowhlA9vQVraQhkudRdzOnK+04ZSP3DUhVSP61YsaLtd/ks7ZgtPcXqPqEafHkdqa84X6aCeL7YWlv6edGFHb+ZFICPlljHhg0bKuk0CSvVznWsotRu433alNdFrqG45ejoaPCaUkWERpLXjzFL2Rpllp7PJU2a/v7Ab8N05/9t27Z16KUqoFGsxnI9EosS2niSYg9SpU6B4JgTrvVW1flt1sT+0ADIJU2maXzcUTraGCRaL1Wp9rUMk16PMom8QhruxzvZIegJjFU7LLCePfS8uaQdPny4jTTL0dbee5mYokQsXTIWNY46kuMbnt8Kmec+LGWtOVIl9cT1rCB0V8WqkjAsRwta93TbwNYoGKsUSChN44lgBNCoHLHzquYKrU6qZ8lolCIN0Rh6cP0Q3U6I6IXILYOQI513hJaSKAorFpuHXJNfVlpRtmYBk1Su1obZr5dnKAO+L10Hrj3WZW+E3qh6IszE37F6EB+68mGpvKm4eb9bFrlzrok7fvr0Kfv727dvWRmdVTJHw0qiiCUSZ6wCK+7XL/AcsgNyL74DQQ730sv78Su7+t/A36MdY0sW5o40ahslXr58aZ5HtZB8GH64m9EmMZ7FpYw4T6QnrZfgenrhFxaSiSGXtPnz57e9TkNZLvTjeqhr734CNtrK41L40sUQckmj1lGKQ0rC37x544r8eNXRpnVE3ZZY7zXo8NomiO0ZUCj2uHz58rbXoZ6gc0uA+F6ZeKS/jhRDUq8MKrTho9fEkihMmhxtBI1DxKFY9XLpVcSkfoi8JGnToZO5sU5aiDQIW716ddt7ZLYtMQlhECdBGXZZMWldY5BHm5xgAroWj4C0hbYkSc/jBmggIrXJWlZM6pSETsEPGqZOndr2uuuR5rF169a2HoHPdurUKZM4CO1WTPqaDaAd+GFGKdIQkxAn9RuEWcTRyN2KSUgiSgF5aWzPTeA/lN5rZubMmR2bE4SIC4nJoltgAV/dVefZm72AtctUCJU2CMJ327hxY9t7EHbkyJFseq+EJSY16RPo3Dkq1kkr7+q0bNmyDuLQcZBEPYmHVdOBiJyIlrRDq41YPWfXOxUysi5fvtyaj+2BpcnsUV/oSoEMOk2CQGlr4ckhBwaetBhjCwH0ZHtJROPJkyc7UjcYLDjmrH7ADTEBXFfOYmB0k9oYBOjJ8b4aOYSe7QkKcYhFlq3QYLQhSidNmtS2RATwy8YOM3EQJsUjKiaWZ+vZToUQgzhkHXudb/PW5YMHD9yZM2faPsMwoc7RciYJXbGuBqJ1UIGKKLv915jsvgtJxCZDubdXr165mzdvtr1Hz5LONA8jrUwKPqsmVesKa49S3Q4WxmRPUEYdTjgiUcfUwLx589ySJUva3oMkP6IYddq6HMS4o55xBJBUeRjzfa4Zdeg56QZ43LhxoyPo7Lf1kNt7oO8wWAbNwaYjIv5lhyS7kRf96dvm5Jah8vfvX3flyhX35cuX6HfzFHOToS1H4BenCaHvO8pr8iDuwoUL7tevX+b5ZdbBair0xkFIlFDlW4ZknEClsp/TzXyAKVOmmHWFVSbDNw1l1+4f90U6IY/q4V27dpnE9bJ+v87QEydjqx/UamVVPRG+mwkNTYN+9tjkwzEx+atCm/X9WvWtDtAb68Wy9LXa1UmvCDDIpPkyOQ5ZwSzJ4jMrvFcr0rSjOUh+GcT4LSg5ugkW1Io0/SCDQBojh0hPlaJdah+tkVYrnTZowP8iq1F1TgMBBauufyB33x1v+NWFYmT5KmppgHC+NkAgbmRkpD3yn9QIseXymoTQFGQmIOKTxiZIWpvAatenVqRVXf2nTrAWMsPnKrMZHz6bJq5jvce6QK8J1cQNgKxlJapMPdZSR64/UivS9NztpkVEdKcrs5alhhWP9NeqlfWopzhZScI6QxseegZRGeg5a8C3Re1Mfl1ScP36ddcUaMuv24iOJtz7sbUjTS4qBvKmstYJoUauiuD3k5qhyr7QdUHMeCgLa1Ear9NquemdXgmum4fvJ6w1lqsuDhNrg1qSpleJK7K3TF0Q2jSd94uSZ60kK1e3qyVpQK6PVWXp2/FC3mp6jBhKKOiY2h3gtUV64TWM6wDETRPLDfSakXmH3w8g9Jlug8ZtTt4kVF0kLUYYmCCtD/DrQ5YhMGbA9L3ucdjh0y8kOHW5gU/VEEmJTcL4Pz/f7mgoAbYkAAAAAElFTkSuQmCC"}}} + }; + auto res4 = llmcli.chat("minicpm-v:8b", messages4, {{"temperature", 0.1}, {"top_k", 1}}); + std::cout << "---------test for chat with images---------" << std::endl; + std::cout << res4 << std::endl; + std::cout << "-------------------------------------------" << std::endl; +} \ No newline at end of file diff --git a/third_party/cpp_llmlib/llmlib_openai_test.cpp b/third_party/cpp_llmlib/llmlib_openai_test.cpp new file mode 100644 index 0000000..381bf36 --- /dev/null +++ b/third_party/cpp_llmlib/llmlib_openai_test.cpp @@ -0,0 +1,79 @@ +#include +#include "llmlib.hpp" +using namespace llmlib; + +/** + * prepared information from https://bailian.console.aliyun.com/#/home (or other OpenAI-compatible Services) + * 1. api base url + * 2. api key +*/ +int main() { + LLMClient llmcli("https://dashscope.aliyuncs.com/compatible-mode/v1", "sk-XXX", LLMBackendType::OpenAI); + /** + * 1. test for simple chat + */ + auto res1 = llmcli.simple_chat("qwen2.5-vl-7b-instruct", "坐飞机为什么要付机票钱？", {}, {{"temperature", 0.1}, {"top_k", 1}}); + std::cout << "------test for simple chat-----" << std::endl; + std::cout << res1 << std::endl; + std::cout << "-------------------------------" << std::endl; + + /** + * 2. test for simple chat with images + */ + auto image1 = cv::imread("/windows2/zhzhi/github/vp_data/test_images/vehicle/0.jpg"); + auto image2 = cv::imread("/windows2/zhzhi/github/vp_data/test_images/vehicle/27.jpg"); + auto res2 = llmcli.simple_chat("qwen2.5-vl-7b-instruct", "描述这两幅图片的差异", {image1, image2}, {}); + std::cout << "-----test for simple chat with images-----" << std::endl; + std::cout << res2 << std::endl; + std::cout << "------------------------------------------" << std::endl; + + /** + * 3. test for chat + */ + auto messages = R"( + [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "你是谁？" + } + ] + )"_json; + auto options = R"( + { + "temperature": 0.4, + "top_k": 2 + } + )"_json; + auto res3 = llmcli.chat("qwen2.5-vl-7b-instruct", messages, options); + std::cout << "---------test for chat---------" << std::endl; + std::cout << res3 << std::endl; + std::cout << "-------------------------------" << std::endl; + + /** + * 4. test for chat with images + */ + json messages4 = { + { + {"role", "system"}, + {"content", "You are a helpful assistant."} + }, + { + {"role", "user"}, + {"content", { + {{"type", "text"}, {"text", "请描述这张图。"}}, + { + {"type", "image_url"}, + {"image_url", {{"url", "data:image/jpg;base64,iVBORw0KGgoAAAANSUhEUgAAAG0AAABmCAYAAADBPx+VAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAA3VSURBVHgB7Z27r0zdG8fX743i1bi1ikMoFMQloXRpKFFIqI7LH4BEQ+NWIkjQuSWCRIEoULk0gsK1kCBI0IhrQVT7tz/7zZo888yz1r7MnDl7z5xvsjkzs2fP3uu71nNfa7lkAsm7d++Sffv2JbNmzUqcc8m0adOSzZs3Z+/XES4ZckAWJEGWPiCxjsQNLWmQsWjRIpMseaxcuTKpG/7HP27I8P79e7dq1ars/yL4/v27S0ejqwv+cUOGEGGpKHR37tzJCEpHV9tnT58+dXXCJDdECBE2Ojrqjh071hpNECjx4cMHVycM1Uhbv359B2F79+51586daxN/+pyRkRFXKyRDAqxEp4yMlDDzXG1NPnnyJKkThoK0VFd1ELZu3TrzXKxKfW7dMBQ6bcuWLW2v0VlHjx41z717927ba22U9APcw7Nnz1oGEPeL3m3p2mTAYYnFmMOMXybPPXv2bNIPpFZr1NHn4HMw0KRBjg9NuRw95s8PEcz/6DZELQd/09C9QGq5RsmSRybqkwHGjh07OsJSsYYm3ijPpyHzoiacg35MLdDSIS/O1yM778jOTwYUkKNHWUzUWaOsylE00MyI0fcnOwIdjvtNdW/HZwNLGg+sR1kMepSNJXmIwxBZiG8tDTpEZzKg0GItNsosY8USkxDhD0Rinuiko2gfL/RbiD2LZAjU9zKQJj8RDR0vJBR1/Phx9+PHj9Z7REF4nTZkxzX4LCXHrV271qXkBAPGfP/atWvu/PnzHe4C97F48eIsRLZ9+3a3f/9+87dwP1JxaF7/3r17ba+5l4EcaVo0lj3SBq5kGTJSQmLWMjgYNei2GPT1MuMqGTDEFHzeQSP2wi/jGnkmPJ/nhccs44jvDAxpVcxnq0F6eT8h4ni/iIWpR5lPyA6ETkNXoSukvpJAD3AsXLiwpZs49+fPn5ke4j10TqYvegSfn0OnafC+Tv9ooA/JPkgQysqQNBzagXY55nO/oa1F7qvIPWkRL12WRpMWUvpVDYmxAPehxWSe8ZEXL20sadYIozfmNch4QJPAfeJgW3rNsnzphBKNJM2KKODo1rVOMRYik5ETy3ix4qWNI81qAAirizgMIc+yhTytx0JWZuNI03qsrgWlGtwjoS9XwgUhWGyhUaRZZQNNIEwCiXD16tXcAHUs79co0vSD8rrJCIW98pzvxpAWyyo3HYwqS0+H0BjStClcZJT5coMm6D2LOF8TolGJtK9fvyZpyiC5ePFi9nc/oJU4eiEP0jVoAnHa9wyJycITMP78+eMeP37sXrx44d6+fdt6f82aNdkx1pg9e3Zb5W+RSRE+n+VjksQWifvVaTKFhn5O8my63K8Qabdv33b379/PiAP//vuvW7BggZszZ072/+TJk91YgkafPn166zXB1rQHFvouAWHq9z3SEevSUerqCn2/dDCeta2jxYbr69evk4MHDyY7d+7MjhMnTiTPnz9Pfv/+nfQT2ggpO2dMF8cghuoM7Ygj5iWCqRlGFml0QC/ftGmTmzt3rmsaKDsgBSPh0/8yPeLLBihLkOKJc0jp8H8vUzcxIA1k6QJ/c78tWEyj5P3o4u9+jywNPdJi5rAH9x0KHcl4Hg570eQp3+vHXGyrmEeigzQsQsjavXt38ujRo44LQuDDhw+TW7duRS1HGgMxhNXHgflaNTOsHyKvHK5Ijo2jbFjJBQK9YwFd6RVMzfgRBmEfP37suBBm/p49e1qjEP2mwTViNRo0VJWH1deMXcNK08uUjVUu7s/zRaL+oLNxz1bpANco4npUgX4G2eFbpDFyQoQxojBCpEGSytmOH8qrH5Q9vuzD6ofQylkCUmh8DBAr+q8JCyVNtWQIidKQE9wNtLSQnS4jDSsxNHogzFuQBw4cyM61UKVsjfr3ooBkPSqqQHesUPWVtzi9/vQi1T+rJj7WiTz4Pt/l3LxUkr5P2VYZaZ4URpsE+st/dujQoaBBYokbrz/8TJNQYLSonrPS9kUaSkPeZyj1AWSj+d+VBoy1pIWVNed8P0Ll/ee5HdGRhrHhR5GGN0r4LGZBaj8oFDJitBTJzIZgFcmU0Y8ytWMZMzJOaXUSrUs5RxKnrxmbb5YXO9VGUhtpXldhEUogFr3IzIsvlpmdosVcGVGXFWp2oU9kLFL3dEkSz6NHEY1sjSRdIuDFWEhd8KxFqsRi1uM/nz9/zpxnwlESONdg6dKlbsaMGS4EHFHtjFIDHwKOo46l4TxSuxgDzi+rE2jg+BaFruOX4HXa0Nnf1lwAPufZeF8/r6zD97WK2qFnGjBxTw5qNGPxT+5T/r7/7RawFC3j4vTp09koCxkeHjqbHJqArmH5UrFKKksnxrK7FuRIs8STfBZv+luugXZ2pR/pP9Ois4z+TiMzUUkUjD0iEi1fzX8GmXyuxUBRcaUfykV0YZnlJGKQpOiGB76x5GeWkWWJc3mOrK6S7xdND+W5N6XyaRgtWJFe13GkaZnKOsYqGdOVVVbGupsyA/l7emTLHi7vwTdirNEt0qxnzAvBFcnQF16xh/TMpUuXHDowhlA9vQVraQhkudRdzOnK+04ZSP3DUhVSP61YsaLtd/ks7ZgtPcXqPqEafHkdqa84X6aCeL7YWlv6edGFHb+ZFICPlljHhg0bKuk0CSvVznWsotRu433alNdFrqG45ejoaPCaUkWERpLXjzFL2Rpllp7PJU2a/v7Ab8N05/9t27Z16KUqoFGsxnI9EosS2niSYg9SpU6B4JgTrvVW1flt1sT+0ADIJU2maXzcUTraGCRaL1Wp9rUMk16PMom8QhruxzvZIegJjFU7LLCePfS8uaQdPny4jTTL0dbee5mYokQsXTIWNY46kuMbnt8Kmec+LGWtOVIl9cT1rCB0V8WqkjAsRwta93TbwNYoGKsUSChN44lgBNCoHLHzquYKrU6qZ8lolCIN0Rh6cP0Q3U6I6IXILYOQI513hJaSKAorFpuHXJNfVlpRtmYBk1Su1obZr5dnKAO+L10Hrj3WZW+E3qh6IszE37F6EB+68mGpvKm4eb9bFrlzrok7fvr0Kfv727dvWRmdVTJHw0qiiCUSZ6wCK+7XL/AcsgNyL74DQQ730sv78Su7+t/A36MdY0sW5o40ahslXr58aZ5HtZB8GH64m9EmMZ7FpYw4T6QnrZfgenrhFxaSiSGXtPnz57e9TkNZLvTjeqhr734CNtrK41L40sUQckmj1lGKQ0rC37x544r8eNXRpnVE3ZZY7zXo8NomiO0ZUCj2uHz58rbXoZ6gc0uA+F6ZeKS/jhRDUq8MKrTho9fEkihMmhxtBI1DxKFY9XLpVcSkfoi8JGnToZO5sU5aiDQIW716ddt7ZLYtMQlhECdBGXZZMWldY5BHm5xgAroWj4C0hbYkSc/jBmggIrXJWlZM6pSETsEPGqZOndr2uuuR5rF169a2HoHPdurUKZM4CO1WTPqaDaAd+GFGKdIQkxAn9RuEWcTRyN2KSUgiSgF5aWzPTeA/lN5rZubMmR2bE4SIC4nJoltgAV/dVefZm72AtctUCJU2CMJ327hxY9t7EHbkyJFseq+EJSY16RPo3Dkq1kkr7+q0bNmyDuLQcZBEPYmHVdOBiJyIlrRDq41YPWfXOxUysi5fvtyaj+2BpcnsUV/oSoEMOk2CQGlr4ckhBwaetBhjCwH0ZHtJROPJkyc7UjcYLDjmrH7ADTEBXFfOYmB0k9oYBOjJ8b4aOYSe7QkKcYhFlq3QYLQhSidNmtS2RATwy8YOM3EQJsUjKiaWZ+vZToUQgzhkHXudb/PW5YMHD9yZM2faPsMwoc7RciYJXbGuBqJ1UIGKKLv915jsvgtJxCZDubdXr165mzdvtr1Hz5LONA8jrUwKPqsmVesKa49S3Q4WxmRPUEYdTjgiUcfUwLx589ySJUva3oMkP6IYddq6HMS4o55xBJBUeRjzfa4Zdeg56QZ43LhxoyPo7Lf1kNt7oO8wWAbNwaYjIv5lhyS7kRf96dvm5Jah8vfvX3flyhX35cuX6HfzFHOToS1H4BenCaHvO8pr8iDuwoUL7tevX+b5ZdbBair0xkFIlFDlW4ZknEClsp/TzXyAKVOmmHWFVSbDNw1l1+4f90U6IY/q4V27dpnE9bJ+v87QEydjqx/UamVVPRG+mwkNTYN+9tjkwzEx+atCm/X9WvWtDtAb68Wy9LXa1UmvCDDIpPkyOQ5ZwSzJ4jMrvFcr0rSjOUh+GcT4LSg5ugkW1Io0/SCDQBojh0hPlaJdah+tkVYrnTZowP8iq1F1TgMBBauufyB33x1v+NWFYmT5KmppgHC+NkAgbmRkpD3yn9QIseXymoTQFGQmIOKTxiZIWpvAatenVqRVXf2nTrAWMsPnKrMZHz6bJq5jvce6QK8J1cQNgKxlJapMPdZSR64/UivS9NztpkVEdKcrs5alhhWP9NeqlfWopzhZScI6QxseegZRGeg5a8C3Re1Mfl1ScP36ddcUaMuv24iOJtz7sbUjTS4qBvKmstYJoUauiuD3k5qhyr7QdUHMeCgLa1Ear9NquemdXgmum4fvJ6w1lqsuDhNrg1qSpleJK7K3TF0Q2jSd94uSZ60kK1e3qyVpQK6PVWXp2/FC3mp6jBhKKOiY2h3gtUV64TWM6wDETRPLDfSakXmH3w8g9Jlug8ZtTt4kVF0kLUYYmCCtD/DrQ5YhMGbA9L3ucdjh0y8kOHW5gU/VEEmJTcL4Pz/f7mgoAbYkAAAAAElFTkSuQmCC"}}} + } + }} + } + }; + auto res4 = llmcli.chat("qwen2.5-vl-7b-instruct", messages4, {{"temperature", 0.1}, {"top_k", 1}}); + std::cout << "---------test for chat with images---------" << std::endl; + std::cout << res4 << std::endl; + std::cout << "-------------------------------------------" << std::endl; +} \ No newline at end of file diff --git a/third_party/kissnet/README.md b/third_party/kissnet/README.md new file mode 100644 index 0000000..9ab349e --- /dev/null +++ b/third_party/kissnet/README.md @@ -0,0 +1,6 @@ +a light weight header-only socket library from https://github.com/Ybalrid/kissnet. + +**NOTE** + +just used for logging & message broking via udp protocal in VideoPipe. + diff --git a/third_party/kissnet/kissnet.hpp b/third_party/kissnet/kissnet.hpp new file mode 100644 index 0000000..ecf3916 --- /dev/null +++ b/third_party/kissnet/kissnet.hpp @@ -0,0 +1,1447 @@ +/* + * MIT License + * + * Copyright (c) 2018-2020 Arthur Brainville (Ybalrid) and with the help of + * Comunity Contributors! + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * INTRODUCTION + * ============ + * + * Kissnet is a simple C++17 layer around the raw OS provided socket API to be + * used on IP networks with the TCP and UDP protocols. + * + * Kissnet is not a networking framework, and it will not process your data or + * assist you in any way. Kissnet's only goal is to provide a simple API to send + * and receive bytes, + * without having to play around with a bunch of structure, file descriptors, + * handles and pointers given to a C-style API. The other goal of kissnet is to + * provide an API that will works in a cross platform setting. + * + * Kissnet will automatically manage the eventual startup/shutdown of the + * library needed to perform socket operations on a particular platform. (e.g. + * the Windows Socket API on MS-Windows. + * + * Kissnet leverages (and expect you to do so), multiple features from C++17, + * including: std::byte, if constexpr, structured bindings, if-initializer and + * template parameter type deduction. + * + * The library is structured across 4 exposed data types: + * + * - buffer : a static array of std::byte implemented via std::array. + * This is what you should use to hold raw data you are getting from a socket, + * before extracting what you need from the bytes + * - port_t : a 16 bit unsigned number. Represent a network port number + * - endpoint : a structure that represent a location where you need to connect + * to. Contains a hostname (as std::string) and a port number (as port_t) + * - socket : a templated class that represents an ipv4 or ipv6 socket. + * Protocol is either TCP or UDP + * + * Kissnet does error handling in 2 ways: + * + * 1: + * When an operation can generate an error that the user should handle by hand + * anyway, a tuple containing the expected type returned, and an object that + * represent the status of what happens is returned. + * + * For example, socket send/receive operation can discover that the connection + * was closed, or was shut down properly. It could also be the fact that a + * socket was configured "non blocking" and would have blocked in this + * situation. On both occasion, these methods will return the fact that 0 bytes + * came across as the transaction size, and the status will indicate either an + * error (socket no longer valid), or an actual status message (connection + * closed, socket would have blocked) + * + * These status objects will behave like a const bool that equals "false" when + * an error occurred, and "true" when it's just a status notification + * + * 2: + * Fatal errors are by default handled by throwing a runtime_error exception. + * But, for many reasons, you may want to + * not use exceptions entirely. + * + * kissnet give you some facilities to get fatal errors information back, and + * to choose how to handle it. Kissnet give you a few levers you can use: + * + * - You can deactivate the exception support by #defining KISSNET_NO_EXCEP + * before #including kissnet.hpp. Instead, kissnet will use a function based + * error handler + * - By default, the error handler prints to stderr the error message, and + * abort the program + * - kissnet::error::callback is a function pointer that gets a string, and a + * context pointer. The string is the error message, and the context pointer + * what ever you gave kissnet for the occasion. This is a global pointer that + * you can set as you want. This will override the "print to stderr" behavior + * at fatal error time. + * - kissnet::error::ctx is a void*, this will be passed to your error handler + * as a "context" pointer. If you need your handler to write to a log, + * or to turn on the HTCPCP enabled teapot on John's desk, you can. + * - kissnet::abortOnFatalError is a boolean that will control the call to + * abort(). This is independent to the fact that you did set or not an error + * callback. please note that any object involved with the operation that + * triggered the fatal error is probably in an invalid state, and probably + * deserve to be thrown away. + */ + +#ifndef KISS_NET +#define KISS_NET + +///Define this to not use exceptions +#ifndef KISSNET_NO_EXCEP +#define kissnet_fatal_error(STR) throw std::runtime_error(STR) +#else +#define kissnet_fatal_error(STR) kissnet::error::handle(STR); +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 + +#define _WINSOCK_DEPRECATED_NO_WARNINGS +#define WIN32_LEAN_AND_MEAN + +#ifndef NOMINMAX +#define NOMINMAX +#endif //endif nominmax + +#include +#include +#include + +using ioctl_setting = u_long; +using buffsize_t = int; + +#define AI_ADDRCONFIG 0x00000400 + +#ifndef SHUT_RDWR +#define SHUT_RDWR SD_BOTH +#endif + +// taken from: https://github.com/rxi/dyad/blob/915ae4939529b9aaaf6ebfd2f65c6cff45fc0eac/src/dyad.c#L58 +inline const char* inet_ntop(int af, const void* src, char* dst, socklen_t size) +{ + union + { + struct sockaddr sa; + struct sockaddr_in sai; + struct sockaddr_in6 sai6; + } addr; + int res; + memset(&addr, 0, sizeof(addr)); + addr.sa.sa_family = af; + if (af == AF_INET6) + { + memcpy(&addr.sai6.sin6_addr, src, sizeof(addr.sai6.sin6_addr)); + } + else + { + memcpy(&addr.sai.sin_addr, src, sizeof(addr.sai.sin_addr)); + } + res = WSAAddressToStringA(&addr.sa, sizeof(addr), 0, dst, reinterpret_cast(&size)); + if (res != 0) return NULL; + return dst; +} + +//Handle WinSock2/Windows Socket API initialization and cleanup +#pragma comment(lib, "Ws2_32.lib") +namespace kissnet +{ + + namespace win32_specific + { + ///Forward declare the object that will permit to manage the WSAStartup/Cleanup automatically + struct WSA; + + ///Enclose the global pointer in this namespace. Only use this inside a shared_ptr + namespace internal_state + { + static WSA* global_WSA = nullptr; + } + + ///WSA object. Only to be constructed with std::make_shared() + struct WSA : std::enable_shared_from_this + { + //For safety, only initialize Windows Socket API once, and delete it once + ///Prevent copy construct + WSA(const WSA&) = delete; + ///Prevent copy assignment + WSA& operator=(const WSA&) = delete; + ///Prevent moving + WSA(WSA&&) = delete; + ///Prevent move assignment + WSA& operator=(WSA&&) = delete; + + ///data storage + WSADATA wsa_data; + + ///Startup + WSA() : + wsa_data {} + { + if (const auto status = WSAStartup(MAKEWORD(2, 2), &wsa_data); status != 0) + { + std::string error_message; + switch (status) // https://docs.microsoft.com/en-us/windows/win32/api/winsock/nf-winsock-wsastartup#return-value + { + default: + error_message = "Unknown error happened."; + break; + case WSASYSNOTREADY: + error_message = "The underlying network subsystem is not ready for network communication."; + break; + case WSAVERNOTSUPPORTED: //unlikely, we specify 2.2! + error_message = " The version of Windows Sockets support requested " + "(2.2)" //we know here the version was 2.2, add that to the error message copied from MSDN + " is not provided by this particular Windows Sockets implementation. "; + break; + case WSAEINPROGRESS: + error_message = "A blocking Windows Sockets 1.1 operation is in progress."; + break; + case WSAEPROCLIM: + error_message = "A limit on the number of tasks supported by the Windows Sockets implementation has been reached."; + break; + case WSAEFAULT: //unlikely, if this ctor is running, wsa_data is part of this object's "stack" data + error_message = "The lpWSAData parameter is not a valid pointer."; + break; + } + kissnet_fatal_error(error_message); + } +#ifdef KISSNET_WSA_DEBUG + std::cerr << "Initialized Windows Socket API\n"; +#endif + } + + ///Cleanup + ~WSA() + { + WSACleanup(); + internal_state::global_WSA = nullptr; +#ifdef KISSNET_WSA_DEBUG + std::cerr << "Cleanup Windows Socket API\n"; +#endif + } + + ///get the shared pointer + std::shared_ptr getPtr() + { + return shared_from_this(); + } + }; + + ///Get-or-create the global pointer + inline std::shared_ptr getWSA() + { + //If it has been created already: + if (internal_state::global_WSA) + return internal_state::global_WSA->getPtr(); //fetch the smart pointer from the naked pointer + + //Create in wsa + auto wsa = std::make_shared(); + + //Save the raw address in the global state + internal_state::global_WSA = wsa.get(); + + //Return the smart pointer + return wsa; + } + } + +#define KISSNET_OS_SPECIFIC_PAYLOAD_NAME wsa_ptr +#define KISSNET_OS_SPECIFIC std::shared_ptr KISSNET_OS_SPECIFIC_PAYLOAD_NAME +#define KISSNET_OS_INIT KISSNET_OS_SPECIFIC_PAYLOAD_NAME = kissnet::win32_specific::getWSA() + + ///Return the last error code + inline int get_error_code() + { + const auto error = WSAGetLastError(); + + //We need to posixify the values that we are actually using inside this header. + switch (error) + { + case WSAEWOULDBLOCK: + return EWOULDBLOCK; + case WSAEBADF: + return EBADF; + case WSAEINTR: + return EINTR; + default: + return error; + } + } +} +#else //UNIX platform + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using ioctl_setting = int; +using buffsize_t = size_t; + +//To get consistent socket API between Windows and Linux: +static const int INVALID_SOCKET = -1; +static const int SOCKET_ERROR = -1; +using SOCKET = int; +using SOCKADDR_IN = sockaddr_in; +using SOCKADDR = sockaddr; +using IN_ADDR = in_addr; + +//Wrap them in their WIN32 names +inline int closesocket(SOCKET in) +{ + return close(in); +} + +template +inline int ioctlsocket(int fd, int request, Params&&... params) +{ + return ioctl(fd, request, params...); +} + +#define KISSNET_OS_SPECIFIC_PAYLOAD_NAME dummy +#define KISSNET_OS_SPECIFIC char dummy +#define KISSNET_OS_INIT dummy = 42; + +namespace unix_specific +{ +} + +inline int get_error_code() +{ + return errno; +} + +#endif //ifdef WIN32 + +#ifdef KISSNET_USE_OPENSSL + +#include +#include + +#include +#include + +#endif //Kissnet use OpenSSL + +#ifndef SOL_TCP +#define SOL_TCP IPPROTO_TCP +#endif + +///Main namespace of kissnet +namespace kissnet +{ + + ///Exception-less error handling infrastructure + namespace error + { + static void (*callback)(const std::string&, void* ctx) = nullptr; + static void* ctx = nullptr; + static bool abortOnFatalError = true; + + inline void handle(const std::string& str) + { + //if the error::callback function has been provided, call that + if (callback) + { + callback(str, ctx); + } + //Print error into the standard error output + else + { + fputs(str.c_str(), stderr); + } + + //If the error abort hasn't been deactivated + if (abortOnFatalError) + { + abort(); + } + } + } + + ///low level protocol used, between TCP\TCP_SSL and UDP + enum class protocol { + tcp, + tcp_ssl, + udp + }; + + ///Address information structs + struct addr_collection { + sockaddr_storage adrinf = {0}; + socklen_t sock_size = 0; + }; + + ///File descriptor set types + static constexpr int fds_read = 0x1; + static constexpr int fds_write = 0x2; + static constexpr int fds_except = 0x4; + + ///buffer is an array of std::byte + template + using buffer = std::array; + + ///port_t is the port + using port_t = uint16_t; + + ///An endpoint is where the network will connect to (address and port) + struct endpoint + { + ///The address to connect to + std::string address {}; + + ///The port to connect to + port_t port {}; + + ///Default constructor, the endpoint is not valid at that point, but you can set the address/port manually + endpoint() = default; + + ///Basically create the endpoint with what you give it + endpoint(std::string addr, port_t prt) : + address { std::move(addr) }, port { prt } + { } + + static bool is_valid_port_number(unsigned long n) + { + return n < 1 << 16; + } + + ///Construct the endpoint from "address:port" + endpoint(std::string addr) + { + const auto separator = addr.find_last_of(':'); + + //Check if input wasn't missformed + if (separator == std::string::npos) + kissnet_fatal_error("string is not of address:port form"); + if (separator == addr.size() - 1) + kissnet_fatal_error("string has ':' as last character. Expected port number here"); + + //Isolate address + address = addr.substr(0, separator); + + //Read from string as unsigned + const auto parsed_port = strtoul(addr.substr(separator + 1).c_str(), nullptr, 10); + + //In all other cases, port was always given as a port_t type, strongly preventing it to be a number outside of the [0; 65535] range. Here it's not the case. + //To detect errors early, check it here : + if (!is_valid_port_number(parsed_port)) + kissnet_fatal_error("Invalid port number " + std::to_string(parsed_port)); + + //Store it + port = static_cast(parsed_port); + } + + ///Construct an endpoint from a SOCKADDR + endpoint(SOCKADDR* addr) + { + + switch (addr->sa_family) + { + case AF_INET: { + auto ip_addr = (SOCKADDR_IN*)(addr); + address = inet_ntoa(ip_addr->sin_addr); + port = ntohs(ip_addr->sin_port); + } + break; + + case AF_INET6: { + auto ip_addr = (sockaddr_in6*)(addr); + char buffer[INET6_ADDRSTRLEN]; + address = inet_ntop(AF_INET6, &(ip_addr->sin6_addr), buffer, INET6_ADDRSTRLEN); + port = ntohs(ip_addr->sin6_port); + } + break; + + default: { + kissnet_fatal_error("Trying to construct an endpoint for a protocol familly that is neither AF_INET or AF_INET6"); + } + } + + if (address.empty()) + kissnet_fatal_error("Couldn't construct endpoint from sockaddr(_storage) struct"); + } + }; + + //Wrap "system calls" here to avoid conflicts with the names used in the socket class + + ///socket() + inline auto syscall_socket = [](int af, int type, int protocol) { + return ::socket(af, type, protocol); + }; + + ///select() + inline auto syscall_select = [](int nfds, fd_set* readfds, fd_set* writefds, fd_set* exceptfds, struct timeval* timeout) { + return ::select(nfds, readfds, writefds, exceptfds, timeout); + }; + + ///recv() + inline auto syscall_recv = [](SOCKET s, char* buff, buffsize_t len, int flags) { + return ::recv(s, buff, len, flags); + }; + + ///send() + inline auto syscall_send = [](SOCKET s, const char* buff, buffsize_t len, int flags) { + return ::send(s, buff, len, flags); + }; + + ///bind() + inline auto syscall_bind = [](SOCKET s, const struct sockaddr* name, socklen_t namelen) { + return ::bind(s, name, namelen); + }; + + ///connect() + inline auto syscall_connect = [](SOCKET s, const struct sockaddr* name, socklen_t namelen) { + return ::connect(s, name, namelen); + }; + + ///listen() + inline auto syscall_listen = [](SOCKET s, int backlog) { + return ::listen(s, backlog); + }; + + ///accept() + inline auto syscall_accept = [](SOCKET s, struct sockaddr* addr, socklen_t* addrlen) { + return ::accept(s, addr, addrlen); + }; + + ///shutdown() + inline auto syscall_shutdown = [](SOCKET s) { + return ::shutdown(s, SHUT_RDWR); + }; + + ///Represent the status of a socket as returned by a socket operation (send, received). Implicitly convertible to bool + struct socket_status + { + ///Enumeration of socket status, with a 1 byte footprint + enum values : int8_t { + errored = 0x0, + valid = 0x1, + cleanly_disconnected = 0x2, + non_blocking_would_have_blocked = 0x3, + timed_out = 0x4 + + /* ... any other info on a "still valid socket" goes here ... */ + + }; + + ///Actual value of the socket_status. + const values value; + + ///Use the default constructor + socket_status() : + value { errored } { } + + ///Construct a "errored/valid" status for a true/false + explicit socket_status(bool state) : + value(values(state ? valid : errored)) { } + + socket_status(values v) : + value(v) { } + + ///Copy socket status by default + socket_status(const socket_status&) = default; + + ///Move socket status by default + socket_status(socket_status&&) = default; + + ///implicitly convert this object to const bool (as the status should not change) + operator bool() const + { + //See the above enum: every value <= 0 correspond to an error, and will return false. Every value > 0 returns true + return value > 0; + } + + int8_t get_value() const + { + return value; + } + + bool operator==(values v) const + { + return v == value; + } + }; + +#ifdef KISSNET_USE_OPENSSL +#if OPENSSL_VERSION_NUMBER < 0x10100000L + static std::shared_ptr> SSL_lock_cs; + + class ThreadSafe_SSL + { + public: + ThreadSafe_SSL() + { + SSL_lock_cs = std::make_shared>(CRYPTO_num_locks()); + + CRYPTO_set_locking_callback((void (*)(int, int, const char*, int)) + win32_locking_callback); + } + + ~ThreadSafe_SSL() { CRYPTO_set_locking_callback(nullptr); } + + private: + static void win32_locking_callback(int mode, int type, const char* file, int line) + { + auto& locks = *SSL_lock_cs; + + if (mode & CRYPTO_LOCK) + { + locks[type].lock(); + } + else + { + locks[type].unlock(); + } + } + }; + +#endif + + class Initialize_SSL + { + public: + Initialize_SSL() + { +#if OPENSSL_VERSION_NUMBER < 0x1010001fL + SSL_load_error_strings(); + SSL_library_init(); +#else + OPENSSL_init_ssl( + OPENSSL_INIT_LOAD_SSL_STRINGS | OPENSSL_INIT_LOAD_CRYPTO_STRINGS, NULL); + + OPENSSL_init_crypto( + OPENSSL_INIT_LOAD_CONFIG | OPENSSL_INIT_ADD_ALL_CIPHERS | OPENSSL_INIT_ADD_ALL_DIGESTS, + nullptr); +#endif + } + + ~Initialize_SSL() + { +#if OPENSSL_VERSION_NUMBER < 0x1010001fL + ERR_free_strings(); +#endif + } + +#if OPENSSL_VERSION_NUMBER < 0x10100000L + private: + ThreadSafe_SSL thread_setup; +#endif + }; + + static Initialize_SSL InitializeSSL; +#endif + + ///Class that represent a socket + template + class socket + { + ///Represent a number of bytes with a status information. Some of the methods of this class returns this. + using bytes_with_status = std::tuple; + + ///OS specific stuff. payload we have to hold onto for RAII management of the Operating System's socket library (e.g. Windows Socket API WinSock2) + KISSNET_OS_SPECIFIC; + + ///operatic-system type for a socket object + SOCKET sock = INVALID_SOCKET; + +#ifdef KISSNET_USE_OPENSSL + SSL* pSSL = nullptr; + SSL_CTX* pContext = nullptr; +#endif + + ///Location where this socket is bound + endpoint bind_loc = {}; + + ///Address information structures + addrinfo getaddrinfo_hints = {}; + addrinfo* getaddrinfo_results = nullptr; + addrinfo* socket_addrinfo = nullptr; + + void initialize_addrinfo() + { + int type {}; + int iprotocol {}; + if constexpr (sock_proto == protocol::tcp || sock_proto == protocol::tcp_ssl) + { + type = SOCK_STREAM; + iprotocol = IPPROTO_TCP; + } + + else if constexpr (sock_proto == protocol::udp) + { + type = SOCK_DGRAM; + iprotocol = IPPROTO_UDP; + } + + getaddrinfo_hints = {}; + getaddrinfo_hints.ai_family = AF_UNSPEC; + getaddrinfo_hints.ai_socktype = type; + getaddrinfo_hints.ai_protocol = iprotocol; + getaddrinfo_hints.ai_flags = AI_ADDRCONFIG; + } + + ///Create and connect to socket + socket_status connect(addrinfo* addr, int64_t timeout, bool createsocket) + { + if constexpr (sock_proto == protocol::tcp || sock_proto == protocol::tcp_ssl) //only TCP is a connected protocol + { + if (createsocket) + { + close(); + socket_addrinfo = nullptr; + sock = syscall_socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol); + } + + if (sock == INVALID_SOCKET) + return socket_status::errored; + + socket_addrinfo = addr; + + if (timeout > 0) + set_non_blocking(true); + + int error = syscall_connect(sock, addr->ai_addr, socklen_t(addr->ai_addrlen)); + if (error == SOCKET_ERROR) + { + error = get_error_code(); + if (error == EWOULDBLOCK || error == EAGAIN || error == EINPROGRESS) + { + struct timeval tv; + tv.tv_sec = static_cast(timeout / 1000); + tv.tv_usec = 1000 * static_cast(timeout % 1000); + + fd_set fd_write, fd_except; + ; + FD_ZERO(&fd_write); + FD_SET(sock, &fd_write); + FD_ZERO(&fd_except); + FD_SET(sock, &fd_except); + + int ret = syscall_select(static_cast(sock) + 1, NULL, &fd_write, &fd_except, &tv); + if (ret == -1) + error = get_error_code(); + else if (ret == 0) + error = ETIMEDOUT; + else + { + socklen_t errlen = sizeof(error); + if (getsockopt(sock, SOL_SOCKET, SO_ERROR, reinterpret_cast(&error), &errlen) != 0) + kissnet_fatal_error("getting socket error returned an error"); + } + } + } + + if (timeout > 0) + set_non_blocking(false); + + if (error == 0) + { + return socket_status::valid; + } + else + { + close(); + socket_addrinfo = nullptr; + return socket_status::errored; + } + } + + kissnet_fatal_error("connect called for non-tcp socket"); + } + + ///sockaddr struct + sockaddr_storage socket_input = {}; + socklen_t socket_input_socklen = 0; + + public: + + ///Construct an invalid socket + socket() = default; + + ///socket<> isn't copyable + socket(const socket&) = delete; + + ///socket<> isn't copyable + socket& operator=(const socket&) = delete; + + ///Move constructor. socket<> isn't copyable + socket(socket&& other) noexcept + { + KISSNET_OS_SPECIFIC_PAYLOAD_NAME = std::move(other.KISSNET_OS_SPECIFIC_PAYLOAD_NAME); + bind_loc = std::move(other.bind_loc); + sock = std::move(other.sock); + socket_input = std::move(other.socket_input); + socket_input_socklen = std::move(other.socket_input_socklen); + getaddrinfo_results = std::move(other.getaddrinfo_results); + socket_addrinfo = std::move(other.socket_addrinfo); + +#ifdef KISSNET_USE_OPENSSL + pSSL = other.pSSL; + pContext = other.pContext; + other.pSSL = nullptr; + other.pContext = nullptr; +#endif + + other.sock = INVALID_SOCKET; + other.getaddrinfo_results = nullptr; + other.socket_addrinfo = nullptr; + } + + ///Move assign operation + socket& operator=(socket&& other) noexcept + { + if (this != &other) + { + if (!(sock < 0) || sock != INVALID_SOCKET) + closesocket(sock); + + KISSNET_OS_SPECIFIC_PAYLOAD_NAME = std::move(other.KISSNET_OS_SPECIFIC_PAYLOAD_NAME); + bind_loc = std::move(other.bind_loc); + sock = std::move(other.sock); + socket_input = std::move(other.socket_input); + socket_input_socklen = std::move(other.socket_input_socklen); + getaddrinfo_results = std::move(other.getaddrinfo_results); + socket_addrinfo = std::move(other.socket_addrinfo); + +#ifdef KISSNET_USE_OPENSSL + pSSL = other.pSSL; + pContext = other.pContext; + other.pSSL = nullptr; + other.pContext = nullptr; +#endif + + other.sock = INVALID_SOCKET; + other.getaddrinfo_results = nullptr; + other.socket_addrinfo = nullptr; + } + return *this; + } + + ///Return true if the underlying OS provided socket representation (file descriptor, handle...). Both socket are pointing to the same thing in this case + bool operator==(const socket& other) const + { + return sock == other.sock; + } + + ///Return true if socket is valid. If this is false, you probably shouldn't attempt to send/receive anything, it will probably explode in your face! + bool is_valid() const + { + return sock != INVALID_SOCKET; + } + + inline operator bool() const + { + return is_valid(); + } + + ///Construct socket and (if applicable) connect to the endpoint + socket(endpoint bind_to) : + bind_loc { std::move(bind_to) } + { + //operating system related housekeeping + KISSNET_OS_INIT; + + //Do we use streams or datagrams + initialize_addrinfo(); + + if (getaddrinfo(bind_loc.address.c_str(), std::to_string(bind_loc.port).c_str(), &getaddrinfo_hints, &getaddrinfo_results) != 0) + { + kissnet_fatal_error("getaddrinfo failed!"); + } + + for (auto* addr = getaddrinfo_results; addr; addr = addr->ai_next) + { + sock = syscall_socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol); + if (sock != INVALID_SOCKET) + { + socket_addrinfo = addr; + break; + } + } + + if (sock == INVALID_SOCKET) + { + kissnet_fatal_error("unable to create socket!"); + } + } + + ///Construct a socket from an operating system socket, an additional endpoint to remember from where we are + socket(SOCKET native_sock, endpoint bind_to) : + sock { native_sock }, bind_loc(std::move(bind_to)) + { + KISSNET_OS_INIT; + + initialize_addrinfo(); + } + + ///Set the socket in non blocking mode + /// \param state By default "true". If put to false, it will set the socket back into blocking, normal mode + void set_non_blocking(bool state = true) const + { +#ifdef _WIN32 + ioctl_setting set = state ? 1 : 0; + if (ioctlsocket(sock, FIONBIO, &set) < 0) +#else + const auto flags = fcntl(sock, F_GETFL, 0); + const auto newflags = state ? (flags | O_NONBLOCK) : (flags & ~O_NONBLOCK); + if (fcntl(sock, F_SETFL, newflags) < 0) +#endif + kissnet_fatal_error("setting socket to nonblock returned an error"); + } + + ///Set the socket option for broadcasts + /// \param state By default "true". If put to false, it will disable broadcasts + void set_broadcast(bool state = true) const + { + const int broadcast = state ? 1 : 0; + if (setsockopt(sock, SOL_SOCKET, SO_BROADCAST, reinterpret_cast(&broadcast), sizeof(broadcast)) != 0) + kissnet_fatal_error("setting socket broadcast mode returned an error"); + } + + /// Set the socket option for TCPNoDelay + /// \param state By default "true". If put to false, it will disable TCPNoDelay + void set_tcp_no_delay(bool state = true) const + { + if constexpr (sock_proto == protocol::tcp) + { + const int tcpnodelay = state ? 1 : 0; + if (setsockopt(sock, SOL_TCP, TCP_NODELAY, reinterpret_cast(&tcpnodelay), sizeof(tcpnodelay)) != 0) + kissnet_fatal_error("setting socket tcpnodelay mode returned an error"); + } + } + + /// Get socket status + socket_status get_status() const + { + int sockerror = 0; + socklen_t errlen = sizeof(sockerror); + if (getsockopt(sock, SOL_SOCKET, SO_ERROR, reinterpret_cast(&sockerror), &errlen) != 0) + kissnet_fatal_error("getting socket error returned an error"); + + return sockerror == SOCKET_ERROR ? socket_status::errored : socket_status::valid; + } + + ///Bind socket locally using the address and port of the endpoint + void bind() + { + if (syscall_bind(sock, static_cast(socket_addrinfo->ai_addr), socklen_t(socket_addrinfo->ai_addrlen)) == SOCKET_ERROR) + { + kissnet_fatal_error("bind() failed\n"); + } + } + + ///Join a multicast group + void join(const endpoint& multi_cast_endpoint, const std::string& interface = "") + { + if (sock_proto != protocol::udp) + { + kissnet_fatal_error("joining a multicast is only possible in UDP mode\n"); + } + + addrinfo *multicast_addr; + addrinfo *local_addr; + addrinfo hints = {0}; + hints.ai_family = PF_UNSPEC; + hints.ai_flags = AI_NUMERICHOST; + if (getaddrinfo(multi_cast_endpoint.address.c_str(), nullptr, &hints, &multicast_addr) != 0) + { + kissnet_fatal_error("getaddrinfo() failed\n"); + } + + hints.ai_family = multicast_addr->ai_family; + hints.ai_socktype = SOCK_DGRAM; + hints.ai_flags = AI_PASSIVE; + if (getaddrinfo(nullptr, std::to_string(multi_cast_endpoint.port).c_str(), &hints, &local_addr) != 0) + { + kissnet_fatal_error("getaddrinfo() failed\n"); + } + + sock = syscall_socket(local_addr->ai_family, local_addr->ai_socktype, local_addr->ai_protocol); + if (sock != INVALID_SOCKET) + { + socket_addrinfo = local_addr; + } else { + kissnet_fatal_error("syscall_socket() failed\n"); + } + + bind(); + + //IPv4 + if (multicast_addr->ai_family == PF_INET && multicast_addr->ai_addrlen == sizeof(struct sockaddr_in)) + { + struct ip_mreq multicastRequest = {0}; + memcpy(&multicastRequest.imr_multiaddr, + &((struct sockaddr_in*)(multicast_addr->ai_addr))->sin_addr, + sizeof(multicastRequest.imr_multiaddr)); + + if (interface.length()) { + multicastRequest.imr_interface.s_addr = inet_addr(interface.c_str());; + } else { + multicastRequest.imr_interface.s_addr = htonl(INADDR_ANY); + } + + if (setsockopt(sock, IPPROTO_IP, IP_ADD_MEMBERSHIP, (char*) &multicastRequest, sizeof(multicastRequest)) != 0) + { + kissnet_fatal_error("setsockopt() failed\n"); + } + } + + //IPv6 + else if (multicast_addr->ai_family == PF_INET6 && multicast_addr->ai_addrlen == sizeof(struct sockaddr_in6)) + { + struct ipv6_mreq multicastRequest = {0}; + memcpy(&multicastRequest.ipv6mr_multiaddr, + &((struct sockaddr_in6*)(multicast_addr->ai_addr))->sin6_addr, + sizeof(multicastRequest.ipv6mr_multiaddr)); + + if (interface.length()) { + struct addrinfo *reslocal; + if (getaddrinfo(interface.c_str(), nullptr, nullptr, &reslocal)){ + kissnet_fatal_error("getaddrinfo() failed\n"); + } + multicastRequest.ipv6mr_interface = ((sockaddr_in6 *)reslocal->ai_addr)->sin6_scope_id; + freeaddrinfo(reslocal); + } else { + multicastRequest.ipv6mr_interface = 0; + } + + + if (setsockopt(sock, IPPROTO_IPV6, IPV6_JOIN_GROUP, (char*) &multicastRequest, sizeof(multicastRequest)) != 0) + { + kissnet_fatal_error("setsockopt() failed\n"); + } + } + else + { + kissnet_fatal_error("unknown AI family.\n"); + } + + freeaddrinfo(multicast_addr); + } + + ///(For TCP) connect to the endpoint as client + socket_status connect(int64_t timeout = 0) + { + if constexpr (sock_proto == protocol::tcp) //only TCP is a connected protocol + { + // try to connect to existing native socket, if any. + auto curr_addr = socket_addrinfo; + if (connect(curr_addr, timeout, false) != socket_status::valid) + { + // try to create/connect native socket for one of the other addrinfo, if any + for (auto* addr = getaddrinfo_results; addr; addr = addr->ai_next) + { + if (addr == curr_addr) + continue; // already checked + + if (connect(addr, timeout, true) == socket_status::valid) + break; // success + } + } + + if (sock == INVALID_SOCKET) + kissnet_fatal_error("unable to create connectable socket!"); + + return socket_status::valid; + } +#ifdef KISSNET_USE_OPENSSL + else if constexpr (sock_proto == protocol::tcp_ssl) //only TCP is a connected protocol + { + // try to connect to existing native socket, if any. + auto curr_addr = socket_addrinfo; + if (connect(curr_addr, timeout, false) != socket_status::valid) + { + // try to create/connect native socket for one of the other addrinfo, if any + for (auto* addr = getaddrinfo_results; addr; addr = addr->ai_next) + { + if (addr == curr_addr) + continue; // already checked + + if (connect(addr, timeout, true) == socket_status::valid) + break; // success + } + } + + if (sock == INVALID_SOCKET) + kissnet_fatal_error("unable to create connectable socket!"); + + auto* pMethod = +#if (OPENSSL_VERSION_NUMBER < 0x10100000L) + TLSv1_2_client_method(); +#else + TLS_client_method(); +#endif + + pContext = SSL_CTX_new(pMethod); + pSSL = SSL_new(pContext); + if (!pSSL) + return socket_status::errored; + + if (!(static_cast(SSL_set_fd(pSSL, sock)))) + return socket_status::errored; + + if (SSL_connect(pSSL) != 1) + return socket_status::errored; + + return socket_status::valid; + } +#endif + } + + ///(for TCP= setup socket to listen to connection. Need to be called on binded socket, before being able to accept() + void listen() + { + if constexpr (sock_proto == protocol::tcp) + { + if (syscall_listen(sock, SOMAXCONN) == SOCKET_ERROR) + { + kissnet_fatal_error("listen failed\n"); + } + } + } + + ///(for TCP) Wait for incoming connection, return socket connect to the client. Blocking. + socket accept() + { + if constexpr (sock_proto != protocol::tcp) + { + return { INVALID_SOCKET, {} }; + } + + sockaddr_storage socket_address; + SOCKET s; + socklen_t size = sizeof socket_address; + + if ((s = syscall_accept(sock, reinterpret_cast(&socket_address), &size)) == INVALID_SOCKET) + { + const auto error = get_error_code(); + switch (error) + { + case EWOULDBLOCK: //if socket "would have blocked" from the call, ignore + case EINTR: //if blocking call got interrupted, ignore; + return {}; + } + + kissnet_fatal_error("accept() returned an invalid socket\n"); + } + + return { s, endpoint(reinterpret_cast(&socket_address)) }; + } + + void close() + { + if (sock != INVALID_SOCKET) + { +#ifdef KISSNET_USE_OPENSSL + if constexpr (sock_proto == protocol::tcp_ssl) + { + if (pSSL) + { + SSL_set_shutdown(pSSL, SSL_RECEIVED_SHUTDOWN | SSL_SENT_SHUTDOWN); + SSL_shutdown(pSSL); + SSL_free(pSSL); + if (pContext) + SSL_CTX_free(pContext); + } + } +#endif + + closesocket(sock); + } + + sock = INVALID_SOCKET; + } + + void shutdown() + { + if (sock != INVALID_SOCKET) + { + syscall_shutdown(sock); + } + } + + ///Close socket on destruction + ~socket() + { + close(); + + if (getaddrinfo_results) + freeaddrinfo(getaddrinfo_results); + } + + ///Select socket with timeout + socket_status select(int fds, int64_t timeout) + { + fd_set fd_read, fd_write, fd_except; + ; + struct timeval tv; + + tv.tv_sec = static_cast(timeout / 1000); + tv.tv_usec = 1000 * static_cast(timeout % 1000); + + if (fds & fds_read) + { + FD_ZERO(&fd_read); + FD_SET(sock, &fd_read); + } + if (fds & fds_write) + { + FD_ZERO(&fd_write); + FD_SET(sock, &fd_write); + } + if (fds & fds_except) + { + FD_ZERO(&fd_except); + FD_SET(sock, &fd_except); + } + + int ret = syscall_select(static_cast(sock) + 1, + fds & fds_read ? &fd_read : NULL, + fds & fds_write ? &fd_write : NULL, + fds & fds_except ? &fd_except : NULL, + &tv); + if (ret == -1) + return socket_status::errored; + else if (ret == 0) + return socket_status::timed_out; + return socket_status::valid; + } + + template + bytes_with_status send(const buffer& buff, const size_t length = buff_size, addr_collection* addr = nullptr) + { + assert(buff_size >= length); + return send(buff.data(), length, addr); + } + + ///Send some bytes through the pipe + bytes_with_status send(const std::byte* read_buff, size_t length, addr_collection* addr = nullptr) + { + auto received_bytes { 0 }; + if constexpr (sock_proto == protocol::tcp) + { + received_bytes = syscall_send(sock, reinterpret_cast(read_buff), static_cast(length), 0); + } +#ifdef KISSNET_USE_OPENSSL + else if constexpr (sock_proto == protocol::tcp_ssl) + { + received_bytes = SSL_write(pSSL, reinterpret_cast(read_buff), static_cast(length)); + } +#endif + else if constexpr (sock_proto == protocol::udp) + { + if (addr) { + received_bytes = sendto(sock, reinterpret_cast(read_buff), static_cast(length), 0, reinterpret_cast(&addr->adrinf) , addr->sock_size); + } else { + received_bytes = sendto(sock, reinterpret_cast(read_buff), static_cast(length), 0, static_cast(socket_addrinfo->ai_addr), socklen_t(socket_addrinfo->ai_addrlen)); + } + } + + if (received_bytes < 0) + { + if (get_error_code() == EWOULDBLOCK) + { + return { 0, socket_status::non_blocking_would_have_blocked }; + } + + return { 0, socket_status::errored }; + } + + return { received_bytes, socket_status::valid }; + } + + ///receive bytes inside the buffer, return the number of bytes you got. You can choose to write inside the buffer at a specific start offset (in number of bytes) + template + bytes_with_status recv(buffer& write_buff, size_t start_offset = 0, addr_collection* addr_info = nullptr) + { + auto received_bytes = 0; + if constexpr (sock_proto == protocol::tcp) + { + received_bytes = syscall_recv(sock, reinterpret_cast(write_buff.data()) + start_offset, static_cast(buff_size - start_offset), 0); + } +#ifdef KISSNET_USE_OPENSSL + else if constexpr (sock_proto == protocol::tcp_ssl) + { + received_bytes = SSL_read(pSSL, reinterpret_cast(write_buff.data()) + start_offset, static_cast(buff_size - start_offset)); + } +#endif + else if constexpr (sock_proto == protocol::udp) + { + socket_input_socklen = sizeof socket_input; + + received_bytes = ::recvfrom(sock, reinterpret_cast(write_buff.data()) + start_offset, static_cast(buff_size - start_offset), 0, reinterpret_cast(&socket_input), &socket_input_socklen); + if (addr_info) { + addr_info->adrinf = socket_input; + addr_info->sock_size = socket_input_socklen; + } + } + + if (received_bytes < 0) + { + const auto error = get_error_code(); + if (error == EWOULDBLOCK) + return { 0, socket_status::non_blocking_would_have_blocked }; + if (error == EAGAIN) + return { 0, socket_status::non_blocking_would_have_blocked }; + return { 0, socket_status::errored }; + } + + if (received_bytes == 0) + { + return { received_bytes, socket_status::cleanly_disconnected }; + } + + return { size_t(received_bytes), socket_status::valid }; + } + + ///receive up-to len bytes inside the memory location pointed by buffer + bytes_with_status recv(std::byte* buffer, size_t len, bool wait = true, addr_collection* addr_info = nullptr) + { + auto received_bytes = 0; + if constexpr (sock_proto == protocol::tcp) + { + int flags; + if (wait) + flags = MSG_WAITALL; + else + { +#ifdef _WIN32 + flags = 0; // MSG_DONTWAIT not avail on windows, need to make socket nonblockingto emulate + set_non_blocking(true); +#else + flags = MSG_DONTWAIT; +#endif + } + received_bytes = syscall_recv(sock, reinterpret_cast(buffer), static_cast(len), flags); +#ifdef _WIN32 + set_non_blocking(false); +#endif + } + +#ifdef KISSNET_USE_OPENSSL + else if constexpr (sock_proto == protocol::tcp_ssl) + { + received_bytes = SSL_read(pSSL, reinterpret_cast(buffer), static_cast(len)); + } +#endif + + else if constexpr (sock_proto == protocol::udp) + { + socket_input_socklen = sizeof socket_input; + + received_bytes = ::recvfrom(sock, reinterpret_cast(buffer), static_cast(len), 0, reinterpret_cast(&socket_input), &socket_input_socklen); + if (addr_info) { + addr_info->adrinf = socket_input; + addr_info->sock_size = socket_input_socklen; + } + } + + if (received_bytes < 0) + { + const auto error = get_error_code(); + if (error == EWOULDBLOCK) + return { 0, socket_status::non_blocking_would_have_blocked }; + if (error == EAGAIN) + return { 0, socket_status::non_blocking_would_have_blocked }; + return { 0, socket_status::errored }; + } + + if (received_bytes == 0) + { + return { received_bytes, socket_status::cleanly_disconnected }; + } + + return { size_t(received_bytes), socket_status::valid }; + } + + ///Return the endpoint where this socket is talking to + endpoint get_bind_loc() const + { + return bind_loc; + } + + ///Return an endpoint that originated the data in the last recv + endpoint get_recv_endpoint() const + { + if constexpr (sock_proto == protocol::tcp) + { + return get_bind_loc(); + } + if constexpr (sock_proto == protocol::udp) + { + const SOCKADDR* addr = reinterpret_cast(&socket_input); + return endpoint(const_cast(addr)); + } + } + + ///Return the number of bytes available inside the socket + size_t bytes_available() const + { + static ioctl_setting size = 0; + const auto status = ioctlsocket(sock, FIONREAD, &size); + + if (status < 0) + { + kissnet_fatal_error("ioctlsocket status is negative when getting FIONREAD\n"); + } + + return size > 0 ? size : 0; + } + + ///Return the protocol used by this socket + static protocol get_protocol() + { + return sock_proto; + } + }; + + ///Alias for socket + using tcp_socket = socket; +#ifdef KISSNET_USE_OPENSSL + ///Alias for socket + using tcp_ssl_socket = socket; +#endif //KISSNET_USE_OPENSSL + ///Alias for socket + using udp_socket = socket; +} + +//cleanup preprocessor macros +#undef KISSNET_OS_SPECIFIC_PAYLOAD_NAME +#undef KISSNET_OS_SPECIFIC +#undef KISSNET_OS_INIT +#undef kissnet_fatal_error + +#endif //KISS_NET \ No newline at end of file diff --git a/third_party/nlohmann/README.md b/third_party/nlohmann/README.md new file mode 100644 index 0000000..1263dba --- /dev/null +++ b/third_party/nlohmann/README.md @@ -0,0 +1,15 @@ +a header-only json library from github: https://github.com/nlohmann/json, easy to use like working in Python. + +**NOTE** + +used for parse/dump json payload when interact with REST API from mLLM service in VideoPipe. + + +``` +compile test: +g++ json_test.cpp -o json_test -O2 + +and run: +./json_test + +``` \ No newline at end of file diff --git a/third_party/nlohmann/json.hpp b/third_party/nlohmann/json.hpp new file mode 100644 index 0000000..c2a3214 --- /dev/null +++ b/third_party/nlohmann/json.hpp @@ -0,0 +1,25677 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + +/****************************************************************************\ + * Note on documentation: The source files contain links to the online * + * documentation of the public API at https://json.nlohmann.me. This URL * + * contains the most recent documentation and should also be applicable to * + * previous versions; documentation for deprecated functions is not * + * removed, but marked deprecated. See "Generate documentation" section in * + * file docs/README.md. * +\****************************************************************************/ + +#ifndef INCLUDE_NLOHMANN_JSON_HPP_ +#define INCLUDE_NLOHMANN_JSON_HPP_ + +#include // all_of, find, for_each +#include // nullptr_t, ptrdiff_t, size_t +#include // hash, less +#include // initializer_list +#ifndef JSON_NO_IO + #include // istream, ostream +#endif // JSON_NO_IO +#include // random_access_iterator_tag +#include // unique_ptr +#include // string, stoi, to_string +#include // declval, forward, move, pair, swap +#include // vector + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +// This file contains all macro definitions affecting or depending on the ABI + +#ifndef JSON_SKIP_LIBRARY_VERSION_CHECK + #if defined(NLOHMANN_JSON_VERSION_MAJOR) && defined(NLOHMANN_JSON_VERSION_MINOR) && defined(NLOHMANN_JSON_VERSION_PATCH) + #if NLOHMANN_JSON_VERSION_MAJOR != 3 || NLOHMANN_JSON_VERSION_MINOR != 12 || NLOHMANN_JSON_VERSION_PATCH != 0 + #warning "Already included a different version of the library!" + #endif + #endif +#endif + +#define NLOHMANN_JSON_VERSION_MAJOR 3 // NOLINT(modernize-macro-to-enum) +#define NLOHMANN_JSON_VERSION_MINOR 12 // NOLINT(modernize-macro-to-enum) +#define NLOHMANN_JSON_VERSION_PATCH 0 // NOLINT(modernize-macro-to-enum) + +#ifndef JSON_DIAGNOSTICS + #define JSON_DIAGNOSTICS 0 +#endif + +#ifndef JSON_DIAGNOSTIC_POSITIONS + #define JSON_DIAGNOSTIC_POSITIONS 0 +#endif + +#ifndef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON + #define JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON 0 +#endif + +#if JSON_DIAGNOSTICS + #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS _diag +#else + #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS +#endif + +#if JSON_DIAGNOSTIC_POSITIONS + #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS _dp +#else + #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS +#endif + +#if JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON + #define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON _ldvcmp +#else + #define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON +#endif + +#ifndef NLOHMANN_JSON_NAMESPACE_NO_VERSION + #define NLOHMANN_JSON_NAMESPACE_NO_VERSION 0 +#endif + +// Construct the namespace ABI tags component +#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c) json_abi ## a ## b ## c +#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b, c) \ + NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c) + +#define NLOHMANN_JSON_ABI_TAGS \ + NLOHMANN_JSON_ABI_TAGS_CONCAT( \ + NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS, \ + NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON, \ + NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS) + +// Construct the namespace version component +#define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) \ + _v ## major ## _ ## minor ## _ ## patch +#define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT(major, minor, patch) \ + NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) + +#if NLOHMANN_JSON_NAMESPACE_NO_VERSION +#define NLOHMANN_JSON_NAMESPACE_VERSION +#else +#define NLOHMANN_JSON_NAMESPACE_VERSION \ + NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT(NLOHMANN_JSON_VERSION_MAJOR, \ + NLOHMANN_JSON_VERSION_MINOR, \ + NLOHMANN_JSON_VERSION_PATCH) +#endif + +// Combine namespace components +#define NLOHMANN_JSON_NAMESPACE_CONCAT_EX(a, b) a ## b +#define NLOHMANN_JSON_NAMESPACE_CONCAT(a, b) \ + NLOHMANN_JSON_NAMESPACE_CONCAT_EX(a, b) + +#ifndef NLOHMANN_JSON_NAMESPACE +#define NLOHMANN_JSON_NAMESPACE \ + nlohmann::NLOHMANN_JSON_NAMESPACE_CONCAT( \ + NLOHMANN_JSON_ABI_TAGS, \ + NLOHMANN_JSON_NAMESPACE_VERSION) +#endif + +#ifndef NLOHMANN_JSON_NAMESPACE_BEGIN +#define NLOHMANN_JSON_NAMESPACE_BEGIN \ + namespace nlohmann \ + { \ + inline namespace NLOHMANN_JSON_NAMESPACE_CONCAT( \ + NLOHMANN_JSON_ABI_TAGS, \ + NLOHMANN_JSON_NAMESPACE_VERSION) \ + { +#endif + +#ifndef NLOHMANN_JSON_NAMESPACE_END +#define NLOHMANN_JSON_NAMESPACE_END \ + } /* namespace (inline namespace) NOLINT(readability/namespace) */ \ + } // namespace nlohmann +#endif + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // transform +#include // array +#include // forward_list +#include // inserter, front_inserter, end +#include // map +#include // string +#include // tuple, make_tuple +#include // is_arithmetic, is_same, is_enum, underlying_type, is_convertible +#include // unordered_map +#include // pair, declval +#include // valarray + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // nullptr_t +#include // exception +#if JSON_DIAGNOSTICS + #include // accumulate +#endif +#include // runtime_error +#include // to_string +#include // vector + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // array +#include // size_t +#include // uint8_t +#include // string + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // declval, pair +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +template struct make_void +{ + using type = void; +}; +template using void_t = typename make_void::type; + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +// https://en.cppreference.com/w/cpp/experimental/is_detected +struct nonesuch +{ + nonesuch() = delete; + ~nonesuch() = delete; + nonesuch(nonesuch const&) = delete; + nonesuch(nonesuch const&&) = delete; + void operator=(nonesuch const&) = delete; + void operator=(nonesuch&&) = delete; +}; + +template class Op, + class... Args> +struct detector +{ + using value_t = std::false_type; + using type = Default; +}; + +template class Op, class... Args> +struct detector>, Op, Args...> +{ + using value_t = std::true_type; + using type = Op; +}; + +template class Op, class... Args> +using is_detected = typename detector::value_t; + +template class Op, class... Args> +struct is_detected_lazy : is_detected { }; + +template class Op, class... Args> +using detected_t = typename detector::type; + +template class Op, class... Args> +using detected_or = detector; + +template class Op, class... Args> +using detected_or_t = typename detected_or::type; + +template class Op, class... Args> +using is_detected_exact = std::is_same>; + +template class Op, class... Args> +using is_detected_convertible = + std::is_convertible, To>; + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include + + +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-FileCopyrightText: 2016 - 2021 Evan Nemerson +// SPDX-License-Identifier: MIT + +/* Hedley - https://nemequ.github.io/hedley + * Created by Evan Nemerson + */ + +#if !defined(JSON_HEDLEY_VERSION) || (JSON_HEDLEY_VERSION < 15) +#if defined(JSON_HEDLEY_VERSION) + #undef JSON_HEDLEY_VERSION +#endif +#define JSON_HEDLEY_VERSION 15 + +#if defined(JSON_HEDLEY_STRINGIFY_EX) + #undef JSON_HEDLEY_STRINGIFY_EX +#endif +#define JSON_HEDLEY_STRINGIFY_EX(x) #x + +#if defined(JSON_HEDLEY_STRINGIFY) + #undef JSON_HEDLEY_STRINGIFY +#endif +#define JSON_HEDLEY_STRINGIFY(x) JSON_HEDLEY_STRINGIFY_EX(x) + +#if defined(JSON_HEDLEY_CONCAT_EX) + #undef JSON_HEDLEY_CONCAT_EX +#endif +#define JSON_HEDLEY_CONCAT_EX(a,b) a##b + +#if defined(JSON_HEDLEY_CONCAT) + #undef JSON_HEDLEY_CONCAT +#endif +#define JSON_HEDLEY_CONCAT(a,b) JSON_HEDLEY_CONCAT_EX(a,b) + +#if defined(JSON_HEDLEY_CONCAT3_EX) + #undef JSON_HEDLEY_CONCAT3_EX +#endif +#define JSON_HEDLEY_CONCAT3_EX(a,b,c) a##b##c + +#if defined(JSON_HEDLEY_CONCAT3) + #undef JSON_HEDLEY_CONCAT3 +#endif +#define JSON_HEDLEY_CONCAT3(a,b,c) JSON_HEDLEY_CONCAT3_EX(a,b,c) + +#if defined(JSON_HEDLEY_VERSION_ENCODE) + #undef JSON_HEDLEY_VERSION_ENCODE +#endif +#define JSON_HEDLEY_VERSION_ENCODE(major,minor,revision) (((major) * 1000000) + ((minor) * 1000) + (revision)) + +#if defined(JSON_HEDLEY_VERSION_DECODE_MAJOR) + #undef JSON_HEDLEY_VERSION_DECODE_MAJOR +#endif +#define JSON_HEDLEY_VERSION_DECODE_MAJOR(version) ((version) / 1000000) + +#if defined(JSON_HEDLEY_VERSION_DECODE_MINOR) + #undef JSON_HEDLEY_VERSION_DECODE_MINOR +#endif +#define JSON_HEDLEY_VERSION_DECODE_MINOR(version) (((version) % 1000000) / 1000) + +#if defined(JSON_HEDLEY_VERSION_DECODE_REVISION) + #undef JSON_HEDLEY_VERSION_DECODE_REVISION +#endif +#define JSON_HEDLEY_VERSION_DECODE_REVISION(version) ((version) % 1000) + +#if defined(JSON_HEDLEY_GNUC_VERSION) + #undef JSON_HEDLEY_GNUC_VERSION +#endif +#if defined(__GNUC__) && defined(__GNUC_PATCHLEVEL__) + #define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__) +#elif defined(__GNUC__) + #define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, 0) +#endif + +#if defined(JSON_HEDLEY_GNUC_VERSION_CHECK) + #undef JSON_HEDLEY_GNUC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_GNUC_VERSION) + #define JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_GNUC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_MSVC_VERSION) + #undef JSON_HEDLEY_MSVC_VERSION +#endif +#if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 140000000) && !defined(__ICL) + #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 10000000, (_MSC_FULL_VER % 10000000) / 100000, (_MSC_FULL_VER % 100000) / 100) +#elif defined(_MSC_FULL_VER) && !defined(__ICL) + #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 1000000, (_MSC_FULL_VER % 1000000) / 10000, (_MSC_FULL_VER % 10000) / 10) +#elif defined(_MSC_VER) && !defined(__ICL) + #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_VER / 100, _MSC_VER % 100, 0) +#endif + +#if defined(JSON_HEDLEY_MSVC_VERSION_CHECK) + #undef JSON_HEDLEY_MSVC_VERSION_CHECK +#endif +#if !defined(JSON_HEDLEY_MSVC_VERSION) + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (0) +#elif defined(_MSC_VER) && (_MSC_VER >= 1400) + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_FULL_VER >= ((major * 10000000) + (minor * 100000) + (patch))) +#elif defined(_MSC_VER) && (_MSC_VER >= 1200) + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_FULL_VER >= ((major * 1000000) + (minor * 10000) + (patch))) +#else + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_VER >= ((major * 100) + (minor))) +#endif + +#if defined(JSON_HEDLEY_INTEL_VERSION) + #undef JSON_HEDLEY_INTEL_VERSION +#endif +#if defined(__INTEL_COMPILER) && defined(__INTEL_COMPILER_UPDATE) && !defined(__ICL) + #define JSON_HEDLEY_INTEL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, __INTEL_COMPILER_UPDATE) +#elif defined(__INTEL_COMPILER) && !defined(__ICL) + #define JSON_HEDLEY_INTEL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, 0) +#endif + +#if defined(JSON_HEDLEY_INTEL_VERSION_CHECK) + #undef JSON_HEDLEY_INTEL_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_INTEL_VERSION) + #define JSON_HEDLEY_INTEL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_INTEL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_INTEL_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_INTEL_CL_VERSION) + #undef JSON_HEDLEY_INTEL_CL_VERSION +#endif +#if defined(__INTEL_COMPILER) && defined(__INTEL_COMPILER_UPDATE) && defined(__ICL) + #define JSON_HEDLEY_INTEL_CL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER, __INTEL_COMPILER_UPDATE, 0) +#endif + +#if defined(JSON_HEDLEY_INTEL_CL_VERSION_CHECK) + #undef JSON_HEDLEY_INTEL_CL_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_INTEL_CL_VERSION) + #define JSON_HEDLEY_INTEL_CL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_INTEL_CL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_INTEL_CL_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_PGI_VERSION) + #undef JSON_HEDLEY_PGI_VERSION +#endif +#if defined(__PGI) && defined(__PGIC__) && defined(__PGIC_MINOR__) && defined(__PGIC_PATCHLEVEL__) + #define JSON_HEDLEY_PGI_VERSION JSON_HEDLEY_VERSION_ENCODE(__PGIC__, __PGIC_MINOR__, __PGIC_PATCHLEVEL__) +#endif + +#if defined(JSON_HEDLEY_PGI_VERSION_CHECK) + #undef JSON_HEDLEY_PGI_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_PGI_VERSION) + #define JSON_HEDLEY_PGI_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_PGI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_PGI_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_SUNPRO_VERSION) + #undef JSON_HEDLEY_SUNPRO_VERSION +#endif +#if defined(__SUNPRO_C) && (__SUNPRO_C > 0x1000) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_C >> 16) & 0xf) * 10) + ((__SUNPRO_C >> 12) & 0xf), (((__SUNPRO_C >> 8) & 0xf) * 10) + ((__SUNPRO_C >> 4) & 0xf), (__SUNPRO_C & 0xf) * 10) +#elif defined(__SUNPRO_C) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_C >> 8) & 0xf, (__SUNPRO_C >> 4) & 0xf, (__SUNPRO_C) & 0xf) +#elif defined(__SUNPRO_CC) && (__SUNPRO_CC > 0x1000) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_CC >> 16) & 0xf) * 10) + ((__SUNPRO_CC >> 12) & 0xf), (((__SUNPRO_CC >> 8) & 0xf) * 10) + ((__SUNPRO_CC >> 4) & 0xf), (__SUNPRO_CC & 0xf) * 10) +#elif defined(__SUNPRO_CC) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_CC >> 8) & 0xf, (__SUNPRO_CC >> 4) & 0xf, (__SUNPRO_CC) & 0xf) +#endif + +#if defined(JSON_HEDLEY_SUNPRO_VERSION_CHECK) + #undef JSON_HEDLEY_SUNPRO_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_SUNPRO_VERSION) + #define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_SUNPRO_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION) + #undef JSON_HEDLEY_EMSCRIPTEN_VERSION +#endif +#if defined(__EMSCRIPTEN__) + #define JSON_HEDLEY_EMSCRIPTEN_VERSION JSON_HEDLEY_VERSION_ENCODE(__EMSCRIPTEN_major__, __EMSCRIPTEN_minor__, __EMSCRIPTEN_tiny__) +#endif + +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK) + #undef JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION) + #define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_EMSCRIPTEN_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_ARM_VERSION) + #undef JSON_HEDLEY_ARM_VERSION +#endif +#if defined(__CC_ARM) && defined(__ARMCOMPILER_VERSION) + #define JSON_HEDLEY_ARM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ARMCOMPILER_VERSION / 1000000, (__ARMCOMPILER_VERSION % 1000000) / 10000, (__ARMCOMPILER_VERSION % 10000) / 100) +#elif defined(__CC_ARM) && defined(__ARMCC_VERSION) + #define JSON_HEDLEY_ARM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ARMCC_VERSION / 1000000, (__ARMCC_VERSION % 1000000) / 10000, (__ARMCC_VERSION % 10000) / 100) +#endif + +#if defined(JSON_HEDLEY_ARM_VERSION_CHECK) + #undef JSON_HEDLEY_ARM_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_ARM_VERSION) + #define JSON_HEDLEY_ARM_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_ARM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_ARM_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_IBM_VERSION) + #undef JSON_HEDLEY_IBM_VERSION +#endif +#if defined(__ibmxl__) + #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ibmxl_version__, __ibmxl_release__, __ibmxl_modification__) +#elif defined(__xlC__) && defined(__xlC_ver__) + #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, (__xlC_ver__ >> 8) & 0xff) +#elif defined(__xlC__) + #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, 0) +#endif + +#if defined(JSON_HEDLEY_IBM_VERSION_CHECK) + #undef JSON_HEDLEY_IBM_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_IBM_VERSION) + #define JSON_HEDLEY_IBM_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_IBM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_IBM_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_VERSION) + #undef JSON_HEDLEY_TI_VERSION +#endif +#if \ + defined(__TI_COMPILER_VERSION__) && \ + ( \ + defined(__TMS470__) || defined(__TI_ARM__) || \ + defined(__MSP430__) || \ + defined(__TMS320C2000__) \ + ) +#if (__TI_COMPILER_VERSION__ >= 16000000) + #define JSON_HEDLEY_TI_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif +#endif + +#if defined(JSON_HEDLEY_TI_VERSION_CHECK) + #undef JSON_HEDLEY_TI_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_VERSION) + #define JSON_HEDLEY_TI_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL2000_VERSION) + #undef JSON_HEDLEY_TI_CL2000_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__TMS320C2000__) + #define JSON_HEDLEY_TI_CL2000_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL2000_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL2000_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL2000_VERSION) + #define JSON_HEDLEY_TI_CL2000_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL2000_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL2000_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL430_VERSION) + #undef JSON_HEDLEY_TI_CL430_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__MSP430__) + #define JSON_HEDLEY_TI_CL430_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL430_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL430_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL430_VERSION) + #define JSON_HEDLEY_TI_CL430_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL430_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL430_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_ARMCL_VERSION) + #undef JSON_HEDLEY_TI_ARMCL_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && (defined(__TMS470__) || defined(__TI_ARM__)) + #define JSON_HEDLEY_TI_ARMCL_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_ARMCL_VERSION_CHECK) + #undef JSON_HEDLEY_TI_ARMCL_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_ARMCL_VERSION) + #define JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_ARMCL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL6X_VERSION) + #undef JSON_HEDLEY_TI_CL6X_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__TMS320C6X__) + #define JSON_HEDLEY_TI_CL6X_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL6X_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL6X_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL6X_VERSION) + #define JSON_HEDLEY_TI_CL6X_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL6X_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL6X_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL7X_VERSION) + #undef JSON_HEDLEY_TI_CL7X_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__C7000__) + #define JSON_HEDLEY_TI_CL7X_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL7X_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL7X_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL7X_VERSION) + #define JSON_HEDLEY_TI_CL7X_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL7X_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL7X_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CLPRU_VERSION) + #undef JSON_HEDLEY_TI_CLPRU_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__PRU__) + #define JSON_HEDLEY_TI_CLPRU_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CLPRU_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CLPRU_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CLPRU_VERSION) + #define JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CLPRU_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_CRAY_VERSION) + #undef JSON_HEDLEY_CRAY_VERSION +#endif +#if defined(_CRAYC) + #if defined(_RELEASE_PATCHLEVEL) + #define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, _RELEASE_PATCHLEVEL) + #else + #define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, 0) + #endif +#endif + +#if defined(JSON_HEDLEY_CRAY_VERSION_CHECK) + #undef JSON_HEDLEY_CRAY_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_CRAY_VERSION) + #define JSON_HEDLEY_CRAY_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_CRAY_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_CRAY_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_IAR_VERSION) + #undef JSON_HEDLEY_IAR_VERSION +#endif +#if defined(__IAR_SYSTEMS_ICC__) + #if __VER__ > 1000 + #define JSON_HEDLEY_IAR_VERSION JSON_HEDLEY_VERSION_ENCODE((__VER__ / 1000000), ((__VER__ / 1000) % 1000), (__VER__ % 1000)) + #else + #define JSON_HEDLEY_IAR_VERSION JSON_HEDLEY_VERSION_ENCODE(__VER__ / 100, __VER__ % 100, 0) + #endif +#endif + +#if defined(JSON_HEDLEY_IAR_VERSION_CHECK) + #undef JSON_HEDLEY_IAR_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_IAR_VERSION) + #define JSON_HEDLEY_IAR_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_IAR_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_IAR_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TINYC_VERSION) + #undef JSON_HEDLEY_TINYC_VERSION +#endif +#if defined(__TINYC__) + #define JSON_HEDLEY_TINYC_VERSION JSON_HEDLEY_VERSION_ENCODE(__TINYC__ / 1000, (__TINYC__ / 100) % 10, __TINYC__ % 100) +#endif + +#if defined(JSON_HEDLEY_TINYC_VERSION_CHECK) + #undef JSON_HEDLEY_TINYC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TINYC_VERSION) + #define JSON_HEDLEY_TINYC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TINYC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TINYC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_DMC_VERSION) + #undef JSON_HEDLEY_DMC_VERSION +#endif +#if defined(__DMC__) + #define JSON_HEDLEY_DMC_VERSION JSON_HEDLEY_VERSION_ENCODE(__DMC__ >> 8, (__DMC__ >> 4) & 0xf, __DMC__ & 0xf) +#endif + +#if defined(JSON_HEDLEY_DMC_VERSION_CHECK) + #undef JSON_HEDLEY_DMC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_DMC_VERSION) + #define JSON_HEDLEY_DMC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_DMC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_DMC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_COMPCERT_VERSION) + #undef JSON_HEDLEY_COMPCERT_VERSION +#endif +#if defined(__COMPCERT_VERSION__) + #define JSON_HEDLEY_COMPCERT_VERSION JSON_HEDLEY_VERSION_ENCODE(__COMPCERT_VERSION__ / 10000, (__COMPCERT_VERSION__ / 100) % 100, __COMPCERT_VERSION__ % 100) +#endif + +#if defined(JSON_HEDLEY_COMPCERT_VERSION_CHECK) + #undef JSON_HEDLEY_COMPCERT_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_COMPCERT_VERSION) + #define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_COMPCERT_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_PELLES_VERSION) + #undef JSON_HEDLEY_PELLES_VERSION +#endif +#if defined(__POCC__) + #define JSON_HEDLEY_PELLES_VERSION JSON_HEDLEY_VERSION_ENCODE(__POCC__ / 100, __POCC__ % 100, 0) +#endif + +#if defined(JSON_HEDLEY_PELLES_VERSION_CHECK) + #undef JSON_HEDLEY_PELLES_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_PELLES_VERSION) + #define JSON_HEDLEY_PELLES_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_PELLES_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_PELLES_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_MCST_LCC_VERSION) + #undef JSON_HEDLEY_MCST_LCC_VERSION +#endif +#if defined(__LCC__) && defined(__LCC_MINOR__) + #define JSON_HEDLEY_MCST_LCC_VERSION JSON_HEDLEY_VERSION_ENCODE(__LCC__ / 100, __LCC__ % 100, __LCC_MINOR__) +#endif + +#if defined(JSON_HEDLEY_MCST_LCC_VERSION_CHECK) + #undef JSON_HEDLEY_MCST_LCC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_MCST_LCC_VERSION) + #define JSON_HEDLEY_MCST_LCC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_MCST_LCC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_MCST_LCC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_GCC_VERSION) + #undef JSON_HEDLEY_GCC_VERSION +#endif +#if \ + defined(JSON_HEDLEY_GNUC_VERSION) && \ + !defined(__clang__) && \ + !defined(JSON_HEDLEY_INTEL_VERSION) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_ARM_VERSION) && \ + !defined(JSON_HEDLEY_CRAY_VERSION) && \ + !defined(JSON_HEDLEY_TI_VERSION) && \ + !defined(JSON_HEDLEY_TI_ARMCL_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL430_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL2000_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL6X_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL7X_VERSION) && \ + !defined(JSON_HEDLEY_TI_CLPRU_VERSION) && \ + !defined(__COMPCERT__) && \ + !defined(JSON_HEDLEY_MCST_LCC_VERSION) + #define JSON_HEDLEY_GCC_VERSION JSON_HEDLEY_GNUC_VERSION +#endif + +#if defined(JSON_HEDLEY_GCC_VERSION_CHECK) + #undef JSON_HEDLEY_GCC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_GCC_VERSION) + #define JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_GCC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_HAS_ATTRIBUTE +#endif +#if \ + defined(__has_attribute) && \ + ( \ + (!defined(JSON_HEDLEY_IAR_VERSION) || JSON_HEDLEY_IAR_VERSION_CHECK(8,5,9)) \ + ) +# define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) __has_attribute(attribute) +#else +# define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_GNUC_HAS_ATTRIBUTE +#endif +#if defined(__has_attribute) + #define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) +#else + #define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_GCC_HAS_ATTRIBUTE +#endif +#if defined(__has_attribute) + #define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) +#else + #define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE +#endif +#if \ + defined(__has_cpp_attribute) && \ + defined(__cplusplus) && \ + (!defined(JSON_HEDLEY_SUNPRO_VERSION) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0)) + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) __has_cpp_attribute(attribute) +#else + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS) + #undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS +#endif +#if !defined(__cplusplus) || !defined(__has_cpp_attribute) + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) (0) +#elif \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_IAR_VERSION) && \ + (!defined(JSON_HEDLEY_SUNPRO_VERSION) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0)) && \ + (!defined(JSON_HEDLEY_MSVC_VERSION) || JSON_HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) JSON_HEDLEY_HAS_CPP_ATTRIBUTE(ns::attribute) +#else + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE +#endif +#if defined(__has_cpp_attribute) && defined(__cplusplus) + #define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) __has_cpp_attribute(attribute) +#else + #define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE +#endif +#if defined(__has_cpp_attribute) && defined(__cplusplus) + #define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) __has_cpp_attribute(attribute) +#else + #define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_BUILTIN) + #undef JSON_HEDLEY_HAS_BUILTIN +#endif +#if defined(__has_builtin) + #define JSON_HEDLEY_HAS_BUILTIN(builtin) __has_builtin(builtin) +#else + #define JSON_HEDLEY_HAS_BUILTIN(builtin) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_BUILTIN) + #undef JSON_HEDLEY_GNUC_HAS_BUILTIN +#endif +#if defined(__has_builtin) + #define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin,major,minor,patch) __has_builtin(builtin) +#else + #define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_BUILTIN) + #undef JSON_HEDLEY_GCC_HAS_BUILTIN +#endif +#if defined(__has_builtin) + #define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin,major,minor,patch) __has_builtin(builtin) +#else + #define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_FEATURE) + #undef JSON_HEDLEY_HAS_FEATURE +#endif +#if defined(__has_feature) + #define JSON_HEDLEY_HAS_FEATURE(feature) __has_feature(feature) +#else + #define JSON_HEDLEY_HAS_FEATURE(feature) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_FEATURE) + #undef JSON_HEDLEY_GNUC_HAS_FEATURE +#endif +#if defined(__has_feature) + #define JSON_HEDLEY_GNUC_HAS_FEATURE(feature,major,minor,patch) __has_feature(feature) +#else + #define JSON_HEDLEY_GNUC_HAS_FEATURE(feature,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_FEATURE) + #undef JSON_HEDLEY_GCC_HAS_FEATURE +#endif +#if defined(__has_feature) + #define JSON_HEDLEY_GCC_HAS_FEATURE(feature,major,minor,patch) __has_feature(feature) +#else + #define JSON_HEDLEY_GCC_HAS_FEATURE(feature,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_EXTENSION) + #undef JSON_HEDLEY_HAS_EXTENSION +#endif +#if defined(__has_extension) + #define JSON_HEDLEY_HAS_EXTENSION(extension) __has_extension(extension) +#else + #define JSON_HEDLEY_HAS_EXTENSION(extension) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_EXTENSION) + #undef JSON_HEDLEY_GNUC_HAS_EXTENSION +#endif +#if defined(__has_extension) + #define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension,major,minor,patch) __has_extension(extension) +#else + #define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_EXTENSION) + #undef JSON_HEDLEY_GCC_HAS_EXTENSION +#endif +#if defined(__has_extension) + #define JSON_HEDLEY_GCC_HAS_EXTENSION(extension,major,minor,patch) __has_extension(extension) +#else + #define JSON_HEDLEY_GCC_HAS_EXTENSION(extension,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) + #define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) __has_declspec_attribute(attribute) +#else + #define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) + #define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) __has_declspec_attribute(attribute) +#else + #define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) + #define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) __has_declspec_attribute(attribute) +#else + #define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_WARNING) + #undef JSON_HEDLEY_HAS_WARNING +#endif +#if defined(__has_warning) + #define JSON_HEDLEY_HAS_WARNING(warning) __has_warning(warning) +#else + #define JSON_HEDLEY_HAS_WARNING(warning) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_WARNING) + #undef JSON_HEDLEY_GNUC_HAS_WARNING +#endif +#if defined(__has_warning) + #define JSON_HEDLEY_GNUC_HAS_WARNING(warning,major,minor,patch) __has_warning(warning) +#else + #define JSON_HEDLEY_GNUC_HAS_WARNING(warning,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_WARNING) + #undef JSON_HEDLEY_GCC_HAS_WARNING +#endif +#if defined(__has_warning) + #define JSON_HEDLEY_GCC_HAS_WARNING(warning,major,minor,patch) __has_warning(warning) +#else + #define JSON_HEDLEY_GCC_HAS_WARNING(warning,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \ + defined(__clang__) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(18,4,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,7,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(2,0,1) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(5,0,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,17) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(8,0,0) || \ + (JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) && defined(__C99_PRAGMA_OPERATOR)) + #define JSON_HEDLEY_PRAGMA(value) _Pragma(#value) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) + #define JSON_HEDLEY_PRAGMA(value) __pragma(value) +#else + #define JSON_HEDLEY_PRAGMA(value) +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_PUSH) + #undef JSON_HEDLEY_DIAGNOSTIC_PUSH +#endif +#if defined(JSON_HEDLEY_DIAGNOSTIC_POP) + #undef JSON_HEDLEY_DIAGNOSTIC_POP +#endif +#if defined(__clang__) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("clang diagnostic push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("clang diagnostic pop") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("GCC diagnostic push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("GCC diagnostic pop") +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH __pragma(warning(push)) + #define JSON_HEDLEY_DIAGNOSTIC_POP __pragma(warning(pop)) +#elif JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("pop") +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,4,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("diag_push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("diag_pop") +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,90,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)") +#else + #define JSON_HEDLEY_DIAGNOSTIC_PUSH + #define JSON_HEDLEY_DIAGNOSTIC_POP +#endif + +/* JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_ is for + HEDLEY INTERNAL USE ONLY. API subject to change without notice. */ +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_ +#endif +#if defined(__cplusplus) +# if JSON_HEDLEY_HAS_WARNING("-Wc++98-compat") +# if JSON_HEDLEY_HAS_WARNING("-Wc++17-extensions") +# if JSON_HEDLEY_HAS_WARNING("-Wc++1z-extensions") +# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ + _Pragma("clang diagnostic ignored \"-Wc++17-extensions\"") \ + _Pragma("clang diagnostic ignored \"-Wc++1z-extensions\"") \ + xpr \ + JSON_HEDLEY_DIAGNOSTIC_POP +# else +# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ + _Pragma("clang diagnostic ignored \"-Wc++17-extensions\"") \ + xpr \ + JSON_HEDLEY_DIAGNOSTIC_POP +# endif +# else +# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ + xpr \ + JSON_HEDLEY_DIAGNOSTIC_POP +# endif +# endif +#endif +#if !defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(x) x +#endif + +#if defined(JSON_HEDLEY_CONST_CAST) + #undef JSON_HEDLEY_CONST_CAST +#endif +#if defined(__cplusplus) +# define JSON_HEDLEY_CONST_CAST(T, expr) (const_cast(expr)) +#elif \ + JSON_HEDLEY_HAS_WARNING("-Wcast-qual") || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) +# define JSON_HEDLEY_CONST_CAST(T, expr) (__extension__ ({ \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL \ + ((T) (expr)); \ + JSON_HEDLEY_DIAGNOSTIC_POP \ + })) +#else +# define JSON_HEDLEY_CONST_CAST(T, expr) ((T) (expr)) +#endif + +#if defined(JSON_HEDLEY_REINTERPRET_CAST) + #undef JSON_HEDLEY_REINTERPRET_CAST +#endif +#if defined(__cplusplus) + #define JSON_HEDLEY_REINTERPRET_CAST(T, expr) (reinterpret_cast(expr)) +#else + #define JSON_HEDLEY_REINTERPRET_CAST(T, expr) ((T) (expr)) +#endif + +#if defined(JSON_HEDLEY_STATIC_CAST) + #undef JSON_HEDLEY_STATIC_CAST +#endif +#if defined(__cplusplus) + #define JSON_HEDLEY_STATIC_CAST(T, expr) (static_cast(expr)) +#else + #define JSON_HEDLEY_STATIC_CAST(T, expr) ((T) (expr)) +#endif + +#if defined(JSON_HEDLEY_CPP_CAST) + #undef JSON_HEDLEY_CPP_CAST +#endif +#if defined(__cplusplus) +# if JSON_HEDLEY_HAS_WARNING("-Wold-style-cast") +# define JSON_HEDLEY_CPP_CAST(T, expr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wold-style-cast\"") \ + ((T) (expr)) \ + JSON_HEDLEY_DIAGNOSTIC_POP +# elif JSON_HEDLEY_IAR_VERSION_CHECK(8,3,0) +# define JSON_HEDLEY_CPP_CAST(T, expr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("diag_suppress=Pe137") \ + JSON_HEDLEY_DIAGNOSTIC_POP +# else +# define JSON_HEDLEY_CPP_CAST(T, expr) ((T) (expr)) +# endif +#else +# define JSON_HEDLEY_CPP_CAST(T, expr) (expr) +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wdeprecated-declarations") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("clang diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warning(disable:1478 1786)") +#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable:1478 1786)) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(20,7,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1216,1444,1445") +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1444") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable:4996)) +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1444") +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1291,1718") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) && !defined(__cplusplus) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,E_DEPRECATED_ATT,E_DEPRECATED_ATT_MESS)") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) && defined(__cplusplus) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,symdeprecated,symdeprecated2)") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress=Pe1444,Pe1215") +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,90,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warn(disable:2241)") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("clang diagnostic ignored \"-Wunknown-pragmas\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("warning(disable:161)") +#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable:161)) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 1675") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("GCC diagnostic ignored \"-Wunknown-pragmas\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable:4068)) +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(16,9,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,3,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 163") +#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 163") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress=Pe161") +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 161") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-attributes") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("clang diagnostic ignored \"-Wunknown-attributes\"") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(17,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("warning(disable:1292)") +#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES __pragma(warning(disable:1292)) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(19,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES __pragma(warning(disable:5030)) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(20,7,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097,1098") +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,14,0) && defined(__cplusplus) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("error_messages(off,attrskipunsup)") +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(18,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,3,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1173") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress=Pe1097") +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wcast-qual") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("clang diagnostic ignored \"-Wcast-qual\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("warning(disable:2203 2331)") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("GCC diagnostic ignored \"-Wcast-qual\"") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunused-function") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("clang diagnostic ignored \"-Wunused-function\"") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("GCC diagnostic ignored \"-Wunused-function\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(1,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION __pragma(warning(disable:4505)) +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("diag_suppress 3142") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION +#endif + +#if defined(JSON_HEDLEY_DEPRECATED) + #undef JSON_HEDLEY_DEPRECATED +#endif +#if defined(JSON_HEDLEY_DEPRECATED_FOR) + #undef JSON_HEDLEY_DEPRECATED_FOR +#endif +#if \ + JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DEPRECATED(since) __declspec(deprecated("Since " # since)) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated("Since " #since "; use " #replacement)) +#elif \ + (JSON_HEDLEY_HAS_EXTENSION(attribute_deprecated_with_message) && !defined(JSON_HEDLEY_IAR_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,5,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(18,1,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(18,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,3,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,3,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__("Since " #since))) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__("Since " #since "; use " #replacement))) +#elif defined(__cplusplus) && (__cplusplus >= 201402L) + #define JSON_HEDLEY_DEPRECATED(since) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[deprecated("Since " #since)]]) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[deprecated("Since " #since "; use " #replacement)]]) +#elif \ + JSON_HEDLEY_HAS_ATTRIBUTE(deprecated) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) + #define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__)) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__)) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_PELLES_VERSION_CHECK(6,50,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DEPRECATED(since) __declspec(deprecated) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated) +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DEPRECATED(since) _Pragma("deprecated") + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) _Pragma("deprecated") +#else + #define JSON_HEDLEY_DEPRECATED(since) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) +#endif + +#if defined(JSON_HEDLEY_UNAVAILABLE) + #undef JSON_HEDLEY_UNAVAILABLE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(warning) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_UNAVAILABLE(available_since) __attribute__((__warning__("Not available until " #available_since))) +#else + #define JSON_HEDLEY_UNAVAILABLE(available_since) +#endif + +#if defined(JSON_HEDLEY_WARN_UNUSED_RESULT) + #undef JSON_HEDLEY_WARN_UNUSED_RESULT +#endif +#if defined(JSON_HEDLEY_WARN_UNUSED_RESULT_MSG) + #undef JSON_HEDLEY_WARN_UNUSED_RESULT_MSG +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(warn_unused_result) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0) && defined(__cplusplus)) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_WARN_UNUSED_RESULT __attribute__((__warn_unused_result__)) + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) __attribute__((__warn_unused_result__)) +#elif (JSON_HEDLEY_HAS_CPP_ATTRIBUTE(nodiscard) >= 201907L) + #define JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard(msg)]]) +#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE(nodiscard) + #define JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) +#elif defined(_Check_return_) /* SAL */ + #define JSON_HEDLEY_WARN_UNUSED_RESULT _Check_return_ + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) _Check_return_ +#else + #define JSON_HEDLEY_WARN_UNUSED_RESULT + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) +#endif + +#if defined(JSON_HEDLEY_SENTINEL) + #undef JSON_HEDLEY_SENTINEL +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(sentinel) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,4,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_SENTINEL(position) __attribute__((__sentinel__(position))) +#else + #define JSON_HEDLEY_SENTINEL(position) +#endif + +#if defined(JSON_HEDLEY_NO_RETURN) + #undef JSON_HEDLEY_NO_RETURN +#endif +#if JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_NO_RETURN __noreturn +#elif \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__)) +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L + #define JSON_HEDLEY_NO_RETURN _Noreturn +#elif defined(__cplusplus) && (__cplusplus >= 201103L) + #define JSON_HEDLEY_NO_RETURN JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[noreturn]]) +#elif \ + JSON_HEDLEY_HAS_ATTRIBUTE(noreturn) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,2,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) + #define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__)) +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) + #define JSON_HEDLEY_NO_RETURN _Pragma("does_not_return") +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_NO_RETURN __declspec(noreturn) +#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,0,0) && defined(__cplusplus) + #define JSON_HEDLEY_NO_RETURN _Pragma("FUNC_NEVER_RETURNS;") +#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3,2,0) + #define JSON_HEDLEY_NO_RETURN __attribute((noreturn)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9,0,0) + #define JSON_HEDLEY_NO_RETURN __declspec(noreturn) +#else + #define JSON_HEDLEY_NO_RETURN +#endif + +#if defined(JSON_HEDLEY_NO_ESCAPE) + #undef JSON_HEDLEY_NO_ESCAPE +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(noescape) + #define JSON_HEDLEY_NO_ESCAPE __attribute__((__noescape__)) +#else + #define JSON_HEDLEY_NO_ESCAPE +#endif + +#if defined(JSON_HEDLEY_UNREACHABLE) + #undef JSON_HEDLEY_UNREACHABLE +#endif +#if defined(JSON_HEDLEY_UNREACHABLE_RETURN) + #undef JSON_HEDLEY_UNREACHABLE_RETURN +#endif +#if defined(JSON_HEDLEY_ASSUME) + #undef JSON_HEDLEY_ASSUME +#endif +#if \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_ASSUME(expr) __assume(expr) +#elif JSON_HEDLEY_HAS_BUILTIN(__builtin_assume) + #define JSON_HEDLEY_ASSUME(expr) __builtin_assume(expr) +#elif \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) + #if defined(__cplusplus) + #define JSON_HEDLEY_ASSUME(expr) std::_nassert(expr) + #else + #define JSON_HEDLEY_ASSUME(expr) _nassert(expr) + #endif +#endif +#if \ + (JSON_HEDLEY_HAS_BUILTIN(__builtin_unreachable) && (!defined(JSON_HEDLEY_ARM_VERSION))) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,5,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(18,10,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,5) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(10,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_UNREACHABLE() __builtin_unreachable() +#elif defined(JSON_HEDLEY_ASSUME) + #define JSON_HEDLEY_UNREACHABLE() JSON_HEDLEY_ASSUME(0) +#endif +#if !defined(JSON_HEDLEY_ASSUME) + #if defined(JSON_HEDLEY_UNREACHABLE) + #define JSON_HEDLEY_ASSUME(expr) JSON_HEDLEY_STATIC_CAST(void, ((expr) ? 1 : (JSON_HEDLEY_UNREACHABLE(), 1))) + #else + #define JSON_HEDLEY_ASSUME(expr) JSON_HEDLEY_STATIC_CAST(void, expr) + #endif +#endif +#if defined(JSON_HEDLEY_UNREACHABLE) + #if \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) + #define JSON_HEDLEY_UNREACHABLE_RETURN(value) return (JSON_HEDLEY_STATIC_CAST(void, JSON_HEDLEY_ASSUME(0)), (value)) + #else + #define JSON_HEDLEY_UNREACHABLE_RETURN(value) JSON_HEDLEY_UNREACHABLE() + #endif +#else + #define JSON_HEDLEY_UNREACHABLE_RETURN(value) return (value) +#endif +#if !defined(JSON_HEDLEY_UNREACHABLE) + #define JSON_HEDLEY_UNREACHABLE() JSON_HEDLEY_ASSUME(0) +#endif + +JSON_HEDLEY_DIAGNOSTIC_PUSH +#if JSON_HEDLEY_HAS_WARNING("-Wpedantic") + #pragma clang diagnostic ignored "-Wpedantic" +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wc++98-compat-pedantic") && defined(__cplusplus) + #pragma clang diagnostic ignored "-Wc++98-compat-pedantic" +#endif +#if JSON_HEDLEY_GCC_HAS_WARNING("-Wvariadic-macros",4,0,0) + #if defined(__clang__) + #pragma clang diagnostic ignored "-Wvariadic-macros" + #elif defined(JSON_HEDLEY_GCC_VERSION) + #pragma GCC diagnostic ignored "-Wvariadic-macros" + #endif +#endif +#if defined(JSON_HEDLEY_NON_NULL) + #undef JSON_HEDLEY_NON_NULL +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(nonnull) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) + #define JSON_HEDLEY_NON_NULL(...) __attribute__((__nonnull__(__VA_ARGS__))) +#else + #define JSON_HEDLEY_NON_NULL(...) +#endif +JSON_HEDLEY_DIAGNOSTIC_POP + +#if defined(JSON_HEDLEY_PRINTF_FORMAT) + #undef JSON_HEDLEY_PRINTF_FORMAT +#endif +#if defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format,4,4,0) && !defined(__USE_MINGW_ANSI_STDIO) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(ms_printf, string_idx, first_to_check))) +#elif defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format,4,4,0) && defined(__USE_MINGW_ANSI_STDIO) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(gnu_printf, string_idx, first_to_check))) +#elif \ + JSON_HEDLEY_HAS_ATTRIBUTE(format) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(__printf__, string_idx, first_to_check))) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(6,0,0) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __declspec(vaformat(printf,string_idx,first_to_check)) +#else + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) +#endif + +#if defined(JSON_HEDLEY_CONSTEXPR) + #undef JSON_HEDLEY_CONSTEXPR +#endif +#if defined(__cplusplus) + #if __cplusplus >= 201103L + #define JSON_HEDLEY_CONSTEXPR JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(constexpr) + #endif +#endif +#if !defined(JSON_HEDLEY_CONSTEXPR) + #define JSON_HEDLEY_CONSTEXPR +#endif + +#if defined(JSON_HEDLEY_PREDICT) + #undef JSON_HEDLEY_PREDICT +#endif +#if defined(JSON_HEDLEY_LIKELY) + #undef JSON_HEDLEY_LIKELY +#endif +#if defined(JSON_HEDLEY_UNLIKELY) + #undef JSON_HEDLEY_UNLIKELY +#endif +#if defined(JSON_HEDLEY_UNPREDICTABLE) + #undef JSON_HEDLEY_UNPREDICTABLE +#endif +#if JSON_HEDLEY_HAS_BUILTIN(__builtin_unpredictable) + #define JSON_HEDLEY_UNPREDICTABLE(expr) __builtin_unpredictable((expr)) +#endif +#if \ + (JSON_HEDLEY_HAS_BUILTIN(__builtin_expect_with_probability) && !defined(JSON_HEDLEY_PGI_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(9,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PREDICT(expr, value, probability) __builtin_expect_with_probability( (expr), (value), (probability)) +# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) __builtin_expect_with_probability(!!(expr), 1 , (probability)) +# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) __builtin_expect_with_probability(!!(expr), 0 , (probability)) +# define JSON_HEDLEY_LIKELY(expr) __builtin_expect (!!(expr), 1 ) +# define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect (!!(expr), 0 ) +#elif \ + (JSON_HEDLEY_HAS_BUILTIN(__builtin_expect) && !defined(JSON_HEDLEY_INTEL_CL_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0) && defined(__cplusplus)) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,7,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,27) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PREDICT(expr, expected, probability) \ + (((probability) >= 0.9) ? __builtin_expect((expr), (expected)) : (JSON_HEDLEY_STATIC_CAST(void, expected), (expr))) +# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) \ + (__extension__ ({ \ + double hedley_probability_ = (probability); \ + ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 1) : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 0) : !!(expr))); \ + })) +# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) \ + (__extension__ ({ \ + double hedley_probability_ = (probability); \ + ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 0) : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 1) : !!(expr))); \ + })) +# define JSON_HEDLEY_LIKELY(expr) __builtin_expect(!!(expr), 1) +# define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect(!!(expr), 0) +#else +# define JSON_HEDLEY_PREDICT(expr, expected, probability) (JSON_HEDLEY_STATIC_CAST(void, expected), (expr)) +# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) (!!(expr)) +# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) (!!(expr)) +# define JSON_HEDLEY_LIKELY(expr) (!!(expr)) +# define JSON_HEDLEY_UNLIKELY(expr) (!!(expr)) +#endif +#if !defined(JSON_HEDLEY_UNPREDICTABLE) + #define JSON_HEDLEY_UNPREDICTABLE(expr) JSON_HEDLEY_PREDICT(expr, 1, 0.5) +#endif + +#if defined(JSON_HEDLEY_MALLOC) + #undef JSON_HEDLEY_MALLOC +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(malloc) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(12,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_MALLOC __attribute__((__malloc__)) +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) + #define JSON_HEDLEY_MALLOC _Pragma("returns_new_memory") +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_MALLOC __declspec(restrict) +#else + #define JSON_HEDLEY_MALLOC +#endif + +#if defined(JSON_HEDLEY_PURE) + #undef JSON_HEDLEY_PURE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(pure) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(2,96,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PURE __attribute__((__pure__)) +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) +# define JSON_HEDLEY_PURE _Pragma("does_not_write_global_data") +#elif defined(__cplusplus) && \ + ( \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(2,0,1) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) \ + ) +# define JSON_HEDLEY_PURE _Pragma("FUNC_IS_PURE;") +#else +# define JSON_HEDLEY_PURE +#endif + +#if defined(JSON_HEDLEY_CONST) + #undef JSON_HEDLEY_CONST +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(const) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(2,5,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_CONST __attribute__((__const__)) +#elif \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) + #define JSON_HEDLEY_CONST _Pragma("no_side_effect") +#else + #define JSON_HEDLEY_CONST JSON_HEDLEY_PURE +#endif + +#if defined(JSON_HEDLEY_RESTRICT) + #undef JSON_HEDLEY_RESTRICT +#endif +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && !defined(__cplusplus) + #define JSON_HEDLEY_RESTRICT restrict +#elif \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,4) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,14,0) && defined(__cplusplus)) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) || \ + defined(__clang__) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_RESTRICT __restrict +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,3,0) && !defined(__cplusplus) + #define JSON_HEDLEY_RESTRICT _Restrict +#else + #define JSON_HEDLEY_RESTRICT +#endif + +#if defined(JSON_HEDLEY_INLINE) + #undef JSON_HEDLEY_INLINE +#endif +#if \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \ + (defined(__cplusplus) && (__cplusplus >= 199711L)) + #define JSON_HEDLEY_INLINE inline +#elif \ + defined(JSON_HEDLEY_GCC_VERSION) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(6,2,0) + #define JSON_HEDLEY_INLINE __inline__ +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(12,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,1,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_INLINE __inline +#else + #define JSON_HEDLEY_INLINE +#endif + +#if defined(JSON_HEDLEY_ALWAYS_INLINE) + #undef JSON_HEDLEY_ALWAYS_INLINE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(always_inline) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) +# define JSON_HEDLEY_ALWAYS_INLINE __attribute__((__always_inline__)) JSON_HEDLEY_INLINE +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(12,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) +# define JSON_HEDLEY_ALWAYS_INLINE __forceinline +#elif defined(__cplusplus) && \ + ( \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) \ + ) +# define JSON_HEDLEY_ALWAYS_INLINE _Pragma("FUNC_ALWAYS_INLINE;") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) +# define JSON_HEDLEY_ALWAYS_INLINE _Pragma("inline=forced") +#else +# define JSON_HEDLEY_ALWAYS_INLINE JSON_HEDLEY_INLINE +#endif + +#if defined(JSON_HEDLEY_NEVER_INLINE) + #undef JSON_HEDLEY_NEVER_INLINE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(noinline) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) + #define JSON_HEDLEY_NEVER_INLINE __attribute__((__noinline__)) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_NEVER_INLINE __declspec(noinline) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(10,2,0) + #define JSON_HEDLEY_NEVER_INLINE _Pragma("noinline") +#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,0,0) && defined(__cplusplus) + #define JSON_HEDLEY_NEVER_INLINE _Pragma("FUNC_CANNOT_INLINE;") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_NEVER_INLINE _Pragma("inline=never") +#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3,2,0) + #define JSON_HEDLEY_NEVER_INLINE __attribute((noinline)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9,0,0) + #define JSON_HEDLEY_NEVER_INLINE __declspec(noinline) +#else + #define JSON_HEDLEY_NEVER_INLINE +#endif + +#if defined(JSON_HEDLEY_PRIVATE) + #undef JSON_HEDLEY_PRIVATE +#endif +#if defined(JSON_HEDLEY_PUBLIC) + #undef JSON_HEDLEY_PUBLIC +#endif +#if defined(JSON_HEDLEY_IMPORT) + #undef JSON_HEDLEY_IMPORT +#endif +#if defined(_WIN32) || defined(__CYGWIN__) +# define JSON_HEDLEY_PRIVATE +# define JSON_HEDLEY_PUBLIC __declspec(dllexport) +# define JSON_HEDLEY_IMPORT __declspec(dllimport) +#else +# if \ + JSON_HEDLEY_HAS_ATTRIBUTE(visibility) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ + ( \ + defined(__TI_EABI__) && \ + ( \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) \ + ) \ + ) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PRIVATE __attribute__((__visibility__("hidden"))) +# define JSON_HEDLEY_PUBLIC __attribute__((__visibility__("default"))) +# else +# define JSON_HEDLEY_PRIVATE +# define JSON_HEDLEY_PUBLIC +# endif +# define JSON_HEDLEY_IMPORT extern +#endif + +#if defined(JSON_HEDLEY_NO_THROW) + #undef JSON_HEDLEY_NO_THROW +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(nothrow) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_NO_THROW __attribute__((__nothrow__)) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,1,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) + #define JSON_HEDLEY_NO_THROW __declspec(nothrow) +#else + #define JSON_HEDLEY_NO_THROW +#endif + +#if defined(JSON_HEDLEY_FALL_THROUGH) + #undef JSON_HEDLEY_FALL_THROUGH +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(fallthrough) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(7,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_FALL_THROUGH __attribute__((__fallthrough__)) +#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(clang,fallthrough) + #define JSON_HEDLEY_FALL_THROUGH JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[clang::fallthrough]]) +#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE(fallthrough) + #define JSON_HEDLEY_FALL_THROUGH JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[fallthrough]]) +#elif defined(__fallthrough) /* SAL */ + #define JSON_HEDLEY_FALL_THROUGH __fallthrough +#else + #define JSON_HEDLEY_FALL_THROUGH +#endif + +#if defined(JSON_HEDLEY_RETURNS_NON_NULL) + #undef JSON_HEDLEY_RETURNS_NON_NULL +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(returns_nonnull) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_RETURNS_NON_NULL __attribute__((__returns_nonnull__)) +#elif defined(_Ret_notnull_) /* SAL */ + #define JSON_HEDLEY_RETURNS_NON_NULL _Ret_notnull_ +#else + #define JSON_HEDLEY_RETURNS_NON_NULL +#endif + +#if defined(JSON_HEDLEY_ARRAY_PARAM) + #undef JSON_HEDLEY_ARRAY_PARAM +#endif +#if \ + defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \ + !defined(__STDC_NO_VLA__) && \ + !defined(__cplusplus) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_TINYC_VERSION) + #define JSON_HEDLEY_ARRAY_PARAM(name) (name) +#else + #define JSON_HEDLEY_ARRAY_PARAM(name) +#endif + +#if defined(JSON_HEDLEY_IS_CONSTANT) + #undef JSON_HEDLEY_IS_CONSTANT +#endif +#if defined(JSON_HEDLEY_REQUIRE_CONSTEXPR) + #undef JSON_HEDLEY_REQUIRE_CONSTEXPR +#endif +/* JSON_HEDLEY_IS_CONSTEXPR_ is for + HEDLEY INTERNAL USE ONLY. API subject to change without notice. */ +#if defined(JSON_HEDLEY_IS_CONSTEXPR_) + #undef JSON_HEDLEY_IS_CONSTEXPR_ +#endif +#if \ + JSON_HEDLEY_HAS_BUILTIN(__builtin_constant_p) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,19) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) && !defined(__cplusplus)) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_IS_CONSTANT(expr) __builtin_constant_p(expr) +#endif +#if !defined(__cplusplus) +# if \ + JSON_HEDLEY_HAS_BUILTIN(__builtin_types_compatible_p) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,4,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,24) +#if defined(__INTPTR_TYPE__) + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) __builtin_types_compatible_p(__typeof__((1 ? (void*) ((__INTPTR_TYPE__) ((expr) * 0)) : (int*) 0)), int*) +#else + #include + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) __builtin_types_compatible_p(__typeof__((1 ? (void*) ((intptr_t) ((expr) * 0)) : (int*) 0)), int*) +#endif +# elif \ + ( \ + defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) && \ + !defined(JSON_HEDLEY_SUNPRO_VERSION) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_IAR_VERSION)) || \ + (JSON_HEDLEY_HAS_EXTENSION(c_generic_selections) && !defined(JSON_HEDLEY_IAR_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(17,0,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(12,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,3,0) +#if defined(__INTPTR_TYPE__) + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) _Generic((1 ? (void*) ((__INTPTR_TYPE__) ((expr) * 0)) : (int*) 0), int*: 1, void*: 0) +#else + #include + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) _Generic((1 ? (void*) ((intptr_t) * 0) : (int*) 0), int*: 1, void*: 0) +#endif +# elif \ + defined(JSON_HEDLEY_GCC_VERSION) || \ + defined(JSON_HEDLEY_INTEL_VERSION) || \ + defined(JSON_HEDLEY_TINYC_VERSION) || \ + defined(JSON_HEDLEY_TI_ARMCL_VERSION) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(18,12,0) || \ + defined(JSON_HEDLEY_TI_CL2000_VERSION) || \ + defined(JSON_HEDLEY_TI_CL6X_VERSION) || \ + defined(JSON_HEDLEY_TI_CL7X_VERSION) || \ + defined(JSON_HEDLEY_TI_CLPRU_VERSION) || \ + defined(__clang__) +# define JSON_HEDLEY_IS_CONSTEXPR_(expr) ( \ + sizeof(void) != \ + sizeof(*( \ + 1 ? \ + ((void*) ((expr) * 0L) ) : \ +((struct { char v[sizeof(void) * 2]; } *) 1) \ + ) \ + ) \ + ) +# endif +#endif +#if defined(JSON_HEDLEY_IS_CONSTEXPR_) + #if !defined(JSON_HEDLEY_IS_CONSTANT) + #define JSON_HEDLEY_IS_CONSTANT(expr) JSON_HEDLEY_IS_CONSTEXPR_(expr) + #endif + #define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (JSON_HEDLEY_IS_CONSTEXPR_(expr) ? (expr) : (-1)) +#else + #if !defined(JSON_HEDLEY_IS_CONSTANT) + #define JSON_HEDLEY_IS_CONSTANT(expr) (0) + #endif + #define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (expr) +#endif + +#if defined(JSON_HEDLEY_BEGIN_C_DECLS) + #undef JSON_HEDLEY_BEGIN_C_DECLS +#endif +#if defined(JSON_HEDLEY_END_C_DECLS) + #undef JSON_HEDLEY_END_C_DECLS +#endif +#if defined(JSON_HEDLEY_C_DECL) + #undef JSON_HEDLEY_C_DECL +#endif +#if defined(__cplusplus) + #define JSON_HEDLEY_BEGIN_C_DECLS extern "C" { + #define JSON_HEDLEY_END_C_DECLS } + #define JSON_HEDLEY_C_DECL extern "C" +#else + #define JSON_HEDLEY_BEGIN_C_DECLS + #define JSON_HEDLEY_END_C_DECLS + #define JSON_HEDLEY_C_DECL +#endif + +#if defined(JSON_HEDLEY_STATIC_ASSERT) + #undef JSON_HEDLEY_STATIC_ASSERT +#endif +#if \ + !defined(__cplusplus) && ( \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) || \ + (JSON_HEDLEY_HAS_FEATURE(c_static_assert) && !defined(JSON_HEDLEY_INTEL_CL_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(6,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + defined(_Static_assert) \ + ) +# define JSON_HEDLEY_STATIC_ASSERT(expr, message) _Static_assert(expr, message) +#elif \ + (defined(__cplusplus) && (__cplusplus >= 201103L)) || \ + JSON_HEDLEY_MSVC_VERSION_CHECK(16,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) +# define JSON_HEDLEY_STATIC_ASSERT(expr, message) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(static_assert(expr, message)) +#else +# define JSON_HEDLEY_STATIC_ASSERT(expr, message) +#endif + +#if defined(JSON_HEDLEY_NULL) + #undef JSON_HEDLEY_NULL +#endif +#if defined(__cplusplus) + #if __cplusplus >= 201103L + #define JSON_HEDLEY_NULL JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(nullptr) + #elif defined(NULL) + #define JSON_HEDLEY_NULL NULL + #else + #define JSON_HEDLEY_NULL JSON_HEDLEY_STATIC_CAST(void*, 0) + #endif +#elif defined(NULL) + #define JSON_HEDLEY_NULL NULL +#else + #define JSON_HEDLEY_NULL ((void*) 0) +#endif + +#if defined(JSON_HEDLEY_MESSAGE) + #undef JSON_HEDLEY_MESSAGE +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") +# define JSON_HEDLEY_MESSAGE(msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \ + JSON_HEDLEY_PRAGMA(message msg) \ + JSON_HEDLEY_DIAGNOSTIC_POP +#elif \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message msg) +#elif JSON_HEDLEY_CRAY_VERSION_CHECK(5,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(_CRI message msg) +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#else +# define JSON_HEDLEY_MESSAGE(msg) +#endif + +#if defined(JSON_HEDLEY_WARNING) + #undef JSON_HEDLEY_WARNING +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") +# define JSON_HEDLEY_WARNING(msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \ + JSON_HEDLEY_PRAGMA(clang warning msg) \ + JSON_HEDLEY_DIAGNOSTIC_POP +#elif \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,8,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(18,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) +# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(GCC warning msg) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) +# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#else +# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_MESSAGE(msg) +#endif + +#if defined(JSON_HEDLEY_REQUIRE) + #undef JSON_HEDLEY_REQUIRE +#endif +#if defined(JSON_HEDLEY_REQUIRE_MSG) + #undef JSON_HEDLEY_REQUIRE_MSG +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(diagnose_if) +# if JSON_HEDLEY_HAS_WARNING("-Wgcc-compat") +# define JSON_HEDLEY_REQUIRE(expr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wgcc-compat\"") \ + __attribute__((diagnose_if(!(expr), #expr, "error"))) \ + JSON_HEDLEY_DIAGNOSTIC_POP +# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wgcc-compat\"") \ + __attribute__((diagnose_if(!(expr), msg, "error"))) \ + JSON_HEDLEY_DIAGNOSTIC_POP +# else +# define JSON_HEDLEY_REQUIRE(expr) __attribute__((diagnose_if(!(expr), #expr, "error"))) +# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) __attribute__((diagnose_if(!(expr), msg, "error"))) +# endif +#else +# define JSON_HEDLEY_REQUIRE(expr) +# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) +#endif + +#if defined(JSON_HEDLEY_FLAGS) + #undef JSON_HEDLEY_FLAGS +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(flag_enum) && (!defined(__cplusplus) || JSON_HEDLEY_HAS_WARNING("-Wbitfield-enum-conversion")) + #define JSON_HEDLEY_FLAGS __attribute__((__flag_enum__)) +#else + #define JSON_HEDLEY_FLAGS +#endif + +#if defined(JSON_HEDLEY_FLAGS_CAST) + #undef JSON_HEDLEY_FLAGS_CAST +#endif +#if JSON_HEDLEY_INTEL_VERSION_CHECK(19,0,0) +# define JSON_HEDLEY_FLAGS_CAST(T, expr) (__extension__ ({ \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("warning(disable:188)") \ + ((T) (expr)); \ + JSON_HEDLEY_DIAGNOSTIC_POP \ + })) +#else +# define JSON_HEDLEY_FLAGS_CAST(T, expr) JSON_HEDLEY_STATIC_CAST(T, expr) +#endif + +#if defined(JSON_HEDLEY_EMPTY_BASES) + #undef JSON_HEDLEY_EMPTY_BASES +#endif +#if \ + (JSON_HEDLEY_MSVC_VERSION_CHECK(19,0,23918) && !JSON_HEDLEY_MSVC_VERSION_CHECK(20,0,0)) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_EMPTY_BASES __declspec(empty_bases) +#else + #define JSON_HEDLEY_EMPTY_BASES +#endif + +/* Remaining macros are deprecated. */ + +#if defined(JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK) + #undef JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK +#endif +#if defined(__clang__) + #define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major,minor,patch) (0) +#else + #define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_CLANG_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_CLANG_HAS_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_BUILTIN) + #undef JSON_HEDLEY_CLANG_HAS_BUILTIN +#endif +#define JSON_HEDLEY_CLANG_HAS_BUILTIN(builtin) JSON_HEDLEY_HAS_BUILTIN(builtin) + +#if defined(JSON_HEDLEY_CLANG_HAS_FEATURE) + #undef JSON_HEDLEY_CLANG_HAS_FEATURE +#endif +#define JSON_HEDLEY_CLANG_HAS_FEATURE(feature) JSON_HEDLEY_HAS_FEATURE(feature) + +#if defined(JSON_HEDLEY_CLANG_HAS_EXTENSION) + #undef JSON_HEDLEY_CLANG_HAS_EXTENSION +#endif +#define JSON_HEDLEY_CLANG_HAS_EXTENSION(extension) JSON_HEDLEY_HAS_EXTENSION(extension) + +#if defined(JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_DECLSPEC_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_WARNING) + #undef JSON_HEDLEY_CLANG_HAS_WARNING +#endif +#define JSON_HEDLEY_CLANG_HAS_WARNING(warning) JSON_HEDLEY_HAS_WARNING(warning) + +#endif /* !defined(JSON_HEDLEY_VERSION) || (JSON_HEDLEY_VERSION < X) */ + + +// This file contains all internal macro definitions (except those affecting ABI) +// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them + +// #include + + +// exclude unsupported compilers +#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) + #if defined(__clang__) + #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 + #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) + #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 + #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #endif +#endif + +// C++ language standard detection +// if the user manually specified the used C++ version, this is skipped +#if !defined(JSON_HAS_CPP_26) && !defined(JSON_HAS_CPP_23) && !defined(JSON_HAS_CPP_20) && !defined(JSON_HAS_CPP_17) && !defined(JSON_HAS_CPP_14) && !defined(JSON_HAS_CPP_11) + #if (defined(__cplusplus) && __cplusplus > 202302L) || (defined(_MSVC_LANG) && _MSVC_LANG > 202302L) + #define JSON_HAS_CPP_26 + #define JSON_HAS_CPP_23 + #define JSON_HAS_CPP_20 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 + #elif (defined(__cplusplus) && __cplusplus > 202002L) || (defined(_MSVC_LANG) && _MSVC_LANG > 202002L) + #define JSON_HAS_CPP_23 + #define JSON_HAS_CPP_20 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 + #elif (defined(__cplusplus) && __cplusplus > 201703L) || (defined(_MSVC_LANG) && _MSVC_LANG > 201703L) + #define JSON_HAS_CPP_20 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 + #elif (defined(__cplusplus) && __cplusplus > 201402L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 + #elif (defined(__cplusplus) && __cplusplus > 201103L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) + #define JSON_HAS_CPP_14 + #endif + // the cpp 11 flag is always specified because it is the minimal required version + #define JSON_HAS_CPP_11 +#endif + +#ifdef __has_include + #if __has_include() + #include + #endif +#endif + +#if !defined(JSON_HAS_FILESYSTEM) && !defined(JSON_HAS_EXPERIMENTAL_FILESYSTEM) + #ifdef JSON_HAS_CPP_17 + #if defined(__cpp_lib_filesystem) + #define JSON_HAS_FILESYSTEM 1 + #elif defined(__cpp_lib_experimental_filesystem) + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1 + #elif !defined(__has_include) + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1 + #elif __has_include() + #define JSON_HAS_FILESYSTEM 1 + #elif __has_include() + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1 + #endif + + // std::filesystem does not work on MinGW GCC 8: https://sourceforge.net/p/mingw-w64/bugs/737/ + #if defined(__MINGW32__) && defined(__GNUC__) && __GNUC__ == 8 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before GCC 8: https://en.cppreference.com/w/cpp/compiler_support + #if defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 8 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before Clang 7: https://en.cppreference.com/w/cpp/compiler_support + #if defined(__clang_major__) && __clang_major__ < 7 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before MSVC 19.14: https://en.cppreference.com/w/cpp/compiler_support + #if defined(_MSC_VER) && _MSC_VER < 1914 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before iOS 13 + #if defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && __IPHONE_OS_VERSION_MIN_REQUIRED < 130000 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before macOS Catalina + #if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED < 101500 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + #endif +#endif + +#ifndef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 0 +#endif + +#ifndef JSON_HAS_FILESYSTEM + #define JSON_HAS_FILESYSTEM 0 +#endif + +#ifndef JSON_HAS_THREE_WAY_COMPARISON + #if defined(__cpp_impl_three_way_comparison) && __cpp_impl_three_way_comparison >= 201907L \ + && defined(__cpp_lib_three_way_comparison) && __cpp_lib_three_way_comparison >= 201907L + #define JSON_HAS_THREE_WAY_COMPARISON 1 + #else + #define JSON_HAS_THREE_WAY_COMPARISON 0 + #endif +#endif + +#ifndef JSON_HAS_RANGES + // ranges header shipping in GCC 11.1.0 (released 2021-04-27) has a syntax error + #if defined(__GLIBCXX__) && __GLIBCXX__ == 20210427 + #define JSON_HAS_RANGES 0 + #elif defined(__cpp_lib_ranges) + #define JSON_HAS_RANGES 1 + #else + #define JSON_HAS_RANGES 0 + #endif +#endif + +#ifndef JSON_HAS_STATIC_RTTI + #if !defined(_HAS_STATIC_RTTI) || _HAS_STATIC_RTTI != 0 + #define JSON_HAS_STATIC_RTTI 1 + #else + #define JSON_HAS_STATIC_RTTI 0 + #endif +#endif + +#ifdef JSON_HAS_CPP_17 + #define JSON_INLINE_VARIABLE inline +#else + #define JSON_INLINE_VARIABLE +#endif + +#if JSON_HEDLEY_HAS_ATTRIBUTE(no_unique_address) + #define JSON_NO_UNIQUE_ADDRESS [[no_unique_address]] +#else + #define JSON_NO_UNIQUE_ADDRESS +#endif + +// disable documentation warnings on clang +#if defined(__clang__) + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wdocumentation" + #pragma clang diagnostic ignored "-Wdocumentation-unknown-command" +#endif + +// allow disabling exceptions +#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) + #define JSON_THROW(exception) throw exception + #define JSON_TRY try + #define JSON_CATCH(exception) catch(exception) + #define JSON_INTERNAL_CATCH(exception) catch(exception) +#else + #include + #define JSON_THROW(exception) std::abort() + #define JSON_TRY if(true) + #define JSON_CATCH(exception) if(false) + #define JSON_INTERNAL_CATCH(exception) if(false) +#endif + +// override exception macros +#if defined(JSON_THROW_USER) + #undef JSON_THROW + #define JSON_THROW JSON_THROW_USER +#endif +#if defined(JSON_TRY_USER) + #undef JSON_TRY + #define JSON_TRY JSON_TRY_USER +#endif +#if defined(JSON_CATCH_USER) + #undef JSON_CATCH + #define JSON_CATCH JSON_CATCH_USER + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_CATCH_USER +#endif +#if defined(JSON_INTERNAL_CATCH_USER) + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER +#endif + +// allow overriding assert +#if !defined(JSON_ASSERT) + #include // assert + #define JSON_ASSERT(x) assert(x) +#endif + +// allow accessing some private functions (needed by the test suite) +#if defined(JSON_TESTS_PRIVATE) + #define JSON_PRIVATE_UNLESS_TESTED public +#else + #define JSON_PRIVATE_UNLESS_TESTED private +#endif + +/*! +@brief macro to briefly define a mapping between an enum and JSON +@def NLOHMANN_JSON_SERIALIZE_ENUM +@since version 3.4.0 +*/ +#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ + template \ + inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ + { \ + /* NOLINTNEXTLINE(modernize-type-traits) we use C++11 */ \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + /* NOLINTNEXTLINE(modernize-avoid-c-arrays) we don't want to depend on */ \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [e](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.first == e; \ + }); \ + j = ((it != std::end(m)) ? it : std::begin(m))->second; \ + } \ + template \ + inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ + { \ + /* NOLINTNEXTLINE(modernize-type-traits) we use C++11 */ \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + /* NOLINTNEXTLINE(modernize-avoid-c-arrays) we don't want to depend on */ \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [&j](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.second == j; \ + }); \ + e = ((it != std::end(m)) ? it : std::begin(m))->first; \ + } + +// Ugly macros to avoid uglier copy-paste when specializing basic_json. They +// may be removed in the future once the class is split. + +#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ + template class ObjectType, \ + template class ArrayType, \ + class StringType, class BooleanType, class NumberIntegerType, \ + class NumberUnsignedType, class NumberFloatType, \ + template class AllocatorType, \ + template class JSONSerializer, \ + class BinaryType, \ + class CustomBaseClass> + +#define NLOHMANN_BASIC_JSON_TPL \ + basic_json + +// Macros to simplify conversion from/to types + +#define NLOHMANN_JSON_EXPAND( x ) x +#define NLOHMANN_JSON_GET_MACRO(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, _25, _26, _27, _28, _29, _30, _31, _32, _33, _34, _35, _36, _37, _38, _39, _40, _41, _42, _43, _44, _45, _46, _47, _48, _49, _50, _51, _52, _53, _54, _55, _56, _57, _58, _59, _60, _61, _62, _63, _64, NAME,...) NAME +#define NLOHMANN_JSON_PASTE(...) NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_GET_MACRO(__VA_ARGS__, \ + NLOHMANN_JSON_PASTE64, \ + NLOHMANN_JSON_PASTE63, \ + NLOHMANN_JSON_PASTE62, \ + NLOHMANN_JSON_PASTE61, \ + NLOHMANN_JSON_PASTE60, \ + NLOHMANN_JSON_PASTE59, \ + NLOHMANN_JSON_PASTE58, \ + NLOHMANN_JSON_PASTE57, \ + NLOHMANN_JSON_PASTE56, \ + NLOHMANN_JSON_PASTE55, \ + NLOHMANN_JSON_PASTE54, \ + NLOHMANN_JSON_PASTE53, \ + NLOHMANN_JSON_PASTE52, \ + NLOHMANN_JSON_PASTE51, \ + NLOHMANN_JSON_PASTE50, \ + NLOHMANN_JSON_PASTE49, \ + NLOHMANN_JSON_PASTE48, \ + NLOHMANN_JSON_PASTE47, \ + NLOHMANN_JSON_PASTE46, \ + NLOHMANN_JSON_PASTE45, \ + NLOHMANN_JSON_PASTE44, \ + NLOHMANN_JSON_PASTE43, \ + NLOHMANN_JSON_PASTE42, \ + NLOHMANN_JSON_PASTE41, \ + NLOHMANN_JSON_PASTE40, \ + NLOHMANN_JSON_PASTE39, \ + NLOHMANN_JSON_PASTE38, \ + NLOHMANN_JSON_PASTE37, \ + NLOHMANN_JSON_PASTE36, \ + NLOHMANN_JSON_PASTE35, \ + NLOHMANN_JSON_PASTE34, \ + NLOHMANN_JSON_PASTE33, \ + NLOHMANN_JSON_PASTE32, \ + NLOHMANN_JSON_PASTE31, \ + NLOHMANN_JSON_PASTE30, \ + NLOHMANN_JSON_PASTE29, \ + NLOHMANN_JSON_PASTE28, \ + NLOHMANN_JSON_PASTE27, \ + NLOHMANN_JSON_PASTE26, \ + NLOHMANN_JSON_PASTE25, \ + NLOHMANN_JSON_PASTE24, \ + NLOHMANN_JSON_PASTE23, \ + NLOHMANN_JSON_PASTE22, \ + NLOHMANN_JSON_PASTE21, \ + NLOHMANN_JSON_PASTE20, \ + NLOHMANN_JSON_PASTE19, \ + NLOHMANN_JSON_PASTE18, \ + NLOHMANN_JSON_PASTE17, \ + NLOHMANN_JSON_PASTE16, \ + NLOHMANN_JSON_PASTE15, \ + NLOHMANN_JSON_PASTE14, \ + NLOHMANN_JSON_PASTE13, \ + NLOHMANN_JSON_PASTE12, \ + NLOHMANN_JSON_PASTE11, \ + NLOHMANN_JSON_PASTE10, \ + NLOHMANN_JSON_PASTE9, \ + NLOHMANN_JSON_PASTE8, \ + NLOHMANN_JSON_PASTE7, \ + NLOHMANN_JSON_PASTE6, \ + NLOHMANN_JSON_PASTE5, \ + NLOHMANN_JSON_PASTE4, \ + NLOHMANN_JSON_PASTE3, \ + NLOHMANN_JSON_PASTE2, \ + NLOHMANN_JSON_PASTE1)(__VA_ARGS__)) +#define NLOHMANN_JSON_PASTE2(func, v1) func(v1) +#define NLOHMANN_JSON_PASTE3(func, v1, v2) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE2(func, v2) +#define NLOHMANN_JSON_PASTE4(func, v1, v2, v3) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE3(func, v2, v3) +#define NLOHMANN_JSON_PASTE5(func, v1, v2, v3, v4) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE4(func, v2, v3, v4) +#define NLOHMANN_JSON_PASTE6(func, v1, v2, v3, v4, v5) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE5(func, v2, v3, v4, v5) +#define NLOHMANN_JSON_PASTE7(func, v1, v2, v3, v4, v5, v6) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE6(func, v2, v3, v4, v5, v6) +#define NLOHMANN_JSON_PASTE8(func, v1, v2, v3, v4, v5, v6, v7) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE7(func, v2, v3, v4, v5, v6, v7) +#define NLOHMANN_JSON_PASTE9(func, v1, v2, v3, v4, v5, v6, v7, v8) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE8(func, v2, v3, v4, v5, v6, v7, v8) +#define NLOHMANN_JSON_PASTE10(func, v1, v2, v3, v4, v5, v6, v7, v8, v9) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE9(func, v2, v3, v4, v5, v6, v7, v8, v9) +#define NLOHMANN_JSON_PASTE11(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE10(func, v2, v3, v4, v5, v6, v7, v8, v9, v10) +#define NLOHMANN_JSON_PASTE12(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE11(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11) +#define NLOHMANN_JSON_PASTE13(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE12(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12) +#define NLOHMANN_JSON_PASTE14(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE13(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13) +#define NLOHMANN_JSON_PASTE15(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE14(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14) +#define NLOHMANN_JSON_PASTE16(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE15(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15) +#define NLOHMANN_JSON_PASTE17(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE16(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16) +#define NLOHMANN_JSON_PASTE18(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE17(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17) +#define NLOHMANN_JSON_PASTE19(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE18(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18) +#define NLOHMANN_JSON_PASTE20(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE19(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19) +#define NLOHMANN_JSON_PASTE21(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE20(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20) +#define NLOHMANN_JSON_PASTE22(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE21(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21) +#define NLOHMANN_JSON_PASTE23(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE22(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22) +#define NLOHMANN_JSON_PASTE24(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE23(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23) +#define NLOHMANN_JSON_PASTE25(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE24(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24) +#define NLOHMANN_JSON_PASTE26(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE25(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25) +#define NLOHMANN_JSON_PASTE27(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE26(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26) +#define NLOHMANN_JSON_PASTE28(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE27(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27) +#define NLOHMANN_JSON_PASTE29(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE28(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28) +#define NLOHMANN_JSON_PASTE30(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE29(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29) +#define NLOHMANN_JSON_PASTE31(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE30(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30) +#define NLOHMANN_JSON_PASTE32(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE31(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31) +#define NLOHMANN_JSON_PASTE33(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE32(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32) +#define NLOHMANN_JSON_PASTE34(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE33(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33) +#define NLOHMANN_JSON_PASTE35(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE34(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34) +#define NLOHMANN_JSON_PASTE36(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE35(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35) +#define NLOHMANN_JSON_PASTE37(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE36(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36) +#define NLOHMANN_JSON_PASTE38(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE37(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37) +#define NLOHMANN_JSON_PASTE39(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE38(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38) +#define NLOHMANN_JSON_PASTE40(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE39(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39) +#define NLOHMANN_JSON_PASTE41(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE40(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40) +#define NLOHMANN_JSON_PASTE42(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE41(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41) +#define NLOHMANN_JSON_PASTE43(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE42(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42) +#define NLOHMANN_JSON_PASTE44(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE43(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43) +#define NLOHMANN_JSON_PASTE45(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE44(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44) +#define NLOHMANN_JSON_PASTE46(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE45(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45) +#define NLOHMANN_JSON_PASTE47(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE46(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46) +#define NLOHMANN_JSON_PASTE48(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE47(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47) +#define NLOHMANN_JSON_PASTE49(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE48(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48) +#define NLOHMANN_JSON_PASTE50(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE49(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49) +#define NLOHMANN_JSON_PASTE51(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE50(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50) +#define NLOHMANN_JSON_PASTE52(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE51(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51) +#define NLOHMANN_JSON_PASTE53(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE52(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52) +#define NLOHMANN_JSON_PASTE54(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE53(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53) +#define NLOHMANN_JSON_PASTE55(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE54(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54) +#define NLOHMANN_JSON_PASTE56(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE55(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55) +#define NLOHMANN_JSON_PASTE57(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE56(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56) +#define NLOHMANN_JSON_PASTE58(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE57(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57) +#define NLOHMANN_JSON_PASTE59(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE58(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58) +#define NLOHMANN_JSON_PASTE60(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE59(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59) +#define NLOHMANN_JSON_PASTE61(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE60(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60) +#define NLOHMANN_JSON_PASTE62(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE61(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61) +#define NLOHMANN_JSON_PASTE63(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE62(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62) +#define NLOHMANN_JSON_PASTE64(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE63(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63) + +#define NLOHMANN_JSON_TO(v1) nlohmann_json_j[#v1] = nlohmann_json_t.v1; +#define NLOHMANN_JSON_FROM(v1) nlohmann_json_j.at(#v1).get_to(nlohmann_json_t.v1); +#define NLOHMANN_JSON_FROM_WITH_DEFAULT(v1) nlohmann_json_t.v1 = !nlohmann_json_j.is_null() ? nlohmann_json_j.value(#v1, nlohmann_json_default_obj.v1) : nlohmann_json_default_obj.v1; + +/*! +@brief macro +@def NLOHMANN_DEFINE_TYPE_INTRUSIVE +@since version 3.9.0 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_type_intrusive/ +*/ +#define NLOHMANN_DEFINE_TYPE_INTRUSIVE(Type, ...) \ + template::value, int> = 0> \ + friend void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + template::value, int> = 0> \ + friend void from_json(const BasicJsonType& nlohmann_json_j, Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT +@since version 3.11.0 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_type_intrusive/ +*/ +#define NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(Type, ...) \ + template::value, int> = 0> \ + friend void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + template::value, int> = 0> \ + friend void from_json(const BasicJsonType& nlohmann_json_j, Type& nlohmann_json_t) { const Type nlohmann_json_default_obj{}; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_TYPE_INTRUSIVE_ONLY_SERIALIZE +@since version 3.11.3 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_type_intrusive/ +*/ +#define NLOHMANN_DEFINE_TYPE_INTRUSIVE_ONLY_SERIALIZE(Type, ...) \ + template::value, int> = 0> \ + friend void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE +@since version 3.9.0 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_type_non_intrusive/ +*/ +#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Type, ...) \ + template::value, int> = 0> \ + void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + template::value, int> = 0> \ + void from_json(const BasicJsonType& nlohmann_json_j, Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT +@since version 3.11.0 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_type_non_intrusive/ +*/ +#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(Type, ...) \ + template::value, int> = 0> \ + void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + template::value, int> = 0> \ + void from_json(const BasicJsonType& nlohmann_json_j, Type& nlohmann_json_t) { const Type nlohmann_json_default_obj{}; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE +@since version 3.11.3 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_type_non_intrusive/ +*/ +#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE(Type, ...) \ + template::value, int> = 0> \ + void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE +@since version 3.12.0 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_derived_type/ +*/ +#define NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE(Type, BaseType, ...) \ + template::value, int> = 0> \ + friend void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { nlohmann::to_json(nlohmann_json_j, static_cast(nlohmann_json_t)); NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + template::value, int> = 0> \ + friend void from_json(const BasicJsonType& nlohmann_json_j, Type& nlohmann_json_t) { nlohmann::from_json(nlohmann_json_j, static_cast(nlohmann_json_t)); NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE_WITH_DEFAULT +@since version 3.12.0 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_derived_type/ +*/ +#define NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE_WITH_DEFAULT(Type, BaseType, ...) \ + template::value, int> = 0> \ + friend void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { nlohmann::to_json(nlohmann_json_j, static_cast(nlohmann_json_t)); NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + template::value, int> = 0> \ + friend void from_json(const BasicJsonType& nlohmann_json_j, Type& nlohmann_json_t) { nlohmann::from_json(nlohmann_json_j, static_cast(nlohmann_json_t)); const Type nlohmann_json_default_obj{}; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE_ONLY_SERIALIZE +@since version 3.12.0 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_derived_type/ +*/ +#define NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE_ONLY_SERIALIZE(Type, BaseType, ...) \ + template::value, int> = 0> \ + friend void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { nlohmann::to_json(nlohmann_json_j, static_cast(nlohmann_json_t)); NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE +@since version 3.12.0 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_derived_type/ +*/ +#define NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE(Type, BaseType, ...) \ + template::value, int> = 0> \ + void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { nlohmann::to_json(nlohmann_json_j, static_cast(nlohmann_json_t)); NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + template::value, int> = 0> \ + void from_json(const BasicJsonType& nlohmann_json_j, Type& nlohmann_json_t) { nlohmann::from_json(nlohmann_json_j, static_cast(nlohmann_json_t)); NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE_WITH_DEFAULT +@since version 3.12.0 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_derived_type/ +*/ +#define NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE_WITH_DEFAULT(Type, BaseType, ...) \ + template::value, int> = 0> \ + void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { nlohmann::to_json(nlohmann_json_j, static_cast(nlohmann_json_t)); NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + template::value, int> = 0> \ + void from_json(const BasicJsonType& nlohmann_json_j, Type& nlohmann_json_t) { nlohmann::from_json(nlohmann_json_j, static_cast(nlohmann_json_t)); const Type nlohmann_json_default_obj{}; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE +@since version 3.12.0 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_derived_type/ +*/ +#define NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE(Type, BaseType, ...) \ + template::value, int> = 0> \ + void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { nlohmann::to_json(nlohmann_json_j, static_cast(nlohmann_json_t)); NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } + +// inspired from https://stackoverflow.com/a/26745591 +// allows calling any std function as if (e.g., with begin): +// using std::begin; begin(x); +// +// it allows using the detected idiom to retrieve the return type +// of such an expression +#define NLOHMANN_CAN_CALL_STD_FUNC_IMPL(std_name) \ + namespace detail { \ + using std::std_name; \ + \ + template \ + using result_of_##std_name = decltype(std_name(std::declval()...)); \ + } \ + \ + namespace detail2 { \ + struct std_name##_tag \ + { \ + }; \ + \ + template \ + std_name##_tag std_name(T&&...); \ + \ + template \ + using result_of_##std_name = decltype(std_name(std::declval()...)); \ + \ + template \ + struct would_call_std_##std_name \ + { \ + static constexpr auto const value = ::nlohmann::detail:: \ + is_detected_exact::value; \ + }; \ + } /* namespace detail2 */ \ + \ + template \ + struct would_call_std_##std_name : detail2::would_call_std_##std_name \ + { \ + } + +#ifndef JSON_USE_IMPLICIT_CONVERSIONS + #define JSON_USE_IMPLICIT_CONVERSIONS 1 +#endif + +#if JSON_USE_IMPLICIT_CONVERSIONS + #define JSON_EXPLICIT +#else + #define JSON_EXPLICIT explicit +#endif + +#ifndef JSON_DISABLE_ENUM_SERIALIZATION + #define JSON_DISABLE_ENUM_SERIALIZATION 0 +#endif + +#ifndef JSON_USE_GLOBAL_UDLS + #define JSON_USE_GLOBAL_UDLS 1 +#endif + +#if JSON_HAS_THREE_WAY_COMPARISON + #include // partial_ordering +#endif + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/////////////////////////// +// JSON type enumeration // +/////////////////////////// + +/*! +@brief the JSON type enumeration + +This enumeration collects the different JSON types. It is internally used to +distinguish the stored values, and the functions @ref basic_json::is_null(), +@ref basic_json::is_object(), @ref basic_json::is_array(), +@ref basic_json::is_string(), @ref basic_json::is_boolean(), +@ref basic_json::is_number() (with @ref basic_json::is_number_integer(), +@ref basic_json::is_number_unsigned(), and @ref basic_json::is_number_float()), +@ref basic_json::is_discarded(), @ref basic_json::is_primitive(), and +@ref basic_json::is_structured() rely on it. + +@note There are three enumeration entries (number_integer, number_unsigned, and +number_float), because the library distinguishes these three types for numbers: +@ref basic_json::number_unsigned_t is used for unsigned integers, +@ref basic_json::number_integer_t is used for signed integers, and +@ref basic_json::number_float_t is used for floating-point numbers or to +approximate integers which do not fit in the limits of their respective type. + +@sa see @ref basic_json::basic_json(const value_t value_type) -- create a JSON +value with the default value for a given type + +@since version 1.0.0 +*/ +enum class value_t : std::uint8_t +{ + null, ///< null value + object, ///< object (unordered set of name/value pairs) + array, ///< array (ordered collection of values) + string, ///< string value + boolean, ///< boolean value + number_integer, ///< number value (signed integer) + number_unsigned, ///< number value (unsigned integer) + number_float, ///< number value (floating-point) + binary, ///< binary array (ordered collection of bytes) + discarded ///< discarded by the parser callback function +}; + +/*! +@brief comparison operator for JSON types + +Returns an ordering that is similar to Python: +- order: null < boolean < number < object < array < string < binary +- furthermore, each type is not smaller than itself +- discarded values are not comparable +- binary is represented as a b"" string in python and directly comparable to a + string; however, making a binary array directly comparable with a string would + be surprising behavior in a JSON file. + +@since version 1.0.0 +*/ +#if JSON_HAS_THREE_WAY_COMPARISON + inline std::partial_ordering operator<=>(const value_t lhs, const value_t rhs) noexcept // *NOPAD* +#else + inline bool operator<(const value_t lhs, const value_t rhs) noexcept +#endif +{ + static constexpr std::array order = {{ + 0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */, + 1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */, + 6 /* binary */ + } + }; + + const auto l_index = static_cast(lhs); + const auto r_index = static_cast(rhs); +#if JSON_HAS_THREE_WAY_COMPARISON + if (l_index < order.size() && r_index < order.size()) + { + return order[l_index] <=> order[r_index]; // *NOPAD* + } + return std::partial_ordering::unordered; +#else + return l_index < order.size() && r_index < order.size() && order[l_index] < order[r_index]; +#endif +} + +// GCC selects the built-in operator< over an operator rewritten from +// a user-defined spaceship operator +// Clang, MSVC, and ICC select the rewritten candidate +// (see GCC bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105200) +#if JSON_HAS_THREE_WAY_COMPARISON && defined(__GNUC__) +inline bool operator<(const value_t lhs, const value_t rhs) noexcept +{ + return std::is_lt(lhs <=> rhs); // *NOPAD* +} +#endif + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/*! +@brief replace all occurrences of a substring by another string + +@param[in,out] s the string to manipulate; changed so that all + occurrences of @a f are replaced with @a t +@param[in] f the substring to replace with @a t +@param[in] t the string to replace @a f + +@pre The search string @a f must not be empty. **This precondition is +enforced with an assertion.** + +@since version 2.0.0 +*/ +template +inline void replace_substring(StringType& s, const StringType& f, + const StringType& t) +{ + JSON_ASSERT(!f.empty()); + for (auto pos = s.find(f); // find the first occurrence of f + pos != StringType::npos; // make sure f was found + s.replace(pos, f.size(), t), // replace with t, and + pos = s.find(f, pos + t.size())) // find the next occurrence of f + {} +} + +/*! + * @brief string escaping as described in RFC 6901 (Sect. 4) + * @param[in] s string to escape + * @return escaped string + * + * Note the order of escaping "~" to "~0" and "/" to "~1" is important. + */ +template +inline StringType escape(StringType s) +{ + replace_substring(s, StringType{"~"}, StringType{"~0"}); + replace_substring(s, StringType{"/"}, StringType{"~1"}); + return s; +} + +/*! + * @brief string unescaping as described in RFC 6901 (Sect. 4) + * @param[in] s string to unescape + * @return unescaped string + * + * Note the order of escaping "~1" to "/" and "~0" to "~" is important. + */ +template +inline void unescape(StringType& s) +{ + replace_substring(s, StringType{"~1"}, StringType{"/"}); + replace_substring(s, StringType{"~0"}, StringType{"~"}); +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // size_t + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/// struct to capture the start position of the current token +struct position_t +{ + /// the total number of characters read + std::size_t chars_read_total = 0; + /// the number of characters read in the current line + std::size_t chars_read_current_line = 0; + /// the number of lines read + std::size_t lines_read = 0; + + /// conversion to size_t to preserve SAX interface + constexpr operator size_t() const + { + return chars_read_total; + } +}; + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-FileCopyrightText: 2018 The Abseil Authors +// SPDX-License-Identifier: MIT + + + +#include // array +#include // size_t +#include // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type +#include // index_sequence, make_index_sequence, index_sequence_for + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +template +using uncvref_t = typename std::remove_cv::type>::type; + +#ifdef JSON_HAS_CPP_14 + +// the following utilities are natively available in C++14 +using std::enable_if_t; +using std::index_sequence; +using std::make_index_sequence; +using std::index_sequence_for; + +#else + +// alias templates to reduce boilerplate +template +using enable_if_t = typename std::enable_if::type; + +// The following code is taken from https://github.com/abseil/abseil-cpp/blob/10cb35e459f5ecca5b2ff107635da0bfa41011b4/absl/utility/utility.h +// which is part of Google Abseil (https://github.com/abseil/abseil-cpp), licensed under the Apache License 2.0. + +//// START OF CODE FROM GOOGLE ABSEIL + +// integer_sequence +// +// Class template representing a compile-time integer sequence. An instantiation +// of `integer_sequence` has a sequence of integers encoded in its +// type through its template arguments (which is a common need when +// working with C++11 variadic templates). `absl::integer_sequence` is designed +// to be a drop-in replacement for C++14's `std::integer_sequence`. +// +// Example: +// +// template< class T, T... Ints > +// void user_function(integer_sequence); +// +// int main() +// { +// // user_function's `T` will be deduced to `int` and `Ints...` +// // will be deduced to `0, 1, 2, 3, 4`. +// user_function(make_integer_sequence()); +// } +template +struct integer_sequence +{ + using value_type = T; + static constexpr std::size_t size() noexcept + { + return sizeof...(Ints); + } +}; + +// index_sequence +// +// A helper template for an `integer_sequence` of `size_t`, +// `absl::index_sequence` is designed to be a drop-in replacement for C++14's +// `std::index_sequence`. +template +using index_sequence = integer_sequence; + +namespace utility_internal +{ + +template +struct Extend; + +// Note that SeqSize == sizeof...(Ints). It's passed explicitly for efficiency. +template +struct Extend, SeqSize, 0> +{ + using type = integer_sequence < T, Ints..., (Ints + SeqSize)... >; +}; + +template +struct Extend, SeqSize, 1> +{ + using type = integer_sequence < T, Ints..., (Ints + SeqSize)..., 2 * SeqSize >; +}; + +// Recursion helper for 'make_integer_sequence'. +// 'Gen::type' is an alias for 'integer_sequence'. +template +struct Gen +{ + using type = + typename Extend < typename Gen < T, N / 2 >::type, N / 2, N % 2 >::type; +}; + +template +struct Gen +{ + using type = integer_sequence; +}; + +} // namespace utility_internal + +// Compile-time sequences of integers + +// make_integer_sequence +// +// This template alias is equivalent to +// `integer_sequence`, and is designed to be a drop-in +// replacement for C++14's `std::make_integer_sequence`. +template +using make_integer_sequence = typename utility_internal::Gen::type; + +// make_index_sequence +// +// This template alias is equivalent to `index_sequence<0, 1, ..., N-1>`, +// and is designed to be a drop-in replacement for C++14's +// `std::make_index_sequence`. +template +using make_index_sequence = make_integer_sequence; + +// index_sequence_for +// +// Converts a typename pack into an index sequence of the same length, and +// is designed to be a drop-in replacement for C++14's +// `std::index_sequence_for()` +template +using index_sequence_for = make_index_sequence; + +//// END OF CODE FROM GOOGLE ABSEIL + +#endif + +// dispatch utility (taken from ranges-v3) +template struct priority_tag : priority_tag < N - 1 > {}; +template<> struct priority_tag<0> {}; + +// taken from ranges-v3 +template +struct static_const +{ + static JSON_INLINE_VARIABLE constexpr T value{}; +}; + +#ifndef JSON_HAS_CPP_17 + template + constexpr T static_const::value; +#endif + +template +constexpr std::array make_array(Args&& ... args) +{ + return std::array {{static_cast(std::forward(args))...}}; +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // numeric_limits +#include // char_traits +#include // tuple +#include // false_type, is_constructible, is_integral, is_same, true_type +#include // declval +#if defined(__cpp_lib_byte) && __cpp_lib_byte >= 201603L + #include // byte +#endif +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // random_access_iterator_tag + +// #include + +// #include + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +template +struct iterator_types {}; + +template +struct iterator_types < + It, + void_t> +{ + using difference_type = typename It::difference_type; + using value_type = typename It::value_type; + using pointer = typename It::pointer; + using reference = typename It::reference; + using iterator_category = typename It::iterator_category; +}; + +// This is required as some compilers implement std::iterator_traits in a way that +// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. +template +struct iterator_traits +{ +}; + +template +struct iterator_traits < T, enable_if_t < !std::is_pointer::value >> + : iterator_types +{ +}; + +template +struct iterator_traits::value>> +{ + using iterator_category = std::random_access_iterator_tag; + using value_type = T; + using difference_type = ptrdiff_t; + using pointer = T*; + using reference = T&; +}; + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN + +NLOHMANN_CAN_CALL_STD_FUNC_IMPL(begin); + +NLOHMANN_JSON_NAMESPACE_END + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN + +NLOHMANN_CAN_CALL_STD_FUNC_IMPL(end); + +NLOHMANN_JSON_NAMESPACE_END + +// #include + +// #include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + +#ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_ + #define INCLUDE_NLOHMANN_JSON_FWD_HPP_ + + #include // int64_t, uint64_t + #include // map + #include // allocator + #include // string + #include // vector + + // #include + + + /*! + @brief namespace for Niels Lohmann + @see https://github.com/nlohmann + @since version 1.0.0 + */ + NLOHMANN_JSON_NAMESPACE_BEGIN + + /*! + @brief default JSONSerializer template argument + + This serializer ignores the template arguments and uses ADL + ([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) + for serialization. + */ + template + struct adl_serializer; + + /// a class to store JSON values + /// @sa https://json.nlohmann.me/api/basic_json/ + template class ObjectType = + std::map, + template class ArrayType = std::vector, + class StringType = std::string, class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template class AllocatorType = std::allocator, + template class JSONSerializer = + adl_serializer, + class BinaryType = std::vector, // cppcheck-suppress syntaxError + class CustomBaseClass = void> + class basic_json; + + /// @brief JSON Pointer defines a string syntax for identifying a specific value within a JSON document + /// @sa https://json.nlohmann.me/api/json_pointer/ + template + class json_pointer; + + /*! + @brief default specialization + @sa https://json.nlohmann.me/api/json/ + */ + using json = basic_json<>; + + /// @brief a minimal map-like container that preserves insertion order + /// @sa https://json.nlohmann.me/api/ordered_map/ + template + struct ordered_map; + + /// @brief specialization that maintains the insertion order of object keys + /// @sa https://json.nlohmann.me/api/ordered_json/ + using ordered_json = basic_json; + + NLOHMANN_JSON_NAMESPACE_END + +#endif // INCLUDE_NLOHMANN_JSON_FWD_HPP_ + + +NLOHMANN_JSON_NAMESPACE_BEGIN +/*! +@brief detail namespace with internal helper functions + +This namespace collects functions that should not be exposed, +implementations of some @ref basic_json methods, and meta-programming helpers. + +@since version 2.1.0 +*/ +namespace detail +{ + +///////////// +// helpers // +///////////// + +// Note to maintainers: +// +// Every trait in this file expects a non-CV-qualified type. +// The only exceptions are in the 'aliases for detected' section +// (i.e., those of the form: decltype(T::member_function(std::declval()))) +// +// In this case, T has to be properly CV-qualified to constraint the function arguments +// (e.g., to_json(BasicJsonType&, const T&)) + +template struct is_basic_json : std::false_type {}; + +NLOHMANN_BASIC_JSON_TPL_DECLARATION +struct is_basic_json : std::true_type {}; + +// used by exceptions create() member functions +// true_type for the pointer to possibly cv-qualified basic_json or std::nullptr_t +// false_type otherwise +template +struct is_basic_json_context : + std::integral_constant < bool, + is_basic_json::type>::type>::value + || std::is_same::value > +{}; + +////////////////////// +// json_ref helpers // +////////////////////// + +template +class json_ref; + +template +struct is_json_ref : std::false_type {}; + +template +struct is_json_ref> : std::true_type {}; + +////////////////////////// +// aliases for detected // +////////////////////////// + +template +using mapped_type_t = typename T::mapped_type; + +template +using key_type_t = typename T::key_type; + +template +using value_type_t = typename T::value_type; + +template +using difference_type_t = typename T::difference_type; + +template +using pointer_t = typename T::pointer; + +template +using reference_t = typename T::reference; + +template +using iterator_category_t = typename T::iterator_category; + +template +using to_json_function = decltype(T::to_json(std::declval()...)); + +template +using from_json_function = decltype(T::from_json(std::declval()...)); + +template +using get_template_function = decltype(std::declval().template get()); + +// trait checking if JSONSerializer::from_json(json const&, udt&) exists +template +struct has_from_json : std::false_type {}; + +// trait checking if j.get is valid +// use this trait instead of std::is_constructible or std::is_convertible, +// both rely on, or make use of implicit conversions, and thus fail when T +// has several constructors/operator= (see https://github.com/nlohmann/json/issues/958) +template +struct is_getable +{ + static constexpr bool value = is_detected::value; +}; + +template +struct has_from_json < BasicJsonType, T, enable_if_t < !is_basic_json::value >> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + +// This trait checks if JSONSerializer::from_json(json const&) exists +// this overload is used for non-default-constructible user-defined-types +template +struct has_non_default_from_json : std::false_type {}; + +template +struct has_non_default_from_json < BasicJsonType, T, enable_if_t < !is_basic_json::value >> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + +// This trait checks if BasicJsonType::json_serializer::to_json exists +// Do not evaluate the trait when T is a basic_json type, to avoid template instantiation infinite recursion. +template +struct has_to_json : std::false_type {}; + +template +struct has_to_json < BasicJsonType, T, enable_if_t < !is_basic_json::value >> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + +template +using detect_key_compare = typename T::key_compare; + +template +struct has_key_compare : std::integral_constant::value> {}; + +// obtains the actual object key comparator +template +struct actual_object_comparator +{ + using object_t = typename BasicJsonType::object_t; + using object_comparator_t = typename BasicJsonType::default_object_comparator_t; + using type = typename std::conditional < has_key_compare::value, + typename object_t::key_compare, object_comparator_t>::type; +}; + +template +using actual_object_comparator_t = typename actual_object_comparator::type; + +///////////////// +// char_traits // +///////////////// + +// Primary template of char_traits calls std char_traits +template +struct char_traits : std::char_traits +{}; + +// Explicitly define char traits for unsigned char since it is not standard +template<> +struct char_traits : std::char_traits +{ + using char_type = unsigned char; + using int_type = uint64_t; + + // Redefine to_int_type function + static int_type to_int_type(char_type c) noexcept + { + return static_cast(c); + } + + static char_type to_char_type(int_type i) noexcept + { + return static_cast(i); + } + + static constexpr int_type eof() noexcept + { + return static_cast(std::char_traits::eof()); + } +}; + +// Explicitly define char traits for signed char since it is not standard +template<> +struct char_traits : std::char_traits +{ + using char_type = signed char; + using int_type = uint64_t; + + // Redefine to_int_type function + static int_type to_int_type(char_type c) noexcept + { + return static_cast(c); + } + + static char_type to_char_type(int_type i) noexcept + { + return static_cast(i); + } + + static constexpr int_type eof() noexcept + { + return static_cast(std::char_traits::eof()); + } +}; + +#if defined(__cpp_lib_byte) && __cpp_lib_byte >= 201603L +template<> +struct char_traits : std::char_traits +{ + using char_type = std::byte; + using int_type = uint64_t; + + static int_type to_int_type(char_type c) noexcept + { + return static_cast(std::to_integer(c)); + } + + static char_type to_char_type(int_type i) noexcept + { + return std::byte(static_cast(i)); + } + + static constexpr int_type eof() noexcept + { + return static_cast(std::char_traits::eof()); + } +}; +#endif + +/////////////////// +// is_ functions // +/////////////////// + +// https://en.cppreference.com/w/cpp/types/conjunction +template struct conjunction : std::true_type { }; +template struct conjunction : B { }; +template +struct conjunction +: std::conditional(B::value), conjunction, B>::type {}; + +// https://en.cppreference.com/w/cpp/types/negation +template struct negation : std::integral_constant < bool, !B::value > { }; + +// Reimplementation of is_constructible and is_default_constructible, due to them being broken for +// std::pair and std::tuple until LWG 2367 fix (see https://cplusplus.github.io/LWG/lwg-defects.html#2367). +// This causes compile errors in e.g., Clang 3.5 or GCC 4.9. +template +struct is_default_constructible : std::is_default_constructible {}; + +template +struct is_default_constructible> + : conjunction, is_default_constructible> {}; + +template +struct is_default_constructible> + : conjunction, is_default_constructible> {}; + +template +struct is_default_constructible> + : conjunction...> {}; + +template +struct is_default_constructible> + : conjunction...> {}; + +template +struct is_constructible : std::is_constructible {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_iterator_traits : std::false_type {}; + +template +struct is_iterator_traits> +{ + private: + using traits = iterator_traits; + + public: + static constexpr auto value = + is_detected::value && + is_detected::value && + is_detected::value && + is_detected::value && + is_detected::value; +}; + +template +struct is_range +{ + private: + using t_ref = typename std::add_lvalue_reference::type; + + using iterator = detected_t; + using sentinel = detected_t; + + // to be 100% correct, it should use https://en.cppreference.com/w/cpp/iterator/input_or_output_iterator + // and https://en.cppreference.com/w/cpp/iterator/sentinel_for + // but reimplementing these would be too much work, as a lot of other concepts are used underneath + static constexpr auto is_iterator_begin = + is_iterator_traits>::value; + + public: + static constexpr bool value = !std::is_same::value && !std::is_same::value && is_iterator_begin; +}; + +template +using iterator_t = enable_if_t::value, result_of_begin())>>; + +template +using range_value_t = value_type_t>>; + +// The following implementation of is_complete_type is taken from +// https://blogs.msdn.microsoft.com/vcblog/2015/12/02/partial-support-for-expression-sfinae-in-vs-2015-update-1/ +// and is written by Xiang Fan who agreed to use it in this library. + +template +struct is_complete_type : std::false_type {}; + +template +struct is_complete_type : std::true_type {}; + +template +struct is_compatible_object_type_impl : std::false_type {}; + +template +struct is_compatible_object_type_impl < + BasicJsonType, CompatibleObjectType, + enable_if_t < is_detected::value&& + is_detected::value >> +{ + using object_t = typename BasicJsonType::object_t; + + // macOS's is_constructible does not play well with nonesuch... + static constexpr bool value = + is_constructible::value && + is_constructible::value; +}; + +template +struct is_compatible_object_type + : is_compatible_object_type_impl {}; + +template +struct is_constructible_object_type_impl : std::false_type {}; + +template +struct is_constructible_object_type_impl < + BasicJsonType, ConstructibleObjectType, + enable_if_t < is_detected::value&& + is_detected::value >> +{ + using object_t = typename BasicJsonType::object_t; + + static constexpr bool value = + (is_default_constructible::value && + (std::is_move_assignable::value || + std::is_copy_assignable::value) && + (is_constructible::value && + std::is_same < + typename object_t::mapped_type, + typename ConstructibleObjectType::mapped_type >::value)) || + (has_from_json::value || + has_non_default_from_json < + BasicJsonType, + typename ConstructibleObjectType::mapped_type >::value); +}; + +template +struct is_constructible_object_type + : is_constructible_object_type_impl {}; + +template +struct is_compatible_string_type +{ + static constexpr auto value = + is_constructible::value; +}; + +template +struct is_constructible_string_type +{ + // launder type through decltype() to fix compilation failure on ICPC +#ifdef __INTEL_COMPILER + using laundered_type = decltype(std::declval()); +#else + using laundered_type = ConstructibleStringType; +#endif + + static constexpr auto value = + conjunction < + is_constructible, + is_detected_exact>::value; +}; + +template +struct is_compatible_array_type_impl : std::false_type {}; + +template +struct is_compatible_array_type_impl < + BasicJsonType, CompatibleArrayType, + enable_if_t < + is_detected::value&& + is_iterator_traits>>::value&& +// special case for types like std::filesystem::path whose iterator's value_type are themselves +// c.f. https://github.com/nlohmann/json/pull/3073 + !std::is_same>::value >> +{ + static constexpr bool value = + is_constructible>::value; +}; + +template +struct is_compatible_array_type + : is_compatible_array_type_impl {}; + +template +struct is_constructible_array_type_impl : std::false_type {}; + +template +struct is_constructible_array_type_impl < + BasicJsonType, ConstructibleArrayType, + enable_if_t::value >> + : std::true_type {}; + +template +struct is_constructible_array_type_impl < + BasicJsonType, ConstructibleArrayType, + enable_if_t < !std::is_same::value&& + !is_compatible_string_type::value&& + is_default_constructible::value&& +(std::is_move_assignable::value || + std::is_copy_assignable::value)&& +is_detected::value&& +is_iterator_traits>>::value&& +is_detected::value&& +// special case for types like std::filesystem::path whose iterator's value_type are themselves +// c.f. https://github.com/nlohmann/json/pull/3073 +!std::is_same>::value&& +is_complete_type < +detected_t>::value >> +{ + using value_type = range_value_t; + + static constexpr bool value = + std::is_same::value || + has_from_json::value || + has_non_default_from_json < + BasicJsonType, + value_type >::value; +}; + +template +struct is_constructible_array_type + : is_constructible_array_type_impl {}; + +template +struct is_compatible_integer_type_impl : std::false_type {}; + +template +struct is_compatible_integer_type_impl < + RealIntegerType, CompatibleNumberIntegerType, + enable_if_t < std::is_integral::value&& + std::is_integral::value&& + !std::is_same::value >> +{ + // is there an assert somewhere on overflows? + using RealLimits = std::numeric_limits; + using CompatibleLimits = std::numeric_limits; + + static constexpr auto value = + is_constructible::value && + CompatibleLimits::is_integer && + RealLimits::is_signed == CompatibleLimits::is_signed; +}; + +template +struct is_compatible_integer_type + : is_compatible_integer_type_impl {}; + +template +struct is_compatible_type_impl: std::false_type {}; + +template +struct is_compatible_type_impl < + BasicJsonType, CompatibleType, + enable_if_t::value >> +{ + static constexpr bool value = + has_to_json::value; +}; + +template +struct is_compatible_type + : is_compatible_type_impl {}; + +template +struct is_constructible_tuple : std::false_type {}; + +template +struct is_constructible_tuple> : conjunction...> {}; + +template +struct is_json_iterator_of : std::false_type {}; + +template +struct is_json_iterator_of : std::true_type {}; + +template +struct is_json_iterator_of : std::true_type +{}; + +// checks if a given type T is a template specialization of Primary +template