gavin/VideoPipe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first commit

Browse Source
This commit is contained in:
陈赣
2026-06-03 12:43:14 +08:00
commit ba76cfae28
608 changed files with 120791 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
utils/logger/README.md Normal file
View File

@@ -0,0 +1,69 @@
## summary ##
-----------
### log format ###
```
[time][level][thread_id][code location] log_content
| auto-generated(support config) | | content |
```
### log example ###
```
[2022-10-21 09:47:36.835][Debug][7fff43722700][../nodes/vp_node.cpp:179] [screen_des_a] after meta flow, in_queue.size()==>12
```
### tips ###
better to add important field using `[]` in log content `Manually`, such as `module`, `type`. below code add name of host node(module) and task(type) in log content.
```
VP_INFO(vp_utils::string_format("[%s] [record] save dir not exists, now creating save dir: `%s`", host_node_name, save_dir));
```
### log api ###
#### log config ####
```c++
// log level
VP_SET_LOG_LEVEL(_log_level);
// log file dir
VP_SET_LOG_DIR(_log_dir);
// log kafka servers and topic
VP_SET_LOG_KAFKA_SERVERS_AND_TOPIC(_kafka_servers_and_topic);
// log to console or not
VP_SET_LOG_TO_CONSOLE(_log_to_console);
// log to file or not
VP_SET_LOG_TO_FILE(_log_to_file);
// log to kafka or not
VP_SET_LOG_TO_KAFKA(_log_to_kafka);
// include log level or not
VP_SET_LOG_INCLUDE_LEVEL(_include_level);
// include code location or not (where the log occurs)
VP_SET_LOG_INCLUDE_CODE_LOCATION(_include_code_location);
// include thread id or not (std::this_thread::get_id())
VP_SET_LOG_INCLUDE_THREAD_ID(_include_thread_id);
// warn if log cache in memory exceed this value
VP_SET_LOG_CACHE_WARN_THRES(_log_cache_warn_threshold);
```
#### write log ####
4 types of log
```c++
// error
VP_ERROR(message);
// warn
VP_WARN(message);
// info
VP_INFO(message);
// debug
VP_DEBUG(message);
```
```c++
// important! call at the begining of main()
VP_LOGGER_INIT();
```

75
utils/logger/vp_log_file_writer.cpp Normal file
View File

@@ -0,0 +1,75 @@
#include "vp_log_file_writer.h"
namespace vp_utils {
vp_log_file_writer::vp_log_file_writer() {
}
vp_log_file_writer::~vp_log_file_writer() {
if (log_writer.is_open()) {
log_writer.close();
}
}
void vp_log_file_writer::init(std::string log_dir, std::string log_file_name_template) {
this->log_dir = log_dir;
this->log_file_name_template = log_file_name_template;
// open log file first time
auto f = create_valid_log_file_name();
log_writer.open(f, std::ofstream::out | std::ofstream::app);
inited = true;
}
void vp_log_file_writer::write(std::string log) {
if (!inited) {
throw "vp_log_file_writer not initialized!";
}
// check if need create new log file
if (get_now_day() != log_day) {
if (log_writer.is_open()) {
log_writer.close();
}
auto f = create_valid_log_file_name();
log_writer.open(f, std::ofstream::out | std::ofstream::app);
}
log_writer << log << std::endl;
}
std::string vp_log_file_writer::create_valid_log_file_name() {
if (!std::experimental::filesystem::exists(log_dir)) {
std::experimental::filesystem::create_directories(log_dir);
}
std::experimental::filesystem::path root_dir(log_dir);
auto f_name = vp_utils::time_format(NOW, log_file_name_template);
auto p = root_dir / f_name;
// cache log start day
log_day = get_now_day();
return p.string();
}
int vp_log_file_writer::get_now_day() {
std::vector<int> time_parts;
vp_utils::time_split(NOW, time_parts);
// refer to vp_utils::time_split(...), indice 2 is day
return time_parts[2];
}
vp_log_file_writer& vp_log_file_writer::operator<<(std::string log) {
write(log);
return *this;
}
}

49
utils/logger/vp_log_file_writer.h Normal file
View File

@@ -0,0 +1,49 @@
#pragma once
#include <string>
#include <fstream>
#include <chrono>
#include <experimental/filesystem>
#include "../vp_utils.h"
namespace vp_utils {
// log file writer, write log to local file. auto create new log file bye date.
// NOT thread safe, MUST use in single thread.
class vp_log_file_writer
{
private:
// ready to go
bool inited = false;
// root dir for log saving
std::string log_dir;
// log file name template, such as `<year>-<mon>-<day>.txt`, refer to vp_utils::time_format(...) for rule details
std::string log_file_name_template;
// file handle
std::ofstream log_writer;
// get valid log file name including path, name and extension
std::string create_valid_log_file_name();
// current log day (1 ~ 31)
int log_day = 0;
// get day of now
int get_now_day();
#define NOW std::chrono::system_clock::now()
public:
vp_log_file_writer();
~vp_log_file_writer();
// write log
void write(std::string log);
// initialize writer
void init(std::string log_dir, std::string log_file_name_template);
// for << operator
vp_log_file_writer& operator<<(std::string log);
};
}

32
utils/logger/vp_log_kafka_writer.cpp Normal file
View File

@@ -0,0 +1,32 @@
#ifdef VP_WITH_KAFKA
#include "vp_log_kafka_writer.h"
namespace vp_utils {
vp_log_kafka_writer::vp_log_kafka_writer(/* args */) {
}
vp_log_kafka_writer::~vp_log_kafka_writer() {
}
void vp_log_kafka_writer::write(std::string log) {
if (!inited) {
throw "vp_log_kafka_writer not initialized!";
}
kafka_producer->pushMessage(log);
}
void vp_log_kafka_writer::init(std::string kafka_servers, std::string topic_name) {
kafka_producer = std::make_shared<KafkaProducer>(kafka_servers, topic_name, 0);
inited = true;
}
// for << operator
vp_log_kafka_writer& vp_log_kafka_writer::operator<<(std::string log) {
write(log);
return *this;
}
}
#endif

31
utils/logger/vp_log_kafka_writer.h Normal file
View File

@@ -0,0 +1,31 @@
#pragma once
#ifdef VP_WITH_KAFKA
#include <memory>
// to-do: refactor for file structure
#include "../../nodes/broker/kafka_utils/KafkaProducer.h"
namespace vp_utils {
class vp_log_kafka_writer
{
private:
// ready to go
bool inited = false;
// wrapper producer
std::shared_ptr<KafkaProducer> kafka_producer = nullptr;
public:
vp_log_kafka_writer(/* args */);
~vp_log_kafka_writer();
// write log
void write(std::string log);
// initialize writer
void init(std::string kafka_servers, std::string topic_name);
// for << operator
vp_log_kafka_writer& operator<<(std::string log);
};
}
#endif

161
utils/logger/vp_logger.cpp Normal file
View File

@@ -0,0 +1,161 @@
#include "vp_logger.h"
namespace vp_utils {
vp_logger::vp_logger(/* args */)
{
}
vp_logger::~vp_logger() {
die();
if (log_writer_th.joinable()) {
log_writer_th.join();
}
}
void vp_logger::die() {
alive = false;
std::lock_guard<std::mutex> guard(log_cache_mutex);
log_cache.push("die");
log_cache_semaphore.signal();
}
void vp_logger::init() {
inited = true;
// initialize file writer
file_writer.init(log_dir, log_file_name_template);
#ifdef VP_WITH_KAFKA
// initialize kafka writer
auto servers_and_topic = vp_utils::string_split(kafka_servers_and_topic, '/');
assert(servers_and_topic.size() == 2);
kafka_writer.init(servers_and_topic[0], servers_and_topic[1]);
#endif
// run thread
auto t = std::thread(&vp_logger::log_write_run, this);
log_writer_th = std::move(t);
}
void vp_logger::log(vp_log_level level, const std::string& message, const char* code_file, int code_line) {
// make sure logger is initialized
if (!inited) {
throw "vp_logger is not initialized yet!";
}
// level filter
if (level > log_level) {
return;
}
// keywords filter for debug level
if (level == vp_log_level::DEBUG && keywords_for_debug_log.size() != 0) {
bool filterd = true;
for(auto& keywords: keywords_for_debug_log) {
if (message.find(keywords) != std::string::npos) {
filterd = false;
break;
}
}
if (filterd) {
return;
}
}
/* create log */
std::string new_log = "";
// 100% true for log time
if (include_time) {
new_log += vp_utils::time_format(NOW, log_time_templete);
}
// log level
if (include_level) {
new_log += "[" + log_level_names.at(level) + "]";
}
// thread id
if (include_thread_id) {
auto id = std::this_thread::get_id();
std::stringstream ss;
ss << std::hex << id; // to hex
auto thread_id = ss.str();
new_log += "[" + thread_id + "]";
}
// code location
if (include_code_location) {
new_log += "[" + std::string(code_file) + ":" + std::to_string(code_line) + "]";
}
new_log += " " + message;
/* write to cache */
// min lock range
std::lock_guard<std::mutex> guard(log_cache_mutex);
log_cache.push(new_log);
// notify
log_cache_semaphore.signal();
}
void vp_logger::log_write_run() {
bool log_thres_warned = false;
/* below code runs in single thread */
while (inited && alive) {
// wait for data
log_cache_semaphore.wait();
auto log = log_cache.front();
log_cache.pop();
if (log == "die") {
continue;
}
/* watch the log cache size */
auto log_cache_size = 0;
{
// min lock range
std::lock_guard<std::mutex> guard(log_cache_mutex);
log_cache_size = log_cache.size();
}
if (!log_thres_warned && log_cache_size > log_cache_warn_threshold) {
VP_WARN(vp_utils::string_format("[logger] log cache size is exceeding threshold! cache size is: [%d], threshold is: [%d]", log_cache_size, log_cache_warn_threshold));
log_thres_warned = true; // warn 1 time
}
if (log_cache_size <= log_cache_warn_threshold) {
log_thres_warned = false;
}
/* write to devices */
if (log_to_console) {
write_to_console(log);
}
if (log_to_file) {
write_to_file(log);
}
if (log_to_kafka) {
write_to_kafka(log);
}
}
}
void vp_logger::write_to_console(const std::string& log) {
std::cout << log << std::endl;
}
void vp_logger::write_to_file(const std::string& log) {
// file_writer.write(log);
file_writer << log;
}
void vp_logger::write_to_kafka(const std::string& log) {
#ifdef VP_WITH_KAFKA
// kafka_writer.write(log);
kafka_writer << log;
#endif
}
}

139
utils/logger/vp_logger.h Normal file
View File

@@ -0,0 +1,139 @@
#pragma once
#include <string>
#include <thread>
#include <queue>
#include <mutex>
#include <chrono>
#include <assert.h>
#include <map>
#include "../vp_semaphore.h"
#include "../vp_utils.h"
#include "vp_log_file_writer.h"
#include "vp_log_kafka_writer.h"
namespace vp_utils {
// log levels
enum vp_log_level {
ERROR = 1,
WARN = 2,
INFO = 3,
DEBUG = 4
};
// a lightweight logger for VideoPipe, architecture: N producer * 1 consumer.
// 1. support 3 types of devices (console, file, kafka)
// 2. multithread safe
// 3. use Macros directly
class vp_logger
{
private:
std::queue<std::string> log_cache;
vp_utils::vp_semaphore log_cache_semaphore;
std::mutex log_cache_mutex;
std::thread log_writer_th;
// initialized or not
bool inited = false;
bool alive = true;
void die();
vp_logger(/* args */);
// write to devices
void write_to_console(const std::string& log);
void write_to_file(const std::string& log);
void write_to_kafka(const std::string& log);
// writing log func
void log_write_run();
// file writer
vp_utils::vp_log_file_writer file_writer;
#ifdef VP_WITH_KAFKA
// kafka writer
vp_utils::vp_log_kafka_writer kafka_writer;
#endif
const std::map<vp_log_level, std::string> log_level_names = {{ERROR, "Error"},
{WARN, "Warn "},
{INFO, "Info "},
{DEBUG, "Debug"}};
public:
// non-copable
vp_logger(const vp_logger&) = delete;
vp_logger& operator=(const vp_logger&) = delete;
// singleton
static vp_logger& get_logger() {
static vp_logger logger;
return logger;
}
~vp_logger();
// CONFIG
vp_log_level log_level = vp_log_level::DEBUG; // filter
std::string log_dir = "./log"; // folder saving log file
std::string kafka_servers_and_topic = "127.0.0.1:9092/vp_log"; // kafka servers and topic, splited by `/`. multiple servers splited by `,`
const std::string log_file_name_template = "<year>-<mon>-<day>.txt";
const std::string log_time_templete = "[<year>-<mon>-<day> <hour>:<min>:<sec>.<mili>]";
/*
* keywords as filters for the log of debug level since it always writes too frequently, empty means no filters.
* use the name of a node is a normal method to apply this function,
* `file_src_0` means write all debug log generated by the node named file_src_0.
*/
std::vector<std::string> keywords_for_debug_log = {};
// watch
int log_cache_warn_threshold = 100; // warning if cache size greater than threshold
// where
bool log_to_console = true; // to console
bool log_to_file = true; // to file
bool log_to_kafka = false; // to kafka
// how
const bool include_time = true; // `log time` part in log content (readonly)
bool include_level = true; // `log level` part in log content
bool include_code_location = true; // `log from` part in log content
bool include_thread_id = true; // 'thread id' part in log content
// END of CONFIG
// better Never call directly
void log(vp_log_level level, const std::string& message, const char* code_file, int code_line);
// init for vp_logger, ready to go
void init();
};
// config Macros
#define VP_SET_LOG_LEVEL(_log_level) vp_utils::vp_logger::get_logger().log_level = _log_level
#define VP_SET_LOG_DIR(_log_dir) vp_utils::vp_logger::get_logger().log_dir = _log_dir
#define VP_SET_LOG_KAFKA_SERVERS_AND_TOPIC(_kafka_servers_and_topic) vp_utils::vp_logger::get_logger().kafka_servers_and_topic = _kafka_servers_and_topic
#define VP_SET_LOG_TO_CONSOLE(_log_to_console) vp_utils::vp_logger::get_logger().log_to_console = _log_to_console
#define VP_SET_LOG_TO_FILE(_log_to_file) vp_utils::vp_logger::get_logger().log_to_console = _log_to_file
#define VP_SET_LOG_TO_KAFKA(_log_to_kafka) vp_utils::vp_logger::get_logger().log_to_kafka = _log_to_kafka
#define VP_SET_LOG_INCLUDE_LEVEL(_include_level) vp_utils::vp_logger::get_logger().include_level = _include_level
#define VP_SET_LOG_INCLUDE_CODE_LOCATION(_include_code_location) vp_utils::vp_logger::get_logger().include_code_location = _include_code_location
#define VP_SET_LOG_INCLUDE_THREAD_ID(_include_thread_id) vp_utils::vp_logger::get_logger().include_thread_id = _include_thread_id
#define VP_SET_LOG_CACHE_WARN_THRES(_log_cache_warn_threshold) vp_utils::vp_logger::get_logger().log_cache_warn_threshold = _log_cache_warn_threshold
#define VP_SET_LOG_KEYWORDS_FOR_DEBUG(_keywords_for_debug_log) vp_utils::vp_logger::get_logger().keywords_for_debug_log = _keywords_for_debug_log
// log Macros
// use vp_utils::string_format to format log content first if need
// example:
// VP_ERROR(vp_utils::string_format("message is %s at %d", s, d));
#define VP_ERROR(message) vp_utils::vp_logger::get_logger().log(vp_utils::vp_log_level::ERROR, message, __FILE__, __LINE__)
#define VP_WARN(message) vp_utils::vp_logger::get_logger().log(vp_utils::vp_log_level::WARN, message, __FILE__, __LINE__)
#define VP_INFO(message) vp_utils::vp_logger::get_logger().log(vp_utils::vp_log_level::INFO, message, __FILE__, __LINE__)
#define VP_DEBUG(message) vp_utils::vp_logger::get_logger().log(vp_utils::vp_log_level::DEBUG, message, __FILE__, __LINE__)
// init Macros
#define VP_LOGGER_INIT() vp_utils::vp_logger::get_logger().init()
#define NOW std::chrono::system_clock::now()
}