gavin/VideoPipe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first commit

Browse Source
This commit is contained in:
陈赣
2026-06-03 12:43:14 +08:00
commit ba76cfae28
608 changed files with 120791 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
nodes/broker/cereal_archive/vp_objects_cereal_archive.h Normal file
View File

@@ -0,0 +1,100 @@
#pragma once
/*
* define EXTERNAL archive functions for objects which need to be serialized by cereal library in VideoPipe.
* refer to `https://uscilab.github.io/cereal/serialization_functions.html` for more details.
*/
// object types
#include "../../../objects/vp_frame_target.h"
#include "../../../objects/vp_frame_face_target.h"
#include "../../../objects/vp_frame_text_target.h"
#include "../../../objects/vp_frame_pose_target.h"
#include "../../../objects/vp_sub_target.h"
/* extend for more types of objects in VideoPipe. */
// headers from cereal
#include "../../../third_party/cereal/cereal.hpp"
#include "../../../third_party/cereal/types/vector.hpp"
#include "../../../third_party/cereal/types/memory.hpp"
#include "../../../third_party/cereal/types/string.hpp"
#include "../../../third_party/cereal/types/utility.hpp"
#include "../../../third_party/cereal/archives/json.hpp"
#include "../../../third_party/cereal/archives/xml.hpp"
/* same namespace as object types */
namespace vp_objects {
/* vp_frame_target */
template<typename Archive>
void serialize(Archive& archive, vp_frame_target& target) {
// define the form of structured data for vp_frame_target
archive(cereal::make_nvp("x", target.x),
cereal::make_nvp("y", target.y),
cereal::make_nvp("width", target.width),
cereal::make_nvp("height", target.height),
cereal::make_nvp("primary_class_id", target.primary_class_id),
cereal::make_nvp("primary_score", target.primary_score),
cereal::make_nvp("primary_label", target.primary_label),
cereal::make_nvp("channel_index", target.channel_index),
cereal::make_nvp("frame_index", target.frame_index),
cereal::make_nvp("track_id", target.track_id),
cereal::make_nvp("secondary_class_ids", target.secondary_class_ids),
cereal::make_nvp("secondary_scores", target.secondary_scores),
cereal::make_nvp("secondary_labels", target.secondary_labels),
cereal::make_nvp("sub_targets", target.sub_targets),
cereal::make_nvp("embeddings", target.embeddings));
}
template<typename Archive>
void serialize(Archive& archive, vp_sub_target& target) {
// define the form of structured data for vp_sub_target
archive(cereal::make_nvp("x", target.x),
cereal::make_nvp("y", target.y),
cereal::make_nvp("width", target.width),
cereal::make_nvp("height", target.height),
cereal::make_nvp("class_id", target.class_id),
cereal::make_nvp("score", target.score),
cereal::make_nvp("label", target.label),
cereal::make_nvp("frame_index", target.frame_index),
cereal::make_nvp("channel_index", target.channel_index),
cereal::make_nvp("attachments", target.attachments));
}
/* END OF vp_frame_target */
/* vp_frame_face_target */
template<typename Archive>
void serialize(Archive& archive, vp_frame_face_target& target) {
// define the form of structured data for vp_frame_face_target
archive(cereal::make_nvp("x", target.x),
cereal::make_nvp("y", target.y),
cereal::make_nvp("width", target.width),
cereal::make_nvp("height", target.height),
cereal::make_nvp("score", target.score),
cereal::make_nvp("embeddings", target.embeddings),
cereal::make_nvp("key_points", target.key_points),
cereal::make_nvp("track_id", target.track_id));
}
/* END OF vp_frame_face_target */
/* vp_frame_text_target */
template<typename Archive>
void serialize(Archive& archive, vp_frame_text_target& target) {
// define the form of structured data for vp_frame_text_target
archive(cereal::make_nvp("text", target.text),
cereal::make_nvp("score", target.score),
cereal::make_nvp("region", target.region_vertexes),
cereal::make_nvp("flags", target.flags));
}
/* END OF vp_frame_text_target */
/* vp_frame_pose_target */
template<typename Archive>
void serialize(Archive& archive, vp_frame_pose_target& target) {
// define the form of structured data for vp_frame_pose_target
}
/* END OF vp_frame_pose_target */
}

167
nodes/broker/kafka_utils/KafkaProducer.cpp Normal file
View File

@@ -0,0 +1,167 @@
#ifdef VP_WITH_KAFKA
#include "KafkaProducer.h"
// callbacks
class ProducerDeliveryReportCb : public RdKafka::DeliveryReportCb {
public:
void dr_cb(RdKafka::Message &message) {
if (message.err())
std::cerr << "Message delivery failed: " << message.errstr() << std::endl;
else {
// Message delivered to topic test [0] at offset 135000
/*
std::cerr << "Message delivered to topic " << message.topic_name()
<< " [" << message.partition() << "] at offset "
<< message.offset() << std::endl;*/
}
}
};
class ProducerEventCb : public RdKafka::EventCb {
public:
void event_cb(RdKafka::Event &event) {
switch (event.type()) {
case RdKafka::Event::EVENT_ERROR:
std::cout << "RdKafka::Event::EVENT_ERROR: " << RdKafka::err2str(event.err()) << std::endl;
break;
case RdKafka::Event::EVENT_STATS:
//std::cout << "RdKafka::Event::EVENT_STATS: " << event.str() << std::endl;
break;
case RdKafka::Event::EVENT_LOG:
//std::cout << "RdKafka::Event::EVENT_LOG " << event.fac() << std::endl;
break;
case RdKafka::Event::EVENT_THROTTLE:
//std::cout << "RdKafka::Event::EVENT_THROTTLE " << event.broker_name() << std::endl;
break;
}
}
};
class HashPartitionerCb : public RdKafka::PartitionerCb {
public:
int32_t partitioner_cb(const RdKafka::Topic *topic, const std::string *key,
int32_t partition_cnt, void *msg_opaque) {
char msg[128] = { 0 };
int32_t partition_id = generate_hash(key->c_str(), key->size()) % partition_cnt;
// [topic][key][partition_cnt][partition_id]
// :[test][6419][2][1]
/*sprintf(msg, "HashPartitionerCb:topic:[%s], key:[%s]partition_cnt:[%d], partition_id:[%d]", topic->name().c_str(),
key->c_str(), partition_cnt, partition_id);
std::cout << msg << std::endl;*/
return partition_id;
}
private:
static inline unsigned int generate_hash(const char *str, size_t len) {
unsigned int hash = 5381;
for (size_t i = 0; i < len; i++)
hash = ((hash << 5) + hash) + str[i];
return hash;
}
};
KafkaProducer::KafkaProducer(const std::string& brokers, const std::string& topic, int partition) {
m_brokers = brokers;
m_topicStr = topic;
m_partition = partition;
m_config = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL);
if(m_config==NULL)
std::cout << "Create RdKafka Conf failed." << std::endl;
m_topicConfig = RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC);
if (m_topicConfig == NULL)
std::cout << "Create RdKafka Topic Conf failed." << std::endl;
RdKafka::Conf::ConfResult errCode;
std::string errorStr;
m_dr_cb = new ProducerDeliveryReportCb;
errCode = m_config->set("dr_cb", m_dr_cb, errorStr);
if (errCode != RdKafka::Conf::CONF_OK)
{
std::cout << "Conf set failed:" << errorStr << std::endl;
}
m_event_cb = new ProducerEventCb;
errCode = m_config->set("event_cb", m_event_cb, errorStr);
if (errCode != RdKafka::Conf::CONF_OK)
{
std::cout << "Conf set failed:" << errorStr << std::endl;
}
m_partitioner_cb = new HashPartitionerCb;
errCode = m_topicConfig->set("partitioner_cb", m_partitioner_cb, errorStr);
if (errCode != RdKafka::Conf::CONF_OK)
{
std::cout << "Conf set failed:" << errorStr << std::endl;
}
errCode = m_config->set("statistics.interval.ms", "10000", errorStr);
if (errCode != RdKafka::Conf::CONF_OK)
{
std::cout << "Conf set failed:" << errorStr << std::endl;
}
errCode = m_config->set("message.max.bytes", "10240000", errorStr);
if (errCode != RdKafka::Conf::CONF_OK)
{
std::cout << "Conf set failed:" << errorStr << std::endl;
}
errCode = m_config->set("bootstrap.servers", m_brokers, errorStr);
if (errCode != RdKafka::Conf::CONF_OK)
{
std::cout << "Conf set failed:" << errorStr << std::endl;
}
m_producer = RdKafka::Producer::create(m_config, errorStr);
if (m_producer == NULL)
{
std::cout << "Create Producer failed:" << errorStr << std::endl;
}
m_topic = RdKafka::Topic::create(m_producer, m_topicStr, m_topicConfig, errorStr);
if (m_topic == NULL)
{
std::cout << "Create Topic failed:" << errorStr << std::endl;
}
}
KafkaProducer::~KafkaProducer() {
while (m_producer->outq_len() > 0) {
std::cerr << "Waiting for " << m_producer->outq_len() << std::endl;
m_producer->flush(5000);
}
delete m_config;
delete m_topicConfig;
delete m_topic;
delete m_producer;
delete m_dr_cb;
delete m_event_cb;
delete m_partitioner_cb;
}
void KafkaProducer::pushMessage(const std::string& str) {
int32_t len = str.length();
void* payload = const_cast<void*>(static_cast<const void*>(str.data()));
RdKafka::ErrorCode errorCode = m_producer->produce(
m_topic,
RdKafka::Topic::PARTITION_UA,
RdKafka::Producer::RK_MSG_COPY,
payload,
len,
NULL,
NULL);
m_producer->poll(0);
if (errorCode != RdKafka::ERR_NO_ERROR) {
std::cerr << "Produce failed: " << RdKafka::err2str(errorCode) << std::endl;
if (errorCode == RdKafka::ERR__QUEUE_FULL) {
m_producer->poll(100);
}
}
}
#endif

35
nodes/broker/kafka_utils/KafkaProducer.h Normal file
View File

@@ -0,0 +1,35 @@
#pragma once
#ifdef VP_WITH_KAFKA
#include <string>
#include <iostream>
// compile tips:
// run `apt-get install librdkafka-dev`, v0.11.3 for ubuntu 18.04 by default.
#include <librdkafka/rdkafkacpp.h>
// wrapper class for Producer in librdkafka
class KafkaProducer
{
public:
explicit KafkaProducer(const std::string& brokers, const std::string& topic, int partition);
void pushMessage(const std::string& str);
~KafkaProducer();
private:
std::string m_brokers;
std::string m_topicStr;
int m_partition;
RdKafka::Conf* m_config;
RdKafka::Conf* m_topicConfig;
RdKafka::Topic* m_topic;
RdKafka::Producer* m_producer;
RdKafka::DeliveryReportCb* m_dr_cb;
RdKafka::EventCb* m_event_cb;
RdKafka::PartitionerCb* m_partitioner_cb;
};
#endif

130
nodes/broker/vp_ba_socket_broker_node.cpp Normal file
View File

@@ -0,0 +1,130 @@
#include "vp_ba_socket_broker_node.h"
namespace vp_nodes {
vp_ba_socket_broker_node::vp_ba_socket_broker_node(std::string node_name,
std::string des_ip,
int des_port,
vp_broke_for broke_for,
int broking_cache_warn_threshold,
int broking_cache_ignore_threshold):
vp_msg_broker_node(node_name, broke_for, broking_cache_warn_threshold, broking_cache_ignore_threshold),
des_ip(des_ip),
des_port(des_port) {
// only for vp_frame_target since BA logic ONLY works on vp_frame_target
assert(broke_for == vp_broke_for::NORMAL);
udp_writer = kissnet::udp_socket(kissnet::endpoint(des_ip, des_port));
VP_INFO(vp_utils::string_format("[%s] [message broker] set des_ip as `%s` and des_port as [%d]", node_name.c_str(), des_ip.c_str(), des_port));
this->initialized();
}
vp_ba_socket_broker_node::~vp_ba_socket_broker_node() {
deinitialized();
stop_broking();
}
void vp_ba_socket_broker_node::format_msg(const std::shared_ptr<vp_objects::vp_frame_meta>& meta, std::string& msg) {
/* format:
line 0, <--
line 1, time
line 2, channel_index, frame_index
line 3, ba_type, ba_label
line 4, ids of involve targets (target_id_1, target_id_2,...)
line 5, vertexs of involve region (x1,y1,x2,y2,...)
line 6, property labels split by '|' and split by ',' between different involve targets
line 7, record image path
line 8, record video path
line 9, -->
line 10, <--
line 11, time
line 12, channel_index, frame_index
line 13, ba_type, ba_label
line 14, ids of involve targets (target_id_1, target_id_2,...)
line 15, vertexs of involve region (x1,y1,x2,y2,...)
line 16, property labels split by '|' and split by ',' between different involve targets
line 17, record image path
linr 18, record video path
line 19, -->
line 20, ...
*/
std::stringstream msg_stream;
auto format_basic_info = [&](int channel_index, int frame_index) {
msg_stream << vp_utils::time_format(NOW) << std::endl; // line1
msg_stream << channel_index << "," << frame_index << std::endl; // line2
};
auto format_ba_info = [&](vp_objects::vp_ba_type ba_type,
std::string ba_label,
const std::vector<int>& involve_target_ids,
const std::vector<vp_objects::vp_point>& involve_region_vertexs) {
msg_stream << int(ba_type) << "," << ba_label << std::endl; // line3
for (int i = 0; i < involve_target_ids.size(); i++) {
msg_stream << involve_target_ids[i]; // line 4
if (i != involve_target_ids.size() - 1) {
msg_stream << ",";
}
}
msg_stream << std::endl;
for (int i = 0; i < involve_region_vertexs.size(); i++) {
msg_stream << involve_region_vertexs[i].x << "," << involve_region_vertexs[i].y; // line 5
if (i != involve_region_vertexs.size() - 1) {
msg_stream << ",";
}
}
msg_stream << std::endl;
auto targets = meta->get_targets_by_ids(involve_target_ids);
for (int i = 0; i < targets.size(); i++) {
auto t = targets[i];
for (int j = 0; j < t->secondary_labels.size(); j++) {
msg_stream << t->secondary_labels[j]; // line 6
if (j != t->secondary_labels.size() - 1) {
msg_stream << "|";
}
}
if (i != targets.size() - 1) {
msg_stream << ",";
}
}
msg_stream << std::endl;
};
auto format_record_info = [&](const std::string& record_image_name, const std::string& record_video_name) {
msg_stream << record_image_name << std::endl; // line7
msg_stream << record_video_name << std::endl; // line8
};
if (broke_for == vp_broke_for::NORMAL) {
for (int i = 0; i < meta->ba_results.size(); i++) {
auto& ba = meta->ba_results[i];
// start flag
msg_stream << "<--" << std::endl;
// basic info
format_basic_info(meta->channel_index, meta->frame_index);
//ba info
format_ba_info(ba->type, ba->ba_label, ba->involve_target_ids_in_frame, ba->involve_region_in_frame);
// record info
format_record_info(ba->record_image_name, ba->record_video_name);
// end flag
msg_stream << "-->";
if (i != meta->ba_results.size() - 1) {
msg_stream << std::endl; // not the last one
}
}
}
msg = msg_stream.str();
}
void vp_ba_socket_broker_node::broke_msg(const std::string& msg) {
// broke msg to socket by udp
auto bytes_2_send = reinterpret_cast<const std::byte*>(msg.c_str());
auto bytes_2_send_len = msg.size();
udp_writer.send(bytes_2_send, bytes_2_send_len);
}
}

37
nodes/broker/vp_ba_socket_broker_node.h Normal file
View File

@@ -0,0 +1,37 @@
#pragma once
#include "vp_msg_broker_node.h"
#include "../../objects/ba/vp_ba_result.h"
#include "cereal_archive/vp_objects_cereal_archive.h"
// light weight socket support
#include "../../third_party/kissnet/kissnet.hpp"
namespace vp_nodes {
// message broker node, broke BA results (ONLY for vp_frame_target) to socket via udp.
// BA results could be used for archive.
class vp_ba_socket_broker_node: public vp_msg_broker_node
{
private:
// host the data sent to via udp
std::string des_ip = "";
// port the data sent to via udp
int des_port = 0;
// udp socket writer
kissnet::udp_socket udp_writer;
protected:
// to custom format
virtual void format_msg(const std::shared_ptr<vp_objects::vp_frame_meta>& meta, std::string& msg) override;
// to socket via udp
virtual void broke_msg(const std::string& msg) override;
public:
vp_ba_socket_broker_node(std::string node_name,
std::string des_ip = "",
int des_port = 0,
vp_broke_for broke_for = vp_broke_for::NORMAL,
int broking_cache_warn_threshold = 50,
int broking_cache_ignore_threshold = 200);
~vp_ba_socket_broker_node();
};
}

137
nodes/broker/vp_embeddings_properties_socket_broker_node.cpp Normal file
View File

@@ -0,0 +1,137 @@
#include "vp_embeddings_properties_socket_broker_node.h"
namespace vp_nodes {
vp_embeddings_properties_socket_broker_node::vp_embeddings_properties_socket_broker_node(std::string node_name,
std::string des_ip,
int des_port,
std::string cropped_dir,
int min_crop_width,
int min_crop_height,
vp_broke_for broke_for,
bool only_for_tracked,
int broking_cache_warn_threshold,
int broking_cache_ignore_threshold):
vp_msg_broker_node(node_name, broke_for, broking_cache_warn_threshold, broking_cache_ignore_threshold),
des_ip(des_ip),
des_port(des_port),
cropped_dir(cropped_dir),
min_crop_width(min_crop_width),
min_crop_height(min_crop_height),
only_for_tracked(only_for_tracked) {
// only for vp_frame_target
assert(broke_for == vp_broke_for::NORMAL);
udp_writer = kissnet::udp_socket(kissnet::endpoint(des_ip, des_port));
VP_INFO(vp_utils::string_format("[%s] [message broker] set des_ip as `%s` and des_port as [%d]", node_name.c_str(), des_ip.c_str(), des_port));
this->initialized();
}
vp_embeddings_properties_socket_broker_node::~vp_embeddings_properties_socket_broker_node() {
deinitialized();
stop_broking();
}
void vp_embeddings_properties_socket_broker_node::format_msg(const std::shared_ptr<vp_objects::vp_frame_meta>& meta, std::string& msg) {
/* format:
line 0, <--
line 1, 1st cropped image's path
line 2, 1st properties, multi property labels splited by ','
line 3, 1st sub target info, empty line if sub target detected (sub target may be vehicle plate)
line 4, 1st embeddings
line 5, -->
line 6, <--
line 7, 2nd cropped image's path
line 8, 2nd properties, multi property labels splited by ','
line 9, 2nd sub target info, empty line if sub target detected (sub target may be vehicle plate)
line 10, 2nd embeddings
line 11, -->
line 12, ...
*/
auto& broked = all_broked[meta->channel_index];
// remove 50 elements every 100 ids
if (broked.size() > 100) {
broked.erase(broked.begin(), broked.begin() + 50);
}
std::stringstream msg_stream;
auto format_embeddings = [&](const std::vector<float>& embeddings) {
for (int i = 0; i < embeddings.size(); i++) {
msg_stream << embeddings[i];
if (i != embeddings.size() - 1) {
msg_stream << ",";
}
}
msg_stream << std::endl;
};
auto format_sub_target = [&](const std::string& sub_target_info) {
msg_stream << sub_target_info << std::endl;
};
auto format_properties = [&](const std::vector<std::string>& secondary_labels) {
for (int i = 0; i < secondary_labels.size(); i++) {
msg_stream << secondary_labels[i];
if (i != secondary_labels.size() - 1) {
msg_stream << ",";
}
}
msg_stream << std::endl;
};
auto save_cropped_image = [&](cv::Mat& frame, cv::Rect rect, std::string name) {
auto cropped = frame(rect);
cv::imwrite(name, cropped);
msg_stream << name << std::endl;
};
if (broke_for == vp_broke_for::NORMAL) {
for (int i = 0; i < meta->targets.size(); i++) {
auto& t = meta->targets[i];
// only broke for tracked targets and have enough frames
if ((only_for_tracked && t->track_id < 0) || (only_for_tracked && t->tracks.size() < min_tracked_frames)) {
continue;
}
// only broke 1 time for specific track id if it has been tracked, or broke many times
if (t->track_id >= 0 &&
std::find(broked.begin(), broked.end(), t->track_id) != broked.end()) {
continue;
}
// size filter
if (t->width < min_crop_width || t->height < min_crop_height) {
continue;
}
if (t->track_id >= 0) {
broked.push_back(t->track_id);
}
auto name = cropped_dir + "/" + std::to_string(t->channel_index) + "_" + std::to_string(t->frame_index) + "_" + std::to_string(t->track_id >= 0 ? t->track_id : i) + ".jpg";
// start flag
msg_stream << "<--" << std::endl;
// save small cropped image
save_cropped_image(meta->frame, cv::Rect(t->x, t->y, t->width, t->height), name);
// format properties
format_properties(t->secondary_labels);
// format sub target (vehicle plate here), just using the first sub target's label is enough.
format_sub_target(t->sub_targets.size() > 0 ? t->sub_targets[0]->label : "");
// format embeddings
format_embeddings(t->embeddings);
// end flag
msg_stream << "-->";
if (i != meta->targets.size() - 1) {
msg_stream << std::endl; // not the last one
}
}
}
msg = msg_stream.str();
}
void vp_embeddings_properties_socket_broker_node::broke_msg(const std::string& msg) {
// broke msg to socket by udp
auto bytes_2_send = reinterpret_cast<const std::byte*>(msg.c_str());
auto bytes_2_send_len = msg.size();
udp_writer.send(bytes_2_send, bytes_2_send_len);
}
}

55
nodes/broker/vp_embeddings_properties_socket_broker_node.h Normal file
View File

@@ -0,0 +1,55 @@
#pragma once
#include "vp_msg_broker_node.h"
#include "cereal_archive/vp_objects_cereal_archive.h"
// light weight socket support
#include "../../third_party/kissnet/kissnet.hpp"
namespace vp_nodes {
// message broker node, broke embeddings, AND properties (ONLY for vp_frame_target) to socket via udp.
// embeddings and properties could be used for similiarity search and property search later.
class vp_embeddings_properties_socket_broker_node: public vp_msg_broker_node
{
private:
// save dir for cropped images, which would be used for embeddings similiarity search or property search later
std::string cropped_dir = "cropped_images";
// min width to crop (embedding/property will be ignored if target's width is smaller than this value)
int min_crop_width = 50;
// min height to crop (embedding/property will be ignored if target's height is smaller than this value)
int min_crop_height = 50;
// only broke for tracked targets (track_id is not -1)
bool only_for_tracked = false;
// min tracked frames if only_for_tracked is true
int min_tracked_frames = 25;
// host the data sent to via udp
std::string des_ip = "";
// port the data sent to via udp
int des_port = 0;
// udp socket writer
kissnet::udp_socket udp_writer;
// support multi-channel
std::map<int, std::vector<int>> all_broked; // channel -> target ids which have been broked
protected:
// to custom format
virtual void format_msg(const std::shared_ptr<vp_objects::vp_frame_meta>& meta, std::string& msg) override;
// to socket via udp
virtual void broke_msg(const std::string& msg) override;
public:
vp_embeddings_properties_socket_broker_node(std::string node_name,
std::string des_ip = "",
int des_port = 0,
std::string cropped_dir = "cropped_images",
int min_crop_width = 50,
int min_crop_height = 50,
vp_broke_for broke_for = vp_broke_for::NORMAL,
bool only_for_tracked = false,
int broking_cache_warn_threshold = 50,
int broking_cache_ignore_threshold = 200);
~vp_embeddings_properties_socket_broker_node();
};
}

156
nodes/broker/vp_embeddings_socket_broker_node.cpp Normal file
View File

@@ -0,0 +1,156 @@
#include "vp_embeddings_socket_broker_node.h"
namespace vp_nodes {
vp_embeddings_socket_broker_node::vp_embeddings_socket_broker_node(std::string node_name,
std::string des_ip,
int des_port,
std::string cropped_dir,
int min_crop_width,
int min_crop_height,
vp_broke_for broke_for,
bool only_for_tracked,
int broking_cache_warn_threshold,
int broking_cache_ignore_threshold):
vp_msg_broker_node(node_name, broke_for, broking_cache_warn_threshold, broking_cache_ignore_threshold),
des_ip(des_ip),
des_port(des_port),
cropped_dir(cropped_dir),
min_crop_width(min_crop_width),
min_crop_height(min_crop_height),
only_for_tracked(only_for_tracked) {
// only for vp_frame_target or vp_frame_face_target
assert(broke_for == vp_broke_for::NORMAL || broke_for == vp_broke_for::FACE);
udp_writer = kissnet::udp_socket(kissnet::endpoint(des_ip, des_port));
VP_INFO(vp_utils::string_format("[%s] [message broker] set des_ip as `%s` and des_port as [%d]", node_name.c_str(), des_ip.c_str(), des_port));
this->initialized();
}
vp_embeddings_socket_broker_node::~vp_embeddings_socket_broker_node() {
deinitialized();
stop_broking();
}
void vp_embeddings_socket_broker_node::format_msg(const std::shared_ptr<vp_objects::vp_frame_meta>& meta, std::string& msg) {
/* format:
line 0, <--
line 1, 1st cropped image's path
line 2, 1st embeddings
line 3, -->
line 4, <--
line 5, 2nd cropped image's path
line 6, 2nd embeddings
line 7, -->
line 8, ...
*/
auto& broked = all_broked[meta->channel_index];
// remove 50 elements every 100 ids
if (broked.size() > 100) {
broked.erase(broked.begin(), broked.begin() + 50);
}
std::stringstream msg_stream;
auto format_embeddings = [&](const std::vector<float>& embeddings) {
for (int i = 0; i < embeddings.size(); i++) {
msg_stream << embeddings[i];
if (i != embeddings.size() - 1) {
msg_stream << ",";
}
}
msg_stream << std::endl;
};
auto save_cropped_image = [&](cv::Mat& frame, cv::Rect rect, std::string name) {
auto cropped = frame(rect);
cv::imwrite(name, cropped);
msg_stream << name << std::endl;
};
if (broke_for == vp_broke_for::NORMAL) {
for (int i = 0; i < meta->targets.size(); i++) {
auto& t = meta->targets[i];
// only broke for tracked targets and have enough frames
if ((only_for_tracked && t->track_id < 0) || (only_for_tracked && t->tracks.size() < min_tracked_frames)) {
continue;
}
// only broke 1 time for specific track id if it has been tracked, or broke many times
if (t->track_id >= 0 &&
std::find(broked.begin(), broked.end(), t->track_id) != broked.end()) {
continue;
}
// size filter
if (t->width < min_crop_width || t->height < min_crop_height) {
continue;
}
if (t->track_id >= 0) {
broked.push_back(t->track_id);
}
auto name = cropped_dir + "/" + std::to_string(t->channel_index) + "_" + std::to_string(t->frame_index) + "_" + std::to_string(t->track_id >= 0 ? t->track_id : i) + ".jpg";
// start flag
msg_stream << "<--" << std::endl;
// save small cropped image
save_cropped_image(meta->frame, cv::Rect(t->x, t->y, t->width, t->height), name);
// format embeddings
format_embeddings(t->embeddings);
// end flag
msg_stream << "-->";
if (i != meta->targets.size() - 1) {
msg_stream << std::endl; // not the last one
}
}
}
if (broke_for == vp_broke_for::FACE) {
for (int i = 0; i < meta->face_targets.size(); i++) {
auto& t = meta->face_targets[i];
// only broke for tracked targets and have enough frames
if ((only_for_tracked && t->track_id < 0) || (only_for_tracked && t->tracks.size() < min_tracked_frames)) {
continue;
}
// only broke 1 time for specific track id if it has been tracked, or broke many times
if (t->track_id >= 0 &&
std::find(broked.begin(), broked.end(), t->track_id) != broked.end()) {
continue;
}
// size filter
if (t->width < min_crop_width || t->height < min_crop_height) {
continue;
}
if (t->track_id >= 0) {
broked.push_back(t->track_id);
}
auto name = cropped_dir + "/" + std::to_string(meta->channel_index) + "_" + std::to_string(meta->frame_index) + "_" + std::to_string(t->track_id >= 0 ? t->track_id : i) + ".jpg";
// start flag
msg_stream << "<--" << std::endl;
// save small cropped image
save_cropped_image(meta->frame, cv::Rect(t->x, t->y, t->width, t->height), name);
// format embeddings
format_embeddings(t->embeddings);
// end flag
msg_stream << "-->";
if (i != meta->face_targets.size() - 1) {
msg_stream << std::endl; // not the last one
}
}
}
msg = msg_stream.str();
}
void vp_embeddings_socket_broker_node::broke_msg(const std::string& msg) {
// broke msg to socket by udp
auto bytes_2_send = reinterpret_cast<const std::byte*>(msg.c_str());
auto bytes_2_send_len = msg.size();
udp_writer.send(bytes_2_send, bytes_2_send_len);
}
}

55
nodes/broker/vp_embeddings_socket_broker_node.h Normal file
View File

@@ -0,0 +1,55 @@
#pragma once
#include "vp_msg_broker_node.h"
#include "cereal_archive/vp_objects_cereal_archive.h"
// light weight socket support
#include "../../third_party/kissnet/kissnet.hpp"
namespace vp_nodes {
// message broker node, broke ONLY embeddings (for vp_frame_target or vp_frame_face_target) to socket via udp.
// embeddings could be used for similiarity search later.
class vp_embeddings_socket_broker_node: public vp_msg_broker_node
{
private:
// save dir for cropped images, which would be used for embeddings similiarity search later
std::string cropped_dir = "cropped_images";
// min width to crop (embedding will be ignored if target's width is smaller than this value)
int min_crop_width = 50;
// min height to crop (embedding will be ignored if target's height is smaller than this value)
int min_crop_height = 50;
// only broke for tracked targets (track_id is not -1)
bool only_for_tracked = false;
// min tracked frames if only_for_tracked is true
int min_tracked_frames = 25;
// host the data sent to via udp
std::string des_ip = "";
// port the data sent to via udp
int des_port = 0;
// udp socket writer
kissnet::udp_socket udp_writer;
// support multi-channel
std::map<int, std::vector<int>> all_broked; // channel -> target ids which have been broked
protected:
// to custom format
virtual void format_msg(const std::shared_ptr<vp_objects::vp_frame_meta>& meta, std::string& msg) override;
// to socket via udp
virtual void broke_msg(const std::string& msg) override;
public:
vp_embeddings_socket_broker_node(std::string node_name,
std::string des_ip = "",
int des_port = 0,
std::string cropped_dir = "cropped_images",
int min_crop_width = 50,
int min_crop_height = 50,
vp_broke_for broke_for = vp_broke_for::NORMAL,
bool only_for_tracked = false,
int broking_cache_warn_threshold = 50,
int broking_cache_ignore_threshold = 200);
~vp_embeddings_socket_broker_node();
};
}

95
nodes/broker/vp_expr_socket_broker_node.cpp Normal file
View File

@@ -0,0 +1,95 @@
#include "vp_expr_socket_broker_node.h"
namespace vp_nodes {
vp_expr_socket_broker_node::vp_expr_socket_broker_node(std::string node_name,
std::string des_ip,
int des_port,
std::string screenshot_dir,
vp_broke_for broke_for,
int broking_cache_warn_threshold,
int broking_cache_ignore_threshold):
vp_msg_broker_node(node_name, broke_for, broking_cache_warn_threshold, broking_cache_ignore_threshold),
des_ip(des_ip),
des_port(des_port),
screenshot_dir(screenshot_dir) {
// only for vp_frame_text_target
assert(broke_for == vp_broke_for::TEXT);
udp_writer = kissnet::udp_socket(kissnet::endpoint(des_ip, des_port));
VP_INFO(vp_utils::string_format("[%s] [message broker] set des_ip as `%s` and des_port as [%d]", node_name.c_str(), des_ip.c_str(), des_port));
this->initialized();
}
vp_expr_socket_broker_node::~vp_expr_socket_broker_node() {
deinitialized();
stop_broking();
}
void vp_expr_socket_broker_node::format_msg(const std::shared_ptr<vp_objects::vp_frame_meta>& meta, std::string& msg) {
/* format:
line 0, <--
line 1, time
line 2, channel_index, frame_index
line 3, total num, num of yes, num of no, number of invalid
line 4, 1st expression string, 2nd expression string, 3rd expression string, ...
line 5, screenshot path
line 6, -->
*/
std::stringstream msg_stream;
auto format_basic_info = [&](int channel_index, int frame_index) {
msg_stream << vp_utils::time_format(NOW) << std::endl; // line1
msg_stream << channel_index << "," << frame_index << std::endl; // line2
};
auto format_expr_info = [&](const std::vector<std::shared_ptr<vp_objects::vp_frame_text_target>>& expr_targets) {
auto num_yes = 0, num_no = 0, num_invalid = 0;
auto expr_strs = std::string();
for (int i = 0; i < expr_targets.size(); i++) {
expr_strs += expr_targets[i]->text;
if (i != expr_targets.size() - 1) {
expr_strs += ",";
}
if (expr_targets[i]->flags.find("yes") != std::string::npos) {
num_yes++;
}
if (expr_targets[i]->flags.find("no") != std::string::npos) {
num_no++;
}
if (expr_targets[i]->flags.find("invalid") != std::string::npos) {
num_invalid++;
}
}
msg_stream << expr_targets.size() << "," << num_yes << "," << num_no << "," << num_invalid << std::endl; // line3
msg_stream << expr_strs << std::endl; // line4
};
auto format_screenshot = [&](cv::Mat& screenshot, const std::string& name) {
cv::imwrite(name, screenshot);
msg_stream << name << std::endl; // line5
};
// at most 1 record for each frame
if (broke_for == vp_broke_for::TEXT) {
// start flag
msg_stream << "<--" << std::endl;
format_basic_info(meta->channel_index, meta->frame_index);
format_expr_info(meta->text_targets);
auto screenshot_name = screenshot_dir + "/" + std::to_string(meta->channel_index) + "_" + std::to_string(meta->frame_index) + ".jpg";
format_screenshot(meta->osd_frame.empty() ? meta->frame : meta->osd_frame, screenshot_name);
// end flag
msg_stream << "-->" << std::endl;
}
msg = msg_stream.str();
}
void vp_expr_socket_broker_node::broke_msg(const std::string& msg) {
// broke msg to socket by udp
auto bytes_2_send = reinterpret_cast<const std::byte*>(msg.c_str());
auto bytes_2_send_len = msg.size();
udp_writer.send(bytes_2_send, bytes_2_send_len);
}
}

40
nodes/broker/vp_expr_socket_broker_node.h Normal file
View File

@@ -0,0 +1,40 @@
#pragma once
#include "vp_msg_broker_node.h"
#include "../../objects/ba/vp_ba_result.h"
#include "cereal_archive/vp_objects_cereal_archive.h"
// light weight socket support
#include "../../third_party/kissnet/kissnet.hpp"
namespace vp_nodes {
// message broker node, broke math expression checking results (ONLY for vp_frame_text_target) to socket via udp.
// math expression checking results could be used for archive.
class vp_expr_socket_broker_node: public vp_msg_broker_node
{
private:
// save dir for screenshot images, which would be used for displaying in web client
std::string screenshot_dir = "screenshot_images";
// host the data sent to via udp
std::string des_ip = "";
// port the data sent to via udp
int des_port = 0;
// udp socket writer
kissnet::udp_socket udp_writer;
protected:
// to custom format
virtual void format_msg(const std::shared_ptr<vp_objects::vp_frame_meta>& meta, std::string& msg) override;
// to socket via udp
virtual void broke_msg(const std::string& msg) override;
public:
vp_expr_socket_broker_node(std::string node_name,
std::string des_ip = "",
int des_port = 0,
std::string screenshot_dir = "screenshot_images",
vp_broke_for broke_for = vp_broke_for::TEXT,
int broking_cache_warn_threshold = 50,
int broking_cache_ignore_threshold = 200);
~vp_expr_socket_broker_node();
};
}

58
nodes/broker/vp_json_console_broker_node.cpp Normal file
View File

@@ -0,0 +1,58 @@
#include "vp_json_console_broker_node.h"
namespace vp_nodes {
vp_json_console_broker_node::vp_json_console_broker_node(std::string node_name,
vp_broke_for broke_for,
int broking_cache_warn_threshold,
int broking_cache_ignore_threshold):
vp_msg_broker_node(node_name, broke_for, broking_cache_warn_threshold, broking_cache_ignore_threshold) {
this->initialized();
}
vp_json_console_broker_node::~vp_json_console_broker_node() {
deinitialized();
stop_broking();
}
void vp_json_console_broker_node::format_msg(const std::shared_ptr<vp_objects::vp_frame_meta>& meta, std::string& msg) {
// serialize objects to json by cereal
std::stringstream msg_stream;
{
cereal::JSONOutputArchive json_archive(msg_stream);
// global values
json_archive(cereal::make_nvp("channel_index", meta->channel_index),
cereal::make_nvp("frame_index", meta->frame_index),
cereal::make_nvp("width", meta->frame.cols),
cereal::make_nvp("height", meta->frame.rows),
cereal::make_nvp("fps", meta->fps),
cereal::make_nvp("broke_for", broke_fors.at(broke_for)));
// serialize values according to broke_for
if (broke_for == vp_broke_for::NORMAL) {
json_archive(cereal::make_nvp("target_size", meta->targets.size()),
cereal::make_nvp("targets", meta->targets));
}
else if (broke_for == vp_broke_for::FACE) {
json_archive(cereal::make_nvp("face_target_size", meta->face_targets.size()),
cereal::make_nvp("face_targets", meta->face_targets));
}
else if (broke_for == vp_broke_for::TEXT) {
json_archive(cereal::make_nvp("text_target_size", meta->text_targets.size()),
cereal::make_nvp("text_targets", meta->text_targets));
}
else {
throw "invalid broke_for!";
}
} // flush
msg = msg_stream.str();
}
void vp_json_console_broker_node::broke_msg(const std::string& msg) {
// broke msg to console by std::cout
std::cout << msg << std::endl;
}
}

26
nodes/broker/vp_json_console_broker_node.h Normal file
View File

@@ -0,0 +1,26 @@
#pragma once
#include <sstream>
#include "vp_msg_broker_node.h"
#include "cereal_archive/vp_objects_cereal_archive.h"
namespace vp_nodes {
// message broker node (for debug purpose), broke json data to console.
class vp_json_console_broker_node: public vp_msg_broker_node
{
private:
/* data */
protected:
// to json
virtual void format_msg(const std::shared_ptr<vp_objects::vp_frame_meta>& meta, std::string& msg) override;
// to console
virtual void broke_msg(const std::string& msg) override;
public:
vp_json_console_broker_node(std::string node_name,
vp_broke_for broke_for = vp_broke_for::NORMAL,
int broking_cache_warn_threshold = 50,
int broking_cache_ignore_threshold = 200);
~vp_json_console_broker_node();
};
}

65
nodes/broker/vp_json_kafka_broker_node.cpp Normal file
View File

@@ -0,0 +1,65 @@
#ifdef VP_WITH_KAFKA
#include "vp_json_kafka_broker_node.h"
namespace vp_nodes {
vp_json_kafka_broker_node::vp_json_kafka_broker_node(std::string node_name,
std::string kafka_servers,
std::string topic_name,
vp_broke_for broke_for,
int broking_cache_warn_threshold,
int broking_cache_ignore_threshold):
vp_msg_broker_node(node_name, broke_for, broking_cache_warn_threshold, broking_cache_ignore_threshold) {
VP_INFO(vp_utils::string_format("[%s] kafka_servers:[%s] topic_name:[%s]", node_name.c_str(), kafka_servers.c_str(), topic_name.c_str()));
kafka_producer = std::make_shared<KafkaProducer>(kafka_servers, topic_name, 0);
this->initialized();
}
vp_json_kafka_broker_node::~vp_json_kafka_broker_node() {
deinitialized();
stop_broking();
}
void vp_json_kafka_broker_node::format_msg(const std::shared_ptr<vp_objects::vp_frame_meta>& meta, std::string& msg) {
// serialize objects to json by cereal
std::stringstream msg_stream;
{
cereal::JSONOutputArchive json_archive(msg_stream);
// global values
json_archive(cereal::make_nvp("channel_index", meta->channel_index),
cereal::make_nvp("frame_index", meta->frame_index),
cereal::make_nvp("width", meta->frame.cols),
cereal::make_nvp("height", meta->frame.rows),
cereal::make_nvp("fps", meta->fps),
cereal::make_nvp("broke_for", broke_fors.at(broke_for)));
// serialize values according to broke_for
if (broke_for == vp_broke_for::NORMAL) {
json_archive(cereal::make_nvp("target_size", meta->targets.size()),
cereal::make_nvp("targets", meta->targets));
}
else if (broke_for == vp_broke_for::FACE) {
json_archive(cereal::make_nvp("face_target_size", meta->face_targets.size()),
cereal::make_nvp("face_targets", meta->face_targets));
}
else if (broke_for == vp_broke_for::TEXT) {
json_archive(cereal::make_nvp("text_target_size", meta->text_targets.size()),
cereal::make_nvp("text_targets", meta->text_targets));
}
else {
throw "invalid broke_for!";
}
} // flush
msg = msg_stream.str();
}
void vp_json_kafka_broker_node::broke_msg(const std::string& msg) {
// broke msg to kafka by kafka client api
if (kafka_producer != nullptr) {
kafka_producer->pushMessage(msg);
}
}
}
#endif

32
nodes/broker/vp_json_kafka_broker_node.h Normal file
View File

@@ -0,0 +1,32 @@
#pragma once
#ifdef VP_WITH_KAFKA
#include <sstream>
#include "vp_msg_broker_node.h"
#include "kafka_utils/KafkaProducer.h"
#include "cereal_archive/vp_objects_cereal_archive.h"
namespace vp_nodes {
// message broker node, broke json data to kafka.
class vp_json_kafka_broker_node: public vp_msg_broker_node
{
private:
/* data */
std::shared_ptr<KafkaProducer> kafka_producer = nullptr;
protected:
// to json
virtual void format_msg(const std::shared_ptr<vp_objects::vp_frame_meta>& meta, std::string& msg) override;
// to console
virtual void broke_msg(const std::string& msg) override;
public:
vp_json_kafka_broker_node(std::string node_name,
std::string kafka_servers = "127.0.0.1:9092",
std::string topic_name = "videopipe_topic",
vp_broke_for broke_for = vp_broke_for::NORMAL,
int broking_cache_warn_threshold = 50,
int broking_cache_ignore_threshold = 200);
~vp_json_kafka_broker_node();
};
}
#endif

85
nodes/broker/vp_msg_broker_node.cpp Normal file
View File

@@ -0,0 +1,85 @@
#include "vp_msg_broker_node.h"
namespace vp_nodes {
vp_msg_broker_node::vp_msg_broker_node(std::string node_name,
vp_broke_for broke_for,
int broking_cache_warn_threshold,
int broking_cache_ignore_threshold):
vp_node(node_name),
broke_for(broke_for),
broking_cache_warn_threshold(broking_cache_warn_threshold),
broking_cache_ignore_threshold(broking_cache_ignore_threshold) {
broking_th = std::thread(&vp_msg_broker_node::broking_run, this);
}
vp_msg_broker_node::~vp_msg_broker_node() {
}
void vp_msg_broker_node::stop_broking() {
broking = false;
frames_to_broke.push(nullptr); // send dead flag to broking_thread
broking_cache_semaphore.signal();
if (broking_th.joinable()) {
broking_th.join();
}
}
std::shared_ptr<vp_objects::vp_meta> vp_msg_broker_node::handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) {
// cache frame meta only if cache size is not greater than threshold
if (frames_to_broke.size() < broking_cache_ignore_threshold) {
// it is a producer
frames_to_broke.push(meta);
broking_cache_semaphore.signal();
}
// warning 1 time in log
auto size = frames_to_broke.size();
if (size > broking_cache_warn_threshold && !broking_cache_warned) {
broking_cache_warned = true;
VP_WARN(vp_utils::string_format("[%s] [message broker] cache size is exceeding threshold! cache size is [%d], threshold is [%d]", node_name.c_str(), size, broking_cache_warn_threshold));
}
if (size <= broking_cache_warn_threshold) {
broking_cache_warned = false;
}
return meta;
}
std::shared_ptr<vp_objects::vp_meta> vp_msg_broker_node::handle_control_meta(std::shared_ptr<vp_objects::vp_control_meta> meta) {
return meta;
}
void vp_msg_broker_node::broking_run() {
while (broking) {
// it is a consumer
broking_cache_semaphore.wait();
auto frame_meta = frames_to_broke.front();
frames_to_broke.pop();
// dead flag
if (frame_meta == nullptr) {
continue;
}
// message to be broked
std::string message;
// step 1, format message
format_msg(frame_meta, message); // MUST be implemented in child class
// ignore if message is empty, because no broking occurs is allowed for some frames if some conditions not satisfied
if (message.empty()) {
continue;
}
// step 2, broke message
broke_msg(message); // MUST be implemented in child class
}
}
}

65
nodes/broker/vp_msg_broker_node.h Normal file
View File

@@ -0,0 +1,65 @@
#pragma once
#include "../vp_node.h"
namespace vp_nodes {
// broke for what type of data (vp_frame_target, vp_frame_face_target or others)
enum class vp_broke_for {
NORMAL, // vp_frame_target
FACE, // vp_frame_face_target
TEXT, // vp_frame_text_target
POSE // vp_frame_pose_target
// others to extend
};
// base node for message brokers,
// used to serialize objects (inside vp_frame_meta) to structured data and then push them to external modules like kafka, file or sockets.
// note:
// 1. this node works asynchronously which would not block pipeline.
// 2. this class can not be initialized directly.
class vp_msg_broker_node: public vp_node
{
private:
// warning if cache size greater than threshold
int broking_cache_warn_threshold = 50;
bool broking_cache_warned = false;
// ignore if cache size greater than threshold (skip directly)
int broking_cache_ignore_threshold = 200;
// cache frames to be broked
std::queue<std::shared_ptr<vp_objects::vp_frame_meta>> frames_to_broke;
vp_utils::vp_semaphore broking_cache_semaphore;
// broking thread
std::thread broking_th;
// broking function
void broking_run();
bool broking = true;
protected:
virtual std::shared_ptr<vp_objects::vp_meta> handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) override final;
virtual std::shared_ptr<vp_objects::vp_meta> handle_control_meta(std::shared_ptr<vp_objects::vp_control_meta> meta) override final;
// serialize objects to message which SHOULD be implemented in child class.
virtual void format_msg(const std::shared_ptr<vp_objects::vp_frame_meta>& meta, std::string& msg) = 0;
// broke message to external modules which SHOULD be implemented in child class.
virtual void broke_msg(const std::string& msg) = 0;
// wait thread exits in vp_msg_broker_node
void stop_broking();
// node applied for what type of target
vp_broke_for broke_for = vp_broke_for::NORMAL;
// string for broke_for
std::map<vp_broke_for, std::string> broke_fors = {{vp_broke_for::NORMAL, "normal"},
{vp_broke_for::FACE, "face"},
{vp_broke_for::TEXT, "text"},
{vp_broke_for::POSE, "pose"}};
public:
vp_msg_broker_node(std::string node_name,
vp_broke_for broke_for = vp_broke_for::NORMAL,
int broking_cache_warn_threshold = 50,
int broking_cache_ignore_threshold = 200);
~vp_msg_broker_node();
};
}

160
nodes/broker/vp_plate_socket_broker_node.cpp Normal file
View File

@@ -0,0 +1,160 @@
#include "vp_plate_socket_broker_node.h"
namespace vp_nodes {
vp_plate_socket_broker_node::vp_plate_socket_broker_node(std::string node_name,
std::string des_ip,
int des_port,
std::string plates_dir,
int min_crop_width,
int min_crop_height,
vp_broke_for broke_for,
bool only_for_tracked,
int broking_cache_warn_threshold,
int broking_cache_ignore_threshold):
vp_msg_broker_node(node_name, broke_for, broking_cache_warn_threshold, broking_cache_ignore_threshold),
des_ip(des_ip),
des_port(des_port),
plates_dir(plates_dir),
min_crop_width(min_crop_width),
min_crop_height(min_crop_height),
only_for_tracked(only_for_tracked) {
// only for vp_frame_target
assert(broke_for == vp_broke_for::NORMAL);
udp_writer = kissnet::udp_socket(kissnet::endpoint(des_ip, des_port));
VP_INFO(vp_utils::string_format("[%s] [message broker] set des_ip as `%s` and des_port as [%d]", node_name.c_str(), des_ip.c_str(), des_port));
this->initialized();
}
vp_plate_socket_broker_node::~vp_plate_socket_broker_node() {
deinitialized();
stop_broking();
}
void vp_plate_socket_broker_node::format_msg(const std::shared_ptr<vp_objects::vp_frame_meta>& meta, std::string& msg) {
/* format:
line 0, <--
line 1, 1st time
line 2, 1st channel index, frame index
line 3, 1st the cropped image's path
line 4, 1st the whole image's path
line 5, 1st plate color
line 6, 1st plate text
line 7, -->
line 8, <--
line 9, 2nd time
line 10, 2nd channel index, frame index
line 11, 2nd the cropped image's path
line 12, 2nd the whole image's path
line 13, 2nd plate color
line 14, 2nd plate text
line 15, -->
line 16, ...
*/
auto& broked_ids = all_broked_ids[meta->channel_index];
auto& broked_texts = all_broked_texts[meta->channel_index];
// remove 50 elements every 100 ids
if (broked_ids.size() > 100) {
broked_ids.erase(broked_ids.begin(), broked_ids.begin() + 50);
}
if (broked_texts.size() > 100) {
broked_texts.erase(broked_texts.begin(), broked_texts.begin() + 50);
}
std::stringstream msg_stream;
auto format_basic_info = [&](int channel_index, int frame_index) {
msg_stream << vp_utils::time_format(NOW) << std::endl; // line1
msg_stream << channel_index << "," << frame_index << std::endl; // line2
};
auto save_cropped_image = [&](cv::Mat& frame, cv::Rect rect, std::string name) {
auto cropped = frame(rect);
cv::imwrite(name, cropped);
msg_stream << name << std::endl; // line3
};
auto save_whole_image = [&](cv::Mat& frame, std::string name) {
cv::imwrite(name, frame);
msg_stream << name << std::endl; // line4
};
auto format_plate = [&](const std::string& plate_color, const std::string& plate_text) {
msg_stream << plate_color << std::endl; // line5
msg_stream << plate_text << std::endl; // line6
};
if (broke_for == vp_broke_for::NORMAL) {
for (int i = 0; i < meta->targets.size(); i++) {
auto& t = meta->targets[i];
// only broke for tracked targets and have enough frames
if ((only_for_tracked && t->track_id < 0) || (only_for_tracked && t->tracks.size() < min_tracked_frames)) {
continue;
}
// only broke 1 time for specific track id if it has been tracked, or broke many times
if (t->track_id >= 0 &&
std::find(broked_ids.begin(), broked_ids.end(), t->track_id) != broked_ids.end()) {
broked_ids.push_back(t->track_id);
continue;
}
// size filter
if (t->width < min_crop_width || t->height < min_crop_height) {
continue;
}
auto color_and_text = vp_utils::string_split(t->primary_label, '_');
// make sure plate text and color detected
if(color_and_text.size() != 2) {
continue;
}
auto& color = color_and_text[0];
auto& text = color_and_text[1];
// check for length, 7 or 3 + 6 or 3 + 7
if (text.length() != 7 && text.length() != 9 && text.length() != 10) {
continue;
}
// only broke 1 time for specific plate text in small period
if (std::find(broked_texts.begin(), broked_texts.end(), text) != broked_texts.end()) {
broked_texts.push_back(text);
continue;
}
// cache for texts
broked_texts.push_back(text);
// cache for ids
if (t->track_id >= 0) {
broked_ids.push_back(t->track_id);
}
auto cropped_name = plates_dir + "/" + std::to_string(t->channel_index) + "_" + std::to_string(t->frame_index) + "_" + color + "_" + text + "_cropped.jpg";
auto whole_name = plates_dir + "/" + std::to_string(t->channel_index) + "_" + std::to_string(t->frame_index) + "_" + color + "_" + text + "_whole.jpg";
// start flag
msg_stream << "<--" << std::endl;
// basic info
format_basic_info(meta->channel_index, meta->frame_index);
// save small cropped image
save_cropped_image(meta->frame, cv::Rect(t->x, t->y, t->width, t->height), cropped_name);
// save whole image
save_whole_image(meta->frame, whole_name);
// format color and text
format_plate(color, text);
// end flag
msg_stream << "-->";
if (i != meta->targets.size() - 1) {
msg_stream << std::endl; // not the last one
}
}
}
msg = msg_stream.str();
}
void vp_plate_socket_broker_node::broke_msg(const std::string& msg) {
// broke msg to socket by udp
auto bytes_2_send = reinterpret_cast<const std::byte*>(msg.c_str());
auto bytes_2_send_len = msg.size();
udp_writer.send(bytes_2_send, bytes_2_send_len);
}
}

56
nodes/broker/vp_plate_socket_broker_node.h Normal file
View File

@@ -0,0 +1,56 @@
#pragma once
#include "vp_msg_broker_node.h"
#include "cereal_archive/vp_objects_cereal_archive.h"
// light weight socket support
#include "../../third_party/kissnet/kissnet.hpp"
namespace vp_nodes {
// message broker node, broke text & color for license plate (hold by vp_frame_target) to socket via udp.
// which could be used for lpr camera.
class vp_plate_socket_broker_node: public vp_msg_broker_node
{
private:
// save dir for cropped images and whole images, which would be used for displaying in lpr camera
std::string plates_dir = "plate_images";
// min width to crop (license plate will be ignored if target's width is smaller than this value)
int min_crop_width = 50;
// min height to crop (license plate will be ignored if target's height is smaller than this value)
int min_crop_height = 50;
// only broke for tracked targets (track_id is not -1)
bool only_for_tracked = false;
// min tracked frames if only_for_tracked is true
int min_tracked_frames = 25;
// host the data sent to via udp
std::string des_ip = "";
// port the data sent to via udp
int des_port = 0;
// udp socket writer
kissnet::udp_socket udp_writer;
// support multi-channel, used for avoid duplicate data
std::map<int, std::vector<int>> all_broked_ids; // channel -> target ids which have been broked
std::map<int, std::vector<std::string>> all_broked_texts; // channel -> plate texts which have been broked (filter using similiarity comparison)
protected:
// to custom format
virtual void format_msg(const std::shared_ptr<vp_objects::vp_frame_meta>& meta, std::string& msg) override;
// to socket via udp
virtual void broke_msg(const std::string& msg) override;
public:
vp_plate_socket_broker_node(std::string node_name,
std::string des_ip = "",
int des_port = 0,
std::string plates_dir = "plate_images",
int min_crop_width = 100,
int min_crop_height = 0,
vp_broke_for broke_for = vp_broke_for::NORMAL,
bool only_for_tracked = true,
int broking_cache_warn_threshold = 50,
int broking_cache_ignore_threshold = 200);
~vp_plate_socket_broker_node();
};
}

70
nodes/broker/vp_xml_file_broker_node.cpp Normal file
View File

@@ -0,0 +1,70 @@
#include "vp_xml_file_broker_node.h"
namespace vp_nodes {
vp_xml_file_broker_node::vp_xml_file_broker_node(std::string node_name,
vp_broke_for broke_for,
std::string file_path_and_name,
int broking_cache_warn_threshold,
int broking_cache_ignore_threshold):
vp_msg_broker_node(node_name, broke_for, broking_cache_warn_threshold, broking_cache_ignore_threshold) {
// open file
if (file_path_and_name.empty()) {
file_path_and_name = "vp_broker_" + vp_utils::time_format(NOW, "<hour>-<min>-<sec>-<mili>") + ".xml";
}
VP_INFO(vp_utils::string_format("[%s] [message broker] set broking file path as `%s`", node_name.c_str(), file_path_and_name.c_str()));
xml_writer.open(file_path_and_name);
this->initialized();
}
vp_xml_file_broker_node::~vp_xml_file_broker_node() {
deinitialized();
stop_broking();
}
void vp_xml_file_broker_node::format_msg(const std::shared_ptr<vp_objects::vp_frame_meta>& meta, std::string& msg) {
// serialize objects to xml by cereal
std::stringstream msg_stream;
{
cereal::XMLOutputArchive xml_archive(msg_stream);
// global values
xml_archive(cereal::make_nvp("channel_index", meta->channel_index),
cereal::make_nvp("frame_index", meta->frame_index),
cereal::make_nvp("width", meta->frame.cols),
cereal::make_nvp("height", meta->frame.rows),
cereal::make_nvp("fps", meta->fps),
cereal::make_nvp("broke_for", broke_fors.at(broke_for)));
// serialize values according to broke_for
if (broke_for == vp_broke_for::NORMAL) {
xml_archive(cereal::make_nvp("target_size", meta->targets.size()),
cereal::make_nvp("targets", meta->targets));
}
else if (broke_for == vp_broke_for::FACE) {
xml_archive(cereal::make_nvp("face_target_size", meta->face_targets.size()),
cereal::make_nvp("face_targets", meta->face_targets));
}
else if (broke_for == vp_broke_for::TEXT) {
xml_archive(cereal::make_nvp("text_target_size", meta->text_targets.size()),
cereal::make_nvp("text_targets", meta->text_targets));
}
else {
throw "invalid broke_for!";
}
} // flush
msg = msg_stream.str();
}
void vp_xml_file_broker_node::broke_msg(const std::string& msg) {
// broke msg to file by ofstream
if (xml_writer.is_open()) {
xml_writer << msg << std::endl;
}
else {
// TO-DO
}
}
}

28
nodes/broker/vp_xml_file_broker_node.h Normal file
View File

@@ -0,0 +1,28 @@
#pragma once
#include <fstream>
#include "vp_msg_broker_node.h"
#include "cereal_archive/vp_objects_cereal_archive.h"
namespace vp_nodes {
// message broker node (for demo/debug purpose), broke xml data to file.
class vp_xml_file_broker_node: public vp_msg_broker_node
{
private:
// xml file writer
ofstream xml_writer;
protected:
// to xml
virtual void format_msg(const std::shared_ptr<vp_objects::vp_frame_meta>& meta, std::string& msg) override;
// to file
virtual void broke_msg(const std::string& msg) override;
public:
vp_xml_file_broker_node(std::string node_name,
vp_broke_for broke_for = vp_broke_for::NORMAL,
std::string file_path_and_name = "",
int broking_cache_warn_threshold = 50,
int broking_cache_ignore_threshold = 200);
~vp_xml_file_broker_node();
};
}

68
nodes/broker/vp_xml_socket_broker_node.cpp Normal file
View File

@@ -0,0 +1,68 @@
#include "vp_xml_socket_broker_node.h"
namespace vp_nodes {
vp_xml_socket_broker_node::vp_xml_socket_broker_node(std::string node_name,
std::string des_ip,
int des_port,
vp_broke_for broke_for,
int broking_cache_warn_threshold,
int broking_cache_ignore_threshold):
vp_msg_broker_node(node_name, broke_for, broking_cache_warn_threshold, broking_cache_ignore_threshold),
des_ip(des_ip),
des_port(des_port) {
udp_writer = kissnet::udp_socket(kissnet::endpoint(des_ip, des_port));
VP_INFO(vp_utils::string_format("[%s] [message broker] set des_ip as `%s` and des_port as [%d]", node_name.c_str(), des_ip.c_str(), des_port));
this->initialized();
}
vp_xml_socket_broker_node::~vp_xml_socket_broker_node() {
deinitialized();
stop_broking();
}
void vp_xml_socket_broker_node::format_msg(const std::shared_ptr<vp_objects::vp_frame_meta>& meta, std::string& msg) {
// serialize objects to xml by cereal
std::stringstream msg_stream;
{
cereal::XMLOutputArchive xml_archive(msg_stream);
// global values
xml_archive(cereal::make_nvp("channel_index", meta->channel_index),
cereal::make_nvp("frame_index", meta->frame_index),
cereal::make_nvp("width", meta->frame.cols),
cereal::make_nvp("height", meta->frame.rows),
cereal::make_nvp("fps", meta->fps),
cereal::make_nvp("broke_for", broke_fors.at(broke_for)));
// serialize values according to broke_for
if (broke_for == vp_broke_for::NORMAL) {
xml_archive(cereal::make_nvp("target_size", meta->targets.size()),
cereal::make_nvp("targets", meta->targets));
}
else if (broke_for == vp_broke_for::FACE) {
xml_archive(cereal::make_nvp("face_target_size", meta->face_targets.size()),
cereal::make_nvp("face_targets", meta->face_targets));
}
else if (broke_for == vp_broke_for::TEXT) {
xml_archive(cereal::make_nvp("text_target_size", meta->text_targets.size()),
cereal::make_nvp("text_targets", meta->text_targets));
}
else {
throw "invalid broke_for!";
}
} // flush
msg = msg_stream.str();
}
void vp_xml_socket_broker_node::broke_msg(const std::string& msg) {
// broke msg to socket by udp
auto bytes_2_send = reinterpret_cast<const std::byte*>(msg.c_str());
auto bytes_2_send_len = msg.size();
udp_writer.send(bytes_2_send, bytes_2_send_len);
}
}

35
nodes/broker/vp_xml_socket_broker_node.h Normal file
View File

@@ -0,0 +1,35 @@
#pragma once
#include "vp_msg_broker_node.h"
#include "cereal_archive/vp_objects_cereal_archive.h"
// light weight socket support
#include "../../third_party/kissnet/kissnet.hpp"
namespace vp_nodes {
// message broker node, broke xml data to socket via udp.
class vp_xml_socket_broker_node: public vp_msg_broker_node
{
private:
// host the data sent to via udp
std::string des_ip = "";
// port the data sent to via udp
int des_port = 0;
// udp socket writer
kissnet::udp_socket udp_writer;
protected:
// to xml
virtual void format_msg(const std::shared_ptr<vp_objects::vp_frame_meta>& meta, std::string& msg) override;
// to socket via udp
virtual void broke_msg(const std::string& msg) override;
public:
vp_xml_socket_broker_node(std::string node_name,
std::string des_ip = "",
int des_port = 0,
vp_broke_for broke_for = vp_broke_for::NORMAL,
int broking_cache_warn_threshold = 50,
int broking_cache_ignore_threshold = 200);
~vp_xml_socket_broker_node();
};
}