gavin/VideoPipe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first commit

Browse Source
This commit is contained in:
陈赣
2026-06-03 12:43:14 +08:00
commit ba76cfae28
608 changed files with 120791 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
nodes/osd/README.md Normal file
View File

@@ -0,0 +1,5 @@
# Summary
Two situations to create a new osd node:
1. Need a new osd style for the same type of target
2. OSD for different types of targets, such as `vp_frame_target` (by default), `vp_frame_pose_target`, `vp_frame_face_target` ...

91
nodes/osd/vp_ba_crossline_osd_node.cpp Normal file
View File

@@ -0,0 +1,91 @@
#include "vp_ba_crossline_osd_node.h"
namespace vp_nodes {
vp_ba_crossline_osd_node::vp_ba_crossline_osd_node(std::string node_name, std::string font): vp_node(node_name) {
if (!font.empty()) {
ft2 = cv::freetype::createFreeType2();
ft2->loadFontData(font, 0);
}
this->initialized();
}
vp_ba_crossline_osd_node::~vp_ba_crossline_osd_node() {
deinitialized();
}
std::shared_ptr<vp_objects::vp_meta> vp_ba_crossline_osd_node::handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) {
// operations on osd_frame
if (meta->osd_frame.empty()) {
meta->osd_frame = meta->frame.clone();
}
auto& canvas = meta->osd_frame;
// scan targets
for (auto& i : meta->targets) {
// track_id
auto id = std::to_string(i->track_id);
auto labels_to_display = i->primary_label;
// tracked
if (i->track_id != -1) {
labels_to_display = "#" + id + " " + labels_to_display;
}
for (auto& label : i->secondary_labels) {
labels_to_display += "|" + label;
}
// draw tracks if size>=2
if (i->tracks.size() >= 2) {
for (int n = 0; n < (i->tracks.size() - 1); n++) {
auto p1 = i->tracks[n].track_point();
auto p2 = i->tracks[n + 1].track_point();
cv::line(canvas, cv::Point(p1.x, p1.y), cv::Point(p2.x, p2.y), cv::Scalar(0, 255, 255), 1, cv::LINE_AA);
}
}
cv::rectangle(canvas, cv::Rect(i->x, i->y, i->width, i->height), cv::Scalar(255, 255, 0), 2);
if (ft2 != nullptr) {
ft2->putText(canvas, labels_to_display, cv::Point(i->x, i->y), 20, cv::Scalar(255, 0, 255), cv::FILLED, cv::LINE_AA, true);
}
else {
//cv::putText(canvas, labels_to_display, cv::Point(i->x, i->y), 1, 1, cv::Scalar(255, 0, 255));
int baseline = 0;
auto size = cv::getTextSize(labels_to_display, 1, 1, 1, &baseline);
vp_utils::put_text_at_center_of_rect(canvas, labels_to_display, cv::Rect(i->x, i->y - size.height, size.width, size.height), true, 1, 1, cv::Scalar(), cv::Scalar(179, 52, 255), cv::Scalar(179, 52, 255));
}
// scan sub targets
for (auto& sub_target: i->sub_targets) {
cv::rectangle(canvas, cv::Rect(sub_target->x, sub_target->y, sub_target->width, sub_target->height), cv::Scalar(255));
if (ft2 != nullptr) {
ft2->putText(canvas, sub_target->label, cv::Point(sub_target->x, sub_target->y), 20, cv::Scalar(0, 0, 255), cv::FILLED, cv::LINE_AA, true);
}
else {
cv::putText(canvas, sub_target->label, cv::Point(sub_target->x, sub_target->y), 1, 1, cv::Scalar(0, 0, 255));
}
}
}
/* crossline draw for current channel */
auto& total_crossline = all_total_crossline[meta->channel_index];
auto& line = all_lines[meta->channel_index];
// scan ba results and ONLY deal with crossline
for (auto& i : meta->ba_results) {
if (i->type == vp_objects::vp_ba_type::CROSSLINE) {
// line has 2 points
assert(i->involve_region_in_frame.size() == 2);
line = vp_objects::vp_line(i->involve_region_in_frame[0], i->involve_region_in_frame[1]);
total_crossline += 1;
}
}
// draw crossline data
cv::line(canvas, cv::Point(line.start.x, line.start.y), cv::Point(line.end.x, line.end.y), cv::Scalar(0, 255, 0), 2, cv::LINE_AA);
cv::putText(canvas, vp_utils::string_format("total crossline targets: [%d]", total_crossline), cv::Point(20, 20), 1, 2, cv::Scalar(0, 0, 255), 2);
return meta;
}
}

26
nodes/osd/vp_ba_crossline_osd_node.h Normal file
View File

@@ -0,0 +1,26 @@
#pragma once
#include <map>
#include <opencv2/freetype.hpp>
#include "../vp_node.h"
#include "../../objects/shapes/vp_point.h"
#include "../../objects/shapes/vp_line.h"
namespace vp_nodes {
// osd node for behaviour analysis of crossline
class vp_ba_crossline_osd_node: public vp_node
{
private:
// support chinese font
cv::Ptr<cv::freetype::FreeType2> ft2;
// support multi channels
std::map<int, int> all_total_crossline;
std::map<int, vp_objects::vp_line> all_lines;
protected:
virtual std::shared_ptr<vp_objects::vp_meta> handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) override;
public:
vp_ba_crossline_osd_node(std::string node_name, std::string font = "");
~vp_ba_crossline_osd_node();
};
}

109
nodes/osd/vp_ba_jam_osd_node.cpp Normal file
View File

@@ -0,0 +1,109 @@
#include "vp_ba_jam_osd_node.h"
namespace vp_nodes {
vp_ba_jam_osd_node::vp_ba_jam_osd_node(std::string node_name, std::string font): vp_node(node_name) {
if (!font.empty()) {
ft2 = cv::freetype::createFreeType2();
ft2->loadFontData(font, 0);
}
this->initialized();
}
vp_ba_jam_osd_node::~vp_ba_jam_osd_node() {
deinitialized();
}
std::shared_ptr<vp_objects::vp_meta> vp_ba_jam_osd_node::handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) {
// operations on osd_frame
if (meta->osd_frame.empty()) {
meta->osd_frame = meta->frame.clone();
}
auto& canvas = meta->osd_frame;
// scan targets
for (auto& i : meta->targets) {
// track_id
auto id = std::to_string(i->track_id);
auto labels_to_display = i->primary_label;
// tracked
if (i->track_id != -1) {
labels_to_display = "#" + id + " " + labels_to_display;
}
for (auto& label : i->secondary_labels) {
labels_to_display += "|" + label;
}
// draw tracks if size>=2
if (i->tracks.size() >= 2) {
for (int n = 0; n < (i->tracks.size() - 1); n++) {
auto p1 = i->tracks[n].track_point();
auto p2 = i->tracks[n + 1].track_point();
cv::line(canvas, cv::Point(p1.x, p1.y), cv::Point(p2.x, p2.y), cv::Scalar(0, 255, 255), 1, cv::LINE_AA);
}
}
cv::rectangle(canvas, cv::Rect(i->x, i->y, i->width, i->height), cv::Scalar(255, 255, 0), 2);
if (ft2 != nullptr) {
ft2->putText(canvas, labels_to_display, cv::Point(i->x, i->y), 20, cv::Scalar(255, 0, 255), cv::FILLED, cv::LINE_AA, true);
}
else {
//cv::putText(canvas, labels_to_display, cv::Point(i->x, i->y), 1, 1, cv::Scalar(255, 0, 255));
int baseline = 0;
auto size = cv::getTextSize(labels_to_display, 1, 1, 1, &baseline);
vp_utils::put_text_at_center_of_rect(canvas, labels_to_display, cv::Rect(i->x, i->y - size.height, size.width, size.height), true, 1, 1, cv::Scalar(), cv::Scalar(179, 52, 255), cv::Scalar(179, 52, 255));
}
// scan sub targets
for (auto& sub_target: i->sub_targets) {
cv::rectangle(canvas, cv::Rect(sub_target->x, sub_target->y, sub_target->width, sub_target->height), cv::Scalar(255));
if (ft2 != nullptr) {
ft2->putText(canvas, sub_target->label, cv::Point(sub_target->x, sub_target->y), 20, cv::Scalar(0, 0, 255), cv::FILLED, cv::LINE_AA, true);
}
else {
cv::putText(canvas, sub_target->label, cv::Point(sub_target->x, sub_target->y), 1, 1, cv::Scalar(0, 0, 255));
}
}
}
/* jam draw for current channel */
auto& region = all_jam_regions[meta->channel_index];
auto& jam_result = all_jam_results[meta->channel_index];
auto& involve_ids = all_involve_ids[meta->channel_index];
// scan ba results and ONLY deal with stop and unstop
for (auto& i : meta->ba_results) {
if (i->type == vp_objects::vp_ba_type::JAM) {
region = i->involve_region_in_frame;
involve_ids = i->involve_target_ids_in_frame;
jam_result = true;
}
if (i->type == vp_objects::vp_ba_type::UNJAM) {
region = i->involve_region_in_frame;
involve_ids = i->involve_target_ids_in_frame; // not used later
jam_result = false;
}
}
// draw jam data
auto poly_vertexs = [&]() {
std::vector<cv::Point> vertexs;
for(auto& p: region) {
vertexs.push_back(cv::Point(p.x, p.y));
}
return vertexs;
};
if (jam_result) {
cv::polylines(canvas, poly_vertexs(), true, cv::Scalar(0, 0, 255), 2, cv::LINE_AA); // region
// targets in jan region, highlight
auto targets = meta->get_targets_by_ids(involve_ids);
for (auto& t: targets) {
cv::rectangle(canvas, cv::Rect(t->x, t->y, t->width, t->height), cv::Scalar(0, 0, 255), 2, cv::LINE_AA);
}
}
return meta;
}
}

27
nodes/osd/vp_ba_jam_osd_node.h Normal file
View File

@@ -0,0 +1,27 @@
#pragma once
#include <map>
#include <opencv2/freetype.hpp>
#include "../vp_node.h"
#include "../../objects/shapes/vp_point.h"
#include "../../objects/shapes/vp_line.h"
namespace vp_nodes {
// osd node for behaviour analysis of stop
class vp_ba_jam_osd_node: public vp_node
{
private:
// support chinese font
cv::Ptr<cv::freetype::FreeType2> ft2;
// support multi channels
std::map<int, std::vector<vp_objects::vp_point>> all_jam_regions; // channel -> jam region
std::map<int, bool> all_jam_results; // channel -> jam status
std::map<int, std::vector<int>> all_involve_ids; // channel -> target ids when enter jam status
protected:
virtual std::shared_ptr<vp_objects::vp_meta> handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) override;
public:
vp_ba_jam_osd_node(std::string node_name, std::string font = "");
~vp_ba_jam_osd_node();
};
}

114
nodes/osd/vp_ba_stop_osd_node.cpp Normal file
View File

@@ -0,0 +1,114 @@
#include "vp_ba_stop_osd_node.h"
namespace vp_nodes {
vp_ba_stop_osd_node::vp_ba_stop_osd_node(std::string node_name, std::string font): vp_node(node_name) {
if (!font.empty()) {
ft2 = cv::freetype::createFreeType2();
ft2->loadFontData(font, 0);
}
this->initialized();
}
vp_ba_stop_osd_node::~vp_ba_stop_osd_node() {
deinitialized();
}
std::shared_ptr<vp_objects::vp_meta> vp_ba_stop_osd_node::handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) {
// operations on osd_frame
if (meta->osd_frame.empty()) {
meta->osd_frame = meta->frame.clone();
}
auto& canvas = meta->osd_frame;
// scan targets
for (auto& i : meta->targets) {
// track_id
auto id = std::to_string(i->track_id);
auto labels_to_display = i->primary_label;
// tracked
if (i->track_id != -1) {
labels_to_display = "#" + id + " " + labels_to_display;
}
for (auto& label : i->secondary_labels) {
labels_to_display += "|" + label;
}
// draw tracks if size>=2
if (i->tracks.size() >= 2) {
for (int n = 0; n < (i->tracks.size() - 1); n++) {
auto p1 = i->tracks[n].track_point();
auto p2 = i->tracks[n + 1].track_point();
cv::line(canvas, cv::Point(p1.x, p1.y), cv::Point(p2.x, p2.y), cv::Scalar(0, 255, 255), 1, cv::LINE_AA);
}
}
cv::rectangle(canvas, cv::Rect(i->x, i->y, i->width, i->height), cv::Scalar(255, 255, 0), 2);
if (ft2 != nullptr) {
ft2->putText(canvas, labels_to_display, cv::Point(i->x, i->y), 20, cv::Scalar(255, 0, 255), cv::FILLED, cv::LINE_AA, true);
}
else {
//cv::putText(canvas, labels_to_display, cv::Point(i->x, i->y), 1, 1, cv::Scalar(255, 0, 255));
int baseline = 0;
auto size = cv::getTextSize(labels_to_display, 1, 1, 1, &baseline);
vp_utils::put_text_at_center_of_rect(canvas, labels_to_display, cv::Rect(i->x, i->y - size.height, size.width, size.height), true, 1, 1, cv::Scalar(), cv::Scalar(179, 52, 255), cv::Scalar(179, 52, 255));
}
// scan sub targets
for (auto& sub_target: i->sub_targets) {
cv::rectangle(canvas, cv::Rect(sub_target->x, sub_target->y, sub_target->width, sub_target->height), cv::Scalar(255));
if (ft2 != nullptr) {
ft2->putText(canvas, sub_target->label, cv::Point(sub_target->x, sub_target->y), 20, cv::Scalar(0, 0, 255), cv::FILLED, cv::LINE_AA, true);
}
else {
cv::putText(canvas, sub_target->label, cv::Point(sub_target->x, sub_target->y), 1, 1, cv::Scalar(0, 0, 255));
}
}
}
/* stop draw for current channel */
auto& region = all_stop_regions[meta->channel_index];
auto& total_stop = all_total_stop[meta->channel_index];
auto& stop_count = all_stop_count[meta->channel_index];
// scan ba results and ONLY deal with stop and unstop
for (auto& i : meta->ba_results) {
if (i->type == vp_objects::vp_ba_type::STOP) {
region = i->involve_region_in_frame;
stop_count++;
total_stop.push_back(i->involve_target_ids_in_frame[0]); // only 1 target
}
if (i->type == vp_objects::vp_ba_type::UNSTOP) {
region = i->involve_region_in_frame;
for (auto j = total_stop.begin(); j < total_stop.end();) {
if (*j == i->involve_target_ids_in_frame[0]) {
j = total_stop.erase(j);
break;
}
j++;
}
}
}
// draw stop data
auto poly_vertexs = [&]() {
std::vector<cv::Point> vertexs;
for(auto& p: region) {
vertexs.push_back(cv::Point(p.x, p.y));
}
return vertexs;
};
cv::polylines(canvas, poly_vertexs(), true, cv::Scalar(0, 255, 0), 2, cv::LINE_AA);
// cv::putText(canvas, vp_utils::string_format("total stop targets: [%d]", stop_count), cv::Point(20, 20), 1, 2, cv::Scalar(0, 0, 255), 2);
// stop targets, highlight
auto stop_targets = meta->get_targets_by_ids(total_stop);
for (auto& t: stop_targets) {
cv::rectangle(canvas, cv::Rect(t->x, t->y, t->width, t->height), cv::Scalar(0, 0, 255), 2, cv::LINE_AA);
}
return meta;
}
}

27
nodes/osd/vp_ba_stop_osd_node.h Normal file
View File

@@ -0,0 +1,27 @@
#pragma once
#include <map>
#include <opencv2/freetype.hpp>
#include "../vp_node.h"
#include "../../objects/shapes/vp_point.h"
#include "../../objects/shapes/vp_line.h"
namespace vp_nodes {
// osd node for behaviour analysis of stop
class vp_ba_stop_osd_node: public vp_node
{
private:
// support chinese font
cv::Ptr<cv::freetype::FreeType2> ft2;
// support multi channels
std::map<int, std::vector<int>> all_total_stop; // channel -> stop target ids
std::map<int, std::vector<vp_objects::vp_point>> all_stop_regions; // channel -> stop region
std::map<int, int> all_stop_count; // channel -> stop count
protected:
virtual std::shared_ptr<vp_objects::vp_meta> handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) override;
public:
vp_ba_stop_osd_node(std::string node_name, std::string font = "");
~vp_ba_stop_osd_node();
};
}

214
nodes/osd/vp_cluster_node.cpp Normal file
View File

@@ -0,0 +1,214 @@
#include "vp_cluster_node.h"
#include "../../third_party/bhtsne/tsne.h"
namespace vp_nodes {
vp_cluster_node::vp_cluster_node(std::string node_name,
bool use_tSNE,
std::vector<std::string> s_labels_to_display,
int sampling_frequency,
int min_sampling_width,
int min_sampling_height,
int max_sample_num_for_tsne,
int max_sample_num_per_category):
vp_node(node_name),
use_tSNE(use_tSNE),
s_labels_to_display(s_labels_to_display),
sampling_frequency(sampling_frequency),
min_sampling_width(min_sampling_width),
min_sampling_height(min_sampling_height),
max_sample_num_for_tsne(max_sample_num_for_tsne),
max_sample_num_per_category(max_sample_num_per_category) {
this->initialized();
}
vp_cluster_node::~vp_cluster_node() {
deinitialized();
}
// please refer to ../../third_party/trt_vehicle/main/vehicle_cluster.cpp
void vp_cluster_node::reduce_dims_using_tsne(std::vector<std::vector<float>>& low_features,
int no_dims, int max_iter, double perplexity,
double theta, int rand_seed, bool skip_random_init,
int stop_lying_iter, int mom_switch_iter) {
assert(cache_high_features.size() != 0);
auto N = cache_high_features.size();
auto D = cache_high_features[0].second.size(); // all the same as the first feature's dims
// prepare input
double data[N * D];
double Y[N * no_dims];
for (int i = 0; i < N; i++) {
for (int j = 0; j < D; j++) {
data[i * D + j] = cache_high_features[i].second[j];
}
}
// call t-SNE
TSNE::run(data, N, D, Y, no_dims, perplexity, theta, rand_seed, skip_random_init, max_iter, stop_lying_iter, mom_switch_iter);
// prepare output
for (int i = 0; i < N; i++) {
std::vector<float> low_dims_feature;
for (int j = 0; j < no_dims; j++) {
low_dims_feature.push_back(float(Y[i * no_dims + j]));
}
low_features.push_back(low_dims_feature);
}
}
std::shared_ptr<vp_objects::vp_meta> vp_cluster_node::handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) {
// sampling frequency
if (std::chrono::duration_cast<std::chrono::milliseconds>(NOW - last_sampling_time) < std::chrono::milliseconds(sampling_frequency)) {
return meta;
}
last_sampling_time = NOW;
/* t-SNE */
if (use_tSNE) {
for (int i = 0; i < meta->targets.size(); i++) {
if (meta->targets[i]->width < min_sampling_width || meta->targets[i]->height < min_sampling_height) {
continue;
}
if (cache_high_features.size() >= max_sample_num_for_tsne) {
// too many cache, we need control number of samples
cache_high_features.erase(cache_high_features.begin());
}
// save mat->feature pair to cache
cv::Mat m;
cv::Mat roi(meta->frame, cv::Rect(meta->targets[i]->x, meta->targets[i]->y, meta->targets[i]->width, meta->targets[i]->height));
roi.copyTo(m);
std::pair<cv::Mat, std::vector<float>> p {m, meta->targets[i]->embeddings};
cache_high_features.push_back(p);
}
// at least 10 samples to t-SNE
if (cache_high_features.size() >= 10) {
// reduce dims first
std::vector<std::vector<float>> low_features;
reduce_dims_using_tsne(low_features);
// now display on screen
// normalize low dims feature to coordinate of [0:1] and display them on 2D screen
auto max_x = 0.0f, max_y = 0.0f, min_x = 0.0f, min_y = 0.0f;
for(int i = 0; i < low_features.size(); i++) {
auto& f = low_features[i];
// 2 values in f
max_x = std::max(max_x, f[0]);
max_y = std::max(max_y, f[1]);
min_x = std::min(min_x, f[0]);
min_y = std::min(min_y, f[1]);
}
auto x_range = max_x - min_x;
auto y_range = max_y - min_y;
// draw on (tsne_canvas_w_h + tsne_thumbnail_w_h) * (tsne_canvas_w_h + tsne_thumbnail_w_h)
cv::Mat canvas(tsne_canvas_w_h + tsne_thumbnail_w_h, tsne_canvas_w_h + tsne_thumbnail_w_h, CV_8UC3, cv::Scalar(127, 127, 127));
for(int i = 0; i < low_features.size(); i++) {
auto& f = low_features[i];
// convert to [0:1]
f[0] = (f[0] - min_x) / x_range;
f[1] = (f[1] - min_y) / y_range;
auto& img = cache_high_features[i].first;
cv::Mat img_tmp;
cv::resize(img, img_tmp, cv::Size(tsne_thumbnail_w_h, tsne_thumbnail_w_h));
cv::rectangle(img_tmp, cv::Rect(0, 0, img_tmp.cols, img_tmp.rows), cv::Scalar(255, 0, 0));
cv::Mat roi(canvas, cv::Rect(int(f[0] * tsne_canvas_w_h), int(f[1] * tsne_canvas_w_h), tsne_thumbnail_w_h, tsne_thumbnail_w_h));
img_tmp.copyTo(roi);
}
cv::imshow("cluster using features powered by t-SNE", canvas);
}
}
/* categories */
for (int i = 0; i < meta->targets.size(); i++) {
auto& t = meta->targets[i];
if (t->width < min_sampling_width || t->height < min_sampling_height) {
continue;
}
cv::Mat m;
cv::Mat roi(meta->frame, cv::Rect(t->x, t->y, t->width, t->height));
roi.copyTo(m);
for (int j = 0; j < t->secondary_labels.size(); j++) {
auto& category = t->secondary_labels[j];
bool filter_pass = false;
// all
if (s_labels_to_display.size() == 0) {
cache_categories[category].push_back(m);
filter_pass = true;
}
else {
// has a filter
if (std::find(s_labels_to_display.begin(), s_labels_to_display.end(), category) != s_labels_to_display.end()) {
cache_categories[category].push_back(m);
filter_pass = true;
}
}
if (filter_pass) {
// too many cache, we need control number of samples
if (cache_categories[category].size() >= max_sample_num_per_category) {
cache_categories[category].erase(cache_categories[category].begin());
}
}
}
}
// display on screen
// calculate total number of rows
auto rows_num = 0;
for (auto& p: cache_categories) {
auto num = p.second.size() / category_num_per_row;
if (p.second.size() % category_num_per_row != 0 || num == 0) {
num++;
}
rows_num += num;
}
// draw on (category_canvas_w) * (category_canvas_h)
auto category_canvas_w = (category_thumbnail_w_h + category_gap) * (category_num_per_row + 1);
auto category_canvas_h = (category_thumbnail_w_h + category_gap) * (rows_num + 1);
cv::Mat canvas(category_canvas_h, category_canvas_w, CV_8UC3, cv::Scalar(127, 127, 127));
auto row_index = 0;
for (auto& p: cache_categories) {
auto col_index = 0;
auto& category_items = p.second;
auto& category_name = p.first;
cv::putText(canvas, category_name, cv::Point(10, (category_gap + category_thumbnail_w_h) * row_index + category_gap), 1, 1, cv::Scalar(0, 0, 255));
bool new_row = false;
for (int i = 0; i < category_items.size(); i++) {
cv::Mat img_tmp;
cv::resize(category_items[i], img_tmp, cv::Size(category_thumbnail_w_h, category_thumbnail_w_h));
cv::Mat roi(canvas, cv::Rect((category_gap + category_thumbnail_w_h) * col_index + category_gap, (category_gap + category_thumbnail_w_h) * row_index + category_gap, category_thumbnail_w_h, category_thumbnail_w_h));
img_tmp.copyTo(roi);
col_index++;
if ((col_index + 1) % category_num_per_row == 0) {
col_index = 0;
row_index ++;
new_row = true;
}
else {
new_row = false;
}
}
if (!new_row) {
row_index++;
}
}
cv::imshow("cluster using labels", canvas);
return meta;
}
std::shared_ptr<vp_objects::vp_meta> vp_cluster_node::handle_control_meta(std::shared_ptr<vp_objects::vp_control_meta> meta) {
return meta;
}
}

57
nodes/osd/vp_cluster_node.h Normal file
View File

@@ -0,0 +1,57 @@
#pragma once
#include "../vp_node.h"
namespace vp_nodes {
// cluster node for vp_frame_targets which has ability to display targets on screen according to its embeddings contained in vp_frame_target::embbedings variable or labels contained in vp_frame_target::secondary_labels vector。
// note!!!
// it is not an osd node which would operates on vp_frame_meta::osd_frame data member.
// it is not a DES node either which can be the last node in pipeline.
// it is just a normal MID node.
class vp_cluster_node: public vp_node
{
private:
// call tSNE algorithm to reduce high dims of feature and display target on 2D screen
bool use_tSNE;
// display target based on categories, empty means for all categories. if you want to disable it, just let vector including just a wrong category name take '123abcd' for example.
std::vector<std::string> s_labels_to_display;
// how often to sampling (miliseconds). since targets differences in adjacent frames are small, we need not sampling continuously.
int sampling_frequency = 1000;
std::chrono::system_clock::time_point last_sampling_time = NOW;
// filter for small targets
int min_sampling_width = 40;
int min_sampling_height = 40;
/* draw parameters for tsne */
int tsne_canvas_w_h = 600;
int tsne_thumbnail_w_h = 60;
int max_sample_num_for_tsne = 100;
std::vector<std::pair<cv::Mat, std::vector<float>>> cache_high_features; // mat -> feature
/* draw parameters for category */
int category_num_per_row = 10;
int category_thumbnail_w_h = 60;
int category_gap = 10;
int max_sample_num_per_category = 10;
std::map<std::string, std::vector<cv::Mat>> cache_categories; // label -> mat list
// reduce dims of features (to xy) so as to display it on 2D screen
void reduce_dims_using_tsne(std::vector<std::vector<float>>& low_features,
/* default parameters for t-SNE algorithm */
int no_dims = 2, int max_iter = 500, double perplexity = 2, double theta = 0.5, int rand_seed = -1, bool skip_random_init = false, int stop_lying_iter = 250, int mom_switch_iter = 250);
protected:
virtual std::shared_ptr<vp_objects::vp_meta> handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) override;
virtual std::shared_ptr<vp_objects::vp_meta> handle_control_meta(std::shared_ptr<vp_objects::vp_control_meta> meta) override;
public:
vp_cluster_node(std::string node_name,
bool use_tSNE = true,
std::vector<std::string> s_labels_to_display = std::vector<std::string>{},
int sampling_frequency = 1000,
int min_sampling_width = 40,
int min_sampling_height = 40,
int max_sample_num_for_tsne = 100,
int max_sample_num_per_category = 10);
~vp_cluster_node();
};
}

56
nodes/osd/vp_expr_osd_node.cpp Normal file
View File

@@ -0,0 +1,56 @@
#include <opencv2/imgproc.hpp>
#include "vp_expr_osd_node.h"
#include "../../utils/vp_utils.h"
namespace vp_nodes {
vp_expr_osd_node::vp_expr_osd_node(std::string node_name, std::string font):vp_node(node_name) {
assert(font != "");
ft2 = cv::freetype::createFreeType2();
ft2->loadFontData(font, 0);
this->initialized();
}
vp_expr_osd_node::~vp_expr_osd_node() {
deinitialized();
}
std::shared_ptr<vp_objects::vp_meta> vp_expr_osd_node::handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) {
// operations on osd_frame
if (meta->osd_frame.empty()) {
meta->osd_frame = meta->frame.clone();
}
auto& canvas = meta->osd_frame;
for (auto& i: meta->text_targets) {
cv::Point rook_points[4];
for (int m = 0; m < i->region_vertexes.size(); m++) {
rook_points[m] =
cv::Point(i->region_vertexes[m].first, i->region_vertexes[m].second);
}
const cv::Point *ppt[1] = {rook_points};
int npt[] = {4};
if (i->flags.find("yes") != std::string::npos) {
cv::polylines(canvas, ppt, npt, 1, 1, CV_RGB(0, 255, 0), 2, cv::LINE_AA, 0); // green
ft2->putText(canvas, "√", rook_points[1], 30, CV_RGB(0, 255, 0), cv::FILLED, cv::LINE_AA, true);
}
else if (i->flags.find("no") != std::string::npos) {
auto right_value = vp_utils::string_split(i->flags, '_')[1];
cv::polylines(canvas, ppt, npt, 1, 1, CV_RGB(255, 0, 0), 2, cv::LINE_AA, 0); // red
ft2->putText(canvas, "×(" + right_value + ")", rook_points[1], 30, CV_RGB(255, 0, 0), cv::FILLED, cv::LINE_AA, true);
}
else if (i->flags == "invalid") {
cv::polylines(canvas, ppt, npt, 1, 1, CV_RGB(255, 165, 0), 2, cv::LINE_AA, 0); // orange
ft2->putText(canvas, "invalid", rook_points[1], 30, CV_RGB(255, 165, 0), cv::FILLED, cv::LINE_AA, true);
}
else {
// to-do
}
}
return meta;
}
}

21
nodes/osd/vp_expr_osd_node.h Normal file
View File

@@ -0,0 +1,21 @@
#pragma once
#include <opencv2/freetype.hpp>
#include "../vp_node.h"
namespace vp_nodes {
// on screen display(short as osd) node.
// mainly used to display math expression which based on vp_frame_text_target on frame.
class vp_expr_osd_node: public vp_node
{
private:
// support chinese font
cv::Ptr<cv::freetype::FreeType2> ft2;
protected:
virtual std::shared_ptr<vp_objects::vp_meta> handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) override;
public:
vp_expr_osd_node(std::string node_name, std::string font);
~vp_expr_osd_node();
};
}

48
nodes/osd/vp_face_osd_node.cpp Normal file
View File

@@ -0,0 +1,48 @@
#include <opencv2/imgproc.hpp>
#include "vp_face_osd_node.h"
namespace vp_nodes {
vp_face_osd_node::vp_face_osd_node(std::string node_name): vp_node(node_name) {
this->initialized();
}
vp_face_osd_node::~vp_face_osd_node() {
deinitialized();
}
std::shared_ptr<vp_objects::vp_meta> vp_face_osd_node::handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) {
// operations on osd_frame
if (meta->osd_frame.empty()) {
meta->osd_frame = meta->frame.clone();
}
auto& canvas = meta->osd_frame;
// scan face targets
for(auto& i : meta->face_targets) {
cv::rectangle(canvas, cv::Rect(i->x, i->y, i->width, i->height), cv::Scalar(0, 255, 0), 2);
// track_id
if (i->track_id != -1) {
auto id = std::to_string(i->track_id);
cv::putText(canvas, id, cv::Point(i->x, i->y), 1, 1.5, cv::Scalar(0, 0, 255));
}
// just handle 5 keypoints
if (i->key_points.size() >= 5) {
cv::circle(canvas, cv::Point(i->key_points[0].first, i->key_points[0].second), 2, cv::Scalar(255, 0, 0), 2);
cv::circle(canvas, cv::Point(i->key_points[1].first, i->key_points[1].second), 2, cv::Scalar(0, 0, 255), 2);
cv::circle(canvas, cv::Point(i->key_points[2].first, i->key_points[2].second), 2, cv::Scalar(0, 255, 0), 2);
cv::circle(canvas, cv::Point(i->key_points[3].first, i->key_points[3].second), 2, cv::Scalar(255, 0, 255), 2);
cv::circle(canvas, cv::Point(i->key_points[4].first, i->key_points[4].second), 2, cv::Scalar(0, 255, 255), 2);
}
}
return meta;
}
std::shared_ptr<vp_objects::vp_meta> vp_face_osd_node::handle_control_meta(std::shared_ptr<vp_objects::vp_control_meta> meta) {
return meta;
}
}

20
nodes/osd/vp_face_osd_node.h Normal file
View File

@@ -0,0 +1,20 @@
#pragma once
#include "../vp_node.h"
namespace vp_nodes {
// on screen display(short as osd) node.
// mainly used to display vp_frame_face_target on frame.
class vp_face_osd_node: public vp_node
{
private:
/* data */
protected:
virtual std::shared_ptr<vp_objects::vp_meta> handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) override;
virtual std::shared_ptr<vp_objects::vp_meta> handle_control_meta(std::shared_ptr<vp_objects::vp_control_meta> meta) override;
public:
vp_face_osd_node(std::string node_name);
~vp_face_osd_node();
};
}

148
nodes/osd/vp_face_osd_node_v2.cpp Normal file
View File

@@ -0,0 +1,148 @@
#include "vp_face_osd_node_v2.h"
#include "../../utils/vp_utils.h"
namespace vp_nodes {
vp_face_osd_node_v2::vp_face_osd_node_v2(std::string node_name): vp_node(node_name) {
this->initialized();
}
vp_face_osd_node_v2::~vp_face_osd_node_v2() {
deinitialized();
}
std::shared_ptr<vp_objects::vp_meta> vp_face_osd_node_v2::handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) {
// operations on osd_frame
if (meta->osd_frame.empty()) {
// add a gap at the bottom of osd frame
meta->osd_frame = cv::Mat(meta->frame.rows + gap_height + padding * 2, meta->frame.cols, meta->frame.type(), cv::Scalar(128, 128, 128));
// initialize by copying frame to osd frame
auto roi = meta->osd_frame(cv::Rect(0, 0, meta->frame.cols, meta->frame.rows));
meta->frame.copyTo(roi);
}
auto& canvas = meta->osd_frame;
// scan face targets in current frame
for(auto& i : meta->face_targets) {
// draw face rect first
cv::rectangle(canvas, cv::Rect(i->x, i->y, i->width, i->height), cv::Scalar(0, 255, 0), 2);
// track_id
if (i->track_id != -1) {
auto id = std::to_string(i->track_id);
cv::putText(canvas, id, cv::Point(i->x, i->y), 1, 1.5, cv::Scalar(0, 0, 255));
}
// just handle 5 keypoints
if (i->key_points.size() >= 5) {
cv::circle(canvas, cv::Point(i->key_points[0].first, i->key_points[0].second), 2, cv::Scalar(255, 0, 0), 2);
cv::circle(canvas, cv::Point(i->key_points[1].first, i->key_points[1].second), 2, cv::Scalar(0, 0, 255), 2);
cv::circle(canvas, cv::Point(i->key_points[2].first, i->key_points[2].second), 2, cv::Scalar(0, 255, 0), 2);
cv::circle(canvas, cv::Point(i->key_points[3].first, i->key_points[3].second), 2, cv::Scalar(255, 0, 255), 2);
cv::circle(canvas, cv::Point(i->key_points[4].first, i->key_points[4].second), 2, cv::Scalar(0, 255, 255), 2);
}
// cache the first face
if (the_baseline_face.empty()) {
auto face = meta->frame(cv::Rect(i->x, i->y, i->width, i->height));
cv::resize(face, the_baseline_face, cv::Size(gap_height, gap_height));
the_baseline_face_feature = i->embeddings;
}
else {
// check if the face has existed in list by calculating distance between 2 faces
bool exist = false;
for(auto& f : face_features) {
if (match(i->embeddings, f, 0) >= cosine_similar_thresh ||
match(i->embeddings, f, 1) <= l2norm_similar_thresh) {
exist = true;
break;
}
}
if (!exist) {
auto cosine_dis = match(i->embeddings, the_baseline_face_feature, 0);
auto l2_dis = match(i->embeddings, the_baseline_face_feature, 1);
if (cosine_dis >= cosine_similar_thresh || l2_dis <= l2norm_similar_thresh) {
// as same as the_baseline_face
}
else {
// new face, add it to list for dispaly
auto face = meta->frame(cv::Rect(i->x, i->y, i->width, i->height));
cv::Mat resized_face;
cv::resize(face, resized_face, cv::Size(gap_height, gap_height));
faces_list.push_back(resized_face);
face_features.push_back(i->embeddings);
cosine_distances.push_back(cosine_dis);
l2_distances.push_back(l2_dis);
}
}
else {
// has existed in list
}
}
}
// too many faces, delete the first ones in list
auto width_need = faces_list.size() * (gap_height + padding) + padding;
while (width_need >= canvas.cols) {
faces_list.erase(faces_list.begin());
face_features.erase(face_features.begin());
cosine_distances.erase(cosine_distances.begin());
l2_distances.erase(l2_distances.begin());
// check again
width_need = faces_list.size() * (gap_height + padding) + padding;
}
// make sure the size for each vector
assert(faces_list.size() == face_features.size());
assert(faces_list.size() == cosine_distances.size());
assert(faces_list.size() == l2_distances.size());
assert(canvas.rows > (gap_height + padding) * 2);
// display the baseline face
if (!the_baseline_face.empty()) {
auto roi = canvas(cv::Rect(padding, canvas.rows - gap_height * 2 - padding * 2, gap_height, gap_height));
the_baseline_face.copyTo(roi);
}
// display faces in list
for (int i = 0; i < faces_list.size(); i++) {
auto roi = canvas(cv::Rect((padding + gap_height) * i + padding, canvas.rows - gap_height - padding, gap_height, gap_height));
faces_list[i].copyTo(roi);
// distance between face and the baseline
cv::line(canvas, cv::Point(padding + gap_height / 2, canvas.rows - gap_height - padding * 2 - gap_height / 2),
cv::Point((padding + gap_height) * i + padding + gap_height / 2, canvas.rows - gap_height - padding), cv::Scalar(255, 0, 0), 1, cv::LINE_AA);
cv::putText(canvas,
"cos_dis:" + vp_utils::round_any(cosine_distances[i], 3),
cv::Point((padding + gap_height) * i + padding, canvas.rows - gap_height - padding + 20), 1, 0.9, cv::Scalar(255, 255, 0));
cv::putText(canvas,
"l2_dis:" + vp_utils::round_any(l2_distances[i], 3),
cv::Point((padding + gap_height) * i + padding, canvas.rows - gap_height - padding + 40), 1, 0.9, cv::Scalar(255, 255, 0));
}
return meta;
}
double vp_face_osd_node_v2::match(std::vector<float>& feature1, std::vector<float>& feature2, int dis_type) {
auto _face_feature1 = cv::Mat(1, feature1.size(), CV_32F, feature1.data());
auto _face_feature2 = cv::Mat(1, feature2.size(), CV_32F, feature2.data());
cv::normalize(_face_feature1, _face_feature1);
cv::normalize(_face_feature2, _face_feature2);
if(dis_type == 0) {
return cv::sum(_face_feature1.mul(_face_feature2))[0];
}
else if(dis_type == 1) {
return cv::norm(_face_feature1, _face_feature2);
}
else {
throw std::invalid_argument("invalid parameter " + std::to_string(dis_type));
}
}
}

40
nodes/osd/vp_face_osd_node_v2.h Normal file
View File

@@ -0,0 +1,40 @@
#pragma once
#include <opencv2/core.hpp>
#include <opencv2/imgproc.hpp>
#include "../vp_node.h"
namespace vp_nodes {
// on screen display(short as osd) node.
// another version for vp_frame_face_target display, including displaying similarity between faces.
class vp_face_osd_node_v2: public vp_node
{
private:
// leave a gap at the bottom of osd frame
int gap_height = 112;
int padding = 5;
// cache for display
cv::Mat the_baseline_face; // the first face detected in frames
std::vector<float> the_baseline_face_feature; // the feature of first face in frames
std::vector<cv::Mat> faces_list; // faces detected except the first one
std::vector<std::vector<float>> face_features; // face features except the first one
std::vector<float> cosine_distances; // cosine distance between face and the first one
std::vector<float> l2_distances; // l2 distance between face and the first one
double cosine_similar_thresh = 0.363; // higher value means higher similarity, max 1.0
double l2norm_similar_thresh = 1.128; // lower value means higher similarity, min 0.0
// calculate distance
// 0 cosine distance
// 1 l2 distance
double match(std::vector<float>& feature1, std::vector<float>& feature2, int dis_type);
protected:
virtual std::shared_ptr<vp_objects::vp_meta> handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) override;
public:
vp_face_osd_node_v2(std::string node_name);
~vp_face_osd_node_v2();
};
}

51
nodes/osd/vp_lane_osd_node.cpp Normal file
View File

@@ -0,0 +1,51 @@
#include <fstream>
#include "vp_lane_osd_node.h"
#include "../../utils/vp_utils.h"
namespace vp_nodes {
vp_lane_osd_node::vp_lane_osd_node(std::string node_name): vp_node(node_name) {
this->initialized();
}
vp_lane_osd_node::~vp_lane_osd_node() {
deinitialized();
}
std::shared_ptr<vp_objects::vp_meta> vp_lane_osd_node::handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) {
// operations on osd_frame
if (meta->osd_frame.empty()) {
meta->osd_frame = meta->frame.clone();
}
auto& canvas = meta->osd_frame;
if (meta->mask.empty()) {
return meta;
}
// resize mask to the same size of canvas
cv::Mat mask(meta->mask.size[2], meta->mask.size[3], CV_32FC1, meta->mask.data);
cv::Mat mask_big;
cv::resize(mask, mask_big, canvas.size());
cv::threshold(mask_big, mask_big, 0.5, 1, cv::THRESH_BINARY);
auto kernel = cv::getStructuringElement(cv::MORPH_RECT, cv::Size(11, 11));
cv::erode(mask_big, mask_big, kernel);
mask_big.convertTo(mask_big, CV_8U, 255);
// merge mask and canvas
for (int y = 0; y < canvas.rows; ++y) {
for (int x = 0; x < canvas.cols; ++x) {
canvas.at<cv::Vec3b>(y, x)[2] = mask_big.at<uchar>(y, x);
canvas.at<cv::Vec3b>(y, x)[1] = cv::saturate_cast<uchar>(
canvas.at<cv::Vec3b>(y, x)[1] * 0.8 + mask_big.at<uchar>(y, x) * 0.2);
canvas.at<cv::Vec3b>(y, x)[0] = cv::saturate_cast<uchar>(
canvas.at<cv::Vec3b>(y, x)[0] * 0.8 + mask_big.at<uchar>(y, x) * 0.2);
}
}
return meta;
}
}

17
nodes/osd/vp_lane_osd_node.h Normal file
View File

@@ -0,0 +1,17 @@
#pragma once
#include <string>
#include "../vp_node.h"
namespace vp_nodes {
class vp_lane_osd_node: public vp_node
{
private:
/* data */
protected:
virtual std::shared_ptr<vp_objects::vp_meta> handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) override;
public:
vp_lane_osd_node(std::string node_name);
~vp_lane_osd_node();
};
}

91
nodes/osd/vp_mllm_osd_node.cpp Normal file
View File

@@ -0,0 +1,91 @@
#include "vp_mllm_osd_node.h"
#ifdef VP_WITH_LLM
namespace vp_nodes {
vp_mllm_osd_node::vp_mllm_osd_node(std::string node_name, std::string font): vp_node(node_name) {
ft2 = cv::freetype::createFreeType2();
ft2->loadFontData(font, 0);
this->initialized();
}
vp_mllm_osd_node::~vp_mllm_osd_node() {
deinitialized();
}
std::shared_ptr<vp_objects::vp_meta> vp_mllm_osd_node::handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) {
if (meta->description.empty()) {
return meta;
}
// operations on osd_frame
if (meta->osd_frame.empty()) {
// add a gap at the bottom of osd frame
meta->osd_frame = cv::Mat(meta->frame.rows + gap_height + padding * 2, meta->frame.cols, meta->frame.type(), cv::Scalar(128, 128, 128));
// initialize by copying frame to osd frame
auto roi = meta->osd_frame(cv::Rect(0, 0, meta->frame.cols, meta->frame.rows));
meta->frame.copyTo(roi);
}
auto& canvas = meta->osd_frame;
//ft2->putText(canvas, meta->description, cv::Point(10, meta->frame.rows + gap_height + padding * 2), 20, cv::Scalar(255, 0, 0), cv::FILLED, cv::LINE_AA, true);
draw_text_in_rect(canvas, meta->description, cv::Rect(5, meta->frame.rows + 5, meta->frame.cols - 10, gap_height + padding * 2 - 10), 20, cv::Scalar(255, 0, 0));
// log using INFO level
VP_INFO(vp_utils::string_format("[%s] [%s]", node_name.c_str(), meta->description.c_str()));
return meta;
}
std::vector<std::string> vp_mllm_osd_node::utf8_split(const std::string& text) {
std::vector<std::string> chars;
for (size_t i = 0; i < text.size();) {
unsigned char c = text[i];
size_t len = 1;
if ((c & 0x80) == 0x00) len = 1; // ASCII
else if ((c & 0xE0) == 0xC0) len = 2; // 2bytes
else if ((c & 0xF0) == 0xE0) len = 3; // 3bytes
else if ((c & 0xF8) == 0xF0) len = 4; // 4bytes
chars.push_back(text.substr(i, len));
i += len;
}
return chars;
}
void vp_mllm_osd_node::draw_text_in_rect(cv::Mat& img,
const std::string& text,
const cv::Rect& rect,
int fontHeight,
cv::Scalar color) {
std::vector<std::string> chars = utf8_split(text);
std::string currentLine;
int baseline = 0;
int y = rect.y;
for (size_t i = 0; i < chars.size(); i++) {
std::string tempLine = currentLine + chars[i];
cv::Size textSize = ft2->getTextSize(tempLine, fontHeight, -1, &baseline);
if (textSize.width > rect.width && !currentLine.empty()) {
int drawY = y + textSize.height;
if (drawY > rect.y + rect.height) break;
ft2->putText(img, currentLine, cv::Point(rect.x, drawY),
fontHeight, color, -1, 8, true);
y += textSize.height + 5;
currentLine.clear();
}
currentLine += chars[i];
}
if (!currentLine.empty()) {
int drawY = y + ft2->getTextSize(currentLine, fontHeight, -1, &baseline).height;
if (drawY <= rect.y + rect.height) {
ft2->putText(img, currentLine, cv::Point(rect.x, drawY),
fontHeight, color, -1, 8, true);
}
}
}
}
#endif

35
nodes/osd/vp_mllm_osd_node.h Normal file
View File

@@ -0,0 +1,35 @@
#pragma once
#ifdef VP_WITH_LLM
#include <opencv2/imgproc.hpp>
#include <opencv2/freetype.hpp>
#include "../vp_node.h"
namespace vp_nodes {
// on screen display(short as osd) node.
// mainly used to display description(output from LLM) on frame.
class vp_mllm_osd_node: public vp_node
{
private:
// leave a gap at the bottom of osd frame
int gap_height = 112;
int padding = 5;
// support chinese font
cv::Ptr<cv::freetype::FreeType2> ft2;
std::vector<std::string> utf8_split(const std::string& text);
void draw_text_in_rect(cv::Mat& img,
const std::string& text,
const cv::Rect& rect,
int fontHeight,
cv::Scalar color);
protected:
virtual std::shared_ptr<vp_objects::vp_meta> handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) override;
public:
vp_mllm_osd_node(std::string node_name, std::string font);
~vp_mllm_osd_node();
};
}
#endif

82
nodes/osd/vp_osd_node.cpp Executable file
View File

@@ -0,0 +1,82 @@
#include <opencv2/imgproc.hpp>
#include "vp_osd_node.h"
namespace vp_nodes {
vp_osd_node::vp_osd_node(std::string node_name, std::string font):
vp_node(node_name) {
if (!font.empty()) {
ft2 = cv::freetype::createFreeType2();
ft2->loadFontData(font, 0);
}
this->initialized();
}
vp_osd_node::~vp_osd_node() {
deinitialized();
}
std::shared_ptr<vp_objects::vp_meta> vp_osd_node::handle_control_meta(std::shared_ptr<vp_objects::vp_control_meta> meta) {
return meta;
}
// display logic
std::shared_ptr<vp_objects::vp_meta> vp_osd_node::handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) {
// operations on osd_frame
if (meta->osd_frame.empty()) {
meta->osd_frame = meta->frame.clone();
}
auto& canvas = meta->osd_frame;
// scan targets
for (auto& i : meta->targets) {
// track_id
auto id = std::to_string(i->track_id);
auto labels_to_display = i->primary_label;
// tracked
if (i->track_id != -1) {
labels_to_display = "#" + id + " " + labels_to_display;
}
for (auto& label : i->secondary_labels) {
labels_to_display += "|" + label;
}
// draw tracks if size>=2
if (i->tracks.size() >= 2) {
for (int n = 0; n < (i->tracks.size() - 1); n++) {
auto p1 = i->tracks[n].track_point();
auto p2 = i->tracks[n + 1].track_point();
cv::line(canvas, cv::Point(p1.x, p1.y), cv::Point(p2.x, p2.y), cv::Scalar(0, 255, 255), 1, cv::LINE_AA);
}
}
cv::rectangle(canvas, cv::Rect(i->x, i->y, i->width, i->height), cv::Scalar(255, 255, 0), 2);
if (ft2 != nullptr) {
ft2->putText(canvas, labels_to_display, cv::Point(i->x, i->y), 20, cv::Scalar(255, 0, 255), cv::FILLED, cv::LINE_AA, true);
}
else {
//cv::putText(canvas, labels_to_display, cv::Point(i->x, i->y), 1, 1, cv::Scalar(255, 0, 255));
int baseline = 0;
auto size = cv::getTextSize(labels_to_display, 1, 1.5, 1, &baseline);
vp_utils::put_text_at_center_of_rect(canvas, labels_to_display, cv::Rect(i->x, i->y - size.height, size.width, size.height), true, 1, 1, cv::Scalar(), cv::Scalar(179, 52, 255), cv::Scalar(179, 52, 255));
}
// scan sub targets
for (auto& sub_target: i->sub_targets) {
cv::rectangle(canvas, cv::Rect(sub_target->x, sub_target->y, sub_target->width, sub_target->height), cv::Scalar(255));
if (ft2 != nullptr) {
ft2->putText(canvas, sub_target->label, cv::Point(sub_target->x, sub_target->y), 20, cv::Scalar(0, 0, 255), cv::FILLED, cv::LINE_AA, true);
}
else {
cv::putText(canvas, sub_target->label, cv::Point(sub_target->x, sub_target->y), 1, 1, cv::Scalar(0, 0, 255));
}
}
}
return meta;
}
}

38
nodes/osd/vp_osd_node.h Executable file
View File

@@ -0,0 +1,38 @@
#pragma once
#include <opencv2/freetype.hpp>
#include "../vp_node.h"
/*
* ################################
* why need osd in our pipeline?
* ################################
* there are several reasons why we need osd,
* 1. we need debug, for the outputs of infer, tracker and ba, displaying is more straightforward than printing.
* 2. in some situations like cast screen, it is a functional requirement that we need display what we have done on screen to show what we can do to others who do not know about our product.
*
* drawing the targets on current frame is the most common operation for osd.
*/
namespace vp_nodes {
// config for vp_osd_node, define how to draw
typedef struct vp_osd_node_option
{
int aaa = 0;
} vp_osd_option;
// on screen display(short as osd) node.
// mainly used to display vp_frame_target on frame.
class vp_osd_node: public vp_node {
private:
// support chinese font
cv::Ptr<cv::freetype::FreeType2> ft2;
protected:
virtual std::shared_ptr<vp_objects::vp_meta> handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) override;
virtual std::shared_ptr<vp_objects::vp_meta> handle_control_meta(std::shared_ptr<vp_objects::vp_control_meta> meta) override;
public:
vp_osd_node(std::string node_name, std::string font = "");
~vp_osd_node();
};
}

96
nodes/osd/vp_osd_node_v2.cpp Normal file
View File

@@ -0,0 +1,96 @@
#include <opencv2/imgproc.hpp>
#include "vp_osd_node_v2.h"
#include "../../utils/vp_utils.h"
namespace vp_nodes {
vp_osd_node_v2::vp_osd_node_v2(std::string node_name, std::string font):vp_node(node_name) {
if (!font.empty()) {
ft2 = cv::freetype::createFreeType2();
ft2->loadFontData(font, 0);
}
this->initialized();
}
vp_osd_node_v2::~vp_osd_node_v2() {
deinitialized();
}
std::shared_ptr<vp_objects::vp_meta> vp_osd_node_v2::handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) {
// operations on osd_frame
if (meta->osd_frame.empty()) {
// add a gap at the bottom of osd frame
meta->osd_frame = cv::Mat(meta->frame.rows + gap_height + padding * 2, meta->frame.cols, meta->frame.type(), cv::Scalar(128, 128, 128));
// initialize by copying frame to osd frame
auto roi = meta->osd_frame(cv::Rect(0, 0, meta->frame.cols, meta->frame.rows));
meta->frame.copyTo(roi);
}
auto& canvas = meta->osd_frame;
auto base_left = padding;
// scan targets
for (int i = 0; i < meta->targets.size(); i++) {
auto& target = meta->targets[i];
// scan sub targets
for (int j = 0; j < target->sub_targets.size(); j++) {
auto& sub_target = target->sub_targets[j];
cv::rectangle(canvas, cv::Rect(sub_target->x, sub_target->y, sub_target->width, sub_target->height), cv::Scalar(255, 255, 255), 2);
auto roi = canvas(cv::Rect(base_left, meta->osd_frame.rows - padding - gap_height, gap_height, gap_height));
// white background
roi = cv::Scalar(255, 255, 255);
auto ori = canvas(cv::Rect(sub_target->x, sub_target->y, sub_target->width, sub_target->height));
cv::Mat tmp = ori.clone();
int offset_w = 0, offset_h = 0;
if (tmp.rows > tmp.cols) {
cv::resize(tmp, tmp, cv::Size(int(float(gap_height) / tmp.rows * tmp.cols) , gap_height));
offset_w = (gap_height - tmp.cols) / 2;
offset_h = 0;
}
else {
cv::resize(tmp, tmp, cv::Size(gap_height, int(float(gap_height) / tmp.cols * tmp.rows)));
offset_h = (gap_height - tmp.rows) / 2;
offset_w = 0;
}
// copy sub target to the bottom of screen
roi = canvas(cv::Rect(base_left + offset_w, meta->osd_frame.rows - padding - gap_height + offset_h, tmp.cols, tmp.rows));
tmp.copyTo(roi);
// line from target to sub target
cv::line(canvas, cv::Point(target->x + target->width / 2, target->y + target->height), cv::Point(base_left + gap_height / 2, meta->osd_frame.rows - padding - gap_height), cv::Scalar(255, 0, 0), 3, cv::LINE_AA);
// label of sub target
auto sub_label = vp_utils::string_split(sub_target->label, '_');
// !!!
// for plate sub target (color_text)
if (sub_label.size() == 2) {
assert(ft2 != nullptr);
ft2->putText(canvas, sub_label[1], cv::Point(base_left + 10, meta->osd_frame.rows - padding - 5), 36, cv::Scalar(0), cv::FILLED, cv::LINE_AA, true);
}
base_left += gap_height + padding;
}
// target
auto labels_to_display = target->primary_label;
for (auto& label : target->secondary_labels) {
labels_to_display += "/" + label;
}
cv::rectangle(canvas, cv::Rect(target->x, target->y, target->width, target->height), cv::Scalar(255, 255, 0), 3);
if (ft2 != nullptr) {
ft2->putText(canvas, labels_to_display, cv::Point(target->x, target->y), 25, cv::Scalar(255, 0, 255), cv::FILLED, cv::LINE_AA, true);
}
else {
cv::putText(canvas, labels_to_display, cv::Point(target->x, target->y), 1, 1, cv::Scalar(255, 0, 255));
}
}
return meta;
}
}

26
nodes/osd/vp_osd_node_v2.h Normal file
View File

@@ -0,0 +1,26 @@
#pragma once
#include <opencv2/freetype.hpp>
#include "../vp_node.h"
namespace vp_nodes {
// on screen display(short as osd) node.
// another version for vp_frame_target display, display vp_sub_target at the bottom of screen.
class vp_osd_node_v2: public vp_node
{
private:
// support chinese font
cv::Ptr<cv::freetype::FreeType2> ft2;
// leave a gap at the bottom of osd frame
int gap_height = 256;
int padding = 10;
protected:
virtual std::shared_ptr<vp_objects::vp_meta> handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) override;
public:
vp_osd_node_v2(std::string node_name, std::string font = "");
~vp_osd_node_v2();
};
}

82
nodes/osd/vp_osd_node_v3.cpp Normal file
View File

@@ -0,0 +1,82 @@
#include <opencv2/imgproc.hpp>
#include "vp_osd_node_v3.h"
namespace vp_nodes {
vp_osd_node_v3::vp_osd_node_v3(std::string node_name, std::string font, bool show_bbox):
vp_node(node_name), show_bbox(show_bbox) {
if (!font.empty()) {
ft2 = cv::freetype::createFreeType2();
ft2->loadFontData(font, 0);
}
this->initialized();
}
vp_osd_node_v3::~vp_osd_node_v3() {
deinitialized();
}
std::shared_ptr<vp_objects::vp_meta> vp_osd_node_v3::handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) {
// operations on osd_frame
if (meta->osd_frame.empty()) {
meta->osd_frame = meta->frame.clone();
}
auto& canvas = meta->osd_frame;
// scan targets
for (auto& i : meta->targets) {
if (show_bbox) {
// track_id
auto id = std::to_string(i->track_id);
auto labels_to_display = i->primary_label;
// tracked
if (i->track_id != -1) {
labels_to_display = "#" + id + " " + labels_to_display;
}
for (auto& label : i->secondary_labels) {
labels_to_display += "|" + label;
}
// draw tracks if size>=2
if (i->tracks.size() >= 2) {
for (int n = 0; n < (i->tracks.size() - 1); n++) {
auto p1 = i->tracks[n].track_point();
auto p2 = i->tracks[n + 1].track_point();
cv::line(canvas, cv::Point(p1.x, p1.y), cv::Point(p2.x, p2.y), cv::Scalar(0, 255, 255), 1, cv::LINE_AA);
}
}
cv::rectangle(canvas, cv::Rect(i->x, i->y, i->width, i->height), cv::Scalar(255, 255, 0), 2);
if (ft2 != nullptr) {
ft2->putText(canvas, labels_to_display, cv::Point(i->x, i->y), 20, cv::Scalar(255, 0, 255), cv::FILLED, cv::LINE_AA, true);
}
else {
//cv::putText(canvas, labels_to_display, cv::Point(i->x, i->y), 1, 1, cv::Scalar(255, 0, 255));
int baseline = 0;
auto size = cv::getTextSize(labels_to_display, 1, 1, 1, &baseline);
vp_utils::put_text_at_center_of_rect(canvas, labels_to_display, cv::Rect(i->x, i->y - size.height, size.width, size.height), true);
}
}
/* mask area */
if (!i->mask.empty()) {
// Resize the mask, threshold, color and apply it on the image
cv::resize(i->mask, i->mask, cv::Size(i->width, i->height));
cv::Mat mask = (i->mask > mask_threshold);
cv::Mat coloredRoi = (0.3 * cv::Scalar(255, 255, 0) + 0.7 * meta->osd_frame(cv::Rect(i->x, i->y, i->width, i->height)));
coloredRoi.convertTo(coloredRoi, CV_8UC3);
// Draw the contours on the image
vector<cv::Mat> contours;
cv::Mat hierarchy;
mask.convertTo(mask, CV_8U);
cv::findContours(mask, contours, hierarchy, cv::RETR_CCOMP, cv::CHAIN_APPROX_SIMPLE);
cv::drawContours(coloredRoi, contours, -1, cv::Scalar(255, 255, 0), 5, cv::LINE_8, hierarchy, 100);
coloredRoi.copyTo(meta->osd_frame(cv::Rect(i->x, i->y, i->width, i->height)), mask);
}
}
return meta;
}
}

24
nodes/osd/vp_osd_node_v3.h Normal file
View File

@@ -0,0 +1,24 @@
#pragma once
#include <opencv2/freetype.hpp>
#include "../vp_node.h"
namespace vp_nodes {
// on screen display(short as osd) node.
// another version for vp_frame_target display, display mask area for image segmentation.
class vp_osd_node_v3: public vp_node
{
private:
// support chinese font
cv::Ptr<cv::freetype::FreeType2> ft2;
float mask_threshold = 0.3;
bool show_bbox = true;
protected:
virtual std::shared_ptr<vp_objects::vp_meta> handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) override;
public:
vp_osd_node_v3(std::string node_name, std::string font = "", bool show_bbox = true);
~vp_osd_node_v3();
};
}

83
nodes/osd/vp_plate_osd_node.cpp Normal file
View File

@@ -0,0 +1,83 @@
#include <opencv2/imgproc.hpp>
#include "vp_plate_osd_node.h"
namespace vp_nodes {
vp_plate_osd_node::vp_plate_osd_node(std::string node_name, std::string font, bool display_his):
vp_node(node_name), display_his(display_his) {
if (!font.empty()) {
ft2 = cv::freetype::createFreeType2();
ft2->loadFontData(font, 0);
}
this->initialized();
}
vp_plate_osd_node::~vp_plate_osd_node() {
deinitialized();
}
std::shared_ptr<vp_objects::vp_meta> vp_plate_osd_node::handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) {
// operations on osd_frame
if (meta->osd_frame.empty()) {
meta->osd_frame = meta->frame.clone();
}
auto& canvas = meta->osd_frame;
if (display_his) {
cv::rectangle(canvas, cv::Rect(0, canvas.rows - height_his, canvas.cols, height_his), cv::Scalar(240, 200, 220), cv::FILLED);
}
// scan targets
for (auto& i : meta->targets) {
// plate color and plate text
auto color_and_text = vp_utils::string_split(i->primary_label, '_');
if(color_and_text.size() != 2) {
continue;
}
auto& color = color_and_text[0];
auto& text = color_and_text[1];
auto text_2_display = text_colors.at(color) + " " + text;
// tracked ?
if (i->track_id != -1) {
text_2_display = "#" + std::to_string(i->track_id) + " " + text_2_display;
}
cv::rectangle(canvas, cv::Rect(i->x, i->y, i->width, i->height), draw_colors.at(color), 2);
if (ft2 != nullptr) {
ft2->putText(canvas, text_2_display, cv::Point(i->x, i->y), 20, draw_colors.at(color), cv::FILLED, cv::LINE_AA, true);
}
else {
// ignore
}
if (display_his) {
// put it to cache
auto plate = meta->frame(cv::Rect(i->x, i->y, i->width, i->height));
cv::Mat resized_plate;
cv::resize(plate, resized_plate, cv::Size(height_his, int((float(height_his) / plate.cols) * plate.rows)));
plates_his.push_back(resized_plate);
}
}
// display history at the bottom of canvas
if (display_his) {
// remove previous history plates if need, since no space to draw
auto width_need = plates_his.size() * (height_his + gap_his) + gap_his;
while(width_need >= canvas.cols) {
plates_his.erase(plates_his.begin());
width_need = plates_his.size() * (height_his + gap_his) + gap_his;
}
// draw history plates at the bottom of screen
for (int i = 0; i < plates_his.size(); i++) {
auto& p = plates_his[i];
auto roi = canvas(cv::Rect((gap_his + height_his) * i + gap_his, canvas.rows - height_his / 2 - p.rows / 2 , height_his, p.rows));
plates_his[i].copyTo(roi);
}
}
return meta;
}
}

42
nodes/osd/vp_plate_osd_node.h Normal file
View File

@@ -0,0 +1,42 @@
#pragma once
#include <opencv2/freetype.hpp>
#include "../vp_node.h"
namespace vp_nodes {
// on screen display(short as osd) node.
// used for displaying vehicle plate on frame, draw rectangle according to plate color
class vp_plate_osd_node: public vp_node
{
private:
// support chinese font
cv::Ptr<cv::freetype::FreeType2> ft2;
float mask_threshold = 0.3;
// draw color on frame
std::map<std::string, cv::Scalar> draw_colors {{"blue", cv::Scalar(255, 0, 0)},
{"green", cv::Scalar(0, 255, 0)},
{"yellow", cv::Scalar(0, 255, 255)},
{"white", cv::Scalar(255, 255, 255)}};
// map to Chinese
std::map<std::string, std::string> text_colors {{"blue", "蓝"},
{"green", "绿"},
{"yellow", "黄"},
{"white", "白"}};
// history plates at the bottom of screen
std::vector<cv::Mat> plates_his;
int height_his = 100;
int gap_his = 10;
bool display_his = true;
protected:
virtual std::shared_ptr<vp_objects::vp_meta> handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) override;
public:
vp_plate_osd_node(std::string node_name, std::string font = "", bool display_his = true);
~vp_plate_osd_node();
};
}

46
nodes/osd/vp_pose_osd_node.cpp Normal file
View File

@@ -0,0 +1,46 @@
#include "vp_pose_osd_node.h"
namespace vp_nodes {
vp_pose_osd_node::vp_pose_osd_node(std::string node_name): vp_node(node_name) {
populateColorPalette(colors, 100); // generate colors
this->initialized();
}
vp_pose_osd_node::~vp_pose_osd_node() {
deinitialized();
}
std::shared_ptr<vp_objects::vp_meta> vp_pose_osd_node::handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) {
// operations on osd_frame
if (meta->osd_frame.empty()) {
meta->osd_frame = meta->frame.clone();
}
// scan pose targets
for (int i = 0; i < meta->pose_targets.size(); i++) {
auto& pose_target = meta->pose_targets[i];
auto nPairs = posePairs_map.at(pose_target->type).size();
for (int j = 0; j < nPairs; j++) {
auto& a = pose_target->key_points[posePairs_map.at(pose_target->type)[j].first];
auto& b = pose_target->key_points[posePairs_map.at(pose_target->type)[j].second];
// some points not detected
if (a.x < 0 || a.y < 0 || b.x < 0 || b.y < 0) {
continue;
}
cv::line(meta->osd_frame, cv::Point(a.x, a.y), cv::Point(b.x, b.y), colors[j], 2, cv::LINE_AA);
cv::circle(meta->osd_frame, cv::Point(a.x, a.y), 3, colors[j], -1, cv::LINE_AA);
cv::circle(meta->osd_frame, cv::Point(b.x, b.y), 3, colors[j], -1, cv::LINE_AA);
}
}
return meta;
}
std::shared_ptr<vp_objects::vp_meta> vp_pose_osd_node::handle_control_meta(std::shared_ptr<vp_objects::vp_control_meta> meta) {
return meta;
}
}

51
nodes/osd/vp_pose_osd_node.h Normal file
View File

@@ -0,0 +1,51 @@
#pragma once
#include <random>
#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>
#include "../vp_node.h"
namespace vp_nodes {
// on screen display(short as osd) node.
// mainly used to display vp_frame_pose_target on frame.
class vp_pose_osd_node: public vp_node
{
private:
// pose pairs for PAFs
const std::map<vp_objects::vp_pose_type, std::vector<std::pair<int,int>>> posePairs_map = {
{vp_objects::vp_pose_type::body_25, {{1,8}, {1,2}, {1,5}, {2,3}, {3,4}, {5,6}, {6,7}, {8,9},
{9,10}, {10,11}, {8,12}, {12,13}, {13,14}, {1,0}, {0,15}, {15,17}, {0,16}, {16,18},
{14,19}, {19,20}, {14,21}, {11,22}, {22,23}, {11,24}}},
{vp_objects::vp_pose_type::coco, {{1,2}, {1,5}, {2,3}, {3,4}, {5,6}, {6,7},
{1,8}, {8,9}, {9,10}, {1,11}, {11,12}, {12,13},
{1,0}, {0,14}, {14,16}, {0,15}, {15,17}, {2,16},
{5,17}}},
{vp_objects::vp_pose_type::mpi_15, {{0,1}, {1,2}, {2,3}, {3,4}, {1,5}, {5,6}, {6,7}, {1,14}, {14,8}, {8,9}, {9,10}, {14,11}, {11,12}, {12,13}, {0, 2}, {0, 5}}},
{vp_objects::vp_pose_type::hand, std::vector<std::pair<int,int>>()},
{vp_objects::vp_pose_type::face, std::vector<std::pair<int,int>>()},
{vp_objects::vp_pose_type::yolov8_pose_17, {{0, 1}, {0, 2}, {0, 5}, {0, 6}, {1, 2}, {1, 3}, {2, 4}, {5, 6}, {5, 7}, {5, 11},
{6, 8}, {6, 12}, {7, 9}, {8, 10}, {11, 12}, {11, 13}, {12, 14}, {13, 15}, {14, 16}}}
};
void populateColorPalette(std::vector<cv::Scalar>& colors, int nColors) {
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_int_distribution<> dis1(64, 200);
std::uniform_int_distribution<> dis2(100, 255);
std::uniform_int_distribution<> dis3(100, 255);
for(int i = 0; i < nColors; ++i){
colors.push_back(cv::Scalar(dis1(gen),dis2(gen),dis3(gen)));
}
}
std::vector<cv::Scalar> colors;
protected:
virtual std::shared_ptr<vp_objects::vp_meta> handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) override;
virtual std::shared_ptr<vp_objects::vp_meta> handle_control_meta(std::shared_ptr<vp_objects::vp_control_meta> meta) override;
public:
vp_pose_osd_node(std::string node_name);
~vp_pose_osd_node();
};
}

127
nodes/osd/vp_seg_osd_node.cpp Normal file
View File

@@ -0,0 +1,127 @@
#include <fstream>
#include "vp_seg_osd_node.h"
#include "../../utils/vp_utils.h"
namespace vp_nodes {
vp_seg_osd_node::vp_seg_osd_node(std::string node_name, std::string classes_file, std::string colors_file): vp_node(node_name) {
// load classes names if possible
if (!classes_file.empty()) {
std::ifstream ifs(classes_file.c_str());
assert(ifs.is_open());
std::string line;
while (std::getline(ifs, line)) {
classes.push_back(line);
}
ifs.close();
}
// load colors if possible
if (!colors_file.empty()) {
std::ifstream ifs(colors_file.c_str());
assert(ifs.is_open());
std::string line;
while (std::getline(ifs, line)) {
auto color_s = vp_utils::string_split(line, ',');
cv::Vec3b color(static_cast<uchar>(std::stoi(color_s[0])), static_cast<uchar>(std::stoi(color_s[1])), static_cast<uchar>(std::stoi(color_s[2])));
colors.push_back(color);
}
ifs.close();
}
this->initialized();
}
vp_seg_osd_node::~vp_seg_osd_node() {
deinitialized();
}
std::shared_ptr<vp_objects::vp_meta> vp_seg_osd_node::handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) {
// operations on osd_frame
if (meta->osd_frame.empty()) {
// add a gap at the left of osd frame
meta->osd_frame = cv::Mat(meta->frame.rows, meta->frame.cols + gap, meta->frame.type(), cv::Scalar(255, 255, 255));
// initialize by copying frame to osd frame
auto roi = meta->osd_frame(cv::Rect(gap, 0, meta->frame.cols, meta->frame.rows));
meta->frame.copyTo(roi);
}
// left for display color/class pairs
auto canvas_left = meta->osd_frame(cv::Rect(0, 0, gap, meta->osd_frame.rows));
// right for display result
auto canvas_right = meta->osd_frame(cv::Rect(gap, 0, meta->frame.cols, meta->osd_frame.rows));
if (!meta->mask.empty()) {
cv::Mat segm;
colorizeSegmentation(meta->mask, segm);
cv::resize(segm, segm, canvas_right.size(), 0, 0, cv::INTER_NEAREST);
cv::addWeighted(canvas_right, 0, segm, 1, 0.0, canvas_right);
}
if (!classes.empty()) {
showLegend(canvas_left);
}
return meta;
}
void vp_seg_osd_node::colorizeSegmentation(const cv::Mat &score, cv::Mat &segm) {
using namespace cv;
const int rows = score.size[2];
const int cols = score.size[3];
const int chns = score.size[1];
if (colors.empty()) {
// Generate colors.
colors.push_back(Vec3b());
for (int i = 1; i < chns; ++i) {
Vec3b color;
for (int j = 0; j < 3; ++j)
color[j] = (colors[i - 1][j] + rand() % 256) / 2;
colors.push_back(color);
}
}
assert(colors.size() == chns);
Mat maxCl = Mat::zeros(rows, cols, CV_8UC1);
Mat maxVal(rows, cols, CV_32FC1, score.data);
for (int ch = 1; ch < chns; ch++) {
for (int row = 0; row < rows; row++) {
const float *ptrScore = score.ptr<float>(0, ch, row);
uint8_t *ptrMaxCl = maxCl.ptr<uint8_t>(row);
float *ptrMaxVal = maxVal.ptr<float>(row);
for (int col = 0; col < cols; col++) {
if (ptrScore[col] > ptrMaxVal[col]) {
ptrMaxVal[col] = ptrScore[col];
ptrMaxCl[col] = (uchar)ch;
}
}
}
}
segm.create(rows, cols, CV_8UC3);
for (int row = 0; row < rows; row++) {
const uchar *ptrMaxCl = maxCl.ptr<uchar>(row);
Vec3b *ptrSegm = segm.ptr<Vec3b>(row);
for (int col = 0; col < cols; col++) {
ptrSegm[col] = colors[ptrMaxCl[col]];
}
}
}
void vp_seg_osd_node::showLegend(cv::Mat& board) {
using namespace cv;
auto kBlockHeight = 30;
const int numClasses = (int)classes.size();
for (int i = 0; i < numClasses; i++) {
Mat block = board.rowRange(i * kBlockHeight, (i + 1) * kBlockHeight);
block.setTo(colors[i]);
putText(block, classes[i], Point(0, kBlockHeight / 2), FONT_HERSHEY_SIMPLEX, 0.5, Vec3b(255, 255, 255));
}
}
}

26
nodes/osd/vp_seg_osd_node.h Normal file
View File

@@ -0,0 +1,26 @@
#pragma once
#include <string>
#include "../vp_node.h"
namespace vp_nodes {
class vp_seg_osd_node: public vp_node
{
private:
/* data */
int gap = 60;
// classs names of semantic segmentation
std::vector<std::string> classes;
// colors of semantic segmentation
std::vector<cv::Vec3b> colors;
void colorizeSegmentation(const cv::Mat &score, cv::Mat &segm);
void showLegend(cv::Mat& board);
protected:
virtual std::shared_ptr<vp_objects::vp_meta> handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) override;
public:
vp_seg_osd_node(std::string node_name, std::string classes_file = "", std::string colors_file = "");
~vp_seg_osd_node();
};
}

53
nodes/osd/vp_text_osd_node.cpp Normal file
View File

@@ -0,0 +1,53 @@
#include "vp_text_osd_node.h"
namespace vp_nodes {
vp_text_osd_node::vp_text_osd_node(std::string node_name, std::string font): vp_node(node_name) {
ft2 = cv::freetype::createFreeType2();
ft2->loadFontData(font, 0);
this->initialized();
}
vp_text_osd_node::~vp_text_osd_node() {
deinitialized();
}
std::shared_ptr<vp_objects::vp_meta> vp_text_osd_node::handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) {
// operations on osd_frame
if (meta->osd_frame.empty()) {
// double times higher than frame
meta->osd_frame = cv::Mat(meta->frame.rows * 2, meta->frame.cols, meta->frame.type());
}
auto canvas1 = meta->frame.clone();
auto canvas2 = cv::Mat(meta->frame.rows, meta->frame.cols, meta->frame.type(), cv::Scalar(255, 255, 255));
for (int i = 0; i < meta->text_targets.size(); i++) {
auto& text = meta->text_targets[i];
cv::Point rook_points[4];
for (int m = 0; m < text->region_vertexes.size(); m++) {
rook_points[m] =
cv::Point(text->region_vertexes[m].first, text->region_vertexes[m].second);
}
const cv::Point *ppt[1] = {rook_points};
int npt[] = {4};
cv::polylines(canvas1, ppt, npt, 1, 1, CV_RGB(0, 255, 0), 2, cv::LINE_AA, 0);
cv::polylines(canvas2, ppt, npt, 1, 1, CV_RGB(0, 255, 0), 1, cv::LINE_AA, 0);
ft2->putText(canvas2, text->text, rook_points[3], 20, cv::Scalar(255, 0, 0), cv::FILLED, cv::LINE_AA, true);
}
// copy back to osd frame
auto roi1 = meta->osd_frame(cv::Rect(0, 0, meta->frame.cols, meta->frame.rows));
auto roi2 = meta->osd_frame(cv::Rect(0, meta->frame.rows, meta->frame.cols, meta->frame.rows));
canvas1.copyTo(roi1);
canvas2.copyTo(roi2);
return meta;
}
}

23
nodes/osd/vp_text_osd_node.h Normal file
View File

@@ -0,0 +1,23 @@
#pragma once
#include <opencv2/imgproc.hpp>
#include <opencv2/freetype.hpp>
#include "../vp_node.h"
namespace vp_nodes {
// on screen display(short as osd) node.
// mainly used to display vp_frame_text_target on frame.
class vp_text_osd_node: public vp_node
{
private:
// support chinese font
cv::Ptr<cv::freetype::FreeType2> ft2;
protected:
virtual std::shared_ptr<vp_objects::vp_meta> handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) override;
public:
vp_text_osd_node(std::string node_name, std::string font);
~vp_text_osd_node();
};
}