#include "vp_cluster_node.h" #include "../../third_party/bhtsne/tsne.h" namespace vp_nodes { vp_cluster_node::vp_cluster_node(std::string node_name, bool use_tSNE, std::vector s_labels_to_display, int sampling_frequency, int min_sampling_width, int min_sampling_height, int max_sample_num_for_tsne, int max_sample_num_per_category): vp_node(node_name), use_tSNE(use_tSNE), s_labels_to_display(s_labels_to_display), sampling_frequency(sampling_frequency), min_sampling_width(min_sampling_width), min_sampling_height(min_sampling_height), max_sample_num_for_tsne(max_sample_num_for_tsne), max_sample_num_per_category(max_sample_num_per_category) { this->initialized(); } vp_cluster_node::~vp_cluster_node() { deinitialized(); } // please refer to ../../third_party/trt_vehicle/main/vehicle_cluster.cpp void vp_cluster_node::reduce_dims_using_tsne(std::vector>& low_features, int no_dims, int max_iter, double perplexity, double theta, int rand_seed, bool skip_random_init, int stop_lying_iter, int mom_switch_iter) { assert(cache_high_features.size() != 0); auto N = cache_high_features.size(); auto D = cache_high_features[0].second.size(); // all the same as the first feature's dims // prepare input double data[N * D]; double Y[N * no_dims]; for (int i = 0; i < N; i++) { for (int j = 0; j < D; j++) { data[i * D + j] = cache_high_features[i].second[j]; } } // call t-SNE TSNE::run(data, N, D, Y, no_dims, perplexity, theta, rand_seed, skip_random_init, max_iter, stop_lying_iter, mom_switch_iter); // prepare output for (int i = 0; i < N; i++) { std::vector low_dims_feature; for (int j = 0; j < no_dims; j++) { low_dims_feature.push_back(float(Y[i * no_dims + j])); } low_features.push_back(low_dims_feature); } } std::shared_ptr vp_cluster_node::handle_frame_meta(std::shared_ptr meta) { // sampling frequency if (std::chrono::duration_cast(NOW - last_sampling_time) < std::chrono::milliseconds(sampling_frequency)) { return meta; } last_sampling_time = NOW; /* t-SNE */ if (use_tSNE) { for (int i = 0; i < meta->targets.size(); i++) { if (meta->targets[i]->width < min_sampling_width || meta->targets[i]->height < min_sampling_height) { continue; } if (cache_high_features.size() >= max_sample_num_for_tsne) { // too many cache, we need control number of samples cache_high_features.erase(cache_high_features.begin()); } // save mat->feature pair to cache cv::Mat m; cv::Mat roi(meta->frame, cv::Rect(meta->targets[i]->x, meta->targets[i]->y, meta->targets[i]->width, meta->targets[i]->height)); roi.copyTo(m); std::pair> p {m, meta->targets[i]->embeddings}; cache_high_features.push_back(p); } // at least 10 samples to t-SNE if (cache_high_features.size() >= 10) { // reduce dims first std::vector> low_features; reduce_dims_using_tsne(low_features); // now display on screen // normalize low dims feature to coordinate of [0:1] and display them on 2D screen auto max_x = 0.0f, max_y = 0.0f, min_x = 0.0f, min_y = 0.0f; for(int i = 0; i < low_features.size(); i++) { auto& f = low_features[i]; // 2 values in f max_x = std::max(max_x, f[0]); max_y = std::max(max_y, f[1]); min_x = std::min(min_x, f[0]); min_y = std::min(min_y, f[1]); } auto x_range = max_x - min_x; auto y_range = max_y - min_y; // draw on (tsne_canvas_w_h + tsne_thumbnail_w_h) * (tsne_canvas_w_h + tsne_thumbnail_w_h) cv::Mat canvas(tsne_canvas_w_h + tsne_thumbnail_w_h, tsne_canvas_w_h + tsne_thumbnail_w_h, CV_8UC3, cv::Scalar(127, 127, 127)); for(int i = 0; i < low_features.size(); i++) { auto& f = low_features[i]; // convert to [0:1] f[0] = (f[0] - min_x) / x_range; f[1] = (f[1] - min_y) / y_range; auto& img = cache_high_features[i].first; cv::Mat img_tmp; cv::resize(img, img_tmp, cv::Size(tsne_thumbnail_w_h, tsne_thumbnail_w_h)); cv::rectangle(img_tmp, cv::Rect(0, 0, img_tmp.cols, img_tmp.rows), cv::Scalar(255, 0, 0)); cv::Mat roi(canvas, cv::Rect(int(f[0] * tsne_canvas_w_h), int(f[1] * tsne_canvas_w_h), tsne_thumbnail_w_h, tsne_thumbnail_w_h)); img_tmp.copyTo(roi); } cv::imshow("cluster using features powered by t-SNE", canvas); } } /* categories */ for (int i = 0; i < meta->targets.size(); i++) { auto& t = meta->targets[i]; if (t->width < min_sampling_width || t->height < min_sampling_height) { continue; } cv::Mat m; cv::Mat roi(meta->frame, cv::Rect(t->x, t->y, t->width, t->height)); roi.copyTo(m); for (int j = 0; j < t->secondary_labels.size(); j++) { auto& category = t->secondary_labels[j]; bool filter_pass = false; // all if (s_labels_to_display.size() == 0) { cache_categories[category].push_back(m); filter_pass = true; } else { // has a filter if (std::find(s_labels_to_display.begin(), s_labels_to_display.end(), category) != s_labels_to_display.end()) { cache_categories[category].push_back(m); filter_pass = true; } } if (filter_pass) { // too many cache, we need control number of samples if (cache_categories[category].size() >= max_sample_num_per_category) { cache_categories[category].erase(cache_categories[category].begin()); } } } } // display on screen // calculate total number of rows auto rows_num = 0; for (auto& p: cache_categories) { auto num = p.second.size() / category_num_per_row; if (p.second.size() % category_num_per_row != 0 || num == 0) { num++; } rows_num += num; } // draw on (category_canvas_w) * (category_canvas_h) auto category_canvas_w = (category_thumbnail_w_h + category_gap) * (category_num_per_row + 1); auto category_canvas_h = (category_thumbnail_w_h + category_gap) * (rows_num + 1); cv::Mat canvas(category_canvas_h, category_canvas_w, CV_8UC3, cv::Scalar(127, 127, 127)); auto row_index = 0; for (auto& p: cache_categories) { auto col_index = 0; auto& category_items = p.second; auto& category_name = p.first; cv::putText(canvas, category_name, cv::Point(10, (category_gap + category_thumbnail_w_h) * row_index + category_gap), 1, 1, cv::Scalar(0, 0, 255)); bool new_row = false; for (int i = 0; i < category_items.size(); i++) { cv::Mat img_tmp; cv::resize(category_items[i], img_tmp, cv::Size(category_thumbnail_w_h, category_thumbnail_w_h)); cv::Mat roi(canvas, cv::Rect((category_gap + category_thumbnail_w_h) * col_index + category_gap, (category_gap + category_thumbnail_w_h) * row_index + category_gap, category_thumbnail_w_h, category_thumbnail_w_h)); img_tmp.copyTo(roi); col_index++; if ((col_index + 1) % category_num_per_row == 0) { col_index = 0; row_index ++; new_row = true; } else { new_row = false; } } if (!new_row) { row_index++; } } cv::imshow("cluster using labels", canvas); return meta; } std::shared_ptr vp_cluster_node::handle_control_meta(std::shared_ptr meta) { return meta; } }