first commit

nodes/osd/vp_cluster_node.cpp

@@ -0,0 +1,214 @@
+#include "vp_cluster_node.h"
+#include "../../third_party/bhtsne/tsne.h"
+
+namespace vp_nodes {
+    vp_cluster_node::vp_cluster_node(std::string node_name, 
+                        bool use_tSNE, 
+                        std::vector<std::string> s_labels_to_display,
+                        int sampling_frequency,
+                        int min_sampling_width,
+                        int min_sampling_height,
+                        int max_sample_num_for_tsne,
+                        int max_sample_num_per_category):
+                        vp_node(node_name),
+                        use_tSNE(use_tSNE),
+                        s_labels_to_display(s_labels_to_display),
+                        sampling_frequency(sampling_frequency),
+                        min_sampling_width(min_sampling_width),
+                        min_sampling_height(min_sampling_height),
+                        max_sample_num_for_tsne(max_sample_num_for_tsne),
+                        max_sample_num_per_category(max_sample_num_per_category) {
+        this->initialized();
+    }
+    
+    vp_cluster_node::~vp_cluster_node() {
+        deinitialized();        
+    }
+
+    // please refer to ../../third_party/trt_vehicle/main/vehicle_cluster.cpp
+    void vp_cluster_node::reduce_dims_using_tsne(std::vector<std::vector<float>>& low_features,
+                                int no_dims, int max_iter, double perplexity, 
+                                double theta, int rand_seed, bool skip_random_init, 
+                                int stop_lying_iter, int mom_switch_iter) {
+        assert(cache_high_features.size() != 0);
+        auto N = cache_high_features.size();
+        auto D = cache_high_features[0].second.size();  // all the same as the first feature's dims
+
+        // prepare input
+        double data[N * D];
+        double Y[N * no_dims];
+        for (int i = 0; i < N; i++) {
+            for (int j = 0; j < D; j++) {
+                data[i * D + j] = cache_high_features[i].second[j];
+            }
+        }
+        
+        // call t-SNE
+        TSNE::run(data, N, D, Y, no_dims, perplexity, theta, rand_seed, skip_random_init, max_iter, stop_lying_iter, mom_switch_iter);
+
+        // prepare output
+        for (int i = 0; i < N; i++) {
+            std::vector<float> low_dims_feature;
+            for (int j = 0; j < no_dims; j++) {
+                low_dims_feature.push_back(float(Y[i * no_dims + j]));
+            }
+            low_features.push_back(low_dims_feature);
+        }
+    }
+
+
+    std::shared_ptr<vp_objects::vp_meta> vp_cluster_node::handle_frame_meta(std::shared_ptr<vp_objects::vp_frame_meta> meta) {
+        // sampling frequency
+        if (std::chrono::duration_cast<std::chrono::milliseconds>(NOW - last_sampling_time) < std::chrono::milliseconds(sampling_frequency)) {
+            return meta;
+        }
+        last_sampling_time = NOW;
+        
+        /* t-SNE */
+        if (use_tSNE) {
+            for (int i = 0; i < meta->targets.size(); i++) {
+                if (meta->targets[i]->width < min_sampling_width || meta->targets[i]->height < min_sampling_height) {
+                    continue;
+                }
+                
+                if (cache_high_features.size() >= max_sample_num_for_tsne) {
+                    // too many cache, we need control number of samples
+                    cache_high_features.erase(cache_high_features.begin());
+                }
+                // save mat->feature pair to cache
+                cv::Mat m;
+                cv::Mat roi(meta->frame, cv::Rect(meta->targets[i]->x, meta->targets[i]->y, meta->targets[i]->width, meta->targets[i]->height));
+                roi.copyTo(m);
+                std::pair<cv::Mat, std::vector<float>> p {m, meta->targets[i]->embeddings};
+                cache_high_features.push_back(p);
+            }
+
+            // at least 10 samples to t-SNE
+            if (cache_high_features.size() >= 10) {
+                // reduce dims first
+                std::vector<std::vector<float>> low_features;
+                reduce_dims_using_tsne(low_features);
+
+                // now display on screen
+                // normalize low dims feature to coordinate of [0:1] and display them on 2D screen
+                auto max_x = 0.0f, max_y = 0.0f, min_x = 0.0f, min_y = 0.0f;
+                for(int i = 0; i < low_features.size(); i++) {
+                    auto& f = low_features[i];
+                    // 2 values in f
+                    max_x = std::max(max_x, f[0]);
+                    max_y = std::max(max_y, f[1]);
+                    min_x = std::min(min_x, f[0]);
+                    min_y = std::min(min_y, f[1]);
+                }
+                auto x_range = max_x - min_x;
+                auto y_range = max_y - min_y;
+
+                // draw on (tsne_canvas_w_h + tsne_thumbnail_w_h) * (tsne_canvas_w_h + tsne_thumbnail_w_h)
+                cv::Mat canvas(tsne_canvas_w_h + tsne_thumbnail_w_h, tsne_canvas_w_h + tsne_thumbnail_w_h, CV_8UC3, cv::Scalar(127, 127, 127));
+                for(int i = 0; i < low_features.size(); i++) {
+                    auto& f = low_features[i];
+                    // convert to [0:1]
+                    f[0] = (f[0] - min_x) / x_range;
+                    f[1] = (f[1] - min_y) / y_range;
+
+                    auto& img = cache_high_features[i].first;
+                    cv::Mat img_tmp;
+                    cv::resize(img, img_tmp, cv::Size(tsne_thumbnail_w_h, tsne_thumbnail_w_h));
+                    cv::rectangle(img_tmp, cv::Rect(0, 0, img_tmp.cols, img_tmp.rows), cv::Scalar(255, 0, 0));
+                    cv::Mat roi(canvas, cv::Rect(int(f[0] * tsne_canvas_w_h), int(f[1] * tsne_canvas_w_h), tsne_thumbnail_w_h, tsne_thumbnail_w_h));
+                    img_tmp.copyTo(roi);
+                }
+
+                cv::imshow("cluster using features powered by t-SNE", canvas);
+            }
+        }
+
+        /* categories */
+        for (int i = 0; i < meta->targets.size(); i++) {
+            auto& t = meta->targets[i];
+            if (t->width < min_sampling_width || t->height < min_sampling_height) {
+                continue;
+            }
+
+            cv::Mat m;
+            cv::Mat roi(meta->frame, cv::Rect(t->x, t->y, t->width, t->height));
+            roi.copyTo(m);
+
+            for (int j = 0; j < t->secondary_labels.size(); j++) {
+                auto& category = t->secondary_labels[j];
+                bool filter_pass = false;
+                // all
+                if (s_labels_to_display.size() == 0) {
+                    cache_categories[category].push_back(m);
+                    filter_pass = true;
+                }
+                else {
+                    // has a filter
+                    if (std::find(s_labels_to_display.begin(), s_labels_to_display.end(), category) != s_labels_to_display.end()) {
+                        cache_categories[category].push_back(m);
+                        filter_pass = true;
+                    }
+                }
+
+                if (filter_pass) {                
+                    // too many cache, we need control number of samples
+                    if (cache_categories[category].size() >= max_sample_num_per_category) {
+                        cache_categories[category].erase(cache_categories[category].begin());
+                    }
+                }
+            }
+        }
+        // display on screen
+        // calculate total number of rows
+        auto rows_num = 0; 
+        for (auto& p: cache_categories) {
+            auto num = p.second.size() / category_num_per_row;
+            if (p.second.size() % category_num_per_row != 0 || num == 0) {
+                num++;
+            }
+            rows_num += num;
+        }
+        
+        // draw on (category_canvas_w) * (category_canvas_h)
+        auto category_canvas_w = (category_thumbnail_w_h + category_gap) * (category_num_per_row + 1);
+        auto category_canvas_h = (category_thumbnail_w_h + category_gap) * (rows_num + 1);
+        cv::Mat canvas(category_canvas_h, category_canvas_w, CV_8UC3, cv::Scalar(127, 127, 127));
+
+        auto row_index = 0;
+        for (auto& p: cache_categories) {
+            auto col_index = 0;
+            auto& category_items = p.second;
+            auto& category_name = p.first;
+            cv::putText(canvas, category_name, cv::Point(10, (category_gap + category_thumbnail_w_h) * row_index + category_gap), 1, 1, cv::Scalar(0, 0, 255));
+
+            bool new_row = false;
+            for (int i = 0; i < category_items.size(); i++) {
+                cv::Mat img_tmp;
+                cv::resize(category_items[i], img_tmp, cv::Size(category_thumbnail_w_h, category_thumbnail_w_h));
+                cv::Mat roi(canvas, cv::Rect((category_gap + category_thumbnail_w_h) * col_index + category_gap, (category_gap + category_thumbnail_w_h) * row_index + category_gap, category_thumbnail_w_h, category_thumbnail_w_h));
+                img_tmp.copyTo(roi);
+
+                col_index++;
+                if ((col_index + 1) % category_num_per_row == 0) {
+                    col_index = 0;
+                    row_index ++;
+                    new_row = true;
+                } 
+                else {
+                    new_row = false;
+                }
+            }
+            if (!new_row) {
+                row_index++;
+            }
+        }
+
+        cv::imshow("cluster using labels", canvas);
+
+        return meta;
+    }
+
+    std::shared_ptr<vp_objects::vp_meta> vp_cluster_node::handle_control_meta(std::shared_ptr<vp_objects::vp_control_meta> meta) {
+        return meta;
+    }
+}