std::cout << std::endl;
if (visualize_delay >= 0) {
- cv::rectangle(image, bb_rect, CV_RGB(0,255,0), 2);
+ cv::Point pt(bb.cx, bb.cy);
+ cv::Size size(bb.w, bb.h);
+ cv::RotatedRect rotatedRectangle(pt, size, bb.a);
+
+ cv::Point2f vertices[4];
+ rotatedRectangle.points(vertices);
+
+ for (int i = 0; i < 4; i++)
+ cv::line(image, vertices[i], vertices[(i + 1) % 4], cv::Scalar(0, 255, 0), 2);
+ // cv::rectangle(image, cv::Rect(bb.cx - bb.w/2., bb.cy - bb.h/2., bb.w, bb.h), CV_RGB(0,255,0),
+ // 2);
+ std::string angle = std::to_string(bb.a);
+ angle.erase(angle.find_last_not_of('0') + 1, std::string::npos);
+ angle.erase(angle.find_last_not_of('.') + 1, std::string::npos);
+ cv::putText(image, "Frame: " + std::to_string(frames) + " " + angle + " angle",
+ cv::Point(0, image.rows - 1), cv::FONT_HERSHEY_SIMPLEX, 0.7, cv::Scalar(0, 255, 0), 2);
cv::imshow("output", image);
int ret = cv::waitKey(visualize_delay);
- if (visualize_delay > 0 && ret != -1 && ret != 255)
- break;
+ if (visualize_delay > 0 && ret != -1 && ret != 255) break;
}
-// std::stringstream s;
-// std::string ss;
-// int countTmp = frames;
-// s << "imgs" << "/img" << (countTmp/10000);
-// countTmp = countTmp%10000;
-// s << (countTmp/1000);
-// countTmp = countTmp%1000;
-// s << (countTmp/100);
-// countTmp = countTmp%100;
-// s << (countTmp/10);
-// countTmp = countTmp%10;
-// s << (countTmp);
-// s << ".jpg";
-// s >> ss;
-// //set image output parameters
-// std::vector<int> compression_params;
-// compression_params.push_back(CV_IMWRITE_JPEG_QUALITY);
-// compression_params.push_back(90);
-// cv::imwrite(ss.c_str(), image, compression_params);
+ // std::stringstream s;
+ // std::string ss;
+ // int countTmp = frames;
+ // s << "imgs" << "/img" << (countTmp/10000);
+ // countTmp = countTmp%10000;
+ // s << (countTmp/1000);
+ // countTmp = countTmp%1000;
+ // s << (countTmp/100);
+ // countTmp = countTmp%100;
+ // s << (countTmp/10);
+ // countTmp = countTmp%10;
+ // s << (countTmp);
+ // s << ".jpg";
+ // s >> ss;
+ // //set image output parameters
+ // std::vector<int> compression_params;
+ // compression_params.push_back(CV_IMWRITE_JPEG_QUALITY);
+ // compression_params.push_back(90);
+ // cv::imwrite(ss.c_str(), image, compression_params);
}
- std::cout << "Average processing speed: " << avg_time/frames << "ms (" << 1./(avg_time/frames)*1000 << " fps)";
+ std::cout << "Average processing speed: " << avg_time / frames << "ms (" << 1. / (avg_time / frames) * 1000 << " fps)";
if (groundtruth_stream.is_open()) {
std::cout << "; Average accuracy: " << sum_accuracy/frames << std::endl;
groundtruth_stream.close();
p_yf.create(p_roi.height, width, 1);
p_xf.create(p_roi.height, width, p_num_of_feats);
- int max1 = m_use_big_batch ? 2 : p_num_scales;
- int max2 = m_use_big_batch ? 1 : p_num_angles;
- int max = BIG_BATCH_MODE ? 2 : p_num_scales;
- for (int i = 0; i < max; ++i) {
- if (BIG_BATCH_MODE && i == 1)
- p_threadctxs.emplace_back(p_roi, p_num_of_feats * p_num_scales, 1, p_num_scales);
- else
- p_threadctxs.emplace_back(p_roi, p_num_of_feats, p_scales[i], 1);
++ int max1 = BIG_BATCH_MODE ? 2 : p_num_scales;
++ int max2 = BIG_BATCH_MODE ? 1 : p_num_angles;
+ for (int i = 0; i < max1; ++i) {
+ for (int j = 0; j < max2; ++j) {
- if (m_use_big_batch && i == 1)
++ if (BIG_BATCH_MODE && i == 1)
+ p_threadctxs.emplace_back(p_roi, p_num_of_feats * p_num_scales * p_num_angles, 1, 0, p_num_scales,
+ p_num_angles);
+ else
+ p_threadctxs.emplace_back(p_roi, p_num_of_feats, p_scales[i], p_angles[j]);
+ }
}
p_current_scale = 1.;
DEBUG_PRINTM(p_yf);
// obtain a sub-window for training initial model
- p_threadctxs.front().patch_feats.clear();
-
- std::vector<cv::Mat> patch_feats = get_features(input_rgb, input_gray, p_pose.cx, p_pose.cy,
- p_windows_size.width, p_windows_size.height);
+ int size_x_scaled = floor(p_windows_size.width);
+ int size_y_scaled = floor(p_windows_size.height);
+
+ cv::Mat patch_gray = get_subwindow(input_gray, this->p_pose.cx, this->p_pose.cy, size_x_scaled, size_y_scaled);
+ geometric_transformations(patch_gray, p_windows_size.width, p_windows_size.height, 0, false);
+
- cv::Mat patch_rgb = cv::Mat::zeros(size_y_scaled, size_x_scaled, CV_32F);
++ cv::Mat patch_rgb;
+ if ((m_use_color || m_use_cnfeat) && input_rgb.channels() == 3) {
+ patch_rgb = get_subwindow(input_rgb, this->p_pose.cx, this->p_pose.cy, size_x_scaled, size_y_scaled);
+ geometric_transformations(patch_rgb, p_windows_size.width, p_windows_size.height, 0, false);
+ }
+
- get_features(patch_rgb, patch_gray, p_threadctxs.front());
- fft.forward_window(p_threadctxs.front().patch_feats, p_model_xf, p_threadctxs.front().fw_all,
++ std::vector<cv::Mat> patch_feats = get_features(patch_rgb, patch_gray);
+ fft.forward_window(patch_feats, p_model_xf, p_threadctxs.front().fw_all,
m_use_cuda ? p_threadctxs.front().data_features.deviceMem() : nullptr,
p_threadctxs.front().stream);
DEBUG_PRINTM(p_model_xf);
});
for (auto const &it : p_threadctxs)
it.async_res.wait();
-
#else // !ASYNC
- // FIXME: Iterate correctly in big batch mode - perhaps have only one element in the list
NORMAL_OMP_PARALLEL_FOR
- for (uint i = m_use_big_batch ? 1 : 0; i < p_threadctxs.size(); ++i)
- for (uint i = 0; i < p_threadctxs.size(); ++i)
++ for (uint i = BIG_BATCH_MODE ? 1 : 0; i < p_threadctxs.size(); ++i)
scale_track(p_threadctxs[i], input_rgb, input_gray);
#endif
new_location = sub_pixel_peak(*max_response_pt, *max_response_map);
DEBUG_PRINT(new_location);
+ if (m_visual_debug) std::cout << "Old p_pose, cx: " << p_pose.cx << " cy: " << p_pose.cy << std::endl;
+
p_pose.cx += p_current_scale * p_cell_size * double(new_location.x);
p_pose.cy += p_current_scale * p_cell_size * double(new_location.y);
+
+ if (m_visual_debug) std::cout << "New p_pose, cx: " << p_pose.cx << " cy: " << p_pose.cy << std::endl;
+
if (p_fit_to_pw2) {
- if (p_pose.cx < 0) p_pose.cx = 0;
- if (p_pose.cx > (img.cols * p_scale_factor_x) - 1) p_pose.cx = (img.cols * p_scale_factor_x) - 1;
- if (p_pose.cy < 0) p_pose.cy = 0;
- if (p_pose.cy > (img.rows * p_scale_factor_y) - 1) p_pose.cy = (img.rows * p_scale_factor_y) - 1;
+ clamp2(p_pose.cx, 0.0, (img.cols * p_scale_factor_x) - 1);
+ clamp2(p_pose.cy, 0.0, (img.rows * p_scale_factor_y) - 1);
} else {
- if (p_pose.cx < 0) p_pose.cx = 0;
- if (p_pose.cx > img.cols - 1) p_pose.cx = img.cols - 1;
- if (p_pose.cy < 0) p_pose.cy = 0;
- if (p_pose.cy > img.rows - 1) p_pose.cy = img.rows - 1;
+ clamp2(p_pose.cx, 0.0, img.cols - 1.0);
+ clamp2(p_pose.cy, 0.0, img.rows - 1.0);
}
// sub grid scale interpolation
p_current_scale *= max->scale;
}
+ clamp2(p_current_scale, p_min_max_scale[0], p_min_max_scale[1]);
- ThreadCtx &ctx = p_threadctxs.front();
+ if (p_current_scale < p_min_max_scale[0]) p_current_scale = p_min_max_scale[0];
+ if (p_current_scale > p_min_max_scale[1]) p_current_scale = p_min_max_scale[1];
+
+ p_current_angle = (p_current_angle + max->angle) < 0
+ ? -std::abs(p_current_angle + max->angle) % 360
+ : (p_current_angle + max->angle) % 360;
+
// obtain a subwindow for training at newly estimated target position
- std::vector<cv::Mat> patch_feats = get_features(input_rgb, input_gray, p_pose.cx, p_pose.cy,
- p_windows_size.width, p_windows_size.height,
- p_current_scale);
- fft.forward_window(patch_feats, p_xf, ctx.fw_all,
- m_use_cuda ? ctx.data_features.deviceMem() : nullptr, ctx.stream);
+ int size_x_scaled = floor(p_windows_size.width * p_current_scale);
+ int size_y_scaled = floor(p_windows_size.height * p_current_scale);
+
+ cv::Mat patch_gray = get_subwindow(input_gray, this->p_pose.cx, this->p_pose.cy, size_x_scaled, size_y_scaled);
+ geometric_transformations(patch_gray, p_windows_size.width, p_windows_size.height, p_current_angle, false);
+
+ cv::Mat patch_rgb = cv::Mat::zeros(size_y_scaled, size_x_scaled, CV_32F);
+ if ((m_use_color || m_use_cnfeat) && input_rgb.channels() == 3) {
+ patch_rgb = get_subwindow(input_rgb, this->p_pose.cx, this->p_pose.cy, size_x_scaled, size_y_scaled);
+ geometric_transformations(patch_rgb, p_windows_size.width, p_windows_size.height, p_current_angle, false);
+ }
+
- p_threadctxs.front().patch_feats.clear();
- get_features(patch_rgb, patch_gray, p_threadctxs.front());
- fft.forward_window(p_threadctxs.front().patch_feats, p_xf, p_threadctxs.front().fw_all,
- m_use_cuda ? p_threadctxs.front().data_features.deviceMem() : nullptr, p_threadctxs.front().stream);
++ ThreadCtx &ctx = p_threadctxs.front();
++ std::vector<cv::Mat> patch_feats = get_features(patch_rgb, patch_gray);
++ fft.forward_window(patch_feats, p_xf, ctx.fw_all, m_use_cuda ? ctx.data_features.deviceMem() : nullptr, ctx.stream);
// subsequent frames, interpolate model
p_model_xf = p_model_xf * float((1. - p_interp_factor)) + p_xf * float(p_interp_factor);
void KCF_Tracker::scale_track(ThreadCtx &vars, cv::Mat &input_rgb, cv::Mat &input_gray)
{
- if (m_use_big_batch) {
- vars.patch_feats.clear();
+ std::vector<cv::Mat> patch_feats;
+ if (BIG_BATCH_MODE) {
BIG_BATCH_OMP_PARALLEL_FOR
- for (uint i = 0; i < p_num_scales; ++i) {
- patch_feats = get_features(input_rgb, input_gray, this->p_pose.cx, this->p_pose.cy,
- this->p_windows_size.width, this->p_windows_size.height,
- this->p_current_scale * this->p_scales[i]);
+ for (uint i = 0; i < this->p_scales.size(); ++i) {
+ for (uint j = 0; j < this->p_angles.size(); ++j) {
+ int size_x_scaled = floor(this->p_windows_size.width * this->p_current_scale * this->p_scales[i]);
+ int size_y_scaled = floor(this->p_windows_size.height * this->p_current_scale * this->p_scales[i]);
+
+ cv::Mat patch_gray =
+ get_subwindow(input_gray, this->p_pose.cx, this->p_pose.cy, size_x_scaled, size_y_scaled);
+ geometric_transformations(patch_gray, p_windows_size.width, p_windows_size.height,
+ p_current_scale * this->p_scales[i], p_current_angle + this->p_angles[j]);
+
- cv::Mat patch_rgb = cv::Mat::zeros(size_y_scaled, size_x_scaled, CV_32F);
++ cv::Mat patch_rgb;
+ if ((m_use_color || m_use_cnfeat) && input_rgb.channels() == 3) {
+ patch_rgb =
+ get_subwindow(input_rgb, this->p_pose.cx, this->p_pose.cy, size_x_scaled, size_y_scaled);
+ geometric_transformations(patch_rgb, p_windows_size.width, p_windows_size.height,
+ p_current_scale * this->p_scales[i], p_current_angle + this->p_angles[j]);
+ }
- get_features(patch_rgb, patch_gray, vars);
++ std::vector<cv::Mat> tmp = get_features(patch_rgb, patch_gray);
++ BIG_BATCH_OMP_ORDERED
++ patch_feats.insert(patch_feats.end(), tmp.begin(), tmp.end());
+ }
}
} else {
- patch_feats = get_features(input_rgb, input_gray, this->p_pose.cx, this->p_pose.cy,
- this->p_windows_size.width, this->p_windows_size.height,
- this->p_current_scale * vars.scale);
+ int size_x_scaled = floor(this->p_windows_size.width * this->p_current_scale * vars.scale);
+ int size_y_scaled = floor(this->p_windows_size.height * this->p_current_scale * vars.scale);
+
+ cv::Mat patch_gray = get_subwindow(input_gray, this->p_pose.cx, this->p_pose.cy, size_x_scaled, size_y_scaled);
+ geometric_transformations(patch_gray, p_windows_size.width, p_windows_size.height, p_current_scale * vars.scale);
+
- cv::Mat patch_rgb = cv::Mat::zeros(size_y_scaled, size_x_scaled, CV_32F);
++ cv::Mat patch_rgb;
+ if ((m_use_color || m_use_cnfeat) && input_rgb.channels() == 3) {
+ patch_rgb = get_subwindow(input_rgb, this->p_pose.cx, this->p_pose.cy, size_x_scaled, size_y_scaled);
+ geometric_transformations(patch_rgb, p_windows_size.width, p_windows_size.height, p_current_scale * vars.scale,
+ p_current_angle + vars.angle);
+ }
- vars.patch_feats.clear();
- get_features(patch_rgb, patch_gray, vars);
++ patch_feats = get_features(patch_rgb, patch_gray);
}
- fft.forward_window(vars.patch_feats, vars.zf, vars.fw_all, m_use_cuda ? vars.data_features.deviceMem() : nullptr,
+ fft.forward_window(patch_feats, vars.zf, vars.fw_all, m_use_cuda ? vars.data_features.deviceMem() : nullptr,
vars.stream);
DEBUG_PRINTM(vars.zf);
// ****************************************************************************
- void KCF_Tracker::get_features(cv::Mat &patch_rgb, cv::Mat &patch_gray, ThreadCtx &vars)
-std::vector<cv::Mat> KCF_Tracker::get_features(cv::Mat & input_rgb, cv::Mat & input_gray, int cx, int cy, int size_x, int size_y, double scale)
++std::vector<cv::Mat> KCF_Tracker::get_features(cv::Mat &patch_rgb, cv::Mat &patch_gray)
{
- int size_x_scaled = floor(size_x * scale);
- int size_y_scaled = floor(size_y * scale);
-
- cv::Mat patch_gray = get_subwindow(input_gray, cx, cy, size_x_scaled, size_y_scaled);
- cv::Mat patch_rgb = get_subwindow(input_rgb, cx, cy, size_x_scaled, size_y_scaled);
-
- // resize to default size
- if (scale > 1.) {
- // if we downsample use INTER_AREA interpolation
- cv::resize(patch_gray, patch_gray, cv::Size(size_x, size_y), 0., 0., cv::INTER_AREA);
- } else {
- cv::resize(patch_gray, patch_gray, cv::Size(size_x, size_y), 0., 0., cv::INTER_LINEAR);
- }
-
// get hog(Histogram of Oriented Gradients) features
- vars.patch_feats = FHoG::extract(patch_gray, 2, p_cell_size, 9);
+ std::vector<cv::Mat> hog_feat = FHoG::extract(patch_gray, 2, p_cell_size, 9);
// get color rgb features (simple r,g,b channels)
std::vector<cv::Mat> color_feat;
return patch;
}
-void KCF_Tracker::gaussian_correlation(struct ThreadCtx &vars, const ComplexMat &xf, const ComplexMat &yf,
- double sigma, bool auto_correlation)
+void KCF_Tracker::geometric_transformations(cv::Mat &patch, int size_x, int size_y, int angle, bool allow_debug)
+{
+ if (m_use_angle) {
+ cv::Point2f center((patch.cols - 1) / 2., (patch.rows - 1) / 2.);
+ cv::Mat r = cv::getRotationMatrix2D(center, angle, 1.0);
+
+ cv::warpAffine(patch, patch, r, cv::Size(patch.cols, patch.rows), cv::INTER_LINEAR, cv::BORDER_REPLICATE);
+ }
+
+ // resize to default size
+ if (patch.channels() != 3) {
+ if (patch.cols / size_x > 1.) {
+ // if we downsample use INTER_AREA interpolation
+ cv::resize(patch, patch, cv::Size(size_x, size_y), 0., 0., cv::INTER_AREA);
+ } else {
+ cv::resize(patch, patch, cv::Size(size_x, size_y), 0., 0., cv::INTER_LINEAR);
+ }
+ } else {
+ if (patch.cols / size_x > 1.) {
+ // if we downsample use INTER_AREA interpolation
+ cv::resize(patch, patch, cv::Size(size_x / p_cell_size, size_y / p_cell_size), 0., 0., cv::INTER_AREA);
+ } else {
+ cv::resize(patch, patch, cv::Size(size_x / p_cell_size, size_y / p_cell_size), 0., 0., cv::INTER_LINEAR);
+ }
+ if (m_visual_debug && allow_debug) {
+ cv::Mat input_clone = patch.clone();
+ cv::resize(input_clone, input_clone, cv::Size(p_debug_image_size, p_debug_image_size), 0., 0.,
+ cv::INTER_LINEAR);
+
+ std::string angle_string = std::to_string(p_current_angle + angle);
+
+ cv::putText(input_clone, angle_string, cv::Point(1, input_clone.rows - 5), cv::FONT_HERSHEY_COMPLEX_SMALL,
+ 0.5, cv::Scalar(0, 255, 0), 1);
+
+ p_debug_subwindows.push_back(input_clone);
+ }
+ }
+}
+
+void KCF_Tracker::gaussian_correlation(struct ThreadCtx &vars, const ComplexMat &xf, const ComplexMat &yf, double sigma,
+ bool auto_correlation)
{
- #ifdef CUFFT
- xf.sqr_norm(vars.xf_sqr_norm.deviceMem());
- if (!auto_correlation) yf.sqr_norm(vars.yf_sqr_norm.deviceMem());
- #else
- xf.sqr_norm(vars.xf_sqr_norm.hostMem());
+ xf.sqr_norm(vars.xf_sqr_norm);
if (auto_correlation) {
vars.yf_sqr_norm.hostMem()[0] = vars.xf_sqr_norm.hostMem()[0];
} else {
inline void scale_y(double factor)
{
cy *= factor;
- h *= factor;
- }
-
- inline cv::Rect get_rect()
- {
- return cv::Rect(int(cx-w/2.), int(cy-h/2.), int(w), int(h));
+ h *= factor;
}
+ inline cv::Rect get_rect() { return cv::Rect(int(cx - w / 2.), int(cy - h / 2.), int(w), int(h)); }
};
-class KCF_Tracker
-{
-public:
- bool m_debug {false};
- bool m_use_scale {true};
- bool m_use_color {true};
+class KCF_Tracker {
+ public:
+ bool m_debug{false};
+ bool m_visual_debug{false};
+ bool m_use_scale{true};
+ bool m_use_angle{false}; // Doesn't work with FFTW-BIG version
+ bool m_use_color{true};
#ifdef ASYNC
- bool m_use_multithreading {true};
+ bool m_use_multithreading{true};
#else
- bool m_use_multithreading{false};
- #endif // ASYNC
- bool m_use_subpixel_localization{true};
- bool m_use_subgrid_scale{true};
- bool m_use_cnfeat{true};
- bool m_use_linearkernel{false};
- #ifdef BIG_BATCH
- bool m_use_big_batch{true};
- #else
- bool m_use_big_batch{false};
- #endif
+ bool m_use_multithreading {false};
+ #endif //ASYNC
+ bool m_use_subpixel_localization {true};
+ bool m_use_subgrid_scale {true};
+ bool m_use_cnfeat {true};
+ bool m_use_linearkernel {false};
#ifdef CUFFT
- bool m_use_cuda {true};
+ bool m_use_cuda{true};
#else
- bool m_use_cuda {false};
+ bool m_use_cuda{false};
#endif
/*
double p_current_scale = 1.;
double p_min_max_scale[2];
std::vector<double> p_scales;
- int p_count = 0;
+ int p_current_angle = 0;
+ uint p_num_angles {5};
+ int p_angle_min = -20, p_angle_max = 20;
+ int p_angle_step = 10;
+ std::vector<int> p_angles;
+
+ // for visual debug
+ int p_debug_image_size = 100;
+ std::vector<cv::Mat> p_debug_scale_responses;
+ std::vector<cv::Mat> p_debug_subwindows;
//for big batch
- int p_num_of_feats;
+ int p_num_of_feats = 31 + (m_use_color ? 3 : 0) + (m_use_cnfeat ? 10 : 0);
cv::Size p_roi;
std::vector<ThreadCtx> p_threadctxs;
void scale_track(ThreadCtx & vars, cv::Mat & input_rgb, cv::Mat & input_gray);
cv::Mat get_subwindow(const cv::Mat & input, int cx, int cy, int size_x, int size_y);
cv::Mat gaussian_shaped_labels(double sigma, int dim1, int dim2);
- void gaussian_correlation(struct ThreadCtx &vars, const ComplexMat & xf, const ComplexMat & yf, double sigma, bool auto_correlation = false);
- cv::Mat circshift(const cv::Mat & patch, int x_rot, int y_rot);
+ void gaussian_correlation(struct ThreadCtx &vars, const ComplexMat &xf, const ComplexMat &yf, double sigma,
+ bool auto_correlation = false);
+ cv::Mat circshift(const cv::Mat &patch, int x_rot, int y_rot);
cv::Mat cosine_window_function(int dim1, int dim2);
- void get_features(cv::Mat &patch_rgb, cv::Mat &patch_gray, ThreadCtx &vars);
- std::vector<cv::Mat> get_features(cv::Mat & input_rgb, cv::Mat & input_gray, int cx, int cy, int size_x, int size_y, double scale = 1.);
- cv::Point2f sub_pixel_peak(cv::Point & max_loc, cv::Mat & response);
++ std::vector<cv::Mat> get_features(cv::Mat &patch_rgb, cv::Mat &patch_gray);
+ void geometric_transformations(cv::Mat &patch, int size_x, int size_y, int angle = 0, bool allow_debug = true);
+ cv::Point2f sub_pixel_peak(cv::Point &max_loc, cv::Mat &response);
double sub_grid_scale(uint index);
-
};
-#endif //KCF_HEADER_6565467831231
+#endif // KCF_HEADER_6565467831231
projects / hercules2020 / kcf.git / commitdiff