accumulate.cpp 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699
  1. /*
  2. * accumulate.cpp
  3. *
  4. * Created on: Mar 7, 2016
  5. * Author: Janai
  6. */
  7. #include <iostream>
  8. #include <fstream>
  9. #include <stdio.h>
  10. #include <string>
  11. #include <cmath>
  12. #include <omp.h>
  13. #include <random>
  14. #include <opencv2/core.hpp>
  15. #include <opencv2/highgui.hpp>
  16. #include <opencv2/imgproc.hpp>
  17. #include <boost/filesystem.hpp>
  18. #include "epic.h"
  19. #include "image.h"
  20. #include "io.h"
  21. #include "variational_mt.h"
  22. #include "utils/edge_detector.h"
  23. #include "utils/utils.h"
  24. #include "../multi_frame_flow/multi_frame_optical_flow.h"
  25. #include "../multi_frame_flow/utils/parameter_list.h"
  26. #include "../libs/middlebury_devkit/cpp/colorcode.h"
  27. #include "../libs/middlebury_devkit/cpp/flowIO.h"
  28. using namespace std;
  29. using namespace cv;
  30. void setDefaultVariational(ParameterList& params) {
  31. // general
  32. params.insert("verbose", "0", true);
  33. params.insert("threads", "1", true);
  34. // params.insert("format", "img_%04d.tif", true);
  35. params.insert("stats", "0", true);
  36. params.insert("S", "2", true);
  37. params.insert("scale", "1.0f", true);
  38. params.insert("dataterm", "1");
  39. params.insert("smoothing", "1");
  40. params.insert("epic_alpha", "1.0f");
  41. params.insert("epic_gamma", "0.72f", true);
  42. params.insert("epic_delta", "0.0f", true);
  43. params.insert("layers", "1", true);
  44. params.insert("p_scale", "0.9f", true);;
  45. params.insert("niter_alter", "1", true);
  46. params.insert("niter_outer", "5", true);
  47. params.insert("thres_outer", "1e-5", true);
  48. params.insert("niter_inner", "1", true);
  49. params.insert("thres_inner", "1e-5", true);
  50. params.insert("niter_solver", "30", true);
  51. params.insert("sor_omega", "1.9f", true);
  52. params.insert("robust_color", "1", true);
  53. params.insert("robust_color_eps", "0.001", true);
  54. params.insert("robust_color_truncation", "0.5", true);
  55. params.insert("robust_reg", "1", true);
  56. params.insert("robust_reg_eps", "0.001", true);
  57. params.insert("robust_reg_truncation", "0.5", true);
  58. params.insert("gradient_sigmoid", "5.0f", true);
  59. }
  60. int main(int argc, char **argv) {
  61. /*
  62. * getting config file location
  63. */
  64. #define isarg(key) !strcmp(a,key)
  65. string file;
  66. string input_path = "";
  67. string output_path = "";
  68. if (argc > 1) {
  69. file = string(argv[1]);
  70. if(boost::filesystem::exists(file)) {
  71. printf("using parameters %s\n", file.c_str());
  72. } else {
  73. cerr << "usage: ./accumulate [cfg-file] -input_path [path] -output_path [path] {optional:Frames}" << endl;
  74. return -1;
  75. }
  76. int selected_jet = -1;
  77. int current_arg = 1;
  78. while(current_arg < argc ){
  79. const char* a = argv[current_arg++];
  80. if(a[0] != '-') {
  81. continue;
  82. }
  83. if( isarg("-h") || isarg("-help") )
  84. cerr << "usage: ./accumulate [cfg-file] -input_path [path] -output_path [path] {optional:Frames}" << endl;
  85. else if( isarg("-input_path") )
  86. input_path = string(argv[current_arg++]);
  87. else if( isarg("-output_path") )
  88. output_path = string(argv[current_arg++]);
  89. else{
  90. fprintf(stderr, "unknown argument %s", a);
  91. cerr << "usage: ./accumulate [cfg-file] -input_path [path] -output_path [path] {optional:Frames}" << endl;
  92. exit(1);
  93. }
  94. }
  95. } else {
  96. cerr << "usage: ./accumulate [cfg-file] -input_path [path] -output_path [path] {optional:Frames}" << endl;
  97. return -1;
  98. }
  99. /*
  100. * read in parameters and print config
  101. */
  102. ParameterList params;
  103. params.read(file);
  104. string path = file.substr(0, file.find_last_of('/'));
  105. bool sintel = params.parameter<bool>("sintel", "0");
  106. // add input path and output path
  107. params.setParameter("flow_file", input_path + params.parameter("flow_file"));
  108. for(int i = 0; i < params.file_list.size(); i++)
  109. params.file_list[i] = input_path + params.file_list[i];
  110. for(int i = 0; i < params.file_gt_list.size(); i++)
  111. params.file_gt_list[i] = input_path + params.file_gt_list[i];
  112. params.output = path + "/";
  113. /*
  114. * decompose sequence in subsets and adjust number of frames if necessary
  115. */
  116. uint32_t Jets = params.Jets;
  117. if (!params.exists("S") || params.parameter<int>("S") < 2)
  118. params.setParameter<int>("S", 2);
  119. int steps = params.parameter<int>("S") - 1;
  120. if(Jets == 0) {
  121. Jets = floor(params.F / steps);
  122. params.Jets = Jets;
  123. }
  124. // int batch_size = floor((1.0f * params.parameter<int>("max_fps")) / (params.parameter<int>("ref_fps")));
  125. int batch_size = floor((1.0f * params.parameter<int>("max_fps")) / (params.parameter<int>("jet_fps")));
  126. // int batches = floor((1.0f * params.Jets) * batch_size);
  127. Jets = floor(batch_size / steps);
  128. int batches = params.parameter<int>("batches", "1");
  129. int skip = floor((1.0f * params.parameter<int>("max_fps")) / (params.parameter<int>("jet_fps")));
  130. if(params.exists("frames")) {
  131. // decompose sequence in batches
  132. skip = ceil(1.0f * batch_size / (params.parameter<int>("frames") - 1.0)); // with ceil we will estimate too far by fractions
  133. Jets = floor(1.0f * Jets / skip);
  134. }
  135. // iterate over experiments
  136. for(uint32_t exp = 0; exp < params.experiments(); exp++) {
  137. ParameterList exp_params;
  138. exp_params = ParameterList(params);
  139. params.nextExp();
  140. // get scaling factor
  141. double scale = params.parameter<float>("scale", "1.0");
  142. string flow_format = exp_params.parameter<string>("flow_format", "frame_%i");
  143. // create folder for accumulated batches
  144. stringstream acc_folder;
  145. acc_folder << exp_params.output << "accumulation/";
  146. boost::filesystem::create_directories(acc_folder.str());
  147. boost::filesystem::create_directories(acc_folder.str() + "gt_occlusions/"); // accumulate folder
  148. exp_params.insert("accumulation", acc_folder.str() , true);
  149. exp_params.Jets = batches;
  150. stringstream news;
  151. news << batch_size*steps + 1;
  152. exp_params.setParameter("S", news.str());
  153. exp_params.print();
  154. ofstream infos;
  155. infos.open((acc_folder.str() + "config.cfg").c_str());
  156. infos << "# SlowFlow variational estimation\n";
  157. infos << exp_params;
  158. infos.close();
  159. uint32_t num_sources = params.file_list.size();
  160. for (uint32_t source = 0; source < num_sources; source++) {
  161. for(int b = 0; b < batches; b++) {
  162. if(b > 0)
  163. exp_params.sequence_start_list[source] += Jets * skip * steps;
  164. // read in sequence, forward and backward flow, and segmentation
  165. Mat* gt = NULL; // plus reference frame
  166. Mat* gt_occlusions = NULL;
  167. Mat *forward_flow = new Mat[Jets];
  168. Mat *backward_flow = new Mat[Jets];
  169. /*
  170. * ################### read in ground truth ###################
  171. */
  172. if(exp_params.file_gt.size() > 0) {
  173. gt = new Mat[batch_size];
  174. for (uint32_t f = 0; f < (batch_size); f++) {
  175. char gtF[200];
  176. if(!sintel)
  177. sprintf(gtF, exp_params.file_gt_list[source].c_str(), exp_params.sequence_start_list[source] + f);
  178. else {
  179. int sintel_frame = exp_params.sequence_start_list[source] / 1000;
  180. int hfr_frame = f + (exp_params.sequence_start_list[source] % 1000);
  181. while(hfr_frame < 0) {
  182. sintel_frame--;
  183. hfr_frame = 42 + hfr_frame;
  184. }
  185. while(hfr_frame > 41) {
  186. sintel_frame++;
  187. hfr_frame = hfr_frame - 42;
  188. }
  189. sprintf(gtF, exp_params.file_gt_list[source].c_str(), sintel_frame, hfr_frame);
  190. }
  191. if (access(gtF, F_OK) == -1) {
  192. continue;
  193. cerr << "Error reading "<< gtF << "!" << endl;
  194. }
  195. cout << "Reading "<< gtF << "!" << endl;
  196. gt[f] = readGTMiddlebury(gtF);
  197. // rescale gt flow
  198. resize(gt[f], gt[f], Size(0, 0), scale, scale, INTER_LINEAR);
  199. gt[f] = scale * gt[f];
  200. }
  201. }
  202. /*
  203. * ################### read in ground truth occlusions ###################
  204. */
  205. if(exp_params.occlusions_list.size() > 0) {
  206. gt_occlusions = new Mat[batch_size];
  207. for (u_int32_t f = 0; f < (uint32_t) (batch_size); f++) {
  208. char oocF[500];
  209. if(!sintel)
  210. sprintf(oocF, exp_params.occlusions_list[source].c_str(), exp_params.sequence_start_list[source] + f);
  211. else {
  212. int sintel_frame = exp_params.sequence_start_list[source] / 1000;
  213. int hfr_frame = f + (exp_params.sequence_start_list[source] % 1000);
  214. while(hfr_frame < 0) {
  215. sintel_frame--;
  216. hfr_frame = 42 + hfr_frame;
  217. }
  218. while(hfr_frame > 41) {
  219. sintel_frame++;
  220. hfr_frame = hfr_frame - 42;
  221. }
  222. sprintf(oocF, exp_params.occlusions_list[source].c_str(), sintel_frame, hfr_frame);
  223. }
  224. // read file only if already processed by first experiment
  225. if(access( oocF, F_OK ) != -1) {
  226. cout << "Reading "<< oocF << "!" << endl;
  227. gt_occlusions[f] = imread(string(oocF));
  228. resize(gt_occlusions[f], gt_occlusions[f], Size(0, 0), scale, scale, INTER_CUBIC);
  229. memset(oocF, 0, 500);
  230. sprintf(oocF, "%s/gt_occlusions/occ_%05i.png", acc_folder.str().c_str(), exp_params.sequence_start + f);
  231. // read file only if already processed by first experiment
  232. if(access( oocF, F_OK ) == -1) {
  233. // write flow and flow image to file
  234. imwrite(oocF, gt_occlusions[f]);
  235. }
  236. if(gt_occlusions[f].channels() > 1)
  237. cvtColor(gt_occlusions[f], gt_occlusions[f], CV_BGR2GRAY);
  238. gt_occlusions[f].convertTo(gt_occlusions[f], CV_8UC1);
  239. } else {
  240. cerr << "Error reading "<< oocF << "!" << endl;
  241. }
  242. }
  243. }
  244. char source_str[1024];
  245. if(params.name_list.size() > 1)
  246. sprintf(source_str, "%s%02i_", params.name_list[source].c_str(), params.id(source));
  247. else
  248. source_str[0] = '\0';
  249. /*
  250. * ################### read in flow fields ###################
  251. */
  252. for (uint32_t f = 0; f < Jets; f++) {
  253. char fFlowF[200];
  254. char bFlowF[200];
  255. // if (exp_params.exists("flow_file")) {
  256. // if(!sintel) {
  257. // sprintf(fFlowF, (exp_params.parameter("flow_file") + ".flo").c_str(), exp_params.sequence_start_list[source] + f * steps);
  258. // sprintf(bFlowF, (exp_params.parameter("flow_file") + "_back.flo").c_str(), exp_params.sequence_start_list[source] + f * steps + steps);
  259. // } else {
  260. // sprintf(fFlowF, (exp_params.parameter("flow_file") + ".flo").c_str(), exp_params.sequence_start_list[source] + f * steps, 0);
  261. // sprintf(bFlowF, (exp_params.parameter("flow_file") + "_back.flo").c_str(), exp_params.sequence_start_list[source] + f * steps + steps, 0);
  262. // }
  263. // } else {
  264. // sprintf(fFlowF, "%sframe_%i.flo", exp_params.output.c_str(), (exp_params.sequence_start_list[source] + f * steps));
  265. // sprintf(bFlowF, "%sframe_%i_back.flo", exp_params.output.c_str(), (exp_params.sequence_start_list[source] + f * steps + steps));
  266. // }
  267. string hfr = "";
  268. if (boost::filesystem::exists( exp_params.output + "hfr/" ))
  269. hfr = "hfr/";
  270. sprintf(fFlowF, ("%s%s" + flow_format + ".flo").c_str(), (exp_params.output + hfr).c_str(), source_str, (exp_params.sequence_start_list[source] + f * steps * skip));
  271. if (exp_params.parameter<bool>("wrong_backward", "0"))
  272. sprintf(bFlowF, ("%s%s" + flow_format + "_back.flo").c_str(), (exp_params.output + hfr).c_str(), source_str,
  273. (exp_params.sequence_start_list[source] + f * steps * skip));
  274. else
  275. sprintf(bFlowF, ("%s%s" + flow_format + "_back.flo").c_str(), (exp_params.output + hfr).c_str(), source_str,
  276. (exp_params.sequence_start_list[source] + f * steps * skip + steps * skip));
  277. if (!boost::filesystem::exists(fFlowF)) {
  278. cerr << fFlowF << " does not exist!" << endl;
  279. return -1;
  280. }
  281. if (!boost::filesystem::exists(bFlowF)) {
  282. cerr << bFlowF << " does not exist!" << endl;
  283. return -1;
  284. }
  285. cout << "Reading " << fFlowF << endl;
  286. cout << "Reading " << bFlowF << endl;
  287. forward_flow[f] = readGTMiddlebury(fFlowF);
  288. backward_flow[f] = readGTMiddlebury(bFlowF);
  289. // crop images
  290. if(exp_params.center.x > 0) {
  291. // NOTE: ROWRANGE IS INDUCING ERRORS IN ACCUMULATION!!!!!!
  292. forward_flow[f] = crop(forward_flow[f], exp_params.center, exp_params.extent);
  293. backward_flow[f] = crop(backward_flow[f], exp_params.center, exp_params.extent);
  294. }
  295. }
  296. Mat lfr_occlusions = Mat::zeros(forward_flow[0].rows, forward_flow[0].cols, CV_8UC1);
  297. if(gt != NULL && gt_occlusions != NULL) lfr_occlusions = fuseOcclusions(gt, gt_occlusions, 0, Jets);
  298. // accumulate ground truth flow
  299. Mat fconf, bconf;
  300. Mat* acc_flow = new Mat[Jets];
  301. // accumulateFlow(acc_flow, &forward_flow[b * Jets], lfr_occlusions, Jets);
  302. accumulateConsistentBatches(acc_flow, forward_flow, backward_flow, fconf, bconf, NULL, Jets, params.parameter<double>("acc_consistency_threshold_slope"), 0, params.parameter<bool>("acc_discard_inconsistent", "1"), 0);
  303. /*
  304. * ########################################### Refine flow ###########################################
  305. */
  306. // if(exp_params.parameter<bool>("refine", "1")) {
  307. // int width = acc_flow->cols;
  308. // int height = acc_flow->rows;
  309. //
  310. // ParameterList refine_params(exp_params);
  311. // setDefaultVariational(refine_params);
  312. // refine_params.F = 2;
  313. // refine_params.Jets = 1;
  314. // refine_params.insert("S", "2", true);
  315. // refine_params.verbose = "1000";
  316. // refine_params.insert("layers", "1", true);
  317. // refine_params.insert("method", "forward", true);
  318. // refine_params.insert("niter_alter", "1", true);
  319. // refine_params.insert("niter_outer", "5", true);
  320. // refine_params.insert("occlusion_reasoning", "0", true);
  321. // refine_params.insert("framerate_reasoning", "0", true);
  322. // refine_params.insert("rho_0", "1", true);
  323. // refine_params.insert("omega_0", "0", true);
  324. //
  325. // color_image_t** im = new color_image_t*[3];
  326. // im[0] = color_image_new(width, height);
  327. // im[1] = color_image_new(width, height);
  328. // im[2] = color_image_new(width, height);
  329. //
  330. // /*
  331. // * ################### read in image sequence ###################
  332. // */
  333. // for (uint32_t f = 0; f < 2; f++) {
  334. // char img_file[1024];
  335. // if (!sintel) {
  336. // sprintf(img_file, (sequence_path + format).c_str(), thread_params.sequence_start_list[source] + f * Jets * steps * skip);
  337. // } else {
  338. // int sintel_frame = thread_params.sequence_start_list[source] / 1000;
  339. // int hfr_frame = f * steps * skip + (thread_params.sequence_start_list[source] % 1000);
  340. //
  341. // while (hfr_frame < 0) {
  342. // sintel_frame--;
  343. // hfr_frame = 42 + hfr_frame;
  344. // }
  345. // while (hfr_frame > 41) {
  346. // sintel_frame++;
  347. // hfr_frame = hfr_frame - 42;
  348. // }
  349. //
  350. // sprintf(img_file, (sequence_path + format).c_str(), sintel_frame, hfr_frame);
  351. // }
  352. // cout << "Reading " << img_file << "..." << endl;
  353. //
  354. // sequence[f] = imread(string(img_file), CV_LOAD_IMAGE_UNCHANGED); // load images
  355. //
  356. // float norm = 1;
  357. // if (sequence[f].type() == 2 || sequence[f].type() == 18)
  358. // norm = 1.0f / 255; // for 16 bit images
  359. //
  360. // // convert to floating point
  361. // sequence[f].convertTo(sequence[f], CV_32FC(sequence[f].channels()));
  362. //
  363. // /*
  364. // * DEMOSAICING
  365. // */
  366. // if (thread_params.exists("raw") && thread_params.parameter<bool>("raw")) {
  367. // Mat tmp = sequence[f].clone();
  368. // color_image_t* tmp_in = color_image_new(sequence[f].cols, sequence[f].rows);
  369. // color_image_t* tmp_out = color_image_new(sequence[f].cols, sequence[f].rows);
  370. //
  371. // switch (thread_params.parameter<int>("raw_demosaicing", "0")) {
  372. // case 0: // use bilinear demosaicing
  373. // sequence[f] = Mat::zeros(tmp.rows, tmp.cols, CV_32FC3);
  374. // // bayer2rgb(tmp, sequence[f], green_start, blue_start); // red green
  375. // bayer2rgbGR(tmp, sequence[f], red_loc[0], red_loc[1]); // red green
  376. // break;
  377. //
  378. // case 1: // use hamilton adams demosaicing
  379. // mat2colorImg<float>(sequence[f], tmp_in);
  380. //
  381. // HamiltonAdamsDemosaic(tmp_out->c1, tmp_in->c1, tmp_in->width, tmp_in->height, red_loc[0], red_loc[1]); // Hamilton-Adams implemented by Pascal Getreuer
  382. //
  383. // sequence[f] = Mat::zeros(sequence[f].rows, sequence[f].cols, CV_32FC3);
  384. // colorImg2colorMat<Vec3f>(tmp_out, sequence[f]);
  385. // break;
  386. //
  387. // case 2: // use opencv demosaicing
  388. // tmp.convertTo(tmp, CV_8UC1);
  389. // sequence[f] = Mat::zeros(tmp.rows, tmp.cols, CV_8UC3);
  390. //
  391. // int code = CV_BayerBG2RGB;
  392. // if (red_loc[1] == 0) // y
  393. // if (red_loc[0] == 0) // x
  394. // code = CV_BayerBG2RGB;
  395. // else
  396. // code = CV_BayerGB2RGB;
  397. // else if (red_loc[0] == 0) // x
  398. // code = CV_BayerGR2RGB;
  399. // else
  400. // code = CV_BayerRG2RGB;
  401. //
  402. // cv::cvtColor(tmp, sequence[f], code); // components from second row, second column !!!!!!!!!!!!!!!!!
  403. // sequence[f].convertTo(sequence[f], CV_32FC(sequence[f].channels()));
  404. // break;
  405. // }
  406. //
  407. // color_image_delete(tmp_in);
  408. // color_image_delete(tmp_out);
  409. // } else {
  410. // // covert to RGB
  411. // cv::cvtColor(sequence[f], sequence[f], CV_BGR2RGB);
  412. // }
  413. //
  414. // /*
  415. // * COLOR CORRECTION
  416. // */
  417. // vector<Mat> channels;
  418. // split(sequence[f], channels);
  419. // float mid_val = (0.5 * 255.0 / norm);
  420. // channels[0] = contrast * (channels[0] + red_balance + brightness - mid_val) + mid_val;
  421. // channels[1] = contrast * (channels[1] + green_balance + brightness - mid_val) + mid_val;
  422. // channels[2] = contrast * (channels[2] + blue_balance + brightness - mid_val) + mid_val;
  423. // merge(channels, sequence[f]);
  424. //
  425. // if (thread_params.parameter<bool>("grayscale", "0"))
  426. // cvtColor(sequence[f], sequence[f], CV_RGB2GRAY);
  427. //
  428. // // use only a part of the images
  429. // if (thread_params.extent.x > 0 || thread_params.extent.y > 0) {
  430. // sequence[f] = sequence[f].rowRange(Range(thread_params.center.y - thread_params.extent.y / 2, thread_params.center.y + thread_params.extent.y / 2));
  431. // sequence[f] = sequence[f].colRange(Range(thread_params.center.x - thread_params.extent.x / 2, thread_params.center.x + thread_params.extent.x / 2));
  432. // }
  433. //
  434. // // rescale image with gaussian blur to avoid anti-aliasing
  435. // double img_scale = thread_params.parameter<double>("scale", "1.0");
  436. // if (img_scale != 1) {
  437. // GaussianBlur(sequence[f], sequence[f], Size(0, 0), 1 / sqrt(2 * img_scale), 1 / sqrt(2 * img_scale), BORDER_REPLICATE);
  438. // resize(sequence[f], sequence[f], Size(0, 0), img_scale, img_scale, INTER_LINEAR);
  439. // }
  440. //
  441. // // print to file
  442. // char file[1024];
  443. // sprintf(file, (acc_folder.str() + "sequence/frame_%s%i.png").c_str(), source_str, thread_params.sequence_start_list[source] - steps * skip + f * skip);
  444. //
  445. // Mat output_img;
  446. // if (thread_params.parameter<bool>("16bit", "0")) {
  447. // sequence[f].convertTo(output_img, CV_16UC(sequence[f].channels()));
  448. // } else {
  449. // sequence[f].convertTo(output_img, CV_8UC(sequence[f].channels()), norm);
  450. // }
  451. //
  452. // if (thread_params.parameter<bool>("grayscale", "0"))
  453. // cv::cvtColor(output_img, output_img, CV_GRAY2BGR); // OpenCV uses BGR
  454. // else
  455. // cv::cvtColor(output_img, output_img, CV_RGB2BGR); // OpenCV uses BGR
  456. //
  457. // if (thread_params.verbosity(WRITE_FILES)) {
  458. // imwrite(file, output_img, compression_params);
  459. // }
  460. //
  461. // data[f] = color_image_new(sequence[f].cols, sequence[f].rows);
  462. // if (thread_params.parameter<bool>("grayscale", "0"))
  463. // mat2colorImg<float>(sequence[f], data[f]);
  464. // else
  465. // colorMat2colorImg<Vec3f>(sequence[f], data[f]);
  466. //
  467. // if (thread_params.verbosity(VER_IN_GT) && (int) f < show_frames) {
  468. // if (source == show_frames_source) {
  469. // stringstream ftitle;
  470. // ftitle << "Frame " << source;
  471. // namedWindow(ftitle.str()); // Create a window for display.
  472. // imshow(ftitle.str(), output_img);
  473. //
  474. // // set the callback function for pixel selection
  475. // int* cbparams = new int[2];
  476. // cbparams[0] = f;
  477. // cbparams[1] = source;
  478. // setMouseCallback(ftitle.str(), CallBackFunc, cbparams);
  479. //
  480. // waitKey(0);
  481. // }
  482. // }
  483. // }
  484. //
  485. // // normalize data terms
  486. // normalize(data, Jets + 1, thread_params);
  487. //
  488. // // rescale image with gaussian blur to avoid anti-aliasing
  489. // double rescale = 1.0f * width / width;
  490. // Mat tmp_1 = sequence[0].clone(),
  491. // tmp_2 = sequence[Jets].clone();
  492. // if (rescale != 1) {
  493. // GaussianBlur(tmp_1, tmp_1, Size(0, 0), 1 / sqrt(2 * rescale), 1 / sqrt(2 * rescale), BORDER_REPLICATE);
  494. // GaussianBlur(tmp_2, tmp_2, Size(0, 0), 1 / sqrt(2 * rescale), 1 / sqrt(2 * rescale), BORDER_REPLICATE);
  495. // }
  496. // resize(tmp_1, tmp_1, Size(width, height), 0, 0, INTER_LINEAR);
  497. // resize(tmp_2, tmp_2, Size(width, height), 0, 0, INTER_LINEAR);
  498. // if (thread_params.parameter<bool>("grayscale", "0")) {
  499. // mat2colorImg<float>(tmp_2, im[0]);// assumes backward frame for symmetric option
  500. // mat2colorImg<float>(tmp_1, im[1]);
  501. // mat2colorImg<float>(tmp_2, im[2]);
  502. // } else {
  503. // colorMat2colorImg<Vec3f>(tmp_2, im[0]);// assumes backward frame for symmetric option
  504. // colorMat2colorImg<Vec3f>(tmp_1, im[1]);
  505. // colorMat2colorImg<Vec3f>(tmp_2, im[2]);
  506. // }
  507. //
  508. // normalize(im, 3, refine_params);
  509. //
  510. // image_t *wx = image_new(im[0]->width, im[0]->height), *wy = image_new(im[0]->width, im[0]->height);
  511. // vector<Mat> vu;
  512. // split(flow[0][Jets - 1], vu);
  513. // mat2img<double>(vu[1], wx);
  514. // mat2img<double>(vu[0], wy);
  515. //
  516. //// bool change = true;
  517. //// while(change) {
  518. //// for(int i=0 ; i<wx->height ; i++){
  519. //// for( int j=0 ; j<wx->width ; j++){
  520. //// change = false;
  521. ////
  522. //// if(wx->data[i*wx->stride+j] > UNKNOWN_FLOW_THRESH) {
  523. //// wx->data[i*wx->stride+j] = 0;
  524. //// int count = 0;
  525. //// if(wx->data[(i+1)*wx->stride+j] < UNKNOWN_FLOW_THRESH) {
  526. //// wx->data[i*wx->stride+j] += wx->data[(i+1)*wx->stride+j];
  527. //// count++;
  528. //// }
  529. //// if(wx->data[(i-1)*wx->stride+j] < UNKNOWN_FLOW_THRESH) {
  530. //// wx->data[i*wx->stride+j] += wx->data[(i-1)*wx->stride+j];
  531. //// count++;
  532. //// }
  533. //// if(wx->data[i*wx->stride+(j+1)] < UNKNOWN_FLOW_THRESH) {
  534. //// wx->data[i*wx->stride+j] += wx->data[i*wx->stride+(j+1)];
  535. //// count++;
  536. //// }
  537. //// if(wx->data[i*wx->stride+(j-1)] < UNKNOWN_FLOW_THRESH) {
  538. //// wx->data[i*wx->stride+j] += wx->data[i*wx->stride+(j-1)];
  539. //// count++;
  540. //// }
  541. ////
  542. //// wx->data[i*wx->stride+j] = wx->data[i*wx->stride+j] / count;
  543. //// change = true;
  544. //// }
  545. ////
  546. //// if(wy->data[i*wy->stride+j] > UNKNOWN_FLOW_THRESH) {
  547. //// wy->data[i*wy->stride+j] = 0;
  548. //// int count = 0;
  549. //// if(wy->data[(i+1)*wy->stride+j] < UNKNOWN_FLOW_THRESH) {
  550. //// wy->data[i*wy->stride+j] += wy->data[(i+1)*wy->stride+j];
  551. //// count++;
  552. //// }
  553. //// if(wy->data[(i-1)*wy->stride+j] < UNKNOWN_FLOW_THRESH) {
  554. //// wy->data[i*wy->stride+j] += wy->data[(i-1)*wy->stride+j];
  555. //// count++;
  556. //// }
  557. //// if(wy->data[i*wy->stride+(j+1)] < UNKNOWN_FLOW_THRESH) {
  558. //// wy->data[i*wy->stride+j] += wy->data[i*wy->stride+(j+1)];
  559. //// count++;
  560. //// }
  561. //// if(wy->data[i*wy->stride+(j-1)] < UNKNOWN_FLOW_THRESH) {
  562. //// wy->data[i*wy->stride+j] += wy->data[i*wy->stride+(j-1)];
  563. //// count++;
  564. //// }
  565. ////
  566. //// wy->data[i*wy->stride+j] = wy->data[i*wy->stride+j] / count;
  567. //// change = true;
  568. //// }
  569. //// }
  570. //// }
  571. //// }
  572. //
  573. // // energy minimization
  574. // // time(&t_start);
  575. // Variational_MT minimzer_f;
  576. // minimzer_f.variational(wx, wy, im, refine_params);
  577. // // time(&t_end);
  578. //
  579. // img2mat<double>(wx, vu[1]);
  580. // img2mat<double>(wy, vu[0]);
  581. // merge(vu, flow[0][Jets - 1]);
  582. //
  583. // color_image_delete(im[0]);
  584. // color_image_delete(im[1]);
  585. // color_image_delete(im[2]);
  586. // delete[] im;
  587. // image_delete(wx);
  588. // image_delete(wy);
  589. // }
  590. // write final estimation
  591. // stringstream flowF;
  592. // flowF << acc_folder.str() << "/" << exp_params.name_list[source].c_str() << source << "frame_" << exp_params.sequence_start_list[source] + steps * batch_size * b;
  593. char flowF[200];
  594. sprintf(flowF, ("%s/%s" + flow_format).c_str(), acc_folder.str().c_str(), source_str, (exp_params.sequence_start_list[source]));
  595. Mat flow_img = flowColorImg(acc_flow[Jets-1], exp_params.verbosity(VER_CMD));
  596. if (!flow_img.data) { // Check for invalid input
  597. cout << "No flow for frame " << exp_params.sequence_start_list[source] << std::endl;
  598. continue;
  599. }
  600. imwrite((string(flowF) + "_vis.png"), flow_img);
  601. writeFlowMiddlebury(acc_flow[Jets-1], (string(flowF) + ".flo"));
  602. //----------------------------------
  603. // ########################################### final clean up ###########################################
  604. //----------------------------------
  605. if(gt != NULL) delete[] gt;
  606. if(gt_occlusions != NULL) delete[] gt_occlusions;
  607. delete[] forward_flow;
  608. delete[] backward_flow;
  609. }
  610. }
  611. }
  612. return 0;
  613. }