/*
* OpenVINS: An Open Platform for Visual-Inertial Research
* Copyright (C) 2018-2022 Patrick Geneva
* Copyright (C) 2018-2022 Guoquan Huang
* Copyright (C) 2018-2022 OpenVINS Contributors
* Copyright (C) 2018-2019 Kevin Eckenhoff
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see .
*/
#include
#include
#include
#include
#include
#include "calc/ResultTrajectory.h"
#include "utils/Loader.h"
#include "utils/colors.h"
#include "utils/print.h"
#ifdef HAVE_PYTHONLIBS
// import the c++ wrapper for matplot lib
// https://github.com/lava/matplotlib-cpp
// sudo apt-get install python-matplotlib python-numpy python2.7-dev
#include "plot/matplotlibcpp.h"
#endif
int main(int argc, char **argv) {
// Verbosity setting
ov_core::Printer::setPrintLevel("INFO");
// Ensure we have a path
if (argc < 4) {
PRINT_ERROR(RED "ERROR: Please specify a align mode, folder, and algorithms\n" RESET);
PRINT_ERROR(RED "ERROR: ./error_comparison \n" RESET);
PRINT_ERROR(RED "ERROR: rosrun ov_eval error_comparison \n" RESET);
std::exit(EXIT_FAILURE);
}
// List the groundtruth files in this folder
std::string path_gts(argv[2]);
std::vector path_groundtruths;
for (const auto &p : boost::filesystem::recursive_directory_iterator(path_gts)) {
if (p.path().extension() == ".txt") {
path_groundtruths.push_back(p.path());
}
}
std::sort(path_groundtruths.begin(), path_groundtruths.end());
// Try to load our paths
for (size_t i = 0; i < path_groundtruths.size(); i++) {
// Load it!
std::vector times;
std::vector> poses;
std::vector cov_ori, cov_pos;
ov_eval::Loader::load_data(path_groundtruths.at(i).string(), times, poses, cov_ori, cov_pos);
// Print its length and stats
double length = ov_eval::Loader::get_total_length(poses);
PRINT_INFO("[COMP]: %d poses in %s => length of %.2f meters\n", (int)times.size(), path_groundtruths.at(i).filename().c_str(), length);
}
// Get the algorithms we will process
// Also create empty statistic objects for each of our datasets
std::string path_algos(argv[3]);
std::vector path_algorithms;
for (const auto &entry : boost::filesystem::directory_iterator(path_algos)) {
if (boost::filesystem::is_directory(entry)) {
path_algorithms.push_back(entry.path());
}
}
std::sort(path_algorithms.begin(), path_algorithms.end());
//===============================================================================
//===============================================================================
//===============================================================================
// ATE summery information
std::map>> algo_ate;
for (const auto &p : path_algorithms) {
std::vector> temp;
for (size_t i = 0; i < path_groundtruths.size(); i++) {
temp.push_back({ov_eval::Statistics(), ov_eval::Statistics()});
}
algo_ate.insert({p.filename().string(), temp});
}
// Relative pose error segment lengths
std::vector segments = {8.0, 16.0, 24.0, 32.0, 40.0, 48.0};
// std::vector segments = {7.0, 14.0, 21.0, 28.0, 35.0};
// std::vector segments = {10.0, 25.0, 50.0, 75.0, 120.0};
// std::vector segments = {5.0, 15.0, 30.0, 45.0, 60.0};
// std::vector segments = {40.0, 60.0, 80.0, 100.0, 120.0};
// The overall RPE error calculation for each algorithm type
std::map>> algo_rpe;
for (const auto &p : path_algorithms) {
std::map> temp;
for (const auto &len : segments) {
temp.insert({len, {ov_eval::Statistics(), ov_eval::Statistics()}});
}
algo_rpe.insert({p.filename().string(), temp});
}
//===============================================================================
//===============================================================================
//===============================================================================
// Loop through each algorithm type
for (size_t i = 0; i < path_algorithms.size(); i++) {
// Debug print
PRINT_DEBUG("======================================\n");
PRINT_DEBUG("[COMP]: processing %s algorithm\n", path_algorithms.at(i).filename().c_str());
// Get the list of datasets this algorithm records
std::map path_algo_datasets;
for (auto &entry : boost::filesystem::directory_iterator(path_algorithms.at(i))) {
if (boost::filesystem::is_directory(entry)) {
path_algo_datasets.insert({entry.path().filename().string(), entry.path()});
}
}
// Loop through our list of groundtruth datasets, and see if we have it
for (size_t j = 0; j < path_groundtruths.size(); j++) {
// Check if we have runs for this dataset
if (path_algo_datasets.find(path_groundtruths.at(j).stem().string()) == path_algo_datasets.end()) {
PRINT_ERROR(RED "[COMP]: %s dataset does not have any runs for %s!!!!!\n" RESET, path_algorithms.at(i).filename().c_str(),
path_groundtruths.at(j).stem().c_str());
continue;
}
// Debug print
PRINT_DEBUG("[COMP]: processing %s algorithm => %s dataset\n", path_algorithms.at(i).filename().c_str(),
path_groundtruths.at(j).stem().c_str());
// Errors for this specific dataset (i.e. our averages over the total runs)
ov_eval::Statistics ate_dataset_ori;
ov_eval::Statistics ate_dataset_pos;
std::map> rpe_dataset;
for (const auto &len : segments) {
rpe_dataset.insert({len, {ov_eval::Statistics(), ov_eval::Statistics()}});
}
// Loop though the different runs for this dataset
std::vector file_paths;
for (auto &entry : boost::filesystem::directory_iterator(path_algo_datasets.at(path_groundtruths.at(j).stem().string()))) {
if (entry.path().extension() != ".txt")
continue;
file_paths.push_back(entry.path().string());
}
std::sort(file_paths.begin(), file_paths.end());
// Now loop through the sorted vector
for (auto &path_esttxt : file_paths) {
// Our paths
std::string dataset = path_groundtruths.at(j).stem().string();
std::string path_gttxt = path_groundtruths.at(j).string();
// Create our trajectory object
ov_eval::ResultTrajectory traj(path_esttxt, path_gttxt, argv[1]);
// Calculate ATE error for this dataset
ov_eval::Statistics error_ori, error_pos;
traj.calculate_ate(error_ori, error_pos);
ate_dataset_ori.values.push_back(error_ori.rmse);
ate_dataset_pos.values.push_back(error_pos.rmse);
// Calculate RPE error for this dataset
std::map> error_rpe;
traj.calculate_rpe(segments, error_rpe);
for (const auto &elm : error_rpe) {
rpe_dataset.at(elm.first).first.values.insert(rpe_dataset.at(elm.first).first.values.end(), elm.second.first.values.begin(),
elm.second.first.values.end());
rpe_dataset.at(elm.first).first.timestamps.insert(rpe_dataset.at(elm.first).first.timestamps.end(),
elm.second.first.timestamps.begin(), elm.second.first.timestamps.end());
rpe_dataset.at(elm.first).second.values.insert(rpe_dataset.at(elm.first).second.values.end(), elm.second.second.values.begin(),
elm.second.second.values.end());
rpe_dataset.at(elm.first).second.timestamps.insert(rpe_dataset.at(elm.first).second.timestamps.end(),
elm.second.second.timestamps.begin(), elm.second.second.timestamps.end());
}
}
// Compute our mean ATE score
ate_dataset_ori.calculate();
ate_dataset_pos.calculate();
// Print stats for this specific dataset
std::string prefix = (ate_dataset_ori.mean > 10 || ate_dataset_pos.mean > 10) ? RED : "";
PRINT_DEBUG("%s\tATE: mean_ori = %.3f | mean_pos = %.3f (%d runs)\n" RESET, prefix.c_str(), ate_dataset_ori.mean,
ate_dataset_pos.mean, (int)ate_dataset_pos.values.size());
PRINT_DEBUG("\tATE: std_ori = %.3f | std_pos = %.3f\n", ate_dataset_ori.std, ate_dataset_pos.std);
for (auto &seg : rpe_dataset) {
seg.second.first.calculate();
seg.second.second.calculate();
// PRINT_DEBUG("\tRPE: seg %d - mean_ori = %.3f | mean_pos = %.3f (%d
// samples)\n",(int)seg.first,seg.second.first.mean,seg.second.second.mean,(int)seg.second.second.values.size());
PRINT_DEBUG("\tRPE: seg %d - median_ori = %.4f | median_pos = %.4f (%d samples)\n", (int)seg.first, seg.second.first.median,
seg.second.second.median, (int)seg.second.second.values.size());
// PRINT_DEBUG("RPE: seg %d - std_ori = %.3f | std_pos = %.3f\n",(int)seg.first,seg.second.first.std,seg.second.second.std);
}
// Update the global ATE error stats
std::string algo = path_algorithms.at(i).filename().string();
algo_ate.at(algo).at(j).first = ate_dataset_ori;
algo_ate.at(algo).at(j).second = ate_dataset_pos;
// Update the global RPE error stats
for (const auto &elm : rpe_dataset) {
algo_rpe.at(algo).at(elm.first).first.values.insert(algo_rpe.at(algo).at(elm.first).first.values.end(),
elm.second.first.values.begin(), elm.second.first.values.end());
algo_rpe.at(algo).at(elm.first).first.timestamps.insert(algo_rpe.at(algo).at(elm.first).first.timestamps.end(),
elm.second.first.timestamps.begin(), elm.second.first.timestamps.end());
algo_rpe.at(algo).at(elm.first).second.values.insert(algo_rpe.at(algo).at(elm.first).second.values.end(),
elm.second.second.values.begin(), elm.second.second.values.end());
algo_rpe.at(algo).at(elm.first).second.timestamps.insert(algo_rpe.at(algo).at(elm.first).second.timestamps.end(),
elm.second.second.timestamps.begin(), elm.second.second.timestamps.end());
}
}
}
PRINT_DEBUG("\n\n");
//===============================================================================
//===============================================================================
//===============================================================================
// Finally print the ATE for all the runs
PRINT_INFO("============================================\n");
PRINT_INFO("ATE LATEX TABLE\n");
PRINT_INFO("============================================\n");
for (size_t i = 0; i < path_groundtruths.size(); i++) {
std::string gtname = path_groundtruths.at(i).stem().string();
boost::replace_all(gtname, "_", "\\_");
PRINT_INFO(" & \\textbf{%s}", gtname.c_str());
}
PRINT_INFO(" & \\textbf{Average} \\\\\\hline\n");
for (auto &algo : algo_ate) {
std::string algoname = algo.first;
boost::replace_all(algoname, "_", "\\_");
PRINT_INFO(algoname.c_str());
double sum_ori = 0.0;
double sum_pos = 0.0;
int sum_ct = 0;
for (auto &seg : algo.second) {
if (seg.first.values.empty() || seg.second.values.empty()) {
PRINT_INFO(" & - / -");
} else {
seg.first.calculate();
seg.second.calculate();
PRINT_INFO(" & %.3f / %.3f", seg.first.mean, seg.second.mean);
sum_ori += seg.first.mean;
sum_pos += seg.second.mean;
sum_ct++;
}
}
PRINT_INFO(" & %.3f / %.3f \\\\\n", sum_ori / sum_ct, sum_pos / sum_ct);
}
PRINT_INFO("============================================\n");
// Finally print the RPE for all the runs
PRINT_INFO("============================================\n");
PRINT_INFO("RPE LATEX TABLE\n");
PRINT_INFO("============================================\n");
for (const auto &len : segments) {
PRINT_INFO(" & \\textbf{%dm}", (int)len);
}
PRINT_INFO(" \\\\\\hline\n");
for (auto &algo : algo_rpe) {
std::string algoname = algo.first;
boost::replace_all(algoname, "_", "\\_");
PRINT_INFO(algoname.c_str());
for (auto &seg : algo.second) {
seg.second.first.calculate();
seg.second.second.calculate();
PRINT_INFO(" & %.3f / %.3f", seg.second.first.mean, seg.second.second.mean);
}
PRINT_INFO(" \\\\\n");
}
PRINT_INFO("============================================\n");
#ifdef HAVE_PYTHONLIBS
// Plot line colors
std::vector colors = {"blue", "red", "black", "green", "cyan", "magenta"};
std::vector linestyle = {"-", "--", "-."};
assert(algo_rpe.size() <= colors.size() * linestyle.size());
// Parameters
std::map params_rpe;
params_rpe.insert({"notch", "false"});
params_rpe.insert({"sym", ""});
// Plot this figure
matplotlibcpp::figure_size(1000, 700);
matplotlibcpp::subplot(2, 1, 1);
matplotlibcpp::title("Relative Orientation Error");
// Plot each RPE next to each other
double width = 1.0 / (algo_rpe.size() + 1);
double spacing = width / (algo_rpe.size() + 1);
std::vector xticks;
std::vector labels;
int ct_algo = 0;
double ct_pos = 0;
for (auto &algo : algo_rpe) {
// Start based on what algorithm we are doing
xticks.clear();
labels.clear();
ct_pos = 1 + ct_algo * (width + spacing);
// Loop through each length type
for (auto &seg : algo.second) {
xticks.push_back(ct_pos - (algo_rpe.size() * (width + spacing) - width) / 2);
labels.push_back(std::to_string((int)seg.first));
matplotlibcpp::boxplot(seg.second.first.values, ct_pos, width, colors.at(ct_algo % colors.size()),
linestyle.at(ct_algo / colors.size()), params_rpe);
ct_pos += 1 + 3 * width;
}
// Move forward
ct_algo++;
}
// Add "fake" plots for our legend
ct_algo = 0;
for (const auto &algo : algo_rpe) {
std::map params_empty;
params_empty.insert({"label", algo.first});
params_empty.insert({"linestyle", linestyle.at(ct_algo / colors.size())});
params_empty.insert({"color", colors.at(ct_algo % colors.size())});
std::vector vec_empty;
matplotlibcpp::plot(vec_empty, vec_empty, params_empty);
ct_algo++;
}
// Plot each RPE next to each other
matplotlibcpp::xlim(0.5, ct_pos - 0.5 - 3 * width);
matplotlibcpp::xticks(xticks, labels);
matplotlibcpp::ylabel("orientation error (deg)");
matplotlibcpp::legend();
matplotlibcpp::subplot(2, 1, 2);
ct_algo = 0;
ct_pos = 0;
for (auto &algo : algo_rpe) {
// Start based on what algorithm we are doing
ct_pos = 1 + ct_algo * (width + spacing);
// Loop through each length type
for (auto &seg : algo.second) {
matplotlibcpp::boxplot(seg.second.second.values, ct_pos, width, colors.at(ct_algo % colors.size()),
linestyle.at(ct_algo / colors.size()), params_rpe);
ct_pos += 1 + 3 * width;
}
// Move forward
ct_algo++;
}
// Add "fake" plots for our legend
ct_algo = 0;
for (const auto &algo : algo_rpe) {
std::map params_empty;
params_empty.insert({"label", algo.first});
params_empty.insert({"linestyle", linestyle.at(ct_algo / colors.size())});
params_empty.insert({"color", colors.at(ct_algo % colors.size())});
std::vector vec_empty;
matplotlibcpp::plot(vec_empty, vec_empty, params_empty);
ct_algo++;
}
// Display to the user
matplotlibcpp::xlim(0.5, ct_pos - 0.5 - 3 * width);
matplotlibcpp::xticks(xticks, labels);
matplotlibcpp::title("Relative Position Error");
matplotlibcpp::ylabel("translational error (m)");
matplotlibcpp::xlabel("sub-segment lengths (m)");
matplotlibcpp::show(true);
#endif
// Done!
return EXIT_SUCCESS;
}