v01

python/basalt/latex/containers.py

@@ -0,0 +1,109 @@
+#
+# BSD 3-Clause License
+#
+# This file is part of the Basalt project.
+# https://gitlab.com/VladyslavUsenko/basalt.git
+#
+# Copyright (c) 2019-2021, Vladyslav Usenko and Nikolaus Demmel.
+# All rights reserved.
+#
+import math
+
+from pylatex import Package
+from pylatex.base_classes import Container
+
+from ..metric import metrics_from_config
+from ..metric import ExperimentSpec
+from ..util import alphanum
+
+
+class MyContainer(Container):
+
+    def __init__(self):
+        super().__init__()
+        # add packages that seem to not propagate properly from added elements
+        self.packages.add(Package("xcolor"))
+        self.packages.add(Package('graphicx'))
+
+    def dumps(self):
+        return self.dumps_content()
+
+
+class ExperimentsContainer(MyContainer):
+
+    def __init__(self, seq_displayname_mapping):
+        super().__init__()
+
+        self.seq_displayname_mapping = seq_displayname_mapping
+
+    def seq_displayname(self, seq):
+        return self.seq_displayname_mapping.get(seq, seq)
+
+
+class ExperimentsTable(ExperimentsContainer):
+
+    def __init__(self, exps, spec, show_values_failed_runs, seq_displayname_mapping, export_basepath):
+        super().__init__(seq_displayname_mapping)
+        self.exps = exps
+        self.spec = spec
+        self.show_values_failed_runs = show_values_failed_runs
+        self.export_basepath = export_basepath
+
+        self.experiment_specs = [ExperimentSpec(s) for s in self.spec.experiments]
+        self.metrics = metrics_from_config(self.spec.metrics)
+
+        self.seq_names = self.sequence_names([s.name for s in self.experiment_specs])
+        self.num_seqs = len(self.seq_names)
+        self.num_metrics = len(self.metrics)
+        self.num_exps = len(self.experiment_specs)
+
+    def sequence_names(self, experiment_names):
+        seq_names = set()
+        for s in experiment_names:
+            seq_names.update(self.exps[s].sequences(filter_regex=self.spec.filter_regex))
+
+        return sorted(seq_names, key=alphanum)
+
+    def is_failed(self, exp, seq):
+        if seq not in exp.runs:
+            return True
+        return exp.runs[seq].is_failed()
+
+    def render_failure(self, exp, seq):
+        if seq in self.spec.override_as_failed:
+            return "x"
+
+        if seq not in exp.runs:
+            return '?'
+        run = exp.runs[seq]
+
+        treat_as_failed = (run.log is None) if self.show_values_failed_runs else run.is_failed()
+
+        if treat_as_failed:
+            return run.failure_str()
+        else:
+            return None
+
+    def get_metrics(self, exp, seq, it):
+        if seq not in exp.runs:
+            return [math.nan for _ in self.metrics]
+        run = exp.runs[seq]
+
+        treat_as_failed = (run.log is None) if self.show_values_failed_runs else run.is_failed()
+
+        if treat_as_failed:
+            return [math.nan for _ in self.metrics]
+
+        return [m.get_value(self.exps, exp, seq, it) for m in self.metrics]
+        # try:
+        #     return [m.get_value(self.exps, exp, seq, it) for m in self.metrics]
+        # except AttributeError as e:
+        #     if e.args[0].startswith("local_error"):
+        #         if not has_imported_sophus():
+        #             print("To use local-error, you need to install sophuspy and flush the cache.")
+        #             sys.exit(1)
+        #         if not exp.runs[seq].log.has_cam_pos:
+        #             print("You cannot use local-error for experiment {}, which has no camera positions in the log.".
+        #                   format(exp.name))
+        #             sys.exit(1)
+        #     raise

python/basalt/latex/plot.py

@@ -0,0 +1,393 @@
+#
+# BSD 3-Clause License
+#
+# This file is part of the Basalt project.
+# https://gitlab.com/VladyslavUsenko/basalt.git
+#
+# Copyright (c) 2019-2021, Vladyslav Usenko and Nikolaus Demmel.
+# All rights reserved.
+#
+import numpy as np
+import os
+import math
+import functools
+
+import matplotlib
+
+matplotlib.use('Agg')  # Not to use X server. For TravisCI.
+import matplotlib.pyplot as plt  # noqa
+from matplotlib.ticker import MaxNLocator
+
+prop_cycle = plt.rcParams['axes.prop_cycle']
+
+#default_cycler = (cycler(linestyle=['-', '--', ':', '-.']) *
+#                  cycler(color=prop_cycle.by_key()['color']))
+
+
+class ModulusList(list):
+
+    def __init__(self, *args, **kwargs):
+        list.__init__(self, *args, **kwargs)
+
+    def __getitem__(self, key):
+        return list.__getitem__(self, key % len(self))
+
+
+default_colors_finite = prop_cycle.by_key()['color']
+default_colors_finite[0] = prop_cycle.by_key()['color'][0]
+default_colors_finite[1] = prop_cycle.by_key()['color'][2]
+default_colors_finite[2] = prop_cycle.by_key()['color'][3]
+default_colors_finite[3] = prop_cycle.by_key()['color'][1]
+
+default_colors = ModulusList(default_colors_finite)
+#default_lines = ModulusList(["-", "-", ":", "--", "-.", ":", "--", "-."])
+#default_markers = ModulusList(["o", "s", "^", "X", "D", "P", "v", "h"])
+default_lines = ModulusList([":", "-", "-.", "--", ":", "--", "-.", "-"])
+default_markers = ModulusList(["o", "s", "^", "X", "D", "P", "v", "h"])
+
+from collections import deque
+from collections import defaultdict
+
+from pylatex import Figure
+from pylatex.utils import NoEscape
+
+from .containers import ExperimentsContainer
+from .util import rotation2d
+
+
+class NoFloatFigure(Figure):
+    pass
+
+
+class Plot(ExperimentsContainer):
+
+    def __init__(self, exps, spec, seq_displayname_mapping, export_basepath):
+        super().__init__(seq_displayname_mapping)
+
+        self.width = None
+
+        plotters = dict(nullspace=self.plot_nullspace,
+                        eigenvalues=self.plot_eigenvalues,
+                        trajectory=self.plot_trajectory)
+
+        plot_fn = plotters[spec.type]
+        plot_fn(exps, spec)
+
+        if spec.width is not None:
+            self.width = spec.width
+        elif self.width is None:
+            self.width = 1
+
+        plt.tight_layout()
+
+        saved_file = self._save_plot(spec, export_basepath)
+
+        if "sequence" in spec:
+            plot_name = '{} {} {}'.format(spec.type, spec.name, spec.sequence).replace("_", " ")
+        else:
+            plot_name = '{} {}'.format(spec.type, spec.name).replace("_", " ")
+
+        #with self.create(Subsection(spec.name, numbering=False)) as p:
+        with self.create(NoFloatFigure()) as f:
+            f.add_image(os.path.abspath(saved_file), width=NoEscape(r'{}\textwidth'.format(self.width)))
+            f.add_caption(plot_name)
+
+        # cleanup
+        plt.close('all')
+
+    def plot_nullspace(self, exps, spec):
+
+        logs = [exps[e].runs[spec.sequence].log for e in spec.experiments]
+        names = [exps[e].display_name for e in spec.experiments]
+
+        num_plots = len(names)
+
+        if num_plots == 4:
+            if True:
+                if spec.figsize is None:
+                    spec.figsize = [10, 2.5]
+                fig, axs = plt.subplots(1, 4, figsize=spec.figsize, sharey=True)
+            else:
+                if spec.figsize is None:
+                    spec.figsize = [10, 4.7]
+                fig, axs = plt.subplots(2, 2, figsize=spec.figsize, sharey=True)
+                axs = axs.flatten()
+        else:
+            if spec.figsize is None:
+                spec.figsize = [6, 2 * num_plots]
+            fig, axs = plt.subplots(num_plots, 1, figsize=spec.figsize, sharey=True)
+
+        if num_plots == 1:
+            axs = [axs]
+
+        for i, (log, name) in enumerate(zip(logs, names)):
+
+            if log is None:
+                continue
+
+            ax = axs[i]
+
+            ns = log.sums.marg_ns[1:]  # skip first prior, which just is all 0
+            ns = np.abs(ns)  # cost change may be negative, we are only interested in the norm
+            ns = np.maximum(ns, 1e-20)  # clamp at very small value
+
+            markerfacecolor = "white"
+
+            markevery = 1000
+            if spec.sequence == "kitti10":
+                markevery = 100
+
+            ax.semilogy(
+                ns[:, 0],
+                ":",
+                # label="x",
+                color="tab:blue")
+            ax.semilogy(
+                ns[:, 1],
+                ":",
+                # label="y",
+                color="tab:blue")
+            ax.semilogy(
+                ns[:, 2],
+                ":",
+                # label="z",
+                label="x, y, z",
+                color="tab:blue",
+                marker="o",
+                markerfacecolor=markerfacecolor,
+                markevery=(markevery // 2, markevery))
+
+            ax.semilogy(
+                ns[:, 3],
+                ":",
+                # label="roll",
+                color="tab:orange")
+            ax.semilogy(
+                ns[:, 4],
+                ":",
+                # label="pitch",
+                label="roll, pitch",
+                color="tab:orange",
+                marker="s",
+                markerfacecolor=markerfacecolor,
+                markevery=(markevery // 2, markevery))
+
+            ax.semilogy(ns[:, 5],
+                        ":",
+                        label="yaw",
+                        color="tab:green",
+                        marker="^",
+                        markerfacecolor=markerfacecolor,
+                        markevery=(0, markevery))
+
+            ax.semilogy(ns[:, 6],
+                        ":",
+                        label="random",
+                        color="tab:red",
+                        marker="D",
+                        markerfacecolor=markerfacecolor,
+                        markevery=(0, markevery))
+
+            # marker on top of lines;
+
+            ax.semilogy(ns[:, 2],
+                        color="None",
+                        marker="o",
+                        markerfacecolor=markerfacecolor,
+                        markeredgecolor="tab:blue",
+                        markevery=(markevery // 2, markevery))
+            ax.semilogy(ns[:, 4],
+                        color="None",
+                        marker="s",
+                        markerfacecolor=markerfacecolor,
+                        markeredgecolor="tab:orange",
+                        markevery=(markevery // 2, markevery))
+
+            #ax.set_yscale("symlog", linthresh=1e-12)
+
+            ax.set_title(name)
+
+            ax.set_yticks([1e-17, 1e-12, 1e-7, 1e-2, 1e3, 1e8])
+
+            if spec.sequence == "kitti10":
+                ax.set_xticks([i * 100 for i in range(4)])
+                #ax.xaxis.set_minor_locator(matplotlib.ticker.FixedLocator([i * 100 + 50 for i in range(4)]))
+
+            if i == 0:
+                ax.set_ylabel("$\\Delta E_m$", rotation=0)
+                ax.yaxis.set_label_coords(-0.05, 1.05)
+
+            if i == num_plots - 1:
+                ax.legend(loc=spec.legend_loc)
+
+            if spec.ylim.top is not None:
+                ax.set_ylim(top=spec.ylim.top)
+            if spec.ylim.bottom is not None:
+                ax.set_ylim(bottom=spec.ylim.bottom)
+
+        if spec.suptitle:
+            fig.suptitle(spec.suptitle)
+
+    def plot_eigenvalues(self, exps, spec):
+
+        logs = [exps[e].runs[spec.sequence].log for e in spec.experiments]
+        names = [exps[e].display_name for e in spec.experiments]
+
+        num_plots = 1
+
+        if spec.figsize is None:
+            spec.figsize = [5.2, 2 * num_plots]
+
+        fig, axs = plt.subplots(num_plots, 1, figsize=spec.figsize)
+
+        ax = axs
+
+        for i, (log, name) in enumerate(zip(logs, names)):
+            if log is not None:
+                min_ev = [np.min(e) for e in log.sums.marg_ev[1:]]
+                #ax.plot(min_ev, ":", label=name, color=default_colors[i])
+                ax.plot(min_ev, default_lines[i], label=name, color=default_colors[i])
+
+        ax.set_yscale("symlog", linthresh=1e-8)
+        ax.legend(loc=spec.legend_loc)
+
+        #ax.set_title("smallest eigenvalue {} {}".format(name, spec.sequence))
+
+        if spec.sequence == "eurocMH01":
+            ax.set_xticks([i * 1000 for i in range(4)])
+            ax.xaxis.set_minor_locator(matplotlib.ticker.FixedLocator([i * 1000 + 500 for i in range(4)]))
+
+        if spec.sequence == "kitti10":
+            ax.set_xticks([i * 100 for i in range(4)])
+            ax.xaxis.set_minor_locator(matplotlib.ticker.FixedLocator([i * 100 + 50 for i in range(4)]))
+
+        ax.set_yticks([-1e4, -1e-4, 0.0, 1e-4, 1e4])
+        ax.set_ylim(bottom=-1e8, top=1e8)
+        # ax.yaxis.tick_right()
+        ax.set_ylabel("$\\sigma_{min}$", rotation=0)
+        ax.yaxis.set_label_coords(0, 1.05)
+
+        if spec.ylim.top is not None:
+            ax.set_ylim(top=spec.ylim.top)
+        if spec.ylim.bottom is not None:
+            ax.set_ylim(bottom=spec.ylim.bottom)
+
+        if spec.suptitle:
+            fig.suptitle(spec.suptitle)
+
+    def plot_trajectory(self, exps, spec):
+
+        #self.width = 1.5
+
+        runs = [exps[e].runs[spec.sequence] for e in spec.experiments]
+        names = [exps[e].display_name for e in spec.experiments]
+
+        linewidth_factor = 3
+
+        R = rotation2d(spec.rotate2d)
+
+        traj_axes_idx = self._axes_spec_to_index(spec.trajectory_axes)
+
+        if spec.figsize is None:
+            spec.figsize = [6.4, 4.8]
+
+        fig, ax = plt.subplots(figsize=spec.figsize)
+
+        ax.axis("equal")
+        ax.axis('off')
+        #ax.set_xlabel("x")
+        #ax.set_ylabel("y")
+
+        gt_color = "tab:grey"
+        #gt_color = "black"
+
+        # take gt-trajectory from first experiment:
+        if runs[0].traj_gt is not None:
+            gt = runs[0].traj_gt[:, traj_axes_idx].T
+            gt = np.matmul(R, gt)
+            ax.plot(gt[0, :],
+                    gt[1, :],
+                    '-',
+                    zorder=1,
+                    linewidth=1 * linewidth_factor,
+                    color=gt_color,
+                    label="ground truth")
+
+        # https://matplotlib.org/stable/gallery/color/named_colors.html
+        linestyles = [":", ":", "--", "-"]
+        colors = [default_colors[1], default_colors[3]]
+        #colors = ["tab:blue", "tab:orange"]
+        linewidths = [2, 1]
+
+        for i, (r, name) in enumerate(zip(runs, names)):
+            # plot in decreasing zorder
+            #zorder = len(runs) - i + 1
+
+            zorder = i + 2
+
+            if r.traj_est is not None:
+                pos = r.traj_est[:, traj_axes_idx].T
+                pos = np.matmul(R, pos)
+                ax.plot(
+                    pos[0, :],
+                    pos[1, :],
+                    linestyles[i],
+                    #default_lines[i],
+                    zorder=zorder,
+                    linewidth=linewidths[i] * linewidth_factor,
+                    label=name,
+                    color=colors[i])
+
+        #ax.set_xlim(np.min(x_gt), np.max(x_gt))
+        #ax.set_ylim(np.min(y_gt), np.max(y_gt))
+
+        #lines = [gt]
+        #colors = ['black']
+        #line_segments = LineCollection(lines, colors=colors, linestyle='solid')
+
+        #ax.add_collection(line_segments)
+
+        ax.legend(loc=spec.legend_loc)
+
+        if spec.title is not None:
+            ax.set_title(spec.title.format(sequence=self.seq_displayname(spec.sequence)))
+
+    @staticmethod
+    def _axes_spec_to_index(axes_spec):
+        index = []
+        assert len(axes_spec) == 2, "Inalid axes_spec {}".format(axes_spec)
+        for c in axes_spec:
+            if c == "x":
+                index.append(0)
+            elif c == "y":
+                index.append(1)
+            elif c == "z":
+                index.append(2)
+            else:
+                assert False, "Inalid axes_spec {}".format(axes_spec)
+        return index
+
+    # static:
+    filename_counters = defaultdict(int)
+
+    def _save_plot(self, spec, basepath, extension=".pdf"):
+
+        os.makedirs(basepath, exist_ok=True)
+
+        if "sequence" in spec:
+            filename = '{}_{}_{}'.format(spec.type, spec.name, spec.sequence)
+        else:
+            filename = '{}_{}'.format(spec.type, spec.name)
+
+        filename = filename.replace(" ", "_").replace("/", "_")
+
+        Plot.filename_counters[filename] += 1
+        counter = Plot.filename_counters[filename]
+        if counter > 1:
+            filename = "{}-{}".format(filename, counter)
+
+        filepath = os.path.join(basepath, "{}.{}".format(filename, extension.strip('.')))
+
+        plt.savefig(filepath)
+
+        return filepath

python/basalt/latex/results_table.py

@@ -0,0 +1,163 @@
+#
+# BSD 3-Clause License
+#
+# This file is part of the Basalt project.
+# https://gitlab.com/VladyslavUsenko/basalt.git
+#
+# Copyright (c) 2019-2021, Vladyslav Usenko and Nikolaus Demmel.
+# All rights reserved.
+#
+import numbers
+import os
+import math
+
+import numpy as np
+
+from pylatex import Subsection, Tabular, TextColor
+from pylatex import MultiRow, FootnoteText
+from pylatex.utils import italic, bold, NoEscape, escape_latex, dumps_list
+
+from .containers import ExperimentsTable
+from .util import format_ratio_percent
+from .util import best_two_non_repeating
+
+
+class ResultsTable(ExperimentsTable):
+
+    def __init__(self, exps, spec, show_values_failed_runs, seq_displayname_mapping, export_basepath):
+        super().__init__(exps, spec, show_values_failed_runs, seq_displayname_mapping, export_basepath)
+
+        self.doit()
+
+    def doit(self):
+
+        is_multirow = self.num_metrics > 1 and self.spec.multirow
+
+        def render_metric(value, best, second, decimals, format_string, highlight_top, relative_to):
+            if isinstance(value, numbers.Number):
+                if relative_to is None or relative_to == 0 or not np.isfinite(relative_to):
+                    # absolute number
+                    rendered = format_string.format(value, prec=decimals)
+                else:
+                    # percent
+                    rendered = format_ratio_percent(value, relative_to, decimals=decimals)
+                if highlight_top:
+                    if value == best:
+                        rendered = bold(rendered)
+                    elif value == second:
+                        rendered = italic(rendered)
+                return rendered
+            else:
+                return value
+
+        if self.spec.export_latex:
+            row_height = None
+        else:
+            row_height = 0.65 if is_multirow and self.num_metrics >= 3 else 1
+
+        column_spec = '|r' if self.spec.vertical_bars else 'r'
+        t = Tabular('l' + column_spec * self.num_exps, row_height=row_height, pos=['t'])
+        escape_header_fun = lambda text: text if self.spec.escape_latex_header else NoEscape(text)
+        if self.spec.rotate_header:
+            t.add_row([''] + [
+                NoEscape(r"\rotatebox{90}{%s}" % escape_latex(escape_header_fun(s.display_name(self.exps[s.name]))))
+                for s in self.experiment_specs
+            ])
+        else:
+            t.add_row([''] + [escape_header_fun(s.display_name(self.exps[s.name])) for s in self.experiment_specs])
+        t.add_hline()
+
+        for seq in self.seq_names:
+            fails = [self.is_failed(self.exps[s.name], seq) for s in self.experiment_specs]
+            failure_strings = [self.render_failure(self.exps[s.name], seq) for s in self.experiment_specs]
+            values = np.array([self.get_metrics(self.exps[s.name], seq, s.it) for s in self.experiment_specs])
+
+            top_values = list(range(self.num_metrics))
+            for c, m in enumerate(self.metrics):
+                try:
+                    values[:, c] = np.around(values[:, c], m.decimals)
+                except IndexError:
+                    pass
+                non_excluded_values = np.array(values[:, c])
+                for i in m.exclude_columns_highlight:
+                    non_excluded_values[i] = math.nan
+                top_values[c] = best_two_non_repeating(non_excluded_values, reverse=m.larger_is_better)
+
+            if is_multirow:
+                rows = [[MultiRow(self.num_metrics, data=self.seq_displayname(seq))]
+                       ] + [list(['']) for _ in range(1, self.num_metrics)]
+            else:
+                rows = [[self.seq_displayname(seq)]]
+            for c, (fail, failure_str, value_col) in enumerate(zip(fails, failure_strings, values)):
+                if failure_str is not None:
+                    if self.spec.color_failed:
+                        failure_str = TextColor(self.spec.color_failed, failure_str)
+                    if is_multirow:
+                        rows[0].append(MultiRow(self.num_metrics, data=failure_str))
+                        for r in range(1, self.num_metrics):
+                            rows[r].append('')
+                    else:
+                        rows[0].append(failure_str)
+                else:
+                    tmp_data = [None] * self.num_metrics
+                    for r, m in enumerate(self.metrics):
+                        if m.failed_threshold and value_col[r] > m.failed_threshold:
+                            obj = "x"
+                            if self.spec.color_failed:
+                                obj = TextColor(self.spec.color_failed, obj)
+                        else:
+                            relative_to = None
+                            if m.relative_to_column is not None and m.relative_to_column != c:
+                                relative_to = values[m.relative_to_column, r]
+                            obj = render_metric(value_col[r],
+                                                top_values[r][0],
+                                                top_values[r][1],
+                                                m.effective_display_decimals(),
+                                                m.format_string,
+                                                m.highlight_top,
+                                                relative_to=relative_to)
+                            if fail and self.spec.color_failed:
+                                obj = TextColor(self.spec.color_failed, obj)
+                        tmp_data[r] = obj
+                    if self.num_metrics == 1 or is_multirow:
+                        for r, obj in enumerate(tmp_data):
+                            rows[r].append(obj)
+                    else:
+                        entry = []
+                        for v in tmp_data:
+                            entry.append(v)
+                            entry.append(NoEscape("~/~"))
+                        entry.pop()
+                        rows[0].append(dumps_list(entry))
+
+            for row in rows:
+                t.add_row(row)
+
+            if is_multirow:
+                t.add_hline()
+
+        if self.spec.export_latex:
+            os.makedirs(self.export_basepath, exist_ok=True)
+            t.generate_tex(os.path.join(self.export_basepath, self.spec.export_latex))
+
+        with self.create(Subsection(self.spec.name, numbering=False)) as p:
+
+            if self.spec.metrics_legend:
+                legend = Tabular('|c|', row_height=row_height, pos=['t'])
+                legend.add_hline()
+                legend.add_row(["Metrics"])
+                legend.add_hline()
+                for m in self.metrics:
+                    legend.add_row([m.display_name])
+                legend.add_hline()
+
+                tab = Tabular("ll")
+                tab.add_row([t, legend])
+                content = tab
+            else:
+                content = t
+
+            if True:
+                content = FootnoteText(content)
+
+            p.append(content)

python/basalt/latex/summarize_sequences_table.py

@@ -0,0 +1,88 @@
+#
+# BSD 3-Clause License
+#
+# This file is part of the Basalt project.
+# https://gitlab.com/VladyslavUsenko/basalt.git
+#
+# Copyright (c) 2019-2021, Vladyslav Usenko and Nikolaus Demmel.
+# All rights reserved.
+#
+import numbers
+import os
+import scipy.stats
+import numpy as np
+
+from pylatex import Subsection, FootnoteText, Tabular, NoEscape, escape_latex
+from pylatex.utils import italic, bold
+
+from .containers import ExperimentsTable
+from .util import best_two_non_repeating
+
+
+class SummarizeSequencesTable(ExperimentsTable):
+
+    def __init__(self, exps, spec, show_values_failed_runs, seq_displayname_mapping, export_basepath):
+        super().__init__(exps, spec, show_values_failed_runs, seq_displayname_mapping, export_basepath)
+
+        self.doit()
+
+    def doit(self):
+
+        def render_metric(value, best, second, decimals, format_string):
+            if isinstance(value, numbers.Number):
+                rendered = format_string.format(value, prec=decimals)
+                if value == best:
+                    rendered = bold(rendered)
+                elif value == second:
+                    rendered = italic(rendered)
+                return rendered
+            else:
+                return value
+
+        values = np.empty((self.num_metrics, self.num_seqs, self.num_exps))
+
+        for i, seq in enumerate(self.seq_names):
+            for j, s in enumerate(self.experiment_specs):
+                values[:, i, j] = np.array(self.get_metrics(self.exps[s.name], seq, s.it))
+
+        means = np.empty((self.num_metrics, self.num_exps))
+        for i, m in enumerate(self.metrics):
+            if m.geometric_mean:
+                means[i, :] = scipy.stats.gmean(values[i, :, :], axis=0)
+            else:
+                means[i, :] = np.mean(values[i, :, :], axis=0)
+
+        t = Tabular('l' + 'c' * self.num_exps)
+
+        t.add_hline()
+        escape_header_fun = lambda text: text if self.spec.escape_latex_header else NoEscape(text)
+        if self.spec.rotate_header:
+            t.add_row([self.spec.header] + [
+                NoEscape(r"\rotatebox{90}{%s}" % escape_latex(escape_header_fun(s.display_name(self.exps[s.name]))))
+                for s in self.experiment_specs
+            ])
+        else:
+            t.add_row([self.spec.header] +
+                      [escape_header_fun(s.display_name(self.exps[s.name])) for s in self.experiment_specs])
+        t.add_hline()
+
+        for i, m in enumerate(self.metrics):
+            row_values = np.around(means[i, :], m.decimals)
+            top_values = best_two_non_repeating(row_values, reverse=m.larger_is_better)
+            row = [m.display_name]
+            for v in row_values:
+                # TODO: use NoEscape only if certain flag is enabled?
+                row.append(
+                    NoEscape(
+                        render_metric(v, top_values[0], top_values[1], m.effective_display_decimals(),
+                                      m.format_string)))
+            t.add_row(row)
+
+        t.add_hline()
+
+        if self.spec.export_latex:
+            os.makedirs(self.export_basepath, exist_ok=True)
+            t.generate_tex(os.path.join(self.export_basepath, self.spec.export_latex))
+
+        with self.create(Subsection(self.spec.name, numbering=False)) as p:
+            p.append(FootnoteText(t))

python/basalt/latex/templates.py

@@ -0,0 +1,93 @@
+#
+# BSD 3-Clause License
+#
+# This file is part of the Basalt project.
+# https://gitlab.com/VladyslavUsenko/basalt.git
+#
+# Copyright (c) 2019-2021, Vladyslav Usenko and Nikolaus Demmel.
+# All rights reserved.
+#
+screenread_sty = r"""
+\ProvidesPackage{screenread}
+% Copyright (C) 2012 John Collins, collins@phys.psu.edu
+% License: LPPL 1.2
+
+% Note:  To avoid compatibility issues between geometry and at least one
+% class file, it may be better to set all the dimensions by hand.
+
+%  20 Nov 2014  - use `pageBreakSection` instead of clobbering `section`
+%               - increase longest page size to 575cm
+%               - make top, right, and left margins something sensible and
+%                 a bit more aesthetically pleasing
+%  24 Jan 2012  Argument to \SetScreen is screen width
+%  23 Jan 2012  Remove package showlayout
+%  22 Jan 2012  Initial version, based on ideas in 
+%               B. Veytsman amd M. Ware, Tugboat 32 (2011) 261.
+
+\RequirePackage{everyshi}
+\RequirePackage{geometry}
+
+%=======================
+
+\pagestyle{empty}
+
+\EveryShipout{%
+    \pdfpageheight=\pagetotal
+    \advance\pdfpageheight by 2in
+    \advance\pdfpageheight by \topmargin
+    \advance\pdfpageheight by \textheight % This and next allow for footnotes
+    \advance\pdfpageheight by -\pagegoal
+}
+
+\AtEndDocument{\pagebreak}
+
+\def\pageBreakSection{\pagebreak\section}
+
+\newlength\screenwidth
+\newlength{\savedscreenwidth}
+
+\newcommand\SetScreen[1]{%
+  % Argument #1 is the screen width.
+  % Set appropriate layout parameters, with only a little white space
+  %   around the text.
+  \setlength\screenwidth{#1}%
+  \setlength\savedscreenwidth{#1}%
+  \setlength\textwidth{#1}%
+  \addtolength\textwidth{-2cm}%
+  \geometry{layoutwidth=\screenwidth,
+            paperwidth=\screenwidth,
+            textwidth=\textwidth,
+            layoutheight=575cm,
+            paperheight=575cm,
+            textheight=575cm,
+            top=1cm,
+            left=1cm,
+            right=1cm,
+            hcentering=true
+  }%
+}
+
+\newcommand\SetPageScreenWidth[1]{%
+  \setlength\savedscreenwidth{\screenwidth}%
+  \setlength\screenwidth{#1}%
+  \pdfpagewidth\screenwidth%
+  \setlength\textwidth{\screenwidth}%
+  \addtolength\textwidth{-2cm}%
+}
+
+\newcommand\RestorePageScreenWidth{%
+  \setlength\screenwidth{\savedscreenwidth}%
+  \pdfpagewidth\screenwidth%
+  \setlength\textwidth{\screenwidth}%
+  \addtolength\textwidth{-2cm}%
+}
+
+
+% Compute a reasonable default screen width, and set it
+\setlength\screenwidth{\textwidth}
+\addtolength\screenwidth{1cm}
+\SetScreen{\screenwidth}
+
+\endinput
+
+"""

python/basalt/latex/util.py

@@ -0,0 +1,61 @@
+#
+# BSD 3-Clause License
+#
+# This file is part of the Basalt project.
+# https://gitlab.com/VladyslavUsenko/basalt.git
+#
+# Copyright (c) 2019-2021, Vladyslav Usenko and Nikolaus Demmel.
+# All rights reserved.
+#
+import math
+import numpy as np
+
+
+def best_two_non_repeating(array, reverse=False):
+    if reverse:
+        best = -math.inf
+        second = -math.inf
+        for v in array:
+            if v > best:
+                second = best
+                best = v
+            elif v < best and v > second:
+                second = v
+    else:
+        best = math.inf
+        second = math.inf
+        for v in array:
+            if v < best:
+                second = best
+                best = v
+            elif v > best and v < second:
+                second = v
+
+    return best, second
+
+
+def format_ratio(val, val_ref=None, decimals=0):
+    if val_ref == 0:
+        return "{}".format(math.inf)
+    else:
+        if val_ref is not None:
+            val = float(val) / float(val_ref)
+        return "{:.{prec}f}".format(val, prec=decimals)
+
+
+def format_ratio_percent(val, val_ref=None, decimals=0):
+    if val_ref == 0:
+        return "{}".format(val)
+    else:
+        if val_ref is not None:
+            val = float(val) / float(val_ref)
+        val = 100 * val
+        return "{:.{prec}f}%".format(val, prec=decimals)
+
+
+def rotation2d(theta_deg):
+    theta = np.radians(theta_deg)
+
+    R = np.array(((np.cos(theta), -np.sin(theta)), (np.sin(theta), np.cos(theta))))
+
+    return R