v01

scripts/batch/generate-batch-configs.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+#
+# BSD 3-Clause License
+#
+# This file is part of the Basalt project.
+# https://gitlab.com/VladyslavUsenko/basalt.git
+#
+# Copyright (c) 2019-2021, Vladyslav Usenko and Nikolaus Demmel.
+# All rights reserved.
+#
+
+#
+# Generate basalt configurations from a batch config file.
+#
+# Example:
+#     ./generate-batch-config.py /path/to/folder
+#
+# It looks for the file named `basalt_batch_config.toml` inside the given folder.
+
+import os
+import toml
+import json
+import argparse
+from pprint import pprint
+from copy import deepcopy
+from collections import OrderedDict
+import itertools
+import shutil
+import datetime
+import sys
+
+
+def isdict(o):
+    return isinstance(o, dict) or isinstance(o, OrderedDict)
+
+
+def merge_config(a, b):
+    "merge b into a"
+    for k, v in b.items():
+        if k in a:
+            if isdict(v) and isdict(a[k]):
+                #print("dict {}".format(k))
+                merge_config(a[k], b[k])
+            elif not isdict(v) and not isdict(a[k]):
+                a[k] = deepcopy(v)
+                #print("not dict {}".format(k))
+            else:
+                raise RuntimeError("Incompatible types for key {}".format(k))
+        else:
+            a[k] = deepcopy(v)
+
+
+def save_config(template, configs, combination, path_prefix):
+    filename = os.path.join(path_prefix, "basalt_config_{}.json".format("_".join(combination)))
+    config = deepcopy(template)
+    #import ipdb; ipdb.set_trace()
+    for override in combination:
+        merge_config(config, configs[override])
+    #import ipdb; ipdb.set_trace()
+    with open(filename, 'w') as f:
+        json.dump(config, f, indent=4)
+    print(filename)
+
+
+def generate_configs(root_path, cmdline=[], overwrite_existing=False, revision_override=None):
+
+    # load and parse batch config file
+    batch_config_path = os.path.join(root_path, "basalt_batch_config.toml")
+    template = toml.load(batch_config_path, OrderedDict)
+    cfg = template["_batch"]
+    del template["_batch"]
+
+    # parse batch configuration
+    revision = str(cfg.get("revision", 0)) if revision_override is None else revision_override
+    configs = cfg["config"]
+    alternatives = cfg.get("alternatives", dict())
+    combinations = cfg["combinations"]
+
+    # prepare output directory
+    date_str = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
+    outdir = root_path if revision is None else os.path.join(root_path, revision)
+    if overwrite_existing and os.path.exists(outdir):
+        print("WARNING: output directory exists, overwriting existing files: {}".format(outdir))
+    else:
+        os.makedirs(outdir)
+    shutil.copy(batch_config_path, outdir)
+    with open(os.path.join(outdir, "timestamp"), 'w') as f:
+        f.write(date_str)
+    with open(os.path.join(outdir, "commandline"), 'w') as f:
+        f.write(cmdline)
+
+    # expand single entry in combination array
+    def expand_one(x):
+        if x in alternatives:
+            return alternatives[x]
+        elif isinstance(x, list):
+            # allow "inline" alternative
+            return x
+        else:
+            return [x]
+
+    def flatten(l):
+        for el in l:
+            if isinstance(el, list):
+                yield from flatten(el)
+            else:
+                yield el
+
+    # generate all configurations
+    for name, description in combinations.items():
+        if True or len(combinations) > 1:
+            path_prefix = os.path.join(outdir, name)
+            if not (overwrite_existing and os.path.exists(path_prefix)):
+                os.mkdir(path_prefix)
+        else:
+            path_prefix = outdir
+        expanded = [expand_one(x) for x in description]
+        for comb in itertools.product(*expanded):
+            # flatten list to allow each alternative to reference multiple configs
+            comb = list(flatten(comb))
+            save_config(template, configs, comb, path_prefix)
+
+
+def main():
+    cmdline = str(sys.argv)
+    parser = argparse.ArgumentParser("Generate basalt configurations from a batch config file.")
+    parser.add_argument("path", help="path to look for config and templates")
+    parser.add_argument("--revision", help="override revision")
+    parser.add_argument("--force", "-f", action="store_true", help="overwrite existing files")
+    args = parser.parse_args()
+    generate_configs(args.path, cmdline, args.force, args.revision)
+
+
+if __name__ == "__main__":
+    main()

scripts/batch/generate-tables.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+#
+# BSD 3-Clause License
+#
+# This file is part of the Basalt project.
+# https://gitlab.com/VladyslavUsenko/basalt.git
+#
+# Copyright (c) 2019-2021, Vladyslav Usenko and Nikolaus Demmel.
+# All rights reserved.
+#
+
+# Dependencies:
+# pip3 install -U --user py_ubjson matplotlib numpy munch scipy pylatex toml
+
+# also: latexmk and latex
+#
+# Ubuntu:
+#     sudo apt install texlive-latex-extra latexmk
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")))
+
+import basalt.generate_tables
+
+basalt.generate_tables.main()

scripts/batch/list-jobs.sh

@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+##
+## BSD 3-Clause License
+##
+## This file is part of the Basalt project.
+## https://gitlab.com/VladyslavUsenko/basalt.git
+##
+## Copyright (c) 2019-2021, Vladyslav Usenko and Nikolaus Demmel.
+## All rights reserved.
+##
+
+#
+# Usage:
+#    list-jobs.sh DIRNAME [DIRNAME ...] [-s|--short] [-o|--only STATUS]
+#
+# Lists all batch jobs found in DIRNAME. If the optional argument
+# STATUS is passed, only lists jobs with that status. Multiple
+# statuses can be passed in a space-separated string.
+#
+# Possible status arguments: queued, running, completed, failed, unknown
+# You can also use 'active' as a synonym for 'queued running unknown'
+
+# exit on error
+set -o errexit -o pipefail
+
+
+# we need GNU getopt...
+GETOPT=getopt
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    if [ -f /usr/local/opt/gnu-getopt/bin/getopt ]; then
+        GETOPT="/usr/local/opt/gnu-getopt/bin/getopt"
+    fi
+fi
+
+# option parsing, see: https://stackoverflow.com/a/29754866/1813258
+usage() { echo "Usage: `basename $0` DIRNAME [DIRNAME ...] [-s|--short] [-o|--only STATUS]" ; exit 1; }
+
+# -allow a command to fail with !’s side effect on errexit
+# -use return value from ${PIPESTATUS[0]}, because ! hosed $?
+! "$GETOPT" --test > /dev/null
+if [[ ${PIPESTATUS[0]} -ne 4 ]]; then
+    echo 'I’m sorry, `getopt --test` failed in this environment.'
+    exit 1
+fi
+
+OPTIONS=hsjo:
+LONGOPTS=help,short,jobids,only:
+
+# -regarding ! and PIPESTATUS see above
+# -temporarily store output to be able to check for errors
+# -activate quoting/enhanced mode (e.g. by writing out “--options”)
+# -pass arguments only via   -- "$@"   to separate them correctly
+! PARSED=$("$GETOPT" --options=$OPTIONS --longoptions=$LONGOPTS --name "`basename $0`" -- "$@")
+if [[ ${PIPESTATUS[0]} -ne 0 ]]; then
+    # e.g. return value is 1
+    #  then getopt has complained about wrong arguments to stdout
+    usage
+fi
+# read getopt’s output this way to handle the quoting right:
+eval set -- "$PARSED"
+
+SHORT=n
+ONLY=""
+JOBIDS=n
+# now enjoy the options in order and nicely split until we see --
+while true; do
+    case "$1" in
+        -h|--help) usage ;;
+        -s|--short) SHORT=y; shift ;;
+        -j|--jobids) JOBIDS=y; shift ;;
+        -o|--only) ONLY="$2"; shift 2 ;;
+        --) shift; break ;;
+        *) echo "Programming error"; exit 3 ;;
+    esac
+done
+
+# handle non-option arguments --> directories
+if [[ $# -lt 1 ]]; then
+    echo "Error: Pass at least one folder"
+    usage
+fi
+DIRS=("$@")
+
+# status aliases:
+ONLY="${ONLY/active/queued running unknown}"
+ONLY="${ONLY/notcompleted/queued running failed unknown}"
+
+contains() {
+    [[ $1 =~ (^| )$2($| ) ]] && return 0 || return 1
+}
+
+display() {
+    if [ -z "$ONLY" ] || contains "$ONLY" $2; then
+        if [ $SHORT = y ]; then
+            echo "$1"
+        else
+            echo -n "$1 : $2"
+            if [ -n "$3" ]; then
+                echo -n " - $3"
+            fi
+            echo ""
+        fi
+    fi
+}
+
+for d in "${DIRS[@]}"; do
+    for f in `find "$d" -name status.log | sort`; do
+        DIR=`dirname "$f"`
+
+        # ignore backup folder from "rerun" scripts
+        if [[ `basename $DIR` = results-backup* ]]; then
+            continue
+        fi
+
+        if ! grep Started "$f" > /dev/null; then
+            display "$DIR" unknown "not started"
+            continue
+        fi
+
+        # job has started:
+
+        if grep Completed "$f" > /dev/null ; then
+            display "$DIR" completed ""
+            continue
+        fi
+
+        # job has started, but not completed (cleanly)
+
+        # check signs of termination
+        if [ -f "$DIR"/output.log ] && grep "Command terminated by signal" "$DIR"/output.log > /dev/null; then
+            display "$DIR" failed killed "`grep -oP 'Command terminated by \Ksignal .+' "$DIR"/output.log`"
+            continue
+        fi
+
+        # might be running or aborted
+        display "$DIR" unknown started
+
+    done
+done

scripts/batch/plot.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+#
+# BSD 3-Clause License
+#
+# This file is part of the Basalt project.
+# https://gitlab.com/VladyslavUsenko/basalt.git
+#
+# Copyright (c) 2019-2021, Vladyslav Usenko and Nikolaus Demmel.
+# All rights reserved.
+#
+
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")))
+
+import basalt.nullspace
+
+basalt.nullspace.main()

scripts/batch/query-config.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+#
+# BSD 3-Clause License
+#
+# This file is part of the Basalt project.
+# https://gitlab.com/VladyslavUsenko/basalt.git
+#
+# Copyright (c) 2019-2021, Vladyslav Usenko and Nikolaus Demmel.
+# All rights reserved.
+#
+
+#
+# Example usage:
+#     $ ./query-config.py path/to/basalt_config.json value0.\"config.vio_debug\"
+#     10G
+
+import json
+import toml
+import argparse
+import sys
+
+
+def parse_query(query):
+    query_list = []
+
+    quote_open_char = None
+    curr = ""
+    for c in query:
+        if quote_open_char:
+            if c == quote_open_char:
+                quote_open_char = None
+            else:
+                curr += c
+        elif c in ['"', "'"]:
+            quote_open_char = c
+        elif c == '.':
+            query_list.append(curr)
+            curr = ""
+        else:
+            curr += c
+    query_list.append(curr)
+
+    return query_list
+
+
+def query_config(path, query, default_value=None, format_env=False, format_cli=False):
+    query_list = parse_query(query)
+    with open(path) as f:
+        cfg = json.load(f)
+    try:
+        curr = cfg
+        for q in query_list:
+            curr = curr[q]
+        result = curr
+    except:
+        if default_value is None:
+            result = ""
+        else:
+            result = default_value
+    if isinstance(result, dict):
+        if format_env:
+            lines = []
+            for k, v in result.items():
+                # NOTE: assumes no special escaping is necessary
+                lines.append("{}='{}'".format(k, v))
+            return "\n".join(lines)
+        elif format_cli:
+            args = ["--{} {}".format(k, v) for k, v in result.items()]
+            return " ".join(args)
+        else:
+            result = toml.dumps(result)
+    else:
+        result = "{}".format(result)
+    return result
+
+
+def main():
+    parser = argparse.ArgumentParser("Parse toml file and print content of query key.")
+    parser.add_argument("config_path", help="path to toml file")
+    parser.add_argument("query", help="query string")
+    parser.add_argument("default_value", help="value printed if query is not successful", nargs='?')
+    parser.add_argument(
+        "--format-env",
+        action="store_true",
+        help="Expect dictionary as query result and output like environment variables, i.e. VAR='VALUE' lines.")
+    parser.add_argument("--format-cli",
+                        action="store_true",
+                        help="Expect dictionary as query result and output like cli arguments, i.e. --VAR 'VALUE'.")
+    args = parser.parse_args()
+
+    res = query_config(args.config_path,
+                       args.query,
+                       default_value=args.default_value,
+                       format_env=args.format_env,
+                       format_cli=args.format_cli)
+    print(res)
+
+
+if __name__ == "__main__":
+    main()

scripts/batch/rerun-failed-in.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+##
+## BSD 3-Clause License
+##
+## This file is part of the Basalt project.
+## https://gitlab.com/VladyslavUsenko/basalt.git
+##
+## Copyright (c) 2019-2021, Vladyslav Usenko and Nikolaus Demmel.
+## All rights reserved.
+##
+
+#
+# Usage:
+#    rerun-failed-in.sh FOLDER
+#
+# Reruns all failed experiments that are found in a given folder.
+
+set -e
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+"$SCRIPT_DIR"/list-jobs.sh "$1" -s -o failed | while read f; do
+    echo "$f"
+    "$SCRIPT_DIR"/rerun-one-in.sh "$f" || true
+done

scripts/batch/rerun-one-in.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+##
+## BSD 3-Clause License
+##
+## This file is part of the Basalt project.
+## https://gitlab.com/VladyslavUsenko/basalt.git
+##
+## Copyright (c) 2019-2021, Vladyslav Usenko and Nikolaus Demmel.
+## All rights reserved.
+##
+
+set -e
+set -x
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+FOLDER="${1}"
+
+cd "$FOLDER"
+
+# backup previous files
+DATE=`date +'%Y%m%d-%H%M%S'`
+BACKUP_FOLDER=results-backup-$DATE
+for f in *.jobid *.log stats*.*json; do
+    if [ -f $f ]; then
+        mkdir -p $BACKUP_FOLDER
+        mv $f $BACKUP_FOLDER/
+    fi
+done
+
+echo "Created" > status.log
+echo "Restarted" >> status.log
+
+echo "Starting run in $PWD"
+"$SCRIPT_DIR"/run-one.sh "$PWD"

scripts/batch/run-all-in.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+##
+## BSD 3-Clause License
+##
+## This file is part of the Basalt project.
+## https://gitlab.com/VladyslavUsenko/basalt.git
+##
+## Copyright (c) 2019-2021, Vladyslav Usenko and Nikolaus Demmel.
+## All rights reserved.
+##
+
+
+# given folder with basalt_config_*.json, run optimization for each config in
+# corresponding subfolder
+
+set -e
+set -x
+
+# number of logical cores on linux and macos
+NUM_CORES=`(which nproc > /dev/null && nproc) || sysctl -n hw.logicalcpu || echo 1`
+
+echo "Running on '`hostname`', nproc: $NUM_CORES"
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+# loop over all arguments, and in each folder find configs and run them
+for FOLDER in "$@"
+do
+
+    pushd "$FOLDER"
+
+    FILE_PATTERN='basalt_config_*.json'
+    FILE_REGEX='basalt_config_(.*)\.json'
+
+    DATE=`date +'%Y%m%d-%H%M%S'`
+    mkdir -p $DATE
+
+    declare -a RUN_DIRS=()
+
+    for f in `find . -name "$FILE_PATTERN" -type f | sort`; do
+        if [[ `basename $f` =~ $FILE_REGEX ]]; then
+            RUN_DIR=${DATE}/`dirname $f`/${BASH_REMATCH[1]}
+            echo "Creating run with config $f in $RUN_DIR"
+            mkdir -p "$RUN_DIR"
+            cp $f "$RUN_DIR"/basalt_config.json
+            echo "Created" > "$RUN_DIR"/status.log
+            RUN_DIRS+=($RUN_DIR)
+        else
+            echo "Skipping $f"
+        fi
+    done
+
+    for RUN_DIR in "${RUN_DIRS[@]}"; do
+        echo "Starting run in $RUN_DIR"
+        "$SCRIPT_DIR"/run-one.sh "$RUN_DIR" || true
+    done
+
+    popd
+
+done

scripts/batch/run-one.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+##
+## BSD 3-Clause License
+##
+## This file is part of the Basalt project.
+## https://gitlab.com/VladyslavUsenko/basalt.git
+##
+## Copyright (c) 2019-2021, Vladyslav Usenko and Nikolaus Demmel.
+## All rights reserved.
+##
+
+#
+# This script runs on the slurm nodes to run rootba for one config.
+
+set -e
+set -o pipefail
+set -x
+
+error() {
+  local parent_lineno="$1"
+  local message="$2"
+  local code="${3:-1}"
+  if [[ -n "$message" ]] ; then
+    echo "Error on or near line ${parent_lineno}: ${message}; exiting with status ${code}"
+  else
+    echo "Error on or near line ${parent_lineno}; exiting with status ${code}"
+  fi
+  echo "Failed" >> status.log
+  exit "${code}"
+}
+trap 'error ${LINENO}' ERR
+
+# number of logical cores on linux and macos
+NUM_CORES=`(which nproc > /dev/null && nproc) || sysctl -n hw.logicalcpu || echo 1`
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+BASALT_BIN_DIR="${BASALT_BIN_DIR:-$SCRIPT_DIR/../../build}"
+
+FOLDER="${1}"
+
+cd "$FOLDER"
+
+if ! which time 2> /dev/null; then
+    echo "Did not find 'time' executable. Not installed?"
+    exit 1
+fi
+
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    TIMECMD="`which time` -lp"
+else
+    TIMECMD="`which time` -v"
+fi
+
+echo "Started" >> status.log
+
+# set environment variables according to config
+while read l; do
+    if [ -n "$l" ]; then
+        eval "export $l"
+    fi
+done <<< `"$SCRIPT_DIR"/query-config.py basalt_config.json batch_run.env --format-env`
+
+# lookup executable to run
+EXECUTABLE=`"$SCRIPT_DIR"/query-config.py basalt_config.json batch_run.executable basalt_vio`
+
+# lookup args
+ARGS=`"$SCRIPT_DIR"/query-config.py basalt_config.json batch_run.args --format-cli`
+
+CMD="$BASALT_BIN_DIR/$EXECUTABLE"
+
+echo "Running on '`hostname`', nproc: $NUM_CORES, bin: $CMD"
+
+# run as many times as specified (for timing tests to make sure filecache is hot); default is once
+rm -f output.log
+NUM_RUNS=`"$SCRIPT_DIR"/query-config.py basalt_config.json batch_run.num_runs 1`
+echo "Will run $NUM_RUNS times."
+for i in $(seq $NUM_RUNS); do
+    echo ">>> Run $i" |& tee -a output.log
+    { $TIMECMD "$CMD" $ARGS; } |& tee -a output.log
+done
+
+echo "Completed" >> status.log