2020-07-30 15:57:32 +02:00
|
|
|
|
#!/usr/bin/env sh
|
2021-05-17 13:40:25 +02:00
|
|
|
|
|
2018-04-30 22:56:14 +02:00
|
|
|
|
GETOPT_BIN=$IN_GETOPT_BIN
|
|
|
|
|
GETOPT_BIN=${GETOPT_BIN:-getopt}
|
|
|
|
|
|
2017-11-03 18:10:09 +01:00
|
|
|
|
__sleep_amount() {
|
2020-07-30 14:48:54 +02:00
|
|
|
|
if [ -n "$constant_sleep" ]; then
|
2017-11-03 18:10:09 +01:00
|
|
|
|
sleep_time=$constant_sleep
|
|
|
|
|
else
|
2017-11-03 21:56:11 +01:00
|
|
|
|
#TODO: check for awk
|
|
|
|
|
#TODO: check if user would rather use one of the other possible dependencies: python, ruby, bc, dc
|
2023-01-14 23:47:20 +01:00
|
|
|
|
sleep_time=$(awk "BEGIN {t = $min_sleep * $((1 << (_retry_attempts - 1))); print (t > $max_sleep ? $max_sleep : t)}")
|
2017-11-03 18:10:09 +01:00
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__log_out() {
|
|
|
|
|
echo "$1" 1>&2
|
|
|
|
|
}
|
|
|
|
|
|
2019-08-14 09:45:29 +02:00
|
|
|
|
# Parameters: max_tries min_sleep max_sleep constant_sleep fail_script EXECUTION_COMMAND
|
2021-05-17 13:40:25 +02:00
|
|
|
|
retry() {
|
|
|
|
|
_retry_max_tries="$1"
|
|
|
|
|
shift
|
|
|
|
|
_retry_min_sleep="$1"
|
|
|
|
|
shift
|
|
|
|
|
_retry_max_sleep="$1"
|
|
|
|
|
shift
|
|
|
|
|
_retry_constant_sleep="$1"
|
|
|
|
|
shift
|
|
|
|
|
_retry_fail_script="$1"
|
|
|
|
|
shift
|
2017-11-03 18:10:09 +01:00
|
|
|
|
if [ -n "$VERBOSE" ]; then
|
2021-05-17 13:40:25 +02:00
|
|
|
|
__log_out "Retry Parameters: max_tries=$_retry_max_tries min_sleep=$_retry_min_sleep max_sleep=$_retry_max_sleep constant_sleep=$_retry_constant_sleep"
|
|
|
|
|
if [ -n "$_retry_fail_script" ]; then __log_out "Fail script: $_retry_fail_script"; fi
|
2017-11-03 18:10:09 +01:00
|
|
|
|
__log_out ""
|
|
|
|
|
__log_out "Execution Command: $*"
|
|
|
|
|
__log_out ""
|
|
|
|
|
fi
|
|
|
|
|
|
2021-05-17 13:40:25 +02:00
|
|
|
|
_retry_attempts=0
|
|
|
|
|
_retry_return_code=1
|
2017-11-03 18:10:09 +01:00
|
|
|
|
|
2021-05-17 13:40:25 +02:00
|
|
|
|
while [ "$_retry_return_code" -ne 0 ] && [ "$_retry_attempts" -le "$_retry_max_tries" ]; do
|
|
|
|
|
if [ $_retry_attempts -gt 0 ]; then
|
2017-11-03 18:10:09 +01:00
|
|
|
|
__sleep_amount
|
2021-05-17 13:40:25 +02:00
|
|
|
|
__log_out "Before retry #$_retry_attempts: sleeping $sleep_time seconds"
|
2020-07-30 15:57:32 +02:00
|
|
|
|
sleep "$sleep_time"
|
2017-11-03 18:10:09 +01:00
|
|
|
|
fi
|
2017-11-03 21:56:11 +01:00
|
|
|
|
|
|
|
|
|
P="$1"
|
2020-07-30 15:57:32 +02:00
|
|
|
|
#for param in "${@:2}"; do P="$P '$param'"; done
|
2017-11-03 21:56:11 +01:00
|
|
|
|
#TODO: replace single quotes in each arg with '"'"' ?
|
2021-05-17 13:40:25 +02:00
|
|
|
|
export RETRY_ATTEMPT=$_retry_attempts
|
2020-07-30 15:57:32 +02:00
|
|
|
|
sh -c "$P"
|
2021-05-17 13:40:25 +02:00
|
|
|
|
_retry_return_code=$?
|
|
|
|
|
#__log_out "Process returned $_retry_return_code on attempt $_retry_attempts"
|
|
|
|
|
if [ $_retry_return_code -eq 127 ]; then
|
2017-11-03 18:10:09 +01:00
|
|
|
|
# command not found
|
2021-05-17 13:40:25 +02:00
|
|
|
|
exit $_retry_return_code
|
|
|
|
|
elif [ $_retry_return_code -ne 0 ]; then
|
|
|
|
|
_retry_attempts=$((_retry_attempts + 1))
|
2017-11-03 18:10:09 +01:00
|
|
|
|
fi
|
|
|
|
|
done
|
|
|
|
|
|
2021-05-17 13:40:25 +02:00
|
|
|
|
if [ "$_retry_attempts" -gt "$_retry_max_tries" ]; then
|
|
|
|
|
if [ -n "$_retry_fail_script" ]; then
|
2017-11-03 18:10:09 +01:00
|
|
|
|
__log_out "Retries exhausted, running fail script"
|
2021-05-17 13:40:25 +02:00
|
|
|
|
eval "$_retry_fail_script"
|
2017-11-03 18:10:09 +01:00
|
|
|
|
else
|
|
|
|
|
__log_out "Retries exhausted"
|
|
|
|
|
fi
|
|
|
|
|
fi
|
|
|
|
|
|
2021-05-17 13:40:25 +02:00
|
|
|
|
exit $_retry_return_code
|
2017-11-03 18:10:09 +01:00
|
|
|
|
}
|
|
|
|
|
|
2021-05-17 13:40:25 +02:00
|
|
|
|
help() {
|
|
|
|
|
_help_retry=$(basename "$0")
|
|
|
|
|
cat <<EOF
|
|
|
|
|
Usage: $_help_retry [options] -- execute command
|
2020-07-30 15:57:32 +02:00
|
|
|
|
-h, -?, --help
|
|
|
|
|
-v, --verbose Verbose output
|
|
|
|
|
-t, --tries=# Set max retries: Default 10
|
|
|
|
|
-s, --sleep=secs Constant sleep amount (seconds)
|
|
|
|
|
-m, --min=secs Exponential Backoff: minimum sleep amount (seconds): Default 0.3
|
|
|
|
|
-x, --max=secs Exponential Backoff: maximum sleep amount (seconds): Default 60
|
|
|
|
|
-f, --fail="script +cmds" Fail Script: run in case of final failure
|
2017-11-03 18:10:09 +01:00
|
|
|
|
EOF
|
2020-07-30 15:57:32 +02:00
|
|
|
|
}
|
2017-11-03 18:10:09 +01:00
|
|
|
|
|
2020-07-30 15:57:32 +02:00
|
|
|
|
# show help for no arguments if stdin is a terminal
|
2021-05-17 13:40:25 +02:00
|
|
|
|
if { [ -z "$1" ] && [ -t 0 ]; } || [ "$1" = '-h' ] || [ "$1" = '-?' ] || [ "$1" = '--help' ]; then
|
|
|
|
|
help
|
|
|
|
|
exit 0
|
2020-07-30 15:57:32 +02:00
|
|
|
|
fi
|
2017-11-03 18:10:09 +01:00
|
|
|
|
|
2021-05-17 13:40:25 +02:00
|
|
|
|
$GETOPT_BIN --test >/dev/null
|
2020-07-30 15:57:32 +02:00
|
|
|
|
if [ $? -ne 4 ]; then
|
2021-05-17 13:40:25 +02:00
|
|
|
|
echo "I’m sorry, 'getopt --test' failed in this environment. Please load GNU getopt."
|
|
|
|
|
exit 1
|
2020-07-30 15:57:32 +02:00
|
|
|
|
fi
|
2017-11-03 18:10:09 +01:00
|
|
|
|
|
2020-07-30 15:57:32 +02:00
|
|
|
|
OPTIONS=vt:s:m:x:f:
|
|
|
|
|
LONGOPTIONS=verbose,tries:,sleep:,min:,max:,fail:
|
2019-08-14 09:45:29 +02:00
|
|
|
|
|
2020-07-30 15:57:32 +02:00
|
|
|
|
PARSED=$($GETOPT_BIN --options="$OPTIONS" --longoptions="$LONGOPTIONS" --name "$0" -- "$@")
|
2021-05-17 13:40:25 +02:00
|
|
|
|
# shellcheck disable=SC2181
|
2020-07-30 15:57:32 +02:00
|
|
|
|
if [ $? -ne 0 ]; then
|
2021-05-17 13:40:25 +02:00
|
|
|
|
# e.g. $? == 1
|
|
|
|
|
# then getopt has complained about wrong arguments to stdout
|
|
|
|
|
exit 2
|
2017-11-03 18:10:09 +01:00
|
|
|
|
fi
|
2020-07-30 15:57:32 +02:00
|
|
|
|
# read getopt’s output this way to handle the quoting right:
|
|
|
|
|
eval set -- "$PARSED"
|
|
|
|
|
|
|
|
|
|
max_tries=10
|
|
|
|
|
min_sleep=0.3
|
|
|
|
|
max_sleep=60.0
|
|
|
|
|
constant_sleep=
|
|
|
|
|
fail_script=
|
|
|
|
|
|
|
|
|
|
# now enjoy the options in order and nicely split until we see --
|
|
|
|
|
while true; do
|
2021-05-17 13:40:25 +02:00
|
|
|
|
case "$1" in
|
|
|
|
|
-v | --verbose)
|
|
|
|
|
VERBOSE=true
|
|
|
|
|
shift
|
|
|
|
|
;;
|
|
|
|
|
-t | --tries)
|
|
|
|
|
max_tries="$2"
|
|
|
|
|
shift 2
|
|
|
|
|
;;
|
|
|
|
|
-s | --sleep)
|
|
|
|
|
constant_sleep="$2"
|
|
|
|
|
shift 2
|
|
|
|
|
;;
|
|
|
|
|
-m | --min)
|
|
|
|
|
min_sleep="$2"
|
|
|
|
|
shift 2
|
|
|
|
|
;;
|
|
|
|
|
-x | --max)
|
|
|
|
|
max_sleep="$2"
|
|
|
|
|
shift 2
|
|
|
|
|
;;
|
|
|
|
|
-f | --fail)
|
|
|
|
|
fail_script="$2"
|
|
|
|
|
shift 2
|
|
|
|
|
;;
|
|
|
|
|
--)
|
|
|
|
|
shift
|
|
|
|
|
break
|
|
|
|
|
;;
|
|
|
|
|
*)
|
|
|
|
|
echo "Programming error"
|
|
|
|
|
exit 3
|
|
|
|
|
;;
|
|
|
|
|
esac
|
2020-07-30 15:57:32 +02:00
|
|
|
|
done
|
|
|
|
|
|
|
|
|
|
retry "$max_tries" "$min_sleep" "$max_sleep" "$constant_sleep" "$fail_script" "$@"
|