Re-try tests if gradle worker crashed

2 files changed, 33 insertions, 1 deletions

diff --git a/.github/actions/test-shard-run/run_tests.sh b/.github/actions/test-shard-run/run_tests.sh
index c4b8a3c63..1f312bbc0 100755
--- a/.github/actions/test-shard-run/run_tests.sh
+++ b/.github/actions/test-shard-run/run_tests.sh
@@ -11,7 +11,8 @@ echo "Will run tests for module '${MODULE}' with extra args '${EXTRA_ARGS}' for
 ./scripts/download_robolectric_jars_to_machine.sh
 
 #extra args needs to come before the coverage task so "--tests" will be passed to the test tasks
-./gradlew "${MODULE}testDebugUnitTest" ${EXTRA_ARGS} "${MODULE}testDebugUnitTestCoverage"
+# we automatically re-try on gradle crash
+./scripts/retry-on-SIGSEGV.sh 2 6,134 ./gradlew "${MODULE}testDebugUnitTest" ${EXTRA_ARGS} "${MODULE}testDebugUnitTestCoverage"
 
 #see https://github.com/actions/cache/issues/133
 [[ -n "${GITHUB_ACTIONS}" ]] && chmod -R a+rwx .
diff --git a/scripts/retry-on-SIGSEGV.sh b/scripts/retry-on-SIGSEGV.sh
new file mode 100755
index 000000000..0f05c7e1d
--- /dev/null
+++ b/scripts/retry-on-SIGSEGV.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+retries=$1
+shift
+IFS=',' read -r -a retry_exit_codes_array <<< "$1"
+retry_exit_codes=" ${retry_exit_codes_array[*]} "
+shift
+
+echo "Will retry '$*' for ${retries} when exit-codes are '${retry_exit_codes}':"
+set +e
+count=0
+EXIT_CODE=0
+pwd
+"$@"
+EXIT_CODE=$?
+echo "EXIT_CODE is ${EXIT_CODE}"
+while [[ "${retry_exit_codes}" =~ .*" ${EXIT_CODE} ".* ]]; do
+  count=$((count + 1))
+  if [[ ${count} -lt ${retries} ]]; then
+    echo "************** Retry ${count}/${retries} exited ${EXIT_CODE}, retrying in ${count} seconds..."
+    ./gradlew --stop
+    sleep ${count}
+    "$@"
+    EXIT_CODE=$?
+    echo "EXIT_CODE is ${EXIT_CODE}"
+  else
+    echo "Retry ${count}/${retries} exited ${EXIT_CODE}, no more retries left."
+  fi
+done
+
+exit ${EXIT_CODE}
\ No newline at end of file