#!/usr/bin/env bash
# Finds flaky jobs and tests in a repository's GitHub Actions history: jobs
# that failed and then passed on a re-run of the same commit, ranked by how
# often and by the minutes they cost. For the worst, reads the failed log and
# pulls out the failing tests' names and first error lines. Needs gh (signed
# in, read access) and jq.
#
#   flaky.sh OWNER/REPO [--days 30] [--workflow ci.yml] [--logs 25] [--out .flaky]
set -euo pipefail

usage() { sed -n '2,8p' "$0" | sed 's/^# \{0,1\}//'; exit 2; }

[ $# -ge 1 ] || usage
repo=$1
shift
days=30
workflow=""
logs=25
out=".flaky"
while [ $# -gt 0 ]; do
    case $1 in
        --days) days=$2; shift 2 ;;
        --workflow) workflow=$2; shift 2 ;;
        --logs) logs=$2; shift 2 ;;
        --out) out=$2; shift 2 ;;
        *) usage ;;
    esac
done

for tool in gh jq; do
    command -v "$tool" >/dev/null || { echo "flaky.sh needs $tool" >&2; exit 1; }
done

# GNU date first, then BSD's (macOS).
since=$(date -u -d "-$days days" +%F 2>/dev/null || date -u -v-"$days"d +%F)
mkdir -p "$out/logs"

echo "Reading runs of $repo since $since…" >&2
args=(-R "$repo" --created ">=$since" --limit 1000 --json databaseId,attempt,headSha,workflowName,event,headBranch,conclusion,status)
[ -n "$workflow" ] && args+=(--workflow "$workflow")
gh run list "${args[@]}" | jq '[.[] | select(.status == "completed")]' >"$out/runs.json"

# Two kinds of evidence that a job failed for no reason in the code:
#   a run re-run on the same commit, whose job failed on one attempt and
#   passed on a later one;
#   two runs of the same workflow on the same commit, one failing the job and
#   one passing it.
# Every re-run run has its jobs read, and the runs of a commit whose workflow
# both failed and passed.
jq -r '
    (map(select(.attempt > 1)) | map({id: .databaseId, attempts: .attempt})),
    (group_by("\(.workflowName) \(.headSha)")
        | map(select(any(.conclusion == "failure") and any(.conclusion == "success"))) | flatten
        | map(select(.attempt == 1)) | map({id: .databaseId, attempts: 1}))
    | .[] | "\(.id) \(.attempts)"
' "$out/runs.json" | sort -u >"$out/to-read.txt"

echo "Reading the jobs of $(wc -l <"$out/to-read.txt" | tr -d ' ') runs with a re-run or a twin…" >&2
# filter=all returns the jobs of every attempt, not just the latest. GitHub
# answers a busy repository's big runs with a 502 now and then: one more try.
jobs_of() {
    gh api "repos/$repo/actions/runs/$1/jobs?filter=all&per_page=100" --paginate \
        -q ".jobs[] | {id, run_id, run_attempt, name, workflow_name, head_sha, conclusion, started_at, completed_at, html_url, labels, steps: [.steps[]? | {name, conclusion}]}" 2>/dev/null
}
while read -r id _; do
    jobs_of "$id" || { sleep 2; jobs_of "$id"; } || echo "Couldn't read the jobs of run $id; left out." >&2
done <"$out/to-read.txt" | jq -s 'unique_by(.id)' >"$out/jobs.json"

# One row per flake: the failed job, and the passing one on the same commit.
jq '
def secs(j): if j.started_at and j.completed_at then ((j.completed_at | fromdate) - (j.started_at | fromdate)) else 0 end;
map(select(.conclusion == "success" or .conclusion == "failure"))
| group_by("\(.workflow_name) / \(.name) / \(.head_sha)")
| map(select(any(.conclusion == "failure") and any(.conclusion == "success")))
| map((map(select(.conclusion == "success")) | first) as $passed
    | map(select(.conclusion == "failure")) | map({
        job: "\(.workflow_name) / \(.name)",
        sha: .head_sha,
        failed_job_id: .id,
        failed_url: .html_url,
        passed_url: $passed.html_url,
        same_run: (.run_id == $passed.run_id),
        failed_step: ([.steps[] | select(.conclusion == "failure") | .name] | first // "?" | split("\n")[0] | rtrimstr(" \\") | .[:50]),
        workflow: .workflow_name,
        lost_seconds: secs(.),
        wait_seconds: (if $passed.completed_at and .completed_at then ((($passed.completed_at | fromdate) - (.completed_at | fromdate)) | if . < 0 then 0 else . end) else 0 end),
        at: .completed_at
    }))
| flatten | sort_by(.at) | reverse
' "$out/jobs.json" >"$out/flakes.json"

total=$(jq length "$out/flakes.json")
if [ "$total" -eq 0 ]; then
    echo "No job failed and then passed on the same commit since $since." | tee "$out/report.txt"
    exit 0
fi

# The failed logs of the most recent flakes, for the tests' names. Each
# framework prints a failing test its own way; these are the usual lines.
echo "Reading the failed logs of the latest $(( logs < total ? logs : total )) of $total flakes…" >&2
jq -r --argjson n "$logs" '.[:$n][] | .failed_job_id' "$out/flakes.json" | while read -r job; do
    log="$out/logs/$job.log"
    # gh won't print a log with colours in it unless told it may.
    [ -s "$log" ] || gh api --allow-escape-sequences "repos/$repo/actions/jobs/$job/logs" >"$log" 2>/dev/null || true
    # Drop the byte-order mark, the timestamp GitHub puts on every line, and
    # colours.
    clean() { sed -E $'s/^\xEF\xBB\xBF//; s/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z ?//; s/\x1b\\[[0-9;]*m//g' "$log" 2>/dev/null; }
    clean | grep -E \
        -e '^\s*--- FAIL: \S+' \
        -e '^panic: test timed out' \
        -e '^WARNING: DATA RACE' \
        -e '^(FAILED|ERROR) \S+::\S+' \
        -e '^\s*● .+ › .+' \
        -e '^\s*(FAIL|×|✗)\s+\S+\.(test|spec)\.[cm]?[jt]sx?\s+>\s+.+' \
        -e '^\s*[✘×]\s+[0-9]+ \[[^]]+\] › .+' \
        -e '^rspec \./\S+:[0-9]+ # .+' \
        -e '^test \S+ \.\.\. FAILED' \
        -e '^not ok [0-9]+ \S+' \
        -e '^(FAIL|ERROR): \S+ \(\S+\)' \
        -e '^\S+ > .+ FAILED$' \
        -e '^\[ERROR\] \S+.* <<< (FAILURE|ERROR)!' \
        | grep -v 'Test suite failed to run' |
        # What changes from one failure of the same test to the next (times,
        # counters, the message) goes, so the same test counts as one.
        sed -E 's/^\s+//; s/ \(?[0-9.]+ ?m?s\)?$//; s/^[✘×]\s+[0-9]+ /✘ /; s/^not ok [0-9]+ /not ok /; s/^(FAILED \S+) - .*/\1/; s/ -- Time elapsed.*//' |
        cut -c1-160 | sort -u | head -12 | sed "s/^/$job	test	/" || true
    # GitHub echoes each step's script in a group starting "Run": it isn't
    # what went wrong.
    clean | awk '
        /^##\[group\]Run / { skip = 1; next }
        /^##\[endgroup\]/ { skip = 0; next }
        skip || /^##\[(group|endgroup)\]|^\[command\]|^Post job cleanup|^\s*$/ { next }
        # The lines just before the first error usually say what went wrong;
        # "Process completed with exit code 1" on its own does not.
        /##\[error\]/ { if (n > 1) print prev2; if (n > 0) print prev1; print; exit }
        { n++; prev2 = prev1; prev1 = $0 }
    ' | cut -c1-160 | sed "s/^/$job	error	/" || true
done >"$out/tests.tsv"

jq -r --rawfile tsv "$out/tests.tsv" --argjson days "$days" --slurpfile runs "$out/runs.json" '
def fmt: if . >= 60 then "\(. / 60 | floor)m\(. % 60 | floor)s" else "\(. | floor)s" end;
def short: if length > 60 then .[:57] + "..." else . end;
def table: (.[0] | keys_unsorted | join("\t")), (.[] | [.[] | tostring] | join("\t"));
def bar(n; total): "\(n) of \(total) runs (\(if total > 0 then (n / total * 100 * 10 | round / 10) else 0 end)%)";
($tsv | split("\n") | map(select(length > 0) | split("\t") | {job: .[0], kind: .[1], line: .[2]})) as $lines
| . as $flakes

| "== Flaky jobs, last \($days) days (failed, then passed on the same commit) ==",
  ($flakes | group_by(.job)
    | map({job: (.[0].job | short), flakes: length,
        # A job that failed before its own commands ran failed on the
        # machine, the network or a download, not in a test.
        where: (if (map(.failed_step) | all(test("^(Set up job|Setup |Set up |Install|Run actions/(checkout|setup-|cache)|Run pnpm/action-setup|Initialize|Checkout|Download)"; "i"))) then "setup" else "tests" end),
        step: (map(.failed_step) | group_by(.) | max_by(length) | .[0] | short),
        lost: (map(.lost_seconds) | add | fmt),
        wait_added: (map(.wait_seconds) | add | fmt),
        last: (map(.at) | max | .[:10])})
    | sort_by(-.flakes) | table),
  "",
  "== Runs that needed a re-run or a second run to pass, by workflow ==",
  ($flakes | group_by(.workflow) | map({workflow: .[0].workflow, flaky_shas: (map(.sha) | unique | length)}) | .[]
    | . as $w | ($runs[0] | map(select(.workflowName == $w.workflow)) | length) as $n
    | "\($w.workflow): \(bar($w.flaky_shas; $n))"),
  "",
  "== Failing tests in the flakes read ==",
  ($lines | map(select(.kind == "test")) | group_by(.line)
    | map({count: length, test: .[0].line}) | sort_by(-.count)
    | if length == 0 then "No test names found in the logs: read the errors below, or the test report artifacts."
      else (.[:25] | map("\(.count)x  \(.test)") | .[]) end),
  "",
  "== Latest flakes ==",
  ($flakes[:15][] | ($lines | map(select(.job == (.failed_job_id | tostring)))) as $_ | . as $f
    | "\($f.at[:16])  \($f.job | short)  step: \($f.failed_step)",
      "  failed: \($f.failed_url)",
      "  passed: \($f.passed_url)\(if $f.same_run then " (re-run)" else "" end)",
      ($lines | map(select(.job == ($f.failed_job_id | tostring) and .kind == "error")) | .[:3][] | "  \(.line)"))
' "$out/flakes.json" |
    # The tables line up; the rest is printed as it is.
    awk -v RS= -F '\n' '/\t/ { print $1; for (i = 2; i <= NF; i++) print $i | "column -t -s \"\t\""; close("column -t -s \"\t\""); print ""; next } { print; print "" }' |
    tee "$out/report.txt"

echo >&2
echo "Raw data in $out/ (runs.json, jobs.json, flakes.json, logs/); this report in $out/report.txt." >&2
