_common

#!/bin/bash -e

# Common shell script snippets to be used when interacting with openQA
# instances, for example over openqa-cli.

errorlog=""

warn() { printf '%s\n' "$@" >&2; }

enable_force_result=${enable_force_result:-false}

client_call=()
client_args=()
curl_args=()
markdown_cmd=$(command -v Markdown.pl) || markdown_cmd=$(command -v markdown) \
    || warn "+++ (Markdown.pl|markdown) not found, please install Text::Markdown +++"
openqa_cli="${openqa_cli:-""}"
# shellcheck disable=SC2034
not_found_regex='404 Not Found'

error-handler() {
    local line=$1
    # shellcheck disable=SC2207
    local c=($(caller))
    warn "${c[1]}: ERROR: line $line"
    [[ -n "$errorlog" ]] && rm -f "$errorlog"
}

# From openqa-cli JSON output filter and return the id/ids of jobs,
# for example from a query to the 'jobs get' route or the result string of a
# 'jobs post' or 'isos post'
job_ids() {
    jq -r '.ids[]' "$@"
}

# Wrapper around openqa-cli that outputs the api arguments in case of
# an error, and the calling command and line
runcli() {
    local args=("$@")
    local rc output errorlog errstr=
    set +e
    errorlog=$(mktemp -t runcli-errorlog-XXXX)
    # shellcheck disable=SC2064
    output=$("${args[@]}" 2> "$errorlog")
    rc=$?
    set -e
    if [[ -s "$errorlog" ]]; then
        read -r errstr < "$errorlog"
        warn "$1 ($(caller)): (${args[*]}) stderr: >>>$errstr<<<"
    fi
    rm "$errorlog"
    [[ "$rc" == 0 ]] && echo "$output" && return
    warn "$1 ($(caller)): (${args[*]}) rc: $rc >>>$output<<<"
    return $rc
}

# Wrapper around jq that outputs the first lines of JSON in case
# jq has a problem with it, and the calling command and line
runjq() {
    local rc output
    local jq_output_limit="${jq_output_limit:-15}"
    input=$(< /dev/stdin)
    set +e
    output=$(echo "$input" | jq "$@" 2>&1)
    rc=$?
    set -e
    [[ "$rc" == 0 ]] && echo "$output" && return
    output=$(echo "$output" | head -"$jq_output_limit")
    warn "jq ($(caller)): $output (rc: $rc Input: >>>$input<<<)"
    return $rc
}

exp_retry() {
    local stop_exponent=$1
    local exponent=$2
    if [[ "$exponent" == "$stop_exponent" ]]; then
        warn "$stop_exponent (re)tries exceeded"
        return 1
    fi
    wait_sec=$((2 ** exponent))
    warn "Waiting ${wait_sec}s until retry #${stop_exponent}"
    sleep "$wait_sec"
}

# shorten a string in the middle, if it is too large
shorten_string() {
    local sstring="$1"
    local max_str_len=${max_str_len:-"120"}
    local max_str_len_half=$((max_str_len / 2))
    if [[ ${#sstring} -gt $max_str_len ]]; then
        sstring=${sstring:0:$max_str_len_half}…${sstring: -$max_str_len_half}
    fi
    echo "$sstring"
}

# Wrapper around curl that reports the HTTP status if it is not 200, and the
# calling command and line
# exp_retries: number of total tries - 0/unset for no retries. Sleep before each retry will double.
runcurl() {
    local rc status_code body response
    local verbose="${verbose:-"false"}"
    local exp_retries="${exp_retries:-"0"}"
    for retry_exponent in $(seq 0 "$exp_retries"); do
        $verbose && warn "[debug] curl: Fetching ($*)"
        set +e
        response=$(curl -w "\n%{http_code}\n" "$@" 2>&1)
        rc=$?
        set -e
        if [[ "$rc" != 0 ]]; then
            response=$(shorten_string "$response")
            warn "curl ($(caller)): Error fetching ($*): $response"
            # shellcheck disable=SC2015
            exp_retry "$exp_retries" "$retry_exponent" && continue || break
        fi
        status_code=$(echo "$response" | tail -1)
        if [[ "$status_code" != 200 ]]; then
            warn "curl ($(caller)): Error fetching url ($*): Got Status $status_code"
            # shellcheck disable=SC2015
            exp_retry "$exp_retries" "$retry_exponent" && continue || break
        fi
        # remove last line
        body=$(echo "$response" | tac | tail -n+2 | tac)
        echo "$body"
        return 0
    done
    return 1
}

openqa-api-get() {
    client_args=(api --header 'User-Agent: openqa-investigate (https://github.com/os-autoinst/scripts)' --host "$host_url")
    local rc response
    set +e
    response=$(openqa-cli "${client_args[@]}" --json "$@" 2>&1)
    rc=$?
    set -e
    # write the response always to stdout (even in error case) so the caller can make sense of it
    echo "$response"
    [[ $rc == 0 ]] && return 0
    # print a warning in the error case including the response (which would otherwise not be logged as the caller is expected
    # to capture the output)
    response=$(shorten_string "$response")
    warn "openqa-cli ($(caller)): Error making API request ($*): $response"
    return 1
}

comment_on_job() {
    local id=$1 comment=$2 force_result=${3:-''}
    if $enable_force_result && [[ -n $force_result ]]; then
        comment="label:force_result:$force_result:$comment"
    fi
    "${client_call[@]}" -X POST jobs/"$id"/comments text="$comment"
}

search_log() {
    local id=$1 search_term=$2 out=$3 grep_timeout=${4:-5}
    local rc=0 grep_output
    local grep_opts="${grep_opts:-"-qPzo"}"
    # shellcheck disable=SC2086
    grep_output=$(timeout "$grep_timeout" grep $grep_opts "$search_term" "$out" 2>&1) || rc=$?
    if [[ "$rc" == 1 ]]; then
        return 1
    elif [[ "$rc" == 124 ]]; then
        warn "grep was killed, possibly timed out: cmd=>grep $grep_opts '$search_term' '$out'< output='$grep_output'"
        return $rc
    elif [[ "$rc" != 0 ]]; then
        # unexpected error, e.g. "exceeded PCRE's backtracking limit"
        warn "grep failed: cmd=>grep $grep_opts '$search_term' '$out'< output='$grep_output'"
        return $rc
    fi
}

label-on-issue() {
    local id=$1 search_term=$2 label=$3 restart=${4:-''} force_result=${5:-''}
    local out=${out:?}
    search_log "$id" "$search_term" "$out" || return
    comment_on_job "$id" "$label" "$force_result"
    if [ "$restart" = "1" ]; then
        "${client_call[@]}" -X POST jobs/"$id"/restart
    fi
}

snip_start="    # --- 8< ---"$'\n'
snip_end="    # --- >8 ---"
handle_unreviewed() {
    local testurl=$1 out=$2 reason=$3 group_id=$4 email_unreviewed=$5 from_email=$6 notification_address=${7:-} job_data=${8:-} dry_run=${9:-}
    local group_data group_name group_description group_mailto header email excerpt=$snip_start
    local job_name job_result info
    to_review+=("$testurl ${reason:0:50}")
    header="[$testurl]($testurl): Unknown test issue, to be reviewed"$'\n'"-> [autoinst-log.txt]($testurl/file/autoinst-log.txt)"$'\n'
    excerpt="Last lines before SUT shutdown:"$'\n'$'\n'"$snip_start"
    # Look for different termination points with likely context
    excerpt+=$( (grep -A 12 'Backend process died, backend errors are reported below in the following lines' "$out" || grep -B 10 'sending magic and exit' "$out" || grep -B 5 'killing command server.*because test execution ended through exception' "$out" || grep -B 5 'EXIT 1' "$out" || grep -B 10 '\(Result: died\|isotovideo failed\)' "$out" || echo '(No log excerpt found)') | sed 's/^/    # /' | head -n -1 | sed 's/\x1b\[[0-9;]*m//g')
    excerpt+=$'\n'"$snip_end"
    echo "$header"
    echo "$excerpt"

    if "$email_unreviewed" && [[ "$group_id" != 'null' ]]; then
        group_data=$(openqa-cli "${client_args[@]}" "job_groups/$group_id")
        group_description=$(echo "$group_data" | runjq -r '.[0].description') || true
        group_mailto=$(echo "$group_description" | perl -n -wE'm/.*MAILTO: (\S+).*/ and say $1' | head -1)
        group_mailto=${group_mailto:-$notification_address}
        # Avoid sending notifications for jobs that were restarted
        clone_id="$(echo "$job_data" | runjq -r '.job.clone_id')"
        if [[ -n "$group_mailto" && "$clone_id" == 'null' ]]; then
            group_name=$(echo "$group_data" | runjq -r '.[0].name')
            job_name=$(echo "$job_data" | runjq -r '.job.name')
            job_result=$(echo "$job_data" | runjq -r '.job.result')
            info="* Name: $job_name
* Result: $job_result
* Reason: $reason

It might be a product bug, an outdated needle, test code needing adaptation or a
test infrastructure related problem.
Adding a [bugref](http://open.qa/docs/#_bug_references) that can be
[carried over](http://open.qa/docs/#carry-over) will prevent these mails for
this issue. If the carry-over is not sufficient, you may want to create a ticket with
[auto-review-regex](https://github.com/os-autoinst/scripts/blob/master/README.md#auto-review---automatically-detect-known-issues-in-openqa-jobs-label-openqa-jobs-with-ticket-references-and-optionally-retrigger)."
            email="[$job_name]($testurl)"$'\n'"$header"$'\n'"$info"$'\n'$'\n'"$excerpt"
            subject="Unreviewed issue (Group $group_id $group_name)"
            email=$(multipart-from-markdown "$email" "$group_mailto" "openqa-label-known-issues <$from_email>" "$subject")
            send-email "$group_mailto" "$email"
        fi
    fi
}

send-email() {
    local mailto=$1 email=$2
    if [[ $dry_run == 1 ]]; then
        echo "Would send email to '$mailto':"$'\n'"$email"
        return
    fi
    echo "$email" | /usr/sbin/sendmail -t "$mailto"
}

multipart-from-markdown() {
    local plain=$1 to_email=$2 from=$3 subject=$4
    local email md body header boundary
    md=$(echo "$plain" | $markdown_cmd)
    boundary="_000_DB6PR0401MB2565_"
    header='Content-Type: multipart/alternative; boundary="'"$boundary"'"'
    body="--$boundary"'
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable

'"$plain

--$boundary"'
Content-Type: text/html; charset="utf-8"
Content-Transfer-Encoding: quoted-printable

<html><body>
'"$md"'
</body></html>

'"--$boundary--"

    email="To: $to_email
From: $from
Subject: $subject
$header

$body
"

    echo "$email"

}

latest_published_tw_builds() {
    local latest_builds group_id=$1
    latest_builds=("$(runcurl -L -sS "$tw_openqa_host/group_overview/$group_id.json" \
        | jq -r '([.build_results[] | select(.tag.description=="published") | select(.version=="Tumbleweed") | .build] | sort | reverse)[]')")
    echo "${latest_builds[@]}"
}

get_image() {
    local image name=$1
    image=$($openqa_cli api --host "$tw_openqa_host" assets \
        | jq -r "[.assets[] | select(.name | test(\"$name\")) | select(.size != null)] | .[0] | .name")
    echo "$image"
}

find_latest_published_tumbleweed_image() {
    local latest_builds build image=null assetname
    local tw_group_id=$1 arch=$2 machine=$3 type=$4
    read -r -a latest_builds <<< "$(latest_published_tw_builds "$tw_group_id")"
    if [[ "${#latest_builds[@]}" -eq 0 ]]; then
        warn "Unable to find latest published Tumbleweed builds."
        exit 1
    fi
    for build in "${latest_builds[@]}"; do
        if [[ "$type" = iso ]]; then
            assetname="Tumbleweed-NET-$arch-Snapshot$build-Media.iso$"
        else
            assetname="Tumbleweed-$arch-$build-minimalx\\\\@$machine.*.qcow"
        fi
        image=$(get_image "$assetname")

        # This published build has an image available
        [[ "$image" != null ]] && break
        # Published but no image available, we'll try and continue
        echo "Unable to determine $type image for Tumbleweed build '$build' (for architecture '$arch' and machine '$machine')."
    done
    # No published image available
    if [[ "$image" = null ]]; then
        exit 2
    fi
    echo "$image"
}

fetch-vars-json() {
    local tid=$1
    runcurl "${curl_args[@]}" -sS "$host_url/tests/$tid"/file/vars.json || return $?
}

client-get-job() {
    local job_id=$1
    openqa-api-get jobs/"$job_id" || return $?
}
client-put-job() {
    local tid=$1 data=$2
    "${client_call[@]}" --json --data "$data" -X PUT jobs/"$tid"
}
client-get-job-state() {
    local tid=$1
    openqa-cli "${client_args[@]}" --json experimental/jobs/"$tid"/status
}
client-get-job-comments() {
    local tid=$1
    "${client_call[@]}" -X GET jobs/"$tid"/comments || return $?
}
client-post-job-comment() {
    local tid=$1 text=$2
    "${client_call[@]}" -X POST jobs/"$tid"/comments text="$text" || return $?
}
client-put-job-comment() {
    local tid=$1 cid=$2 text=$3
    "${client_call[@]}" -X PUT jobs/"$tid"/comments/"$cid" text="$text"
}
client-delete-job-comment() {
    local tid=$1 cid=$2
    "${client_call[@]}" -X DELETE jobs/"$id"/comments/"$cid" || return $?
}
get-dependencies-ajax() {
    local tid=$1
    openqa-cli "${client_args[@]}" --apibase '' --json tests/"$tid"/dependencies_ajax
}

list_packages() {
    local obs_project=$1
    osc ls "$obs_project" | grep -v '\-test$'
}

delete_packages_from_obs_project() {
    local obs_project=$1
    local packages
    packages=$(list_packages "$obs_project") || true # osc ls returns non-zero return code for empty projects
    for package in $packages; do
        osc rdelete -m "Cleaning up $package to allow triggering new jobs" "$obs_project" "$package"
    done
}