analyze

#!/bin/bash

#compute time-bound stats from a autodevstats datadir
#input env vars:
#SPAN_DAYS is the desired number of days for analysis
#EARLIEST_PR a "Z" terminated iso-8601 date string indicating
#   earliest date to consider in analysis
#DATADIR the location of autodevstats data fetched and prepped
#stats will be written to stdout

#allow passing some environment vairables to override some automated steps

#enter safe-mode (no more undefined variables!)
set -eu -o pipefail

# cross-OS compatibility (greadlink, gsed, gzcat are GNU implementations for OS X)
[[ $(uname) == 'Darwin' ]] && {
    shopt -s expand_aliases
    which greadlink gsed gzcat gjoin gmktemp gdate gwc > /dev/null && {
        unalias readlink sed zcat join mktemp date wc >/dev/null 2>/dev/null
        alias readlink=greadlink sed=gsed zcat=gzcat join=gjoin mktemp=gmktemp date=gdate wc=gwc
    } || {
        echo 'ERROR: GNU utils required for Mac. You may use homebrew to install them: brew install coreutils gnu-sed'
        exit 1
    }
}

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

echo "performing timespan analysis $SPAN_DAYS days since $EARLIEST_PR" > /dev/stderr

#set up state from datadir

if [ -z "${DATADIR}" ] || [ ! -e "${DATADIR}" ] || [ ! -d "${DATADIR}" ]; then
    echo "DATDIR, \"${DATADIR}\", must exist and be a directory to perform autodevstats analysis" > /dev/stderr
    exit 1
fi

ANALYSISDIR=${DATADIR}/analysis

REPO=$(cat ${DATADIR}/repo | jq -r '.full_name')

rm -rf ${ANALYSISDIR}
mkdir ${ANALYSISDIR}

#set up some necessary analysis data
echo "preparing analysis data..." > /dev/stderr

#gather PR status
#limit by EARLIEST_PR (using closed date, or created date if we have to)
#also drop pulls that are FROM the default branch on the main repo
#   we'll assume prs from other branches are bound for mainline eventually
echo "prepping pr statuses..." > /dev/stderr
pv ${DATADIR}/pulls.gz | zcat |\
    jq -r --arg default_branch $DEFAULT_BRANCH --arg full_name $REPO '.[] | select(.head.ref != $default_branch or .head.repo.full_name != $full_name) | [.number, if .merged_at != null then "merged" else .state end, .created_at, .closed_at, .merge_commit_sha] | @tsv' |\
    gawk -F\\t '($4!="" && $4 >= "'${EARLIEST_PR}'") || $3 >= "'${EARLIEST_PR}'"' |\
    LC_ALL=C sort \
    > ${ANALYSISDIR}/pr_status

echo "preparing commit pulls..." > /dev/stderr

#drop commit pulls FROM default branch, as above
cat ${DATADIR}/commit_pulls |\
    jq -r -R -c  --arg default_branch $DEFAULT_BRANCH --arg full_name $REPO 'split("\t") | (.[1] | fromjson | [.[] | select(.head.ref != $default_branch or .head.repo.full_name != $full_name)])' |\
    paste <(cat ${DATADIR}/commit_pulls | cut -f1) -\
    > ${ANALYSISDIR}/commit_pulls

echo "preparing a sample of reviewed and unreviewed commits..." > /dev/stderr

> ${ANALYSISDIR}/reviewed_commits.tmp

#commits with known GH templates
cat ${DATADIR}/commit_messages |\
    ag -A1 '^__commit__ [0-9a-f]{40}$' |\
    gawk 'BEGIN {OFS="\t"} /^__commit__ [a-f0-9]{40}$/ {commit=$2} !/^__commit__ [a-f0-9]{40}$/ {print commit, $0}' |\
    gawk -F\\t 'BEGIN {OFS="\t"} {if(match($2, /(Merge pull request #([0-9]+))|(\(#([0-9]+)\)$)/, m) > 0) { if(m[2] == "") { print m[4], $1} else { print m[2], $1}}}' |\
    LC_ALL=C sort | LC_ALL=C join -t$'\t' - ${ANALYSISDIR}/pr_status |\
    gawk -F\\t 'BEGIN {OFS="\t"} {print $2, $1, "commit_message"}' | LC_ALL=C sort -u\
    >> ${ANALYSISDIR}/reviewed_commits.tmp

#commits from external merge tools
cat ${DATADIR}/commit_autolinks |\
    (ag 'closes' || true) |\
    gawk -F\\t 'BEGIN {OFS="\t"} {print $2,$1}' |\
    LC_ALL=C sort | LC_ALL=C join -t$'\t' - ${ANALYSISDIR}/pr_status |\
    gawk -F\\t 'BEGIN {OFS="\t"} {print $2,$1, "autolink"}' | LC_ALL=C sort -u\
    >> ${ANALYSISDIR}/reviewed_commits.tmp

#commits listed in merge_commit_sha for merged PRs
cat ${ANALYSISDIR}/pr_status |\
    (ag 'merged' || true) |\
    gawk -F\\t 'BEGIN {OFS="\t"} {print $5,$1,"merge_commit"}' |\
    LC_ALL=C sort | LC_ALL=C join -t$'\t' - ${DATADIR}/commitdates |\
    cut -f 1,2,3\
    >> ${ANALYSISDIR}/reviewed_commits.tmp

#cascade review to commits with same committer and commit time
zcat ${DATADIR}/commit_graph.gz |\
    gawk -F\\t '{printf("%s\t%d\n", $0, NR)}' |\
    LC_ALL=C sort |\
    join -t$'\t' -a2 -o0,1.2,2.3,2.4,2.5,2.7 <(cat ${ANALYSISDIR}/reviewed_commits.tmp | LC_ALL=C sort) - |\
    sort -t$'\t' -k4r,4r -k3,3 -k6n,6n |\
    gawk -F\\t -f ${DIR}/cascade_review.awk |\
    LC_ALL=C sort -u -k1,1 \
    > ${ANALYSISDIR}/reviewed_commits

#rm ${ANALYSISDIR}/reviewed_commits.tmp

#the complement, but during the right period
zcat ${DATADIR}/commit_graph.gz |\
    gawk -F\\t '$4 >='$(date -d ${EARLIEST_PR} +%s) | cut -f1 |\
    LC_ALL=C sort |\
    LC_ALL=C join -v1 - ${ANALYSISDIR}/reviewed_commits \
    > ${ANALYSISDIR}/unreviewed_commits


#partition reviewed commits into zero-comment reviews and non-zero-comment reviews
echo "partitioning reviewed commits by zero-comment reviews" > /dev/stderr
cat ${ANALYSISDIR}/reviewed_commits |\
    LC_ALL=C sort -k2,2 |\
    LC_ALL=C join -t$'\t' -1 2 -2 1 - <(cat ${DATADIR}/commentcounts | ag 'allcommentswzero' | LC_ALL=C sort) |\
    gawk -F\\t '$4 > 0 {printf("%s\t\n", $2)}' |\
    LC_ALL=C sort -u -k1,1 \
    > ${ANALYSISDIR}/reviewed_commits.nzc

cat ${ANALYSISDIR}/reviewed_commits |\
    LC_ALL=C sort -k2,2 |\
    LC_ALL=C join -t$'\t' -1 2 -2 1 - <(cat ${DATADIR}/commentcounts | ag 'allcommentswzero' | LC_ALL=C sort) |\
    gawk -F\\t '$4 == 0 {printf("%s\n", $2)}' |\
    LC_ALL=C sort -u -k1,1 \
    > ${ANALYSISDIR}/reviewed_commits.zc

cat ${ANALYSISDIR}/reviewed_commits.zc ${ANALYSISDIR}/unreviewed_commits |\
    LC_ALL=C sort \
    > ${ANALYSISDIR}/unreviewed_commits.zc


echo "computing average comment time..." > /dev/stderr
AVG_COMMENT_TIME=$(\
    cat ${DATADIR}/pr_comments_data |\
    gawk -F\\t '$6 >= "'${EARLIEST_PR}'"' |\
    gawk -F\\t -i ${DIR}/date.awk -i ${DIR}/reduce.awk -f ${DIR}/extractplies.awk |\
    gawk -F\\t -i ${DIR}/reduce.awk 'BEGIN {OFS="\t";setkey("1\t2\t3");} function startrun(key) {state=$6;startts=$4;comments=0;sumtime=0;lastts=$4} function reduce(key) {if(comments>0) {print $4-lastts;} comments+=1; lastts=$4} function endrun(key) { }' |\
    gawk '{s+=$1;n+=1} END {if(n>0) { print s/n } else { print 0} }')

echo "filtering LOC metadata..." > /dev/stderr
pv ${DATADIR}/metadata.gz | zcat |\
    gawk -F\\t '$7 >='$(date -d "${EARLIEST_PR}" +%s) |\
    gzip -c\
    > ${ANALYSISDIR}/metadata.gz


#start computing stats
echo "doing analysis..." > /dev/stderr

#record some analysis params
printf "{\"stat\":\"analysis_params\", \"data\": {\"earliest_date\": \"%s\", \"span_days\": %d}}\n" ${EARLIEST_PR} ${SPAN_DAYS}

echo "code birthdate summary (during analysis period)" > /dev/stderr
pv ${ANALYSISDIR}/metadata.gz | zcat |\
    cut -f 4,7 |\
    gawk -M -f ${DIR}/groupstats.awk |\
    jq -c --slurp --raw-input --arg stat_name during_pr_code_birthdate_summary -f ${DIR}/gs2json.jq

echo "code lifetime summary (during analysis period)" > /dev/stderr
pv ${ANALYSISDIR}/metadata.gz | zcat |\
    cut -f4,5 |\
    gawk -M -f ${DIR}/groupstats.awk |\
    jq -c --slurp --raw-input --arg stat_name during_pr_code_lifetime_summary -f ${DIR}/gs2json.jq

echo "dead code lifetime distribution (during analysis period)" > /dev/stderr
pv ${ANALYSISDIR}/metadata.gz | zcat |\
    (ag 'died' || true) | cut -f 5 | sort -n |\
    gawk -f ${DIR}/cdf.awk <(echo -n "86400_604800_1209600_2592000_5184000_7776000_15552000_31104000" | tr '_' '\n') - |\
    jq -c --slurp --raw-input --arg stat_name during_pr_code_lifetime_died_cdf -f ${DIR}/cdf2json.jq

echo "live code lifetime distribution (during analysis period)" > /dev/stderr
pv ${ANALYSISDIR}/metadata.gz | zcat |\
    (ag 'live' || true) | cut -f 5 | sort -n |\
    gawk -f ${DIR}/cdf.awk <(echo -n "86400_604800_1209600_2592000_5184000_7776000_15552000_31104000" | tr '_' '\n') - |\
    jq -c --slurp --raw-input --arg stat_name during_pr_code_lifetime_live_cdf -f ${DIR}/cdf2json.jq

echo "comments per dev-PR" > /dev/stderr
cat ${DATADIR}/pr_comments_data | cut -f 1,2,4 |\
    gawk 'BEGIN {OFS="\t"} { print $1,$2,$3; print $1,$2,"any"}' |\
    cut -f 1,3 | LC_ALL=C sort | uniq -c |\
    gawk 'BEGIN {OFS="\t"} {print $2,$3,$1}' |\
    LC_ALL=C join -t$'\t' - ${ANALYSISDIR}/pr_status | gawk -F\\t '{printf("%s-%s\t%d\n", $4,$2,$3)}' |\
    gawk -M -f ${DIR}/groupstats.awk |\
    jq -c --slurp --raw-input --arg stat_name comment_per_dev_pr -f ${DIR}/gs2json.jq

echo "comments per dev" > /dev/stderr
cat ${DATADIR}/pr_comments_data | cut -f 1,2,4 |\
    gawk -F\\t 'BEGIN {OFS="\t"} { print $1,$2,$3; print $1,$2,"any"}' |\
    LC_ALL=C join -t$'\t' - ${ANALYSISDIR}/pr_status |\
    gawk -F\\t '$2!="" {printf("%s\t%s-%s\n", $2,$4,$3)}' | sort | uniq -c | gawk '{print $3,$1}' |\
    gawk -M -f ${DIR}/groupstats.awk |\
    jq -c --slurp --raw-input --arg stat_name comment_per_dev -f ${DIR}/gs2json.jq

echo "overlap in files for reviewed vs unreviewed commits" > /dev/stderr
function file_overlap() {
    local unreviewed_commits=$1
    local reviewed_commits=$2
    local stat_name=$3

    cat\
        <(cat $unreviewed_commits | gawk -v commits=$(cat $unreviewed_commits | wc -l) '{printf("%s\t%s\t%f\n", $1, "unreviewed", 1.0/commits)}')\
        <(cat $reviewed_commits | gawk -v commits=$(cat $reviewed_commits | wc -l) '{printf("%s\t%s\t%f\n", $1, "reviewed", 1.0/commits)}') |\
        LC_ALL=C sort | LC_ALL=C join -t$'\t' -\
            <(cat ${DATADIR}/filestatus | cut -f 1,4 | LC_ALL=C sort) |\
        gawk 'BEGIN {OFS="\t"} {print $4,$2,$3}' |\
        LC_ALL=C sort |\
        gawk -F\\t -i ${DIR}/reduce.awk -f ${DIR}/jaccard.awk |\
        jq -c --slurp --raw-input --arg stat_name $stat_name -f ${DIR}/gs2json.jq
}

file_overlap "${ANALYSISDIR}/unreviewed_commits" "${ANALYSISDIR}/reviewed_commits" "commit_review_file_overlap"
file_overlap "${ANALYSISDIR}/unreviewed_commits.zc" "${ANALYSISDIR}/reviewed_commits.nzc" "ur_v_nzc_commit_review_file_overlap"
file_overlap "${ANALYSISDIR}/reviewed_commits.zc" "${ANALYSISDIR}/reviewed_commits.nzc" "zc_v_nzc_commit_review_file_overlap"

echo "overlap in files for reviewed vs unreviewed commits (total commits normalized)" > /dev/stderr
function file_overlap_by_commits() {
    local unreviewed_commits=$1
    local reviewed_commits=$2
    local stat_name=$3

    cat\
        <(cat $unreviewed_commits | gawk -v commits=$(cat $unreviewed_commits $reviewed_commits| wc -l) '{printf("%s\t%s\t%f\n", $1, "unreviewed", 1.0/commits)}')\
        <(cat $reviewed_commits | gawk -v commits=$(cat $unreviewed_commits $reviewed_commits | wc -l) '{printf("%s\t%s\t%f\n", $1, "reviewed", 1.0/commits)}') |\
        LC_ALL=C sort | LC_ALL=C join -t$'\t' -\
            <(cat ${DATADIR}/filestatus | cut -f 1,4 | LC_ALL=C sort) |\
        gawk 'BEGIN {OFS="\t"} {print $4,$2,$3}' |\
        LC_ALL=C sort |\
        gawk -F\\t -i ${DIR}/reduce.awk -f ${DIR}/jaccard.awk |\
        jq -c --slurp --raw-input --arg stat_name $stat_name -f ${DIR}/gs2json.jq
}

file_overlap_by_commits "${ANALYSISDIR}/unreviewed_commits" "${ANALYSISDIR}/reviewed_commits" "commit_review_file_overlap_by_commits"
file_overlap_by_commits "${ANALYSISDIR}/unreviewed_commits.zc" "${ANALYSISDIR}/reviewed_commits.nzc" "ur_v_nzc_commit_review_file_overlap_by_commits"
file_overlap_by_commits "${ANALYSISDIR}/reviewed_commits.zc" "${ANALYSISDIR}/reviewed_commits.nzc" "zc_v_nzc_commit_review_file_overlap_by_commits"

echo "lines of code for reviewed vs unreviewed commits by outcome" > /dev/stderr
function commit_review_size_by_outcome() {
    local unreviewed_commits=$1
    local reviewed_commits=$2
    local stat_name=$3

    cat\
        <(cat $reviewed_commits | cut -f1 | gawk '{printf("%s\treviewed\n", $1)}')\
        <(cat $unreviewed_commits | cut -f1 | gawk '{printf("%s\tunreviewed\n", $1)}') |\
        LC_ALL=C sort |\
        LC_ALL=C join <(pv ${ANALYSISDIR}/metadata.gz | zcat | cut -f 2,4 | LC_ALL=C sort) - |\
        LC_ALL=C sort | uniq -c |\
        gawk '{printf("%s-%s\t%d\n",$4,$3,$1)}' |\
        gawk -M -f ${DIR}/groupstats.awk |\
        jq -c --slurp --raw-input --arg stat_name $stat_name -f ${DIR}/gs2json.jq
}

commit_review_size_by_outcome "${ANALYSISDIR}/unreviewed_commits" "${ANALYSISDIR}/reviewed_commits" "commit_review_size_by_outcome"
commit_review_size_by_outcome "${ANALYSISDIR}/unreviewed_commits.zc" "${ANALYSISDIR}/reviewed_commits.nzc" "ur_v_nzc_commit_review_size_by_outcome"
commit_review_size_by_outcome "${ANALYSISDIR}/reviewed_commits.zc" "${ANALYSISDIR}/reviewed_commits.nzc" "zc_v_nzc_commit_review_size_by_outcome"

echo "dates for reviewed vs unreviewed commits" > /dev/stderr
function commit_review_vs_date {
    local unreviewed_commits=$1
    local reviewed_commits=$2
    local stat_name=$3

    cat\
        <(cat $reviewed_commits | cut -f 1 |\
            LC_ALL=C join -t$'\t' - ${DATADIR}/commitdates |\
            gawk -F\\t '{printf("reviewed\t%f\n", $2)}')\
        <(cat $unreviewed_commits |\
            LC_ALL=C join -t$'\t' - ${DATADIR}/commitdates |\
            gawk -F\\t '{printf("unreviewed\t%f\n", $2)}') |\
        gawk -M -f ${DIR}/groupstats.awk |\
        jq -c --slurp --raw-input --arg stat_name $stat_name -f ${DIR}/gs2json.jq
}

commit_review_vs_date "${ANALYSISDIR}/unreviewed_commits" "${ANALYSISDIR}/reviewed_commits" "commit_review_vs_date"
commit_review_vs_date "${ANALYSISDIR}/unreviewed_commits.zc" "${ANALYSISDIR}/reviewed_commits.nzc" "ur_v_nzc_commit_review_vs_date"
commit_review_vs_date "${ANALYSISDIR}/reviewed_commits.zc" "${ANALYSISDIR}/reviewed_commits.nzc" "zc_v_nzc_commit_review_vs_date"

echo "lifetime for code from reviewed vs unreviewed commits" > /dev/stderr
function commit_review_vs_lifetime() {
    local unreviewed_commits=$1
    local reviewed_commits=$2
    local stat_name=$3

    cat\
        <(cat $reviewed_commits | cut -f 1 |\
            gawk -F\\t '{printf("%s\treviewed\n", $1)}')\
        <(cat $unreviewed_commits |\
            gawk -F\\t '{printf("%s\tunreviewed\n", $1)}') |\
        LC_ALL=C sort |\
        LC_ALL=C join -t$'\t' - <(pv ${ANALYSISDIR}/metadata.gz | zcat | cut -f 2,5 | LC_ALL=C sort) | cut -f 2,3 |\
        gawk -M -f ${DIR}/groupstats.awk |\
        jq -c --slurp --raw-input --arg stat_name $stat_name -f ${DIR}/gs2json.jq
}

commit_review_vs_lifetime "${ANALYSISDIR}/unreviewed_commits" "${ANALYSISDIR}/reviewed_commits" "commit_review_vs_lifetime"
commit_review_vs_lifetime "${ANALYSISDIR}/unreviewed_commits.zc" "${ANALYSISDIR}/reviewed_commits.nzc" "ur_v_nzc_commit_review_vs_lifetime"
commit_review_vs_lifetime "${ANALYSISDIR}/reviewed_commits.zc" "${ANALYSISDIR}/reviewed_commits.nzc" "zc_v_nzc_commit_review_vs_lifetime"

function commit_review_vs_lifetime_died() {
    local unreviewed_commits=$1
    local reviewed_commits=$2
    local stat_name=$3

    cat\
        <(cat $reviewed_commits | cut -f 1 |\
            gawk -F\\t '{printf("%s\treviewed\n", $1)}')\
        <(cat $unreviewed_commits |\
            gawk -F\\t '{printf("%s\tunreviewed\n", $1)}') |\
        LC_ALL=C sort |\
        LC_ALL=C join -t$'\t' - <(pv ${ANALYSISDIR}/metadata.gz | zcat | (ag 'died' || true) | cut -f 2,5 | LC_ALL=C sort) | cut -f 2,3 |\
        gawk -M -f ${DIR}/groupstats.awk |\
        jq -c --slurp --raw-input --arg stat_name $stat_name -f ${DIR}/gs2json.jq
}

commit_review_vs_lifetime_died "${ANALYSISDIR}/unreviewed_commits" "${ANALYSISDIR}/reviewed_commits" "commit_review_vs_lifetime_died"
commit_review_vs_lifetime_died "${ANALYSISDIR}/unreviewed_commits.zc" "${ANALYSISDIR}/reviewed_commits.nzc" "ur_v_nzc_commit_review_vs_lifetime_died"
commit_review_vs_lifetime_died "${ANALYSISDIR}/reviewed_commits.zc" "${ANALYSISDIR}/reviewed_commits.nzc" "zc_v_nzc_commit_review_vs_lifetime_died"

function commit_review_vs_lifetime_live() {
    local unreviewed_commits=$1
    local reviewed_commits=$2
    local stat_name=$3

    cat\
        <(cat $reviewed_commits | cut -f 1 |\
            gawk -F\\t '{printf("%s\treviewed\n", $1)}')\
        <(cat $unreviewed_commits |\
            gawk -F\\t '{printf("%s\tunreviewed\n", $1)}') |\
        LC_ALL=C sort |\
        LC_ALL=C join -t$'\t' - <(pv ${ANALYSISDIR}/metadata.gz | zcat | (ag 'live' || true) | cut -f 2,5 | LC_ALL=C sort) | cut -f 2,3 |\
        gawk -M -f ${DIR}/groupstats.awk |\
        jq -c --slurp --raw-input --arg stat_name $stat_name -f ${DIR}/gs2json.jq
}

commit_review_vs_lifetime_live "${ANALYSISDIR}/unreviewed_commits" "${ANALYSISDIR}/reviewed_commits" "commit_review_vs_lifetime_live"
commit_review_vs_lifetime_live "${ANALYSISDIR}/unreviewed_commits.zc" "${ANALYSISDIR}/reviewed_commits.nzc" "ur_v_nzc_commit_review_vs_lifetime_live"
commit_review_vs_lifetime_live "${ANALYSISDIR}/reviewed_commits.zc" "${ANALYSISDIR}/reviewed_commits.nzc" "zc_v_nzc_commit_review_vs_lifetime_live"

echo "commit distribution across authors (during analysis period)" > /dev/stderr
function during_pr_commits_proportion_by_dev_cdf() {
    local unreviewed_commits=$1
    local reviewed_commits=$2
    local stat_name=$3

    cat ${DATADIR}/commits_with_author |\
        LC_ALL=C join -t$'\t' - <(cat $reviewed_commits $unreviewed_commits | LC_ALL=C sort) |\
        gawk -F\\t '{print $2}' | sort | uniq -c | sort -rn |\
        gawk '{d[NR]=$1;s+=$1;} END {c=0; for (x in d) { c+=d[x]/s; print c}}' |\
        gawk -f ${DIR}/cdf.awk <(echo "0.1_0.25_0.5_0.75_0.8_0.9_0.95_0.99_1" | tr '_' '\n') - |\
        jq --slurp --raw-input --arg stat_name $stat_name -f ${DIR}/cdf2json.jq
}

during_pr_commits_proportion_by_dev_cdf "${ANALYSISDIR}/unreviewed_commits" "${ANALYSISDIR}/reviewed_commits" "during_pr_commits_proportion_by_dev_cdf"
during_pr_commits_proportion_by_dev_cdf "${ANALYSISDIR}/unreviewed_commits.zc" "${ANALYSISDIR}/reviewed_commits.nzc" "ur_v_nzc_during_pr_commits_proportion_by_dev_cdf"
during_pr_commits_proportion_by_dev_cdf "${ANALYSISDIR}/reviewed_commits.zc" "${ANALYSISDIR}/reviewed_commits.nzc" "zc_v_nzc_during_pr_commits_proportion_by_dev_cdf"


echo "commit distribution across authors (reviewed)" > /dev/stderr
function rev_commits_proportion_by_dev_cdf() {
    local unreviewed_commits=$1
    local reviewed_commits=$2
    local stat_name=$3

    cat ${DATADIR}/commits_with_author |\
        LC_ALL=C join -t$'\t' - $reviewed_commits |\
        gawk -F\\t '{print $2}' | sort | uniq -c | sort -rn |\
        gawk '{d[NR]=$1;s+=$1;} END {c=0; for (x in d) { c+=d[x]/s; print c}}' |\
        gawk -f ${DIR}/cdf.awk <(echo "0.1_0.25_0.5_0.75_0.8_0.9_0.95_0.99_1" | tr '_' '\n') - |\
        jq --slurp --raw-input --arg stat_name $stat_name -f ${DIR}/cdf2json.jq
}

rev_commits_proportion_by_dev_cdf "${ANALYSISDIR}/unreviewed_commits" "${ANALYSISDIR}/reviewed_commits" "rev_commits_proportion_by_dev_cdf"
rev_commits_proportion_by_dev_cdf "${ANALYSISDIR}/unreviewed_commits.zc" "${ANALYSISDIR}/reviewed_commits.nzc" "ur_v_nzc_rev_commits_proportion_by_dev_cdf"
rev_commits_proportion_by_dev_cdf "${ANALYSISDIR}/reviewed_commits.zc" "${ANALYSISDIR}/reviewed_commits.nzc" "zc_v_nzc_rev_commits_proportion_by_dev_cdf"


echo "commit distribution across authors (unreviewed)" > /dev/stderr
function unrev_commits_proportion_by_dev_cdf() {
        local unreviewed_commits=$1
        local reviewed_commits=$2
        local stat_name=$3

    cat ${DATADIR}/commits_with_author |\
        LC_ALL=C join -t$'\t' - $unreviewed_commits |\
        gawk -F\\t '{print $2}' | sort | uniq -c | sort -rn |\
        gawk '{d[NR]=$1;s+=$1;} END {c=0; for (x in d) { c+=d[x]/s; print c}}' |\
        gawk -f ${DIR}/cdf.awk <(echo "0.1_0.25_0.5_0.75_0.8_0.9_0.95_0.99_1" | tr '_' '\n') - |\
        jq --slurp --raw-input --arg stat_name $stat_name -f ${DIR}/cdf2json.jq
}

unrev_commits_proportion_by_dev_cdf "${ANALYSISDIR}/unreviewed_commits" "${ANALYSISDIR}/reviewed_commits" "unrev_commits_proportion_by_dev_cdf"
unrev_commits_proportion_by_dev_cdf "${ANALYSISDIR}/unreviewed_commits.zc" "${ANALYSISDIR}/reviewed_commits.nzc" "ur_v_nzc_unrev_commits_proportion_by_dev_cdf"
unrev_commits_proportion_by_dev_cdf "${ANALYSISDIR}/reviewed_commits.zc" "${ANALYSISDIR}/reviewed_commits.nzc" "zc_v_nzc_unrev_commits_proportion_by_dev_cdf"


echo "devs per PR" > /dev/stderr
cat ${DATADIR}/pr_comments_data | cut -f 1,2,4 | sort -u |\
    gawk -F\\t 'BEGIN {OFS="\t"} { print $0; print $1,$2,"any"}' |\
    cut -f 1,3 | LC_ALL=C sort | uniq -c |\
    gawk 'BEGIN {OFS="\t"} {print $2,$3,$1}' |\
    LC_ALL=C join -t$'\t' - ${ANALYSISDIR}/pr_status | gawk -F\\t '{printf("%s-%s\t%d\n", $4,$2,$3)}' |\
    gawk -M -f ${DIR}/groupstats.awk |\
    jq -c --slurp --raw-input --arg stat_name dev_per_pr -f ${DIR}/gs2json.jq

echo "PR merge commits during analysis period" > /dev/stderr
printf "%d\t%d\t%d\t%d\n"\
    $(cat ${DATADIR}/commit_messages | ag '__commit__ [a-f0-9]{40}' | gawk '{print $2}' | LC_ALL=C sort | LC_ALL=C join - ${DATADIR}/commitdates | gawk '$2 >= '$(date -d ${EARLIEST_PR} +%s) | wc -l)\
    $(cat ${DATADIR}/commit_messages | (grep -E -o 'Merge pull request #[0-9]+ from' || true) | grep -o '[0-9]*' | LC_ALL=C sort | LC_ALL=C join - ${ANALYSISDIR}/pr_status | wc -l)\
    $(cat ${DATADIR}/commit_messages | (grep -E -A1 '^__commit__ [a-f0-9]{40}$' || true) | (grep -E -o ' \(#[0-9]+\)$' || true) | grep -o '[0-9]*' | LC_ALL=C sort | LC_ALL=C join - ${ANALYSISDIR}/pr_status | wc -l)\
    $(cat ${DATADIR}/commit_autolinks | grep 'close' | cut -f 2 | LC_ALL=C sort | LC_ALL=C join - ${ANALYSISDIR}/pr_status | wc -l) |\
    jq -c --slurp --raw-input 'split("\t") | {"stat":"gh_merges_during_prs", "data":{"commits":(.[0]|tonumber), "gh_merges":(.[1]|tonumber), "gh_likely_merge":(.[2]|tonumber), "likely_external_merge":(.[3]|tonumber)}}'

echo "comparing GH commit pull association with commit message analysis" > /dev/stderr
cat ${ANALYSISDIR}/commit_pulls |\
    sed -E 's/^https:\/\/api.github.com\/repos\/[^\/]*\/[^\/]*\/commits\/([a-f0-9]{40})\/pulls/\1/' |\
    LC_ALL=C sort | LC_ALL=C join -t$'\t' - <(cat ${ANALYSISDIR}/reviewed_commits ${ANALYSISDIR}/unreviewed_commits | LC_ALL=C sort) |\
    jq -r -R 'split("\t") | [.[0], (.[1] | fromjson | length), .[2]] | @tsv' |\
    gawk -F\\t '$3=="" {printf("unreviewed\t%d\n",$2>0)} $3!="" {printf("reviewed\t%d\n",$2>0)}' |\
    gawk -M -f ${DIR}/groupstats.awk |\
    jq -c --slurp --raw-input --arg stat_name gh_rev_vs_commit_rev -f ${DIR}/gs2json.jq

cat ${ANALYSISDIR}/commit_pulls |\
    sed -E 's/^https:\/\/api.github.com\/repos\/[^\/]*\/[^\/]*\/commits\/([a-f0-9]{40})\/pulls/\1/' |\
    LC_ALL=C sort | LC_ALL=C join -t$'\t' - <(cat ${ANALYSISDIR}/reviewed_commits ${ANALYSISDIR}/unreviewed_commits | LC_ALL=C sort) |\
    jq -r -R 'split("\t") | [.[0], (.[2] as $prnumber | .[1] | fromjson | map(.number) | select(($prnumber // "0" | tonumber))|length), .[2]] | @tsv' |\
    gawk -F\\t '$3=="" {printf("unreviewed\t%d\n",$2>0)} $3!="" {printf("reviewed\t%d\n",$2>0)}' |\
    gawk -M -f ${DIR}/groupstats.awk |\
    jq -c --slurp --raw-input --arg stat_name gh_rev_vs_commit_rev_strict -f ${DIR}/gs2json.jq

echo "PR comment count summary" > /dev/stderr
cat ${DATADIR}/commentcounts | LC_ALL=C sort | join -t$'\t' ${ANALYSISDIR}/pr_status - | gawk -F\\t '{printf("%s-%s\t%d\n", $2, $6, $7)}' |\
    gawk -F\\t -M -f ${DIR}/groupstats.awk |\
    jq -c --slurp --raw-input --arg stat_name pr_comment_summary -f ${DIR}/gs2json.jq

echo "PR comment distribution" > /dev/stderr
LC_ALL=C join -t $'\t' -o 0,1.2,1.3 ${DATADIR}/commentcounts ${ANALYSISDIR}/pr_status  |\
    (ag 'allcommentswzero' || true) | cut -f 3 | sort -n |\
    gawk -f ${DIR}/cdf.awk <(echo "0_1_2_3_4_5_7_10_15_20_30_50_75_100" | tr '_' '\n') - |\
    jq -c --slurp --raw-input --arg stat_name pr_comment_cdf -f ${DIR}/cdf2json.jq

echo "PR comment distribution (open)" > /dev/stderr
LC_ALL=C join -t $'\t' -o 0,1.2,1.3 ${DATADIR}/commentcounts <(cat ${ANALYSISDIR}/pr_status | ag 'open') |\
    (ag 'allcommentswzero' || true) | cut -f 3 | sort -n |\
    gawk -f ${DIR}/cdf.awk <(echo "0_1_2_3_4_5_7_10_15_20_30_50_75_100" | tr '_' '\n') - |\
    jq -c --slurp --raw-input --arg stat_name open_pr_comment_cdf -f ${DIR}/cdf2json.jq

echo "PR comment distribution (merged)" > /dev/stderr
LC_ALL=C join -t $'\t' -o 0,1.2,1.3 ${DATADIR}/commentcounts <(cat ${ANALYSISDIR}/pr_status | ag 'merged') |\
    (ag 'allcommentswzero' || true) | cut -f 3 | sort -n |\
    gawk -f ${DIR}/cdf.awk <(echo "0_1_2_3_4_5_7_10_15_20_30_50_75_100" | tr '_' '\n') - |\
    jq -c --slurp --raw-input --arg stat_name merged_pr_comment_cdf -f ${DIR}/cdf2json.jq

echo "PR comment distribution (closed)" > /dev/stderr
LC_ALL=C join -t $'\t' -o 0,1.2,1.3 ${DATADIR}/commentcounts <(cat ${ANALYSISDIR}/pr_status | ag 'closed') |\
    (ag 'allcommentswzero' || true) | cut -f 3 | sort -n |\
    gawk -f ${DIR}/cdf.awk <(echo "0_1_2_3_4_5_7_10_15_20_30_50_75_100" | tr '_' '\n') - |\
    jq -c --slurp --raw-input --arg stat_name closed_pr_comment_cdf -f ${DIR}/cdf2json.jq


echo "PR lifetime summary" > /dev/stderr
cat ${ANALYSISDIR}/pr_status | gawk -F\\t -i ${DIR}/date.awk 'BEGIN {OFS="\t"} $2=="open" {print $2, systime() - parsedate($3)} $2!="open" { print $2, parsedate($4) - parsedate($3)}' |\
    gawk -M -f ${DIR}/groupstats.awk |\
    jq -c --slurp --raw-input --arg stat_name pr_lifetime_summary -f ${DIR}/gs2json.jq

echo "PR lifetime distribution" > /dev/stderr
cat ${ANALYSISDIR}/pr_status |\
    gawk -F\\t -i ${DIR}/date.awk 'BEGIN {OFS="\t"} $2=="merged" { print parsedate($4) - parsedate($3)}' |\
    sort -n |\
    gawk -f ${DIR}/cdf.awk <(echo "0_600_1800_3600_7200_14400_28800_43200_86400_172800_259200_432000_604800_864000_1209600_1814400_2592000_5184000_7776000_15552000_31536000" | tr '_' '\n') - |\
    jq -c --slurp --raw-input --arg stat_name merged_pr_lifetime_cdf -f ${DIR}/cdf2json.jq

cat ${ANALYSISDIR}/pr_status |\
    gawk -F\\t -i ${DIR}/date.awk 'BEGIN {OFS="\t"} $2=="closed" { print parsedate($4) - parsedate($3)}' |\
    sort -n |\
    gawk -f ${DIR}/cdf.awk <(echo "0_600_1800_3600_7200_14400_28800_43200_86400_172800_259200_432000_604800_864000_1209600_1814400_2592000_5184000_7776000_15552000_31536000" | tr '_' '\n') - |\
    jq -c --slurp --raw-input --arg stat_name closed_pr_lifetime_cdf -f ${DIR}/cdf2json.jq

echo "PR cycle count per PR by outcome" > /dev/stderr
cat ${DATADIR}/pr_comments_data |\
    gawk -F\\t '$6 >= "'${EARLIEST_PR}'"' |\
    gawk -F\\t -i ${DIR}/date.awk -i ${DIR}/reduce.awk -f ${DIR}/extractplies.awk |\
    cut -f 1,2,3,6 | uniq | cut -f1,4 | uniq -c | gawk '{print $3,$1}' |\
    gawk -M -f ${DIR}/groupstats.awk |\
    jq -c --slurp --raw-input --arg stat_name pr_plies_per_pr -f ${DIR}/gs2json.jq

echo "PR active review time by outcome" > /dev/stderr
cat ${DATADIR}/pr_comments_data |\
    gawk -F\\t '$6 >= "'${EARLIEST_PR}'"' |\
    gawk -F\\t -i ${DIR}/date.awk -i ${DIR}/reduce.awk -f ${DIR}/extractplies.awk |\
    gawk -F\\t -i ${DIR}/reduce.awk -v avgctime=$AVG_COMMENT_TIME 'BEGIN {OFS="\t";setkey("1\t2\t3");} function startrun(key) {state=$6;startts=$4;comments=0;lastts=$4} function reduce(key) { comments+=1;lastts=$4} function endrun(key) { print key[1], key[2], key[3], comments, comments*avgctime, lastts-startts, state}' |\
    gawk -F\\t -i ${DIR}/reduce.awk 'BEGIN {OFS="\t";setkey("1");} function startrun(key) {estimate=0;flr=0;state=$7} function reduce(key) {estimate+=$5;flr+=$6} function endrun(key) { printf("%s-estimate\t%f\n", state, estimate);printf("%s-floorwzero\t%f\n", state, flr);}' |\
    gawk -M -f ${DIR}/groupstats.awk |\
    jq -c --slurp --raw-input --arg stat_name pr_time_per_pr -f ${DIR}/gs2json.jq

echo "PR active review time by outcome including zero engagement reviews" > /dev/stderr
cat ${DATADIR}/pr_comments_data |\
    gawk -F\\t '$6 >= "'${EARLIEST_PR}'"' |\
    gawk -F\\t -i ${DIR}/date.awk -i ${DIR}/reduce.awk -f ${DIR}/extractplies.awk |\
    gawk -F\\t -i ${DIR}/reduce.awk -v avgctime=$AVG_COMMENT_TIME 'BEGIN {OFS="\t";setkey("1\t2\t3");} function startrun(key) {state=$6;startts=$4;comments=0;lastts=$4} function reduce(key) { comments+=1;lastts=$4} function endrun(key) { print key[1], key[2], key[3], comments, comments*avgctime, lastts-startts, state}' |\
    LC_ALL=C join -t$'\t' -o 0,2.2,2.3,2.4,2.5,2.6,1.2 -a1 ${ANALYSISDIR}/pr_status - |\
    gawk -F\\t -i ${DIR}/reduce.awk 'BEGIN {OFS="\t";setkey("1");} function startrun(key) {estimate=0;flr=0;state=$7} function reduce(key) {estimate+=$5;flr+=$6} function endrun(key) { printf("%s-estimate\t%f\n", state, estimate);printf("%s-floorwzero\t%f\n", state, flr);}' |\
    gawk -M -f ${DIR}/groupstats.awk |\
    jq -c --slurp --raw-input --arg stat_name pr_time_per_pr_wzero -f ${DIR}/gs2json.jq

echo "done with analysis for $SPAN_DAYS days back to ${EARLIEST_PR}" > /dev/stderr
echo > /dev/stderr