#!/bin/sh -e

PROGNAME=`basename $0`
CUSTOMTMP="yes"
OUTDIR="./"
NUM_CORES="1"

usage() {
	echo "usage: $PROGNAME [OPTIONS] [buildarch hostarch buildpackages sources]" >&2
	echo >&2
	echo " -h, --help       Show this help message and exit" >&2
	echo " -k, --keep       Keep the temporary files" >&2
	echo " -w, --online     Produce stat.html for online consumption" >&2
	echo " -o, --output=DIR Output directory. Default is the current directory" >&2
	echo " -t, --tmp=DIR    Temporary directory. Default is created by mktemp(1)" >&2
	echo " -G, --optgraph   Calculate a dependency graph where each installation" >&2
	echo "                  set contains the minimal number of unavailable" >&2
	echo "                  binary packages" >&2
	echo " -j, --jobs=NUM   Number of threads or processes to run in parallel" >&2
	echo "                  where applicable" >&2
	echo " -I, --drop-b-d-indep Drop Build-Depends-Indep dependencies" >&2
	echo " -v, --verbose    Be more verbose" >&2
	echo " -d, --debug      Maximum verbosity" >&2
	echo " -D, --develop    Execute tools from the source checkout instead of \$PATH" >&2
}

die() {
        echo " + error $@"
        if [ -n "$KEEP" ] || [ "$CUSTOMTMP" = "yes" ]; then
                echo " + temporary files stored in $tmp" >&2
        else
                rm -f "$tmp"/*
                rm -fd $tmp
        fi
        exit 1
}

run() {
        if [ -n "$TIMERS" ]; then
                time --format "%e seconds" "$@"
        else
                "$@"
        fi
}

# clean up when sighup, sigint or sigterm is received
trap "die \"got signal\"" 1 2 15

getopt -T > /dev/null && exitcode=0 || exitcode=$?
if [ $exitcode -eq 4 ]; then
	# GNU enhanced getopt is available
	ARGS=`getopt --long help,online,output:,tmp:,optgraph,jobs:,drop-b-d-indep,verbose,debug,develop --options hwo:t:Gj:IvdD -- "$@"`
else
	# Original getopt is available
	ARGS=`getopt hwo:t:Gj:IvdD "$@"`
fi

if [ $? -ne 0 ]; then
	exit 1
fi

eval set -- $ARGS

while [ $# -gt 0 ]; do
	case "$1" in
		-h | --help)    usage; exit 0;;
		-k | --keep)    KEEP=true;;
		-w | --online)  ONLINE="--online";;
		-o | --output)  OUTDIR="$2"; shift;;
		-t | --tmp)     tmp="$2"; shift;;
		-G | --optgraph) OPTGRAPH=--optgraph;;
		-j | --jobs)    NUM_CORES="$2"; shift;;
		-I | --drop-b-d-indep) DROPBDINDEP=--drop-b-d-indep;;
		-v | --verbose) VERBOSE=--verbose;;
		-d | --debug)   set -x;;
		-D | --develop) DEVELOP=true;;
		--)             shift; break;;
	esac
	shift
done

bin_coinstall=dose-deb-coinstall
bin_buildcheck=dose-builddebcheck
if [ -n "$DEVELOP" ]; then
	droppable=./droppable
	[ -f ./bin2src.native ] && bin_bin2src=./bin2src.native || bin_bin2src=./bin2src.d.byte
	[ -f ./build-fixpoint.native ] && bin_build_fixpoint=./build-fixpoint.native || bin_build_fixpoint=./build-fixpoint.d.byte
	[ -f ./buildgraph2srcgraph.native ] && bin_buildgraph_to_srcgraph=./buildgraph2srcgraph.native || bin_buildgraph_to_srcgraph=./buildgraph2srcgraph.d.byte
	[ -f ./clean-repository.native ] && bin_clean_repository=./clean-repository.native || bin_clean_repository=./clean-repository.d.byte
	[ -f ./create-graph.native ] && bin_create_graph=./create-graph.native || bin_create_graph=./create-graph.d.byte
	[ -f ./find-fvs.native ] && bin_find_fvs=./find-fvs.native || bin_find_fvs=./find-fvs.d.byte
	[ -f ./partial-order.native ] && bin_partial_order=./partial-order.native || bin_partial_order=./partial-order.d.byte
	[ -f ./print-stats.native ] && bin_print_stats=./print-stats.native || bin_print_stats=./print-stats.d.byte
	[ -f ./src2bin.native ] && bin_src2bin=./src2bin.native || bin_src2bin=./src2bin.d.byte
	bin_add_arch=./tools/add-arch.py
	bin_apply_ma_diff=./tools/apply-ma-diff.py
	bin_convert_arch=./tools/convert-arch.py
	bin_dose2html=./tools/dose2html.py
	bin_extract_scc=./tools/extract-scc.py
	bin_fasofstats=./tools/fasofstats.py
	bin_fix_cross_problems=./tools/fix-cross-problems.py
	bin_graph_info=./tools/graph-info.py
	bin_ma_diff=./tools/ma-diff.py
	bin_packages_difference=./tools/packages-difference.py
	bin_packages_intersection=./tools/packages-intersection.py
	bin_packages_union=./tools/packages-union.py
	bin_profile_build_fvs=./tools/profile-build-fvs.py
	bin_stat_html=./tools/stat-html.py
	bin_download_pkgsrc=./tools/download-pkgsrc.sh
	bin_buildgraph2packages=./tools/buildgraph2packages.py
else
	droppable=/usr/share/botch/droppable
	bin_bin2src=botch-bin2src
	bin_build_fixpoint=botch-build-fixpoint
	bin_buildgraph_to_srcgraph=botch-buildgraph2srcgraph
	bin_clean_repository=botch-clean-repository
	bin_create_graph=botch-create-graph
	bin_find_fvs=botch-find-fvs
	bin_partial_order=botch-partial-order
	bin_print_stats=botch-print-stats
	bin_src2bin=botch-src2bin
	bin_add_arch=botch-add-arch
	bin_apply_ma_diff=botch-apply-ma-diff
	bin_convert_arch=botch-convert-arch
	bin_dose2html=botch-dose2html
	bin_extract_scc=botch-extract-scc
	bin_fasofstats=botch-fasofstats
	bin_fix_cross_problems=botch-fix-cross-problems
	bin_graph_info=botch-graph-info
	bin_ma_diff=botch-ma-diff
	bin_packages_difference=botch-packages-difference
	bin_packages_intersection=botch-packages-intersection
	bin_packages_union=botch-packages-union
	bin_profile_build_fvs=botch-profile-build-fvs
	bin_stat_html=botch-stat-html
	bin_download_pkgsrc=botch-download-pkgsrc
	bin_buildgraph2packages=botch-buildgraph2packages
fi


mkdir -p "$OUTDIR"

case $# in
	0)
		BUILDARCH=amd64
		HOSTARCH=armhf
		suite=sid
		date=20140101T000000Z
		$bin_download_pkgsrc $BUILDARCH 2014
		buildpackages=$suite-$BUILDARCH-packages-$date.gz
		hostpackages=$suite-$HOSTARCH-packages-$date.gz
		sources=$suite-sources-$date.gz
		;;
	4)
		BUILDARCH=$1
		HOSTARCH=$2
		buildpackages=$3
		sources=$4
		;;
	*)
		usage
		exit 1
		;;
esac

if [ -n "$tmp" ]; then
	mkdir -p "$tmp"
else
	tmp=`mktemp --directory`
	CUSTOMTMP="no"
fi

if [ "$KEEP" != "true" ] && [ -n "$(ls -A $tmp)" ]; then
       echo "$tmp is not empty and you did not specify --keep" >&2
       echo "refusing to run and delete that directory" >&2
       exit 1
fi

echo " + running clean repository" >&2

# clean up buildpackages repository
$bin_clean_repository $VERBOSE $DROPBDINDEP --deb-native-arch=$BUILDARCH \
	$buildpackages $sources \
	> $tmp/buildpackages_clean
buildpackages="$tmp/buildpackages_clean"

echo " + running bin2src" >&2

run $bin_bin2src $VERBOSE --noarchall --deb-native-arch=$BUILDARCH $buildpackages $sources \
	> $tmp/sources_clean || die "bin2src failed"
sources="$tmp/sources_clean"

hostpackages="$tmp/hostpackages"

# we do this because version skew kills
run $bin_convert_arch $VERBOSE $BUILDARCH $HOSTARCH $buildpackages $hostpackages || die "convert arch failed"
# we do this because some of the source packages might not compile for our hostarch
run $bin_add_arch $VERBOSE $HOSTARCH $sources $tmp/tmp_sources || die "add arch failed"
mv $tmp/tmp_sources $sources

# we start off with making a smaller package selection by first calculating a
# selfcontained repository

zcat -f "$hostpackages" | \
    run grep-dctrl -X \( \
    -FPackage build-essential --or \
    -FPackage apt --or \
    -FPackage sysvinit-core --or \
    -FPackage debhelper --or \
    -FEssential yes \
\) > "$tmp/minimal" || die "zcat failed"

echo " + running coinstall..." >&2

run $bin_coinstall $VERBOSE \
    --bg="$hostpackages" --fg="$tmp/minimal" --deb-native-arch=$HOSTARCH \
    > "$tmp/minimal-$HOSTARCH" || die "coinstall failed"

echo " + running bin2src..." >&2

run $bin_bin2src $VERBOSE --noarchall --deb-native-arch=$HOSTARCH $ALLOWSRCMISMATCH \
        "$tmp/minimal-$HOSTARCH" "$sources" \
        > "$tmp/crossed-src" || die "bin2src failed"

echo " + running grep-dctrl..." >&2

run grep-dctrl -FArchitecture all "$hostpackages" \
	> "$tmp/available-all" || die "grep-dctrl failed"

echo " + creating a self-contained repository graph..." >&2

# we ignore essential because it is already part of $tmp/minimal-$HOSTARCH
# this also means that we have to create the union of the results of this with
# $tmp/minimal-$HOSTARCH
# the graph is started from all source packages that have been crossed before
# and are thus creating the binary packages in $tmp/minimal-$HOSTARCH
run $bin_create_graph $VERBOSE $DROPBDINDEP --num_cores=$NUM_CORES \
	--deb-ignore-essential --progress --timers \
	-A "$tmp/available-all" $ALLOWSRCMISMATCH --deb-native-arch=$HOSTARCH \
	--bg $sources $hostpackages $tmp/crossed-src \
	> "${OUTDIR}/selfcontained_repo.xml" || die "create graph failed"

echo " + running buildgraph2packages" >&2

run $bin_buildgraph2packages "${OUTDIR}/selfcontained_repo.xml" "$hostpackages" \
	> "$tmp/minimal-closure" || die "buildgraph2packages failed"

echo " + union..." >&2

run $bin_packages_union $VERBOSE "$tmp/minimal-closure" \
	"$tmp/minimal-$HOSTARCH" "$tmp/minimal-closure" \
	|| die "union failed"

echo " + running bin2src..." >&2

run $bin_bin2src $VERBOSE --noarchall --deb-native-arch=$HOSTARCH $ALLOWSRCMISMATCH \
       "$tmp/minimal-closure" "$sources" \
       > "$tmp/minimal-closure-src" || die "bin2src failed"

hostpackages="$tmp/minimal-closure"
sources="$tmp/minimal-closure-src"

echo " + running buildcheck..." >&2

# if buildcheck succeeds and since the source list was generated from the
# binary list, at this point we know that the pair of minimal-closure and
# minimal-closure-src is indeed a valid self-contained repository
run $bin_buildcheck $VERBOSE --explain-minimal \
	--failures --deb-native-arch=$HOSTARCH "$hostpackages" \
	"$sources" || die "buildcheck failed"

# at this point the calculation of the selfcontained repository is complete
# we now alter the repository until all its source packages can satisfy their
# cross build dependencies

cat > "$tmp/crossbuild-essential-${HOSTARCH}" << END
Package: crossbuild-essential-${HOSTARCH}
Version: 1.0
Architecture: ${BUILDARCH}
END

# save the location of the old buildpackages to do a diff later
oldbuildpackages=$buildpackages

# if ma.diff already exists, apply it
if [ -s "${OUTDIR}/ma.diff" ]; then
	run $bin_apply_ma_diff $VERBOSE "${OUTDIR}/ma.diff" $buildpackages \
		$tmp/buildpackages || die "apply-ma-diff failed"
	buildpackages=$tmp/buildpackages
	run $bin_apply_ma_diff $VERBOSE "${OUTDIR}/ma.diff" $hostpackages \
		$tmp/minimal-closure-crossable || die "apply-ma-diff failed"
	hostpackages=$tmp/minimal-closure-crossable
	#run $bin_buildcheck --explain --failures --deb-native-arch=$BUILDARCH --deb-host-arch=$HOSTARCH $hostpackages $buildpackages "$tmp/crossbuild-essential-${HOSTARCH}" $sources
fi

# fix the remaining problems
cp $buildpackages $tmp/buildpackages-fixed
cp $hostpackages $tmp/minimal-closure-crossable-fixed
i=0
while true; do
	echo $i
	i=$((i+1))
	run $bin_buildcheck $VERBOSE --explain --failures \
		--deb-native-arch=$BUILDARCH --deb-host-arch=$HOSTARCH \
		$tmp/minimal-closure-crossable-fixed $tmp/buildpackages-fixed \
		"$tmp/crossbuild-essential-${HOSTARCH}" $sources \
		> $tmp/buildcheck.yaml || [ $? -lt 64 ] || die "buildcheck failed"
	run $bin_packages_union $VERBOSE $tmp/buildpackages-fixed $tmp/buildpackages-fixed || die "union failed"
	run $bin_packages_union $VERBOSE $tmp/minimal-closure-crossable-fixed $tmp/minimal-closure-crossable-fixed || die "union failed"
	oldmd5=`cat $tmp/buildpackages-fixed $tmp/minimal-closure-crossable-fixed | md5sum`
	run $bin_fix_cross_problems $VERBOSE $tmp/buildcheck.yaml $tmp/buildpackages-fixed \
		$tmp/minimal-closure-crossable-fixed || die "fix-cross-problems failed"
	run $bin_ma_diff $VERBOSE $oldbuildpackages $tmp/buildpackages-fixed \
		| sort > "${OUTDIR}/ma.diff" || true # non-zero exit is not fatal here
	run $bin_packages_union $VERBOSE $tmp/buildpackages-fixed $tmp/buildpackages-fixed || die "union failed"
	run $bin_packages_union $VERBOSE $tmp/minimal-closure-crossable-fixed $tmp/minimal-closure-crossable-fixed || die "union failed"
	newmd5=`cat $tmp/buildpackages-fixed $tmp/minimal-closure-crossable-fixed | md5sum`
	[ "$oldmd5" = "$newmd5" ] && break
done
buildpackages=$tmp/buildpackages-fixed
hostpackages=$tmp/minimal-closure-crossable-fixed

echo " + running grep-dctrl..." >&2

run grep-dctrl -FArchitecture all "$hostpackages" \
	> "$tmp/available-all" || die "grep-dctrl failed"

# calculate tocompile
echo " + difference..." >&2
run $bin_packages_difference $VERBOSE "$hostpackages" "$tmp/available-all" \
	"$tmp/not-available" || die "packages difference failed"

# after we have found the packages that have to be compiled, we do not
# need host architecture packages that are m-a:foreign anymore
# from this point on, if there are any foreign/host architecture and
# m-a:foreign binary packages around, then they will be used to satisfy
# dependencies instead of using the native/build architecture m-a:forein
# binary packages
# therefore we delete them so that they are not used for any dependency
# satisfaction anymore
# we delete all m-a:foreign packages from the host package list because
# arch:all packages come from the build package list
echo " + running grep-dctrl..." >&2
run grep-dctrl -X \( -FArchitecture all --or --not -FMulti-Arch foreign \) "$hostpackages" \
	> "$tmp/hostpackages-no-ma-foreign" || die "grep-dctrl failed"
hostpackages="$tmp/hostpackages-no-ma-foreign"

echo " + bin2src..." >&2
run $bin_bin2src $VERBOSE --noarchall --deb-native-arch=$BUILDARCH --deb-host-arch=$HOSTARCH $ALLOWSRCMISMATCH \
        "$tmp/not-available" "$sources" \
        > "$tmp/tocompile" || die "bin2src failed"

# calculate overall available (incl. build arch)
echo " + union..." >&2
run $bin_packages_union $VERBOSE "$tmp/available-all" $buildpackages \
	"$tmp/crossbuild-essential-${HOSTARCH}" "$tmp/available-all-build" || die "union failed"

# running fixpoint
echo " + running build-fixpoint..." >&2

run $bin_build_fixpoint $VERBOSE $DROPBDINDEP --deb-native-arch=$BUILDARCH --deb-host-arch=$HOSTARCH \
	$ALLOWSRCMISMATCH -A "$tmp/available-all-build" \
	--output-order="${OUTDIR}/order1.lst" "$buildpackages" \
	"$tmp/crossbuild-essential-${HOSTARCH}" "$hostpackages" \
	"$tmp/tocompile" \
	> "$tmp/compilable-src" || die "fixpoint failed"

echo " + running src2bin..." >&2

run $bin_src2bin $VERBOSE --deb-native-arch=$BUILDARCH --deb-host-arch=$HOSTARCH $ALLOWSRCMISMATCH \
    $IGNORESRCLESSBIN -A "$tmp/available-all" --bg "$sources" \
    "$hostpackages" "$tmp/compilable-src" \
    > "$tmp/compilable" || die "src2bin failed"

echo " + union..." >&2

run $bin_packages_union $VERBOSE "$tmp/available-all-build" "$tmp/compilable" \
	"$tmp/available" || die "union failed"

echo " + difference..." >&2
run $bin_packages_difference $VERBOSE "$tmp/tocompile" "$tmp/compilable-src" \
	"$tmp/tocompile" || die "difference failed"


# from this point on, everything that reads GraphML must read in the same
# Packages and Sources files to create the exact same cudf universe
UNIVERSE="--deb-native-arch=$BUILDARCH --deb-host-arch=$HOSTARCH --bg $sources $buildpackages "$tmp/crossbuild-essential-${HOSTARCH}" $hostpackages $tmp/tocompile"

# we can use deb-ignore-essential here because the set of available packages
# was made sure to contain a coinstallation set of essential earlier in this
# script
run $bin_create_graph $VERBOSE $DROPBDINDEP $OPTGRAPH --progress --timers \
	--num_cores=$NUM_CORES --deb-ignore-essential $STRONGARG \
	-A "$tmp/available" $ALLOWSRCMISMATCH $UNIVERSE \
	> "${OUTDIR}/buildgraph.xml" || die "create graph failed"

echo " + running buildgraph2srcgraph..." >&2

run $bin_buildgraph_to_srcgraph $VERBOSE $DROPBDINDEP "${OUTDIR}/buildgraph.xml" $UNIVERSE \
	> "${OUTDIR}/srcgraph.xml" || die "buildgraph2srcgraph failed"

droplist="--remove-weak=${droppable}/weak-build-dependencies.list --remove-reduced=${droppable}/pehjota.list,${droppable}/thorsten-glaser.list,${droppable}/daniel-schepler.list,${droppable}/gentoo.list,${droppable}/wookey.list,${droppable}/selfcycles.list"

run $bin_print_stats $VERBOSE $DROPBDINDEP --max-length=4 --max-length-fas=8 $ALLOWSRCMISMATCH \
	$SAPSB $droplist -A "$tmp/available" "${OUTDIR}/buildgraph.xml" \
	"${OUTDIR}/srcgraph.xml" $UNIVERSE > "${OUTDIR}/stats.json" || die "print stats failed"

echo " + generating html..." >&2

run $bin_stat_html $VERBOSE $ONLINE "${OUTDIR}/stats.json" > "${OUTDIR}/stats.html" || die "stat-html failed"

echo " + extracting the calculated feedback arc set..." >&2

run $bin_fasofstats $VERBOSE "${OUTDIR}/stats.json" | sort > "${OUTDIR}/remove.list" || die "fasofstats failed"

echo " + extract strongly connected components..." >&2

run $bin_extract_scc $VERBOSE --outdir=${OUTDIR} "${OUTDIR}/buildgraph.xml" \
	cyclic --outfnameattr=cudfname --outfnameverts type:src \
	| sort > "${OUTDIR}/fas" || die "extract scc failed"

echo " + calculate feedback vertex sets..." >&2

alldrop="$droplist,${OUTDIR}/remove.list"

echo > "${OUTDIR}/feedback_vertex_set.list"
while read g; do
	run $bin_find_fvs $VERBOSE $DROPBDINDEP $alldrop "${OUTDIR}/$g" $UNIVERSE \
		|| die "find fvs failed"
done < "${OUTDIR}/fas" | sort > "${OUTDIR}/feedback_vertex_set.list"

echo " + profile build selected source packages..." >&2


run $bin_profile_build_fvs $VERBOSE $alldrop "${OUTDIR}/buildgraph.xml" \
	"${OUTDIR}/feedback_vertex_set.list" \
	> "${OUTDIR}/buildgraph_acyclic.xml" || die "profile-build-fvs failed"

echo " + convert to srcgraph..." >&2

run $bin_buildgraph_to_srcgraph $VERBOSE $DROPBDINDEP "${OUTDIR}/buildgraph_acyclic.xml" $UNIVERSE \
	> "${OUTDIR}/srcgraph_acyclic.xml" || die "buildgraph2srcgraph failed"

echo " + calculate partial order..." >&2

run $bin_partial_order $VERBOSE $DROPBDINDEP "${OUTDIR}/srcgraph_acyclic.xml" $UNIVERSE \
	> "${OUTDIR}/order2.lst" || die "partial order failed"

echo " + build order is saved in ${OUTDIR}/order1.lst and ${OUTDIR}/order2.lst" >&2
echo " + an overview of the dependency situation is now available in ${OUTDIR}/stats.html" >&2
echo " + the data from which stats.html was generated is saved in ${OUTDIR}/stats.json" >&2
echo " + the build order was generated using the feedback arc set in ${OUTDIR}/remove.list" >&2
echo " + without the content of remove.list, the remaining strongly connected components are:" >&2

echo -e "graph.xml\ttype\tvertices\tedges\tscc"
while read g; do
	run $bin_graph_info $VERBOSE "${OUTDIR}/$g" || die "graph-info failed"
done < "${OUTDIR}/fas"

if [ -n "$KEEP" ] || [ "$CUSTOMTMP" = "yes" ]; then
	echo " + temporary files stored in $tmp" >&2
else
	rm -f "$tmp"/*
	rm -fd $tmp
fi
