#!/bin/sh
#
# cook-pkgdb - SliTaz cook: rebuild package databases for a packages
# directory plus the wok-wide split.db and maint.db.
#
# Produces in $PKGS: packages.list, packages.md5, packages.info,
# packages.equiv, descriptions.txt, bdeps.txt, files-list.lzma,
# bundle.tar.lzma. Produces in $cache: split.db, maint.db.
#
# Usage:
#   cook-pkgdb              # full rebuild (wok DBs + $PKGS lists)
#   cook-pkgdb <dir>        # full rebuild against packages dir <dir>
#   cook-pkgdb --flavors    # full rebuild + regenerate TazLiTo flavors
#   cook-pkgdb --rmpkg      # full rebuild + remove unused tazpkg files
#   cook-pkgdb --splitdb    # only refresh $cache/split.db
#   cook-pkgdb --maintdb    # only refresh $cache/maint.db
#
# Copyright (C) SliTaz GNU/Linux - GNU GPL v3
#

. /usr/lib/slitaz/libcook.sh


#
# Functions
#

usage() {
	cat <<EOT

$(boldify "$(_ 'Usage:')") cook-pkgdb [command|<packages-dir>]

$(boldify "$(_ 'Commands:')")
  usage|help|--help|-h $(_ 'Display this short usage.')
  --splitdb            $(_ 'Refresh $cache/split.db only.')
  --maintdb            $(_ 'Refresh $cache/maint.db only.')
  --flavors            $(_ 'Full rebuild + regenerate TazLiTo flavors.')
  --rmpkg              $(_ 'Full rebuild + remove unused tazpkg files.')

$(boldify "$(_ 'Default:')")
  cook-pkgdb           $(_ 'Refresh split.db + maint.db + lists in $PKGS.')
  cook-pkgdb <dir>     $(_ 'Same, against the packages dir <dir>.')

$(boldify "$(_ 'cook.conf:')")
  MAKE_BUNDLE="yes"    $(_ 'Also build bundle.tar.lzma (build host only).')

EOT
	exit 0
}


dblog() { tee -a $LOGS/pkgdb.log; }


# Human-readable file size (vs `du`, which rounds to block size).

filesize() { busybox ls -lh "$1" | awk '{print $5 "B"}'; }


# Rebuild $cache/split.db: one line per receipt mapping package name
# to its all_names() list (main + SPLIT).

recreate_split_db() {
	local db="$cache/split.db"

	cd $WOK
	for pkg in *; do
		[ -f "$WOK/$pkg/receipt" ] || continue
		unset PACKAGE SPLIT
		. $WOK/$pkg/receipt
		echo -e "$PACKAGE\t$(all_names)"
	done > $db
}


# Rebuild $cache/maint.db: one line per receipt mapping maintainer
# email to package name, sorted.

recreate_maint_db() {
	local db="$cache/maint.db"

	cd $WOK
	for pkg in *; do
		[ -f "$WOK/$pkg/receipt" ] || continue
		unset PACKAGE MAINTAINER
		. $WOK/$pkg/receipt
		# Strip "Name <" prefix and "> ..." suffix without forking sed:
		# keep only the address between the angle brackets.
		MAINTAINER=${MAINTAINER##*<}; MAINTAINER=${MAINTAINER%%>*}
		echo -e "$MAINTAINER\t$PACKAGE"
	done | sort > $db
}


# all_names: list main pkg + every name from SPLIT, with v1/v2 handling.
# Copied verbatim from cook so this tool is self-contained.

all_names() {
	# Get package names from $SPLIT variable. Skip the awk|tr fork when
	# there's no SPLIT (the common case) -- this runs ~6000x per pass.
	local split='' split_space first
	if [ -n "$SPLIT" ]; then
		split=$(echo -n $SPLIT \
			| awk '
				BEGIN { RS = " "; FS = ":"; }
				{ print $1; }' \
			| tr '\n' ' ')
	fi
	split_space=" $split "
	# Receipt format version sits on the first line; read it with the
	# shell instead of `head ... | fgrep` (2 forks/receipt -> 50x faster).
	read -r first < "$WOK/$pkg/receipt" 2>/dev/null
	case "$first" in
		*v2*) : ;;	# v2: honour $SPLIT ordering below
		*)
			# For receipts v1: $SPLIT may present in the $WANTED package,
			# but split packages have their own receipts
			echo $PACKAGE; return ;;
	esac
	if [ "${split_space/ $PACKAGE /}" != "$split_space" ]; then
		# $PACKAGE included somewhere in $SPLIT (probably in the end).
		# We should build packages in the order defined in the $SPLIT.
		echo $split
	else
		# We'll build the $PACKAGE, then all defined in the $SPLIT.
		echo $PACKAGE $split
	fi
}


# scan_wok: one pass over every receipt feeding ALL the wok-wide outputs
# at once -- split.db + maint.db ($cache) + bdeps.txt + keep.list ($dbs).
# Replaces what used to be 4 separate full-wok scans (split, maint, bdeps
# and the toremove "second pass"), each sourcing ~6000 receipts. keep.list
# collects, per receipt, the tazpkg filenames a receipt still vouches for
# (every all_names() x {-$ARCH, -any}); packages.toremove is later derived
# from it with a single awk instead of thousands of in-loop `sed -i`.
# Needs $dbs and $arch set, so it runs from the main flow (not up here).

scan_wok() {
	local m names name
	: > "$cache/split.db"
	: > "$dbs/bdeps.txt"
	: > "$dbs/keep.list"
	: > "$dbs/maint.tmp"
	cd $WOK
	for pkg in *; do
		[ -s "$WOK/$pkg/receipt" ] || continue
		unset PACKAGE SPLIT MAINTAINER BUILD_DEPENDS VERSION EXTRAVERSION
		. $WOK/$pkg/receipt

		names=$(all_names)
		echo -e "$PACKAGE\t$names" >> "$cache/split.db"

		# maintainer email between the angle brackets, no sed fork
		m=${MAINTAINER##*<}; m=${m%%>*}
		echo -e "$m\t$PACKAGE" >> "$dbs/maint.tmp"

		printf '%s\t%s\n' "$pkg" "$(echo $BUILD_DEPENDS)" >> "$dbs/bdeps.txt"

		for name in $names; do
			printf '%s\n%s\n' \
				"$name-$VERSION$EXTRAVERSION$arch.tazpkg" \
				"$name-$VERSION$EXTRAVERSION-any.tazpkg" >> "$dbs/keep.list"
		done
	done
	sort "$dbs/maint.tmp" > "$cache/maint.db"
	rm -f "$dbs/maint.tmp"
}


#
# Dedicated subcommands: exit early.
#

case "$1" in
	usage|help|--help|-h)  usage ;;
	--splitdb)  recreate_split_db; exit 0 ;;
	--maintdb)  recreate_maint_db; exit 0 ;;
esac


#
# Default mode: the wok-wide DBs (split.db, maint.db, bdeps.txt) and the
# toremove keep.list are all built in a single scan_wok pass below, once
# $dbs and $arch are set. Then the $PKGS lists are generated.
#
# Below: legacy `modules/pkgdb` body — packages dir lists generation.
#

# Find how much time was spent the last time (for web interface)
lastcooktime=$(sed '/Time:/!d; s|.*: *\([0-9]*\)s.*|\1|' $LOGS/pkgdb.log 2>/dev/null | sed '$!d')
[ -n "$lastcooktime" ] && echo "cook:pkgdb $lastcooktime $(date +%s)" >> $cooktime
while read cmd duration start; do
	[ $(($start + $duration)) -lt $(date +%s) ] &&
	echo "sed -i '/^$cmd $duration/d' $cooktime"
done < $cooktime | sh

rm $LOGS/pkgdb.log 2>/dev/null

case "$1" in
	--flavors|--rmpkg) ;;
	*)
		[ -n "$1" ] && PKGS="$1"
		if [ ! -d "$PKGS" ]; then
			{ newline; _ "Packages directory \"%s\" doesn't exist" "$PKGS"; newline; } | dblog
			exit 1
		fi ;;
esac

time=$(date +%s)
flavors="$SLITAZ/flavors"
live="$SLITAZ/live"

arch=''
case "$ARCH" in
	arm*|x86_64) arch="-$ARCH" ;;
esac

echo 'cook:pkgdb' > $command
_ 'Cook pkgdb: Creating all packages lists' | log
newline; { _ 'Creating lists for "%s"' "$PKGS"; separator; } | dblog

{ _ 'Cook pkgdb started: %s' "$(date "$(_ '+%%F %%R')")"; newline; } | dblog

cd $PKGS

# Web interface database files should exist on the mirror1
>.folderlist; >.filelist
chmod 666 .folderlist .filelist

# Command `cook pkgdb` may be executed by cron. Creating a packages database
# takes some time, during which previously pending packages may continue to
# be created. This will result in an error due to the lack of a database. This in
# turn can lead to errors in the creating of many subsequent packages and will
# require manual intervention.
# Solution is an atomic update of the packages database: first we create new files
# (in the separate temp dir), and then instantly (well, almost) replace the old
# files by new ones.

dbs=$(mktemp -d)

# One scan over the whole wok: split.db + maint.db + bdeps.txt + keep.list.
action 'Refreshing wok databases...'
scan_wok
cd $PKGS		# scan_wok cd'd into $WOK; the find below needs $PKGS
status

_n 'Creating file "%s"' 'packages.list' | dblog
find . -name '*.tazpkg' | sed 's|^./||; s|.tazpkg$||' > $dbs/packages.list
echo " ($(filesize $dbs/packages.list))" | dblog

_n 'Creating file "%s"' 'packages.md5' | dblog
find . -name '*.tazpkg' -exec md5sum '{}' \; | sed 's|./||' > $dbs/packages.md5
echo " ($(filesize $dbs/packages.md5))" | dblog

md5sum $dbs/packages.md5 | cut -d' ' -f1 > $dbs/ID
( cat $dbs/ID | tr $'\n' ' '; date -ur $dbs/ID +%s ) > $dbs/IDs	# md5 and timestamp

_n 'Creating file "%s"' 'descriptions.txt' | dblog
for i in $(ls $WOK | sort); do
	[ -d "$WOK/$i/taz" ] || continue

	for j in $(ls $WOK/$i/taz | sort); do
		[ -e "$WOK/$i/taz/$j/description.txt" ] || continue

		pkgname=$(. $WOK/$i/taz/$j/receipt; echo $PACKAGE)
		echo "$pkgname"
		sed 's|^$| |' "$WOK/$i/taz/$j/description.txt"
		# if description.txt doesn't end with \n then add one
		[ -z "$(tail -c1 $WOK/$i/taz/$j/description.txt)" ] || echo
		echo
	done >> $dbs/descriptions.txt
done
echo " ($(filesize $dbs/descriptions.txt))" | dblog

_n 'Creating file "%s"' 'bdeps.txt' | dblog		# built by scan_wok above
echo " ($(filesize $dbs/bdeps.txt))" | dblog


_ 'Creating lists from "%s"' "$WOK" | dblog
cd $WOK
rsumf=$(mktemp)
#touch $dbs/packages.desc $dbs/packages.txt $dbs/packages.info $dbs/packages.equiv
touch $dbs/packages.info $dbs/packages.equiv
for i in *; do
	[ -d "$WOK/$i/taz" ] || continue

	for j in $(ls $WOK/$i/taz | sort); do
		pack="$i/taz/$j"
		[ -f "$WOK/$pack/receipt" ] || continue
		unset_receipt
		unset PACKED_SIZE UNPACKED_SIZE RSUM PACKAGE VERSION EXTRAVERSION CATEGORY SHORT_DESC MAINTAINER LICENSE WEB_SITE DEPENDS TAGS PROVIDE
		. ./$pack/receipt

		# $arch is "" on i486 (no suffix, e.g. busybox-1.37.0.tazpkg) and
		# "-$ARCH" on x86_64/arm. -any.tazpkg covers noarch packages on
		# every arch.
		if [ -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION$arch.tazpkg" -o \
		     -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION-any.tazpkg" ]; then

			# packages.info combines TazPkg separate files
			# and will substitute them all
			SIZES=$(echo $PACKED_SIZE $UNPACKED_SIZE | sed 's|\.0||g')
			DEPENDS=$(echo $DEPENDS) # remove newlines from some receipts

#			#MD5="$(fgrep " $PACKAGE-$VERSION$EXTRAVERSION.tazpkg" $PKGS/packages.md5 | awk '{print $1}')"

			# RSUM defined in new "small" receipts, but may be absent in old
			if [ -z "$RSUM" ]; then
				cp $pack/md5sum $rsumf
				md5sum $pack/receipt | sed 's| [^ ]*/| |' >> $rsumf
				[ -e "$pack/description.txt" ] && md5sum $pack/description.txt | sed 's| [^ ]*/| |' >> $rsumf
				RSUM=$(md5sum $rsumf | awk '{print $1}')
			fi

			# i486 .tazpkg carry no arch suffix -> classic 10-field
			# record. x86_64/arm append field 11, the noarch flag:
			# "0" = -any.tazpkg, "1" = -$ARCH.tazpkg. tazpkg/get reads it
			# to rebuild the correct filename suffix (i486 needs none).
			noarch=''
			if [ -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION-any.tazpkg" ]; then
				noarch='\t0'
			else
				if [ -n "$arch" ]; then
					noarch='\t1'
				fi
			fi
			printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s%b\n' \
				"$PACKAGE" "$VERSION$EXTRAVERSION" "$CATEGORY" \
				"$SHORT_DESC" "$WEB_SITE" "$TAGS" "$SIZES" \
				"$DEPENDS" "$RSUM" "$PROVIDE" "$noarch" \
				>> $dbs/packages.info

			# packages.equiv is used by tazpkg install to check depends.
			for k in $PROVIDE; do
				DEST=''
				echo $k | fgrep -q : && DEST="${k#*:}:"
				if grep -qs ^${k%:*}= $dbs/packages.equiv; then
					sed -i "s/^${k%:*}=/${k%:*}=$DEST$PACKAGE /" \
						$dbs/packages.equiv
				else
					echo "${k%:*}=$DEST$PACKAGE" >> $dbs/packages.equiv
				fi
			done

			# files.list provides a list of all packages files.
			sed "s|^|$PACKAGE: \0|" $i/taz/$j/files.list >> $dbs/files.list

			# vouch for this built tazpkg (its taz/ version) so the
			# toremove awk below keeps it -- covers a version-skew case
			# scan_wok (source-receipt version) would miss.
			echo "$PACKAGE-$VERSION$EXTRAVERSION$arch.tazpkg" >> $dbs/keep.list
		else
			# The taz/ built-receipt copy strips WANTED and HOST_ARCH, so
			# consult the source receipt to judge arch relevance.
			unset WANTED HOST_ARCH
			eval "$(grep -E '^(WANTED|HOST_ARCH)=' "$WOK/$i/receipt" 2>/dev/null)"

			# A WANTED sub-package with no HOST_ARCH of its own is emitted
			# (or not) by its parent's per-arch build, not cooked directly --
			# e.g. the linux586* kernel splits only exist on i486. Missing
			# such a split here is the parent's per-arch decision, not a real
			# absence, so don't flag it (kills the linux586* noise flood on
			# x86_64/arm). Top-level packages and splits that explicitly claim
			# this arch via HOST_ARCH are still reported.
			if [ -n "$WANTED" -a -z "$HOST_ARCH" ]; then
				:
			# if receipt variable HOST_ARCH absent/empty or contains ARCH
			elif [ -z "$HOST_ARCH" -o "${HOST_ARCH/$ARCH/}" != "$HOST_ARCH" ]; then
				_ '  - absent: %s (%s)' "$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" "$ARCH" | dblog
			fi
		fi
	done
done
rm $rsumf


# packages.toremove = every tazpkg in packages.md5 that no receipt vouches
# for. keep.list was filled by scan_wok (all receipts' all_names x {-$ARCH,
# -any} -- this is the old "second pass" that kept packages whose taz/ was
# clean-wok'ed or pre-existing in $PKGS) plus the main loop (built taz/
# versions). One awk replaces the old ~7000 in-loop `sed -i` rewrites.
touch $dbs/keep.list
awk 'NR==FNR { keep[$0]=1; next } !($2 in keep)' \
	$dbs/keep.list $dbs/packages.md5 > $dbs/packages.toremove
rm -f $dbs/keep.list


# Display list size.
_ 'Done: %s (%s)' 'packages.info'  "$(filesize $dbs/packages.info)"  | dblog
_ 'Done: %s (%s)' 'packages.equiv' "$(filesize $dbs/packages.equiv)" | dblog

cd $PKGS


# Check for unnecessary packages
if [ -s "$dbs/packages.toremove" ]; then
	newline | dblog
	case x$rmpkg in
		x) _ 'Found unnecessary packages (use `cook-pkgdb --rmpkg` to remove):' | dblog;;
		*) _ 'Removing unnecessary packages:' | dblog;;
	esac
	while read pkgsum pkgfile; do
		echo "  - $pkgfile" | dblog
		sed -i "/${pkgfile%.tazpkg}/d" $dbs/packages.list
		sed -i "/ $pkgfile/d" $dbs/packages.md5
		[ -n "$rmpkg" ] && rm $PKGS/$pkgfile	# remove packages only with --rmpkg
	done < $dbs/packages.toremove
	newline | dblog
fi
rm $dbs/packages.toremove


# files.list.lzma
_n 'Creating file "%s"' 'files.list.lzma' | dblog
touch $dbs/files.list
# pkgs.slitaz.org strongly depends on list sorted by packages names
#lzma e $dbs/files.list $dbs/files.list.lzma
lzma -c $dbs/files.list > $dbs/files.list.lzma
echo " ($(filesize $dbs/files.list.lzma))" | dblog

# Pre-sorting filenames causes 10% smaller resulting lzma file
_n 'Creating file "%s"' 'files-list.lzma' | dblog
cat $dbs/files.list | sort -k2 -o $dbs/files.list.sorted
#lzma e $dbs/files.list.sorted $dbs/files-list.lzma
lzma -c $dbs/files.list.sorted > $dbs/files-list.lzma
rm -f $dbs/files.list $dbs/files.list.sorted
echo " ($(filesize $dbs/files-list.lzma))" | dblog

md5sum $dbs/files-list.lzma | cut -d' ' -f1 | tr -d $'\n' > $dbs/files-list.md5

# Bundle for `tazpkg recharge` clients — bakes upstream mirrors +
# extra.list into the local DB so clients fetch everything in one
# round-trip. Only useful on the official build host; gated by
# MAKE_BUNDLE in cook.conf. If mirror1 is unreachable we still skip
# gracefully rather than hang.
if [ "$MAKE_BUNDLE" = 'yes' ]; then
	_n 'Creating file "%s"' 'bundle.tar.lzma' | dblog

	# Fetch mirrors first with a SHORT timeout: it doubles as a
	# reachability probe. When mirror1 is down (chronic) the old
	# 2-files x 3-tries x 10s retry loops burned up to ~60s only to fall
	# back to the empty placeholders below anyway. If this single probe
	# fails, skip the rest of the fetches and go straight to placeholders.
	wget -q -T 5 -O $dbs/mirrors http://mirror1.slitaz.org/mirrors 2>/dev/null
	if [ -s "$dbs/mirrors" ]; then
		i=3
		while [ $i -gt 0 ] && [ ! -s "$dbs/extra.list" ]; do
			wget -q -T 10 -O $dbs/extra.list http://mirror1.slitaz.org/packages/get.list
			echo -n '.' | dblog
			i=$((i - 1))
		done
	fi

	# Fall back to empty placeholders if mirror1 is unreachable: spk
	# (next-gen pkg manager) still wants bundle.tar.lzma and the local
	# DB files are what tazpkg/spk actually consume — mirrors and
	# extra.list are just upstream metadata that can be empty.
	[ -s "$dbs/mirrors" ]    || : > $dbs/mirrors
	[ -s "$dbs/extra.list" ] || : > $dbs/extra.list

	(
		cd $dbs
		busybox tar -chf bundle.tar \
			mirrors extra.list files-list.md5 packages.info \
			descriptions.txt packages.md5 packages.list packages.equiv
	)
	lzma -c $dbs/bundle.tar > $dbs/bundle.tar.lzma
	rm $dbs/bundle.tar $dbs/mirrors
	echo " ($(filesize $dbs/bundle.tar.lzma))" | dblog
fi

# Display some info.
separator | dblog
nb=$(ls $PKGS/*.tazpkg 2>/dev/null | wc -l)
time=$(($(date +%s) - $time))
# L10n: 's' is for seconds (cooking time)
{ _ 'Packages: %s - Time: %ss' "$nb" "$time"; newline; } | dblog

# "Atomic" update now
ls -l $dbs
cp -f $dbs/* $PKGS
rm -r $dbs


# Create all flavors files at once. Do we really need code to monitor
# flavors changes? Let's just build them with packages lists before
# syncing the mirror.
[ "$1" != '--flavors' ] && rm $command && exit 1

if [ ! -d "$flavors" ]; then
	{ _ 'Missing flavors folder "%s"' "$flavors"; newline; } | dblog
	rm $command
	exit 1
fi

[ ! -d "$live" ] && mkdir -p $live
_ 'Creating flavors files in "%s"' "$live" | dblog
_ 'Cook pkgdb: Creating all flavors' | log
separator | dblog

_ 'Recharging lists to use latest packages...' | dblog
tazpkg recharge >/dev/null 2>/dev/null

# We need a custom tazlito config to set working dir to /home/slitaz.
if [ ! -f "$live/tazlito.conf" ]; then
	_ 'Creating configuration file "%s"' 'tazlito.conf' | dblog
	cp /etc/tazlito/tazlito.conf $live
	sed -i s@WORK_DIR=.*@WORK_DIR=\"/home/slitaz\"@ \
		$live/tazlito.conf
fi

# Update Hg flavors repo and pack.
if [ -d "$flavors/.hg" ]; then
	cd $flavors; hg pull -u
fi

cd $live
_ 'Starting to generate flavors...' | dblog
rm -f flavors.list *.flavor
for i in $flavors/*; do
	fl=$(basename $i)
	_ 'Packing flavor "%s"' "$fl" | dblog
	tazlito pack-flavor $fl >/dev/null || exit 1
	tazlito show-flavor $fl --brief --noheader 2>/dev/null >> flavors.list
done
cp -f $live/*.flavor $live/flavors.list $PKGS
separator | dblog
{ _ 'Total flavors size: %s' "$(du -sh $live | awk '{print $1}')"; newline; } | dblog
separator | dblog
_ 'Cook pkgdb end: %s' "$(date "$(_ '+%%F %%R')")" | dblog

rm $command


exit 0
