#!/bin/sh
#
# Spk-mirror - Handle SliTaz packages mirrors. Read the README before
# adding or modifying any code in spk!
#
# Copyright (C) SliTaz GNU/Linux - BSD License
# Author: See AUTHORS files
#
. /usr/lib/slitaz/libspk.sh

# gendb a depot for another arch than the host (e.g. the i486 depot on an
# x86_64 build host): --arch= wins over the host /etc/slitaz/slitaz.conf.
# The arch drives the noarch column of packages.info (i486 = 10 fields).
[ -n "$arch" ] && SLITAZ_ARCH="$arch"

# NOTE: I want to use 'extra' mirrors since we can add any extra mirror
# URLs and not only undigest. See also undigest command - Pankso

#
# Functions
#

# Help and usage
usage() {
	name=$(basename $0)
	cat << EOT

$(boldify $(gettext "Usage:")) $name [urls|info|--options]

$(gettext "Handle SliTaz packages mirrors")

$(boldify $(gettext "Commands:"))
  gendb [dir]  $(gettext "Build package lists from .tazpkg (no wok needed)")

$(boldify $(gettext "Options:"))
  --list       $(gettext "List configured mirrors")
  --rm         $(gettext "Remove an extra mirror")
  --gendb      $(gettext "With a dir: build lists, then register it")
  --fresh      $(gettext "gendb: ignore the cache and re-extract everything")
  --arch=      $(gettext "gendb: target arch for a cross-arch depot (default: host)")
  --debug

$(boldify $(gettext "Examples:"))
  $name http://mirror.slitaz.org/
  $name gendb /home/slitaz/packages
  $name /path/to/packages --gendb   $(gettext "(build lists + register)")
  $name /path/to/packages           $(gettext "(register, lists already built)")

EOT
	exit 0
}

# Check if we have packages ID and lists on mirror.
check_mirror() {
	for file in packages.list packages.$SUM packages.info ID
	do
		echo $(gettext "Checking file:") $file
		# busybox wget spells the existence probe "--spider"; the old "-s"
		# is not a valid option in either busybox OR GNU wget ("invalid
		# option -- 's'"), so check_mirror failed on every file and no extra
		# mirror could ever be added. Use busybox explicitly (consistent with
		# the rest of spk; GNU wget in PATH lacks --no-check-certificate).
		if ! busybox wget $WGET_OPT -q --spider -T 6 ${1}${file} 2>/dev/null; then
			newline
			gettext "Unable to reach:"; colorize 31 " $file"
			newline && return 1
		fi
		status
	done
}

# Get extra mirror DB path
extra_db() {
	db=$(echo ${url#*://} | cut -d "/" -f1 )
	debug "extra mirror: $url"
	debug "extra DB: $db"
}

# Display info about an extra repo
extra_info() {
	local db=$1
	debug "extra DB: $extradb/$db"
	nb=$(cat $extradb/$db/packages.$SUM 2>/dev/null | wc -l)
	echo "Extra DB  : "$(boldify $db)
	echo "Extra URL : "$(cat $extradb/$db/mirror)
	gettext "Packages  :"; colorize 32 " $nb"
}

# Generate the package DB lists from the .tazpkg files in $1 — the
# consumer-side counterpart to cook-pkgdb (which builds the same files
# from the wok on the build host). No wok needed: receipts are read
# straight from each .tazpkg, so this works on deployed / minimal envs.
# Output is kept byte-compatible with cook-pkgdb, incl. the field-11
# noarch flag on x86_64/arm ("0" = -any.tazpkg, "1" = -$ARCH.tazpkg).
# Also emits descriptions.txt and bundle.tar.lzma so the resulting depot
# is rechargeable by tazpkg (whose recharge fetches only the bundle),
# not just by spk — a dual spk+tazpkg mirror from a wok-less box.
gen_pkgdb() {
	local pkgsdir pkg base sizes noarch rsum t cache fmt md5 prov P cf m \
		reused built newid
	cd "$1" || return 1
	pkgsdir=$(pwd)
	# Refuse to run two gendb on the same depot at once (e.g. a cron rebuild
	# and a manual one): they share .gendb-cache and rewrite the lists in
	# place, so concurrent runs corrupt both. mkdir is atomic; the lock is
	# released on any exit (incl. Ctrl-C) via the trap.
	if ! mkdir "$pkgsdir/.gendb.lock" 2>/dev/null; then
		gettext "ERROR: another gendb is running here"; newline
		gettext "(remove"; echo " $pkgsdir/.gendb.lock if it is stale)"
		return 1
	fi
	trap 'rmdir "'"$pkgsdir"'/.gendb.lock" 2>/dev/null' EXIT INT TERM
	newline
	boldify $(gettext "Generating package lists")
	separator

	# packages.list + packages.md5 — trivial, from the filenames. ID is the
	# change marker clients poll (spk-up: "same ID -> up to date"); it must
	# be the LAST thing written, once every list is complete. Writing it here
	# — before the minutes-long receipt indexing below — let a client that
	# recharged mid-run record the new ID against a still-empty packages.info
	# and then never re-fetch, keeping a corrupt DB until the depot next
	# changed. Compute it now (packages.md5 is final), publish it at the end.
	gettext "Creating packages.list / packages.md5..."
	find . -name '*.tazpkg' | sed 's|^\./||; s|\.tazpkg$||' | sort > packages.list
	find . -name '*.tazpkg' -exec $CHECKSUM '{}' \; | sed 's|\./||' > packages.md5
	newid=$($CHECKSUM packages.md5 | cut -d' ' -f1)
	status

	# packages.info + packages.equiv + files.list — need each receipt.
	# Incremental: each package's contribution is cached keyed by its md5
	# (.gendb-cache/<md5>.i = packages.info line, .f = files.list lines).
	# Re-running gendb after adding/changing a few packages only extracts
	# those — unchanged ones are replayed straight from the cache. The
	# cache is wiped with --fresh or when the format/arch marker changes.
	gettext "Indexing receipts..."; newline
	cache=".gendb-cache"; fmt="1 ${SLITAZ_ARCH}"; reused=0; built=0
	if [ "$fresh" ] || [ "$(cat $cache/.fmt 2>/dev/null)" != "$fmt" ]; then
		rm -rf "$cache"
	fi
	mkdir -p "$cache"; echo "$fmt" > "$cache/.fmt"

	: > packages.info; : > packages.equiv; : > files.list
	t=$(mktemp -d)
	# Map filename->md5 once, then emit "md5<TAB>pkg" in glob order (the
	# sorted order the old `for pkg in *.tazpkg` loop used). One awk pass
	# instead of an awk-per-package scan of packages.md5 — that lookup
	# alone was ~0.75s on a 600-pkg depot.
	ordered=$(mktemp)
	printf '%s\n' *.tazpkg | awk 'NR==FNR{m[$2]=$1;next} m[$0]!=""{print m[$0]"\t"$0}' \
		packages.md5 - > "$ordered"

	# Extract cache misses only. On a warm run the test below is a shell
	# builtin (no fork per package), so unchanged packages cost almost
	# nothing; only new/changed ones spawn cpio.
	while IFS='	' read md5 pkg; do
		# Reuse only a COMPLETE cache entry (all three fragments). .i is
		# written last so its presence normally implies .f/.d, but a depot
		# carrying a half-written set from an older gendb still gets
		# re-extracted rather than served a missing files.list.
		if [ -s "$cache/$md5.i" ] && [ -f "$cache/$md5.f" ] && \
			[ -f "$cache/$md5.d" ]; then
			reused=$((reused + 1)); continue
		fi
		rm -rf "$t"/* 2>/dev/null
		( cd "$t" && cpio -i --quiet receipt files.list md5sum \
			description.txt ) < "$pkgsdir/$pkg" 2>/dev/null
		[ -f "$t/receipt" ] || continue
		# The receipt is sourced and every fragment written inside ONE
		# subshell: a broken receipt (a syntax error aborts the shell
		# that sources it) kills the subshell, not gendb — the package
		# is skipped and, having no .i, gets retried on the next run.
		if ! (
		unset PACKAGE VERSION EXTRAVERSION CATEGORY SHORT_DESC WEB_SITE \
			TAGS PACKED_SIZE UNPACKED_SIZE DEPENDS RSUM PROVIDE
		. "$t/receipt" >/dev/null 2>&1 || exit 1
		[ -n "$PACKAGE" ] || exit 1
		sizes=$(echo $PACKED_SIZE $UNPACKED_SIZE | sed 's|\.0||g')
		# RSUM fallback (older receipts lack it) — same recipe as cook-pkgdb.
		rsum="$RSUM"
		if [ -z "$rsum" ] && [ -f "$t/md5sum" ]; then
			{ cat "$t/md5sum"; md5sum "$t/receipt" | sed 's| [^ ]*/| |'
			  [ -f "$t/description.txt" ] && \
				md5sum "$t/description.txt" | sed 's| [^ ]*/| |'
			} > "$t/.rsum"
			rsum=$(md5sum "$t/.rsum" | awk '{print $1}')
		fi
		# Field 11 (noarch flag) only on x86_64/arm; i486 = 10 fields.
		noarch=''
		case "$SLITAZ_ARCH" in
			arm*|x86_64)
				case "$pkg" in
					*-any.tazpkg) noarch='\t0' ;;
					*)            noarch='\t1' ;;
				esac ;;
		esac
		# Write .f and .d FIRST, then .i LAST (atomically). The warm-run reuse
		# test keys on .i alone, so making .i the last fragment written turns
		# it into this entry's commit marker: a gendb interrupted mid-package
		# (Ctrl-C, OOM) leaves no .i, and the package is re-extracted next run
		# instead of being served with a missing/partial files.list or
		# description (the old order wrote .i first, so an interrupt after it
		# stranded a valid-looking .i beside absent .f/.d).
		if [ -f "$t/files.list" ]; then
			sed "s|^|$PACKAGE: |" "$t/files.list" > "$cache/$md5.f"
		else
			: > "$cache/$md5.f"
		fi
		# descriptions.txt fragment — same block layout as cook-pkgdb:
		# a PACKAGE header line, the body with blank lines blanked to a
		# single space, a guaranteed trailing newline, then a blank
		# separator. Empty when the package ships no description.txt.
		if [ -f "$t/description.txt" ]; then
			{ echo "$PACKAGE"
			  sed 's|^$| |' "$t/description.txt"
			  [ -z "$(tail -c1 "$t/description.txt")" ] || echo
			  echo
			} > "$cache/$md5.d"
		else
			: > "$cache/$md5.d"
		fi
		# .i last, via temp+rename so it is all-or-nothing (a truncated .i
		# from an interrupt mid-write would otherwise be reused as valid).
		printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s%b\n' \
			"$PACKAGE" "${VERSION}${EXTRAVERSION}" "$CATEGORY" \
			"$SHORT_DESC" "$WEB_SITE" "$(echo $TAGS)" "$sizes" \
			"$(echo $DEPENDS)" "$rsum" "$(echo $PROVIDE)" "$noarch" \
			> "$cache/$md5.i.tmp" && mv "$cache/$md5.i.tmp" "$cache/$md5.i"
		); then
			rm -f "$cache/$md5.f" "$cache/$md5.d" "$cache/$md5.i"
			colorize 31 "  Skipping broken receipt: $pkg" >&2
			continue
		fi
		built=$((built + 1))
	done < "$ordered"
	rm -rf "$t"

	# Assemble packages.info + files.list from the cache in glob order with
	# one xargs cat each (a couple of execs total, not a cat+cut per
	# package — that loop body was ~4.3s on a 600-pkg depot). Byte-identical
	# to a full rebuild since the order is unchanged.
	awk -v c="$cache" '{print c"/"$1".i"}' "$ordered" | tr '\n' '\0' \
		| xargs -0 cat > packages.info 2>/dev/null
	awk -v c="$cache" '{print c"/"$1".f"}' "$ordered" | tr '\n' '\0' \
		| xargs -0 cat > files.list 2>/dev/null
	# descriptions.txt — needed by tazpkg (it goes into bundle.tar.lzma)
	# and by spk-search. Same glob order; empty fragments contribute
	# nothing, matching cook-pkgdb which only emits a block per package
	# that actually ships a description.txt.
	awk -v c="$cache" '{print c"/"$1".d"}' "$ordered" | tr '\n' '\0' \
		| xargs -0 cat > descriptions.txt 2>/dev/null
	rm -f "$ordered"

	# packages.equiv in a single awk over packages.info (field 1 = PACKAGE,
	# field 10 = PROVIDE) instead of a grep + in-place sed per PROVIDE
	# (O(n^2), ~2s). Same output: providers prepended (reverse glob order),
	# keys in first-seen order, "alias:provider" kept for name:dest forms.
	awk -F'\t' '
		{
			pkg=$1; np=split($10, pr, " ")
			for(j=1;j<=np;j++){
				p=pr[j]; ci=index(p,":")
				if(ci){ key=substr(p,1,ci-1); ent=substr(p,ci+1)":"pkg }
				else  { key=p; ent=pkg }
				if(key in seen) map[key]=ent" "map[key]
				else { map[key]=ent; seen[key]=1; ord[++k]=key }
			}
		}
		END{ for(i=1;i<=k;i++) print ord[i]"="map[ord[i]] }
	' packages.info > packages.equiv

	# Prune cache fragments whose package is no longer in the depot.
	for cf in "$cache"/*.i "$cache"/*.f "$cache"/*.d; do
		[ -f "$cf" ] || continue
		m=$(basename "$cf"); m=${m%.*}
		grep -q "^$m  " packages.md5 || rm -f "$cf"
	done

	# files-list.lzma needs an lzma compressor — optional for spk (only
	# spk-search --file uses it). Skip with a warning on minimal envs.
	if command -v lzma >/dev/null 2>&1; then
		gettext "Compressing files-list..."
		sort -k2 files.list > files.list.sorted
		lzma -c files.list.sorted > files-list.lzma
		rm -f files.list.sorted
		$CHECKSUM files-list.lzma | cut -d ' ' -f1 | tr -d '\n' > files-list.md5
		status
		# bundle.tar.lzma — what tazpkg's recharge actually downloads (it
		# no longer fetches the loose files; a missing bundle = recharge
		# failure). Same member set as cook-pkgdb's MAKE_BUNDLE. We can't
		# reach mirror1 for the real mirrors/extra.list from a consumer
		# box, so ship empty placeholders like cook-pkgdb's offline path.
		gettext "Bundling DB for tazpkg..."
		# Only synthesize empty placeholders when the depot doesn't already
		# ship these lists: a real extra.list from cook-pkgdb was being blanked
		# by `: > extra.list` and then bundled empty, so clients recharged an
		# empty extra list. Remove afterwards only the placeholders we created.
		local made_mirrors=
		[ -f mirrors ]    || { : > mirrors; made_mirrors=yes; }
		# extra.list: create an empty one only if the depot ships none, and
		# leave it on disk afterwards (clients fetch it loose at recharge).
		# The old `: > extra.list` blanked a real cook-pkgdb list.
		[ -f extra.list ] || : > extra.list
		busybox tar -chf bundle.tar mirrors extra.list files-list.md5 \
			packages.info descriptions.txt packages.md5 packages.list \
			packages.equiv 2>/dev/null
		lzma -c bundle.tar > bundle.tar.lzma
		rm -f bundle.tar
		# mirrors was historically a transient placeholder; only remove the
		# one we synthesized, never a real file already in the depot.
		[ "$made_mirrors" ] && rm -f mirrors
		status
	else
		gettext "No lzma: skipping files-list.lzma + bundle (tazpkg recharge off)"
		newline
	fi
	rm -f files.list

	# Publish the change marker last: every list above is now in place, so a
	# client that sees this new ID is guaranteed a complete, consistent DB.
	printf '%s\n' "$newid" > ID
	{ printf '%s ' "$newid"; date +%s; } > IDs
	separator
	gettext "Packages indexed:"; colorize 32 " $(grep -c . packages.info)"
	[ "$reused" -gt 0 ] && { gettext " (extracted"; colorize 32 " $built"
		gettext ", reused from cache"; colorize 32 " $reused"; echo ")"; }
	newline
	cd - >/dev/null
}

#
# Commands
#

cmd=$1
[ -d "$cmd" ] && cmd=continue

case "$cmd" in
	*usage|*help) usage ;;
	http://*|ftp://|'continue') continue ;;
	undigest)
		echo "TODO: enable official undigest repo" ;;
	gendb)
		# Build the package lists from a dir of .tazpkg (no wok). Default
		# to the configured main mirror path when no dir is given.
		dir="$2"
		[ "$dir" ] || dir=$(cat $mirrorurl 2>/dev/null)
		case "$dir" in file://*) dir=${dir#file://} ;; esac
		dir=${dir%/}
		if [ ! -d "$dir" ]; then
			gettext "Not a local directory:"; colorize 31 " $dir"
			newline && exit 1
		fi
		if [ -z "$(ls $dir/*.tazpkg 2>/dev/null)" ]; then
			gettext "No packages found in:"; colorize 31 " $dir"
			newline && exit 1
		fi
		gen_pkgdb "$dir"
		# Propagate gen_pkgdb's status so a refused run (lock held) or a
		# failure is visible to scripts/cron, instead of a blanket exit 0.
		exit $? ;;
	""|--list)
		extra=0
		excolor=32
		newline
		boldify "Mirror info"
		separator
		gettext "Main URL :"
		boldify " $(cat $mirrorurl)"
		if [ -f "$mirrors" ]; then
			nb=$(cat $mirrors | wc -l)
			gettext "Mirrors  :"; colorize $excolor " $nb "
		fi
		if [ -d "$extradb" ]; then
			extra=$(ls $extradb | wc -l)
		fi
		[ "$extra" != 0 ] || excolor=31
		gettext "Extra    :"; colorize $excolor " $extra"
		separator
		newline
		if [ "$list" ]; then
			boldify $(gettext "Official mirrors")
			separator
			cat $mirrors
			separator && newline
			if [ "$extra" != "0" ]; then
				boldify $(gettext "Extra mirrors")
				separator
				cat $extradb/*/mirror
				separator && newline
			fi
		fi
		exit 0 ;;
esac

#
# Handle all urls
#

count=0

for url in $@
do
	[ -d "$url" ] && url="dir:$url"
	case "$url" in
		http://*|https://*|ftp://*|local)
			[ "$count" = 0 ] && newline
			count=$(($count + 1))
			url="${url%/}/"
			debug "checking: $url"
			# Remove extra on --rm
			if [ "$rm" ]; then
				extra_db
				if [ -d "$extradb/$db" ]; then
					gettext "Removing extra mirror:"
					echo -n " $(boldify $db)"
					rm -rf $extradb/$db && status
				else
					gettext "Not an extra mirror:"; colorize 31 " $db"
				fi
				newline && continue
			fi
			# Official mirror
			if fgrep -q "$url" $mirrors; then
				gettext "Enabling: official mirror"; newline
				gettext "Architecture:"; echo " $SLITAZ_ARCH"
				# Handle arch type
				case "$SLITAZ_ARCH" in
					arm|x86_64) arch="$SLITAZ_ARCH/" ;;
					i486) arch="/" ;;
				esac
				url=${url}packages/${SLITAZ_RELEASE}${arch}
				echo "URL: $(boldify $url)"
				echo "$url" > $mirrorurl
				newline && continue
			fi
			# Extra mirror
			extra_db
			if [ -d "$extradb/$db" ]; then
				gettext "Extra mirror already exists"; newline
				extra_info $db
			else
				boldify $(gettext "Enabling: extra mirror")
				separator
				echo "URL: $(boldify $url)"
				if ! check_mirror $url; then
					continue
				fi
				mkdir -p $extradb/$db
				echo "$url" > $extradb/$db/mirror
				separator
				gettext "New extra mirror is ready to use"; newline
			fi
			newline ;;
		dir:*)
			# Register a local directory as a mirror by symlinking its
			# package lists. The lists come from cook-pkgdb (build host,
			# from the wok) or from 'spk-mirror gendb <dir>' (consumer
			# side, from the .tazpkg) — this just consumes them.
			dir=${url#dir:}
			pkgsdir=${dir%/}
			# Normalize to an absolute path. `spk-mirror ./packages` (or a
			# bare relative name) otherwise wrote "./packages/" into the
			# mirror file and created symlinks with relative targets resolved
			# from $extradb/local (dead links), and download() treats only
			# /*-style paths as local — so every install from the depot ran
			# wget on "./packages/…" and failed.
			case "$pkgsdir" in
				/*) ;;
				*)  pkgsdir=$(cd "$pkgsdir" 2>/dev/null && pwd) || pkgsdir=${dir%/} ;;
			esac
			debug "dir: $dir"
			nb=$(ls $pkgsdir/*.tazpkg 2>/dev/null | wc -l)
			if [ "$nb" = 0 ]; then
				gettext "No packages found in:"; boldify " $pkgsdir"
				newline && continue
			fi
			# --gendb: (re)build the lists from the .tazpkg before registering.
			[ "$gendb" ] && gen_pkgdb "$pkgsdir"
			if [ ! -f "$pkgsdir/packages.info" ]; then
				gettext "No package lists in:"; boldify " $pkgsdir"
				newline
				gettext "Build them first: spk-mirror gendb $pkgsdir"
				newline && continue
			fi
			newline
			gettext "Directory:"; boldify " $pkgsdir"
			gettext "Packages :"; colorize 32 " $nb"
			newline
			# Enable it as the local extra mirror via symlinks to the
			# cook-pkgdb lists (kept fresh in the chroot on demand).
			boldify $(gettext "Enabling extra local mirror")
			separator
			db=$extradb/local
			debug "database: $db"
			mkdir -p $db && rm -f $db/*
			echo "$pkgsdir/" > $db/mirror
			for list in packages.list packages.$SUM packages.info \
				packages.equiv files-list.lzma files-list.md5 \
				descriptions.txt extra.list ID
			do
				[ -f "$pkgsdir/$list" ] || continue
				echo -n $(gettext "Linking:") $list
				ln -sf $pkgsdir/$list $db/$list
				status
			done
			separator && newline ;;
		--*) continue ;;
		*)
			[ "$count" = 0 ] && newline
			count=$(($count + 1))
			# Should we handle --rm also here ?
			if [ -d "$extradb/$url" ]; then
				extra_info $(basename $extradb/$url)
				newline && continue
			fi
			gettext "Can't handle:"
			colorize 31 " $url"; newline ;;
	esac
done

exit 0
