Repo management: rework to increase reliability and speed

- introduce common repo so we only add packages once
- publish via snapshots to increase reliability
- overwrite packages in case of conflicts

tools/repository/aptly.conf

@@ -9,7 +9,7 @@
   "dependencyFollowSource": false,
   "gpgDisableSign": false,
   "gpgDisableVerify": false,
-  "gpgProvider": "internal",
+  "gpgProvider": "gpg2",
   "downloadSourcePackages": false,
   "ppaDistributorID": "ubuntu",
   "ppaCodename": "",

tools/repository/repo

@@ -3,11 +3,15 @@
 # Drop unsupported releases
 drop_unsupported_releases() {
 
-	echo "Cleanup: dropping unsupported"
+	if [[ "$1" == "all" ]]; then
-
+		echo "Cleanup: dropping published repositories" | sudo tee -a "${DEBUGFILE}"
+		BUILD_FW=()
+	else
+		echo "Cleanup: dropping unsupported" | sudo tee -a "${DEBUGFILE}"
 		BUILD_FW=($(grep -rw config/distributions/*/support -ve 'eos' | cut -d"/" -f3))
-	REPO=($(aptly publish list -config="${CONFIG}" --raw | sed "s/. //g"))
+	fi
 
+	REPO=($(aptly publish list -config="${CONFIG}" --raw | sed "s/. //g"))
 	DROP=()
 	for i in "${REPO[@]}"; do
 	skip=
@@ -19,11 +23,26 @@ drop_unsupported_releases() {
 
 	# drop
 	for i in "${DROP[@]}"; do
-		aptly publish drop -config="${CONFIG}" "${i}"
+		aptly publish drop -config="${CONFIG}" "${i}" | sudo tee -a "${DEBUGFILE}" >/dev/null
 	done
 }
 
 
+# Display repository content
+#
+showall(){
+
+		echo "Displaying common repository contents"
+		aptly repo show -with-packages -config="${CONFIG}" common | tail -n +7
+		for release in "${DISTROS[@]}"; do
+			echo "Displaying repository contents for $release"
+			aptly repo show -with-packages -config="${CONFIG}" "${release}" | tail -n +7
+			echo "Displaying repository contents for $release-utils"
+			aptly repo show -with-packages -config="${CONFIG}" "${release}-utils" | tail -n +7
+			echo "Displaying repository contents for $release-desktop"
+			aptly repo show -with-packages -config="${CONFIG}" "${release}-desktop" | tail -n +7
+		done
+}
 
 
 # Adding package
@@ -35,45 +54,15 @@ drop_unsupported_releases() {
 #
 adding_packages() {
 # add deb files to repository if they are not already there
-for f in "${4}${2}"/*.deb; do
+	if ! find "${4}${2}" -maxdepth 1 -type f -name "*.deb" 2> /dev/null | grep -q .; then
-
+	return 0
-	local package name version arch
-	# read package
-	package=$(dpkg-deb -I "${f}")
-	name=$(echo "${package}" | awk /Package/'{print $2}')
-	version=$(echo "${package}" | awk /Version/'{print $2}')
-	arch=$(echo "${package}" | awk /Architecture/'{print $2}')
-	# add if not already there
-	aptly repo search -architectures="${arch}" -config="${CONFIG}" "${1}" \
-	'Name (% '${name}'), $Version (='${version}'), $Architecture (='${arch}')' &> /dev/null
-	if [[ $? -ne 0 ]]; then
-		echo -e "Checking and adding \x1B[92m$name\x1B[0m to repository \x1B[92m$release $3\x1B[0m"
-		aptly repo add -config="${CONFIG}" "${1}" "${f}" &> /dev/null
 	fi
-
+	for f in "${4}${2}"/*.deb; do
-done
+			aptly repo add -remove-files -force-replace -config="${CONFIG}" "${1}" "${f}"
+	done
 }
 
 
-fake_package()
-{
-fake_package_dir=$2
-tmp_dir=$(mktemp -d)
-mkdir -p "${tmp_dir}/${fake_package_dir}"/DEBIAN/
-# set up control file
-cat <<- END > "${tmp_dir}/${fake_package_dir}"/DEBIAN/control
-Package: empty
-Version: $3
-Architecture: all
-Description: Fake pacakge
-Maintainer: Armbian
-END
-dpkg-deb --build "${tmp_dir}/${fake_package_dir}" &> /dev/null
-aptly repo add -force-replace=true -config="${CONFIG}" "${1}" "${tmp_dir}/${fake_package_dir}" &> /dev/null
-}
-
-
-
 # publishing repository
 #
 # $1: Input folder
@@ -81,85 +70,62 @@ aptly repo add -force-replace=true -config="${CONFIG}" "${1}" "${tmp_dir}/${fake
 # $3: Command
 # $4: GPG password
 # $5: jammy,sid
-#
-publishing() {
 
-# read comma delimited distros into array
+publishing(){
-IFS=', ' read -r -a DISTROS <<< "$5"
-local errors=0
-# publish all, update selected
-local distributions=($(grep -rw config/distributions/*/support -ve '' | cut -d"/" -f3))
-for release in "${distributions[@]}"; do
 
-	local forceoverwrite=""
+	# this repository contains packages that are the same in all releases.
-
+	if [[ -z $(aptly repo list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep common) ]]; then
-	ADDING_PACKAGES="false"
+		aptly repo create -config="${CONFIG}" -distribution="common" -component="main" -comment="Armbian common packages" "common" | sudo tee -a "${DEBUGFILE}" >/dev/null
-# shellcheck disable=SC2207,2199
-	if [[ " ${DISTROS[@]} " =~ " ${release} " ]] ; then
-		ADDING_PACKAGES="true"
 	fi
 
-	# don't add packages to eos
+	# add packages from main folder
-	if [[ ${status} == $(cat config/distributions/${release}/support) ]]; then
+	adding_packages "common" "" "main" "$1"
-		ADDING_PACKAGES="false"
-	fi
 
-	if [[ ${ADDING_PACKAGES} == true ]]; then
+	# create snapshot
-		echo "Adding: ${release}"
+	UNIQUE_NAME=$(date +%s)
+	if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "common") ]]; then
+	aptly -config="${CONFIG}" snapshot drop common | sudo tee -a "${DEBUGFILE}" >/dev/null
 	fi
+	aptly -config="${CONFIG}" snapshot create common from repo common | sudo tee -a "${DEBUGFILE}" >/dev/null
 
-	# let's drop from publish if exits
+	# make it for all that exists. It costs little extra time
-	if [[ -n $(aptly publish list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}") ]]; then
+	local distributions=($(grep -rw config/distributions/*/support -ve '' | cut -d"/" -f3))
-		aptly publish drop -config="${CONFIG}" "${release}" > /dev/null 2>&1
+	for release in "${distributions[@]}"; do
-	fi
+
-	# create local repository if not exist
+		# create for each one
 		if [[ -z $(aptly repo list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}") ]]; then
-		aptly repo create -config="${CONFIG}" -distribution="${release}" \
+		aptly repo create -config="${CONFIG}" -component="${release}" -distribution="${release}" -comment="Armbian ${release} repository" "${release}" | sudo tee -a "${DEBUGFILE}" >/dev/null
-		-component="main,${release}-utils,${release}-desktop" -comment="Armbian main repository" "${release}"  > /dev/null 2>&1
-		fake_package "${release}" test 1234
 		fi
 		if [[ -z $(aptly repo list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-utils") ]]; then
-		aptly repo create -config="${CONFIG}" -distribution="${release}" \
+		aptly repo create -config="${CONFIG}" -component="${release}-utils" -distribution="${release}" -comment="Armbian ${release}-utils repository" "${release}-utils" | sudo tee -a "${DEBUGFILE}" >/dev/null
-		-component="${release}-utils" -comment="Armbian ${release} utilities" "${release}-utils" > /dev/null 2>&1
-		fake_package "${release}" test 1234
 		fi
 		if [[ -z $(aptly repo list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-desktop") ]]; then
-		aptly repo create -config="${CONFIG}" -distribution="${release}" \
+		aptly repo create -config="${CONFIG}" -component="${release}-desktop" -distribution="${release}" -comment="Armbian ma${release}-desktop repository" "${release}-desktop" | sudo tee -a "${DEBUGFILE}" >/dev/null
-		-component="${release}-desktop" -comment="Armbian ${release} desktop" "${release}-desktop" > /dev/null 2>&1
-		fake_package "${release}" test 1234
-	fi
-	# adding main
-	if find "$1"/ -maxdepth 1 -type f -name "*.deb" 2> /dev/null | grep -q .; then
-		[[ "${ADDING_PACKAGES}" == true ]] && adding_packages "$release" "" "main" "$1"
 		fi
 
-	local COMPONENTS="main"
+		adding_packages "${release}" "/${release}" "release packages" "$1"
-	# adding release-specific main
+		adding_packages "${release}-utils" "/extra/${release}-utils" "release utils" "$1"
-	if find "${1}/${release}" -maxdepth 1 -type f -name "*.deb" 2> /dev/null | grep -q .; then
+		adding_packages "${release}-desktop" "/extra/${release}-desktop" "release desktop" "$1"
-		[[ "${ADDING_PACKAGES}" == true ]] && adding_packages "${release}" "/${release}" "release packages" "$1"
+
+		# drop release snapshot
+		if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}") ]]; then
+			aptly -config="${CONFIG}" snapshot drop ${release} | sudo tee -a "${DEBUGFILE}" 2>/dev/null
+		fi
+		# drop release utils snapshot
+		if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-utils") ]]; then
+			aptly -config="${CONFIG}" snapshot drop ${release}-utils | sudo tee -a "${DEBUGFILE}" 2>/dev/null
+		fi
+		# drop release desktop snapshot
+		if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-desktop") ]]; then
+			aptly -config="${CONFIG}" snapshot drop ${release}-desktop | sudo tee -a "${DEBUGFILE}" 2>/dev/null
 		fi
 
-	# adding release-specific utils
+		aptly -config="${CONFIG}" snapshot create ${release} from repo ${release} | sudo tee -a "${DEBUGFILE}" >/dev/null
-	if find "${1}/extra/${release}-utils" -maxdepth 1 -type f -name "*.deb" 2> /dev/null | grep -q .; then
+		aptly -config="${CONFIG}" snapshot create ${release}-utils from repo ${release}-utils | sudo tee -a "${DEBUGFILE}" >/dev/null
-		[[ "${ADDING_PACKAGES}" == true ]] && adding_packages "${release}-utils" "/extra/${release}-utils" "release utils" "$1"
+		aptly -config="${CONFIG}" snapshot create ${release}-desktop from repo ${release}-desktop | sudo tee -a "${DEBUGFILE}" >/dev/null
-	fi
+
-	COMPONENTS="${COMPONENTS} ${release}-utils"
+		echo "Publishing $release"
-	# adding release-specific desktop
-	if find "${1}/extra/${release}-desktop" -maxdepth 1 -type f -name "*.deb" 2> /dev/null | grep -q .; then
-		[[ "${ADDING_PACKAGES}" == true ]] && adding_packages "${release}-desktop" "/extra/${release}-desktop" "desktop" "$1"
-	fi
 
-	COMPONENTS="${COMPONENTS} ${release}-desktop"
-	local mainnum utilnum desknum
-	mainnum=$(aptly repo show -with-packages -config="${CONFIG}" "${release}" | grep "Number of packages" | awk '{print $NF}')
-	utilnum=$(aptly repo show -with-packages -config="${CONFIG}" "${release}-desktop" | grep "Number of packages" | awk '{print $NF}')
-	desknum=$(aptly repo show -with-packages -config="${CONFIG}" "${release}-utils" | grep "Number of packages" | awk '{print $NF}')
-	#if [ $mainnum -gt 0 ] && [ $utilnum -gt 0 ] && [ $desknum -gt 0 ]; then
-		# write repo sync control file
-		mkdir -p ${2}/public/
-		sudo date +%s > ${2}/public/control
-		# publish
-		echo "Publishing: ${release}"
 		aptly publish \
 			-acquire-by-hash \
 			-architectures="armhf,arm64,amd64,riscv64,i386,all" \
@@ -167,32 +133,18 @@ for release in "${distributions[@]}"; do
 			-origin="Armbian" \
 			-label="Armbian" \
 			-config="${CONFIG}" \
-			-component="${COMPONENTS// /,}" \
+			-component=main,${release},${release}-utils,${release}-desktop \
-			-distribution="${release}" repo "${release}" ${COMPONENTS//main/} > /dev/null
+			-distribution="${release}" snapshot common ${release} ${release}-utils ${release}-desktop > /dev/null
-		if [[ $? -ne 0 ]]; then
-			echo "Publishing failed ${release}"
-			errors=$((errors + 1))
-			exit 0
-		fi
-	#else
-	#	errors=$((errors + 1))
-	#	local err_txt=": All components must be present: main, utils and desktop for first build"
-	#fi
 done
 # cleanup
-aptly db cleanup -config="${CONFIG}" > /dev/null
+aptly db cleanup -config="${CONFIG}"
 # key
 mkdir -p "${2}"/public/
 cp config/armbian.key "${2}"/public/
+# write repo sync control file
+sudo date +%s > ${2}/public/control
 # display what we have
-(aptly repo list -config="${CONFIG}") | grep -E packages
+showall
-# remove debs if no errors found
-if [[ $errors -eq 0 ]]; then
-		echo "Purging incoming debs"
-		sudo find "${1}" -name "*.deb" -type f -delete
-else
-	echo "There were some problems $err_txt - leaving incoming directory intact" "err"
-fi
 }
 
 
@@ -233,6 +185,8 @@ case $3 in
 	;;
 
 	delete)
+			echo "Deleting $6 from common"
+			aptly -config="${CONFIG}" repo remove common "$6"
 			for release in "${DISTROS[@]}"; do
 				echo "Deleting $6 from $release"
 				aptly -config="${CONFIG}" repo remove "${release}" "$6"
@@ -246,15 +200,9 @@ case $3 in
 
 	show)
 
-		for release in "${DISTROS[@]}"; do
+		showall
-			echo "Displaying repository contents for $release"
-			aptly repo show -with-packages -config="${CONFIG}" "${release}" | tail -n +7
-			echo "Displaying repository contents for $release-utils"
-			aptly repo show -with-packages -config="${CONFIG}" "${release}-utils" | tail -n +7
-			echo "Displaying repository contents for $release-desktop"
-			aptly repo show -with-packages -config="${CONFIG}" "${release}-desktop" | tail -n +7
-		done
 		return 0
+
 	;;
 
 	unique)
@@ -262,6 +210,7 @@ case $3 in
 		IFS=$'\n'
 		while true; do
 			LIST=()
+			LIST+=($(aptly repo show -with-packages -config="${CONFIG}" common | tail -n +7))
 			for release in "${DISTROS[@]}"; do
 				LIST+=($(aptly repo show -with-packages -config="${CONFIG}" "${release}" | tail -n +7))
 				LIST+=($(aptly repo show -with-packages -config="${CONFIG}" "${release}-utils" | tail -n +7))
@@ -282,6 +231,7 @@ case $3 in
 			exitstatus=$?
 			exec 3>&-
 			if [[ $exitstatus -eq 0 ]]; then
+				aptly repo remove -config="${CONFIG}" "common" "$TARGET_VERSION"
 				for release in "${DISTROS[@]}"; do
 					aptly repo remove -config="${CONFIG}" "${release}" "$TARGET_VERSION"
 					aptly repo remove -config="${CONFIG}" "${release}-utils" "$TARGET_VERSION"
@@ -298,7 +248,7 @@ case $3 in
 
 	update)
 		# remove old releases from publishing
-		drop_unsupported_releases
+		drop_unsupported_releases "all"
 		publishing "$1" "$2" "$3" "$4" "$5"
 		;;
 
@@ -347,6 +297,7 @@ Usage: $0 [ -short | --long ]
 SHORT=i:,l:,o:,c:,p:,r:,h
 LONG=input:,list:,output:,command:,password:,releases:,help
 OPTS=$(getopt -a -n repo --options $SHORT --longoptions $LONG -- "$@")
+DEBUGFILE="/var/log/repo-management.log"
 
 VALID_ARGUMENTS=$# # Returns the count of arguments that are in short or long options