mirror of https://github.com/armbian/build.git
Repo management: rework to increase reliability and speed
- introduce common repo so we only add packages once - publish via snapshots to increase reliability - overwrite packages in case of conflicts
This commit is contained in:
Igor Pecovnik
Igor
parent
0621b9008f
commit
239f3a5dc3
|
|
@ -9,7 +9,7 @@
|
|||
"dependencyFollowSource": false,
|
||||
"gpgDisableSign": false,
|
||||
"gpgDisableVerify": false,
|
||||
"gpgProvider": "internal",
|
||||
"gpgProvider": "gpg2",
|
||||
"downloadSourcePackages": false,
|
||||
"ppaDistributorID": "ubuntu",
|
||||
"ppaCodename": "",
|
||||
|
|
|
|||
|
|
@ -3,11 +3,15 @@
|
|||
# Drop unsupported releases
|
||||
drop_unsupported_releases() {
|
||||
|
||||
echo "Cleanup: dropping unsupported"
|
||||
if [[ "$1" == "all" ]]; then
|
||||
echo "Cleanup: dropping published repositories" | sudo tee -a "${DEBUGFILE}"
|
||||
BUILD_FW=()
|
||||
else
|
||||
echo "Cleanup: dropping unsupported" | sudo tee -a "${DEBUGFILE}"
|
||||
BUILD_FW=($(grep -rw config/distributions/*/support -ve 'eos' | cut -d"/" -f3))
|
||||
fi
|
||||
|
||||
BUILD_FW=($(grep -rw config/distributions/*/support -ve 'eos' | cut -d"/" -f3))
|
||||
REPO=($(aptly publish list -config="${CONFIG}" --raw | sed "s/. //g"))
|
||||
|
||||
DROP=()
|
||||
for i in "${REPO[@]}"; do
|
||||
skip=
|
||||
|
|
@ -19,11 +23,26 @@ drop_unsupported_releases() {
|
|||
|
||||
# drop
|
||||
for i in "${DROP[@]}"; do
|
||||
aptly publish drop -config="${CONFIG}" "${i}"
|
||||
aptly publish drop -config="${CONFIG}" "${i}" | sudo tee -a "${DEBUGFILE}" >/dev/null
|
||||
done
|
||||
}
|
||||
|
||||
|
||||
# Display repository content
|
||||
#
|
||||
showall(){
|
||||
|
||||
echo "Displaying common repository contents"
|
||||
aptly repo show -with-packages -config="${CONFIG}" common | tail -n +7
|
||||
for release in "${DISTROS[@]}"; do
|
||||
echo "Displaying repository contents for $release"
|
||||
aptly repo show -with-packages -config="${CONFIG}" "${release}" | tail -n +7
|
||||
echo "Displaying repository contents for $release-utils"
|
||||
aptly repo show -with-packages -config="${CONFIG}" "${release}-utils" | tail -n +7
|
||||
echo "Displaying repository contents for $release-desktop"
|
||||
aptly repo show -with-packages -config="${CONFIG}" "${release}-desktop" | tail -n +7
|
||||
done
|
||||
}
|
||||
|
||||
|
||||
# Adding package
|
||||
|
|
@ -35,45 +54,15 @@ drop_unsupported_releases() {
|
|||
#
|
||||
adding_packages() {
|
||||
# add deb files to repository if they are not already there
|
||||
for f in "${4}${2}"/*.deb; do
|
||||
|
||||
local package name version arch
|
||||
# read package
|
||||
package=$(dpkg-deb -I "${f}")
|
||||
name=$(echo "${package}" | awk /Package/'{print $2}')
|
||||
version=$(echo "${package}" | awk /Version/'{print $2}')
|
||||
arch=$(echo "${package}" | awk /Architecture/'{print $2}')
|
||||
# add if not already there
|
||||
aptly repo search -architectures="${arch}" -config="${CONFIG}" "${1}" \
|
||||
'Name (% '${name}'), $Version (='${version}'), $Architecture (='${arch}')' &> /dev/null
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo -e "Checking and adding \x1B[92m$name\x1B[0m to repository \x1B[92m$release $3\x1B[0m"
|
||||
aptly repo add -config="${CONFIG}" "${1}" "${f}" &> /dev/null
|
||||
if ! find "${4}${2}" -maxdepth 1 -type f -name "*.deb" 2> /dev/null | grep -q .; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
done
|
||||
for f in "${4}${2}"/*.deb; do
|
||||
aptly repo add -remove-files -force-replace -config="${CONFIG}" "${1}" "${f}"
|
||||
done
|
||||
}
|
||||
|
||||
|
||||
fake_package()
|
||||
{
|
||||
fake_package_dir=$2
|
||||
tmp_dir=$(mktemp -d)
|
||||
mkdir -p "${tmp_dir}/${fake_package_dir}"/DEBIAN/
|
||||
# set up control file
|
||||
cat <<- END > "${tmp_dir}/${fake_package_dir}"/DEBIAN/control
|
||||
Package: empty
|
||||
Version: $3
|
||||
Architecture: all
|
||||
Description: Fake pacakge
|
||||
Maintainer: Armbian
|
||||
END
|
||||
dpkg-deb --build "${tmp_dir}/${fake_package_dir}" &> /dev/null
|
||||
aptly repo add -force-replace=true -config="${CONFIG}" "${1}" "${tmp_dir}/${fake_package_dir}" &> /dev/null
|
||||
}
|
||||
|
||||
|
||||
|
||||
# publishing repository
|
||||
#
|
||||
# $1: Input folder
|
||||
|
|
@ -81,85 +70,62 @@ aptly repo add -force-replace=true -config="${CONFIG}" "${1}" "${tmp_dir}/${fake
|
|||
# $3: Command
|
||||
# $4: GPG password
|
||||
# $5: jammy,sid
|
||||
#
|
||||
publishing() {
|
||||
|
||||
# read comma delimited distros into array
|
||||
IFS=', ' read -r -a DISTROS <<< "$5"
|
||||
local errors=0
|
||||
# publish all, update selected
|
||||
local distributions=($(grep -rw config/distributions/*/support -ve '' | cut -d"/" -f3))
|
||||
for release in "${distributions[@]}"; do
|
||||
publishing(){
|
||||
|
||||
local forceoverwrite=""
|
||||
|
||||
ADDING_PACKAGES="false"
|
||||
# shellcheck disable=SC2207,2199
|
||||
if [[ " ${DISTROS[@]} " =~ " ${release} " ]] ; then
|
||||
ADDING_PACKAGES="true"
|
||||
# this repository contains packages that are the same in all releases.
|
||||
if [[ -z $(aptly repo list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep common) ]]; then
|
||||
aptly repo create -config="${CONFIG}" -distribution="common" -component="main" -comment="Armbian common packages" "common" | sudo tee -a "${DEBUGFILE}" >/dev/null
|
||||
fi
|
||||
|
||||
# don't add packages to eos
|
||||
if [[ ${status} == $(cat config/distributions/${release}/support) ]]; then
|
||||
ADDING_PACKAGES="false"
|
||||
fi
|
||||
# add packages from main folder
|
||||
adding_packages "common" "" "main" "$1"
|
||||
|
||||
if [[ ${ADDING_PACKAGES} == true ]]; then
|
||||
echo "Adding: ${release}"
|
||||
# create snapshot
|
||||
UNIQUE_NAME=$(date +%s)
|
||||
if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "common") ]]; then
|
||||
aptly -config="${CONFIG}" snapshot drop common | sudo tee -a "${DEBUGFILE}" >/dev/null
|
||||
fi
|
||||
aptly -config="${CONFIG}" snapshot create common from repo common | sudo tee -a "${DEBUGFILE}" >/dev/null
|
||||
|
||||
# let's drop from publish if exits
|
||||
if [[ -n $(aptly publish list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}") ]]; then
|
||||
aptly publish drop -config="${CONFIG}" "${release}" > /dev/null 2>&1
|
||||
fi
|
||||
# create local repository if not exist
|
||||
if [[ -z $(aptly repo list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}") ]]; then
|
||||
aptly repo create -config="${CONFIG}" -distribution="${release}" \
|
||||
-component="main,${release}-utils,${release}-desktop" -comment="Armbian main repository" "${release}" > /dev/null 2>&1
|
||||
fake_package "${release}" test 1234
|
||||
fi
|
||||
if [[ -z $(aptly repo list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-utils") ]]; then
|
||||
aptly repo create -config="${CONFIG}" -distribution="${release}" \
|
||||
-component="${release}-utils" -comment="Armbian ${release} utilities" "${release}-utils" > /dev/null 2>&1
|
||||
fake_package "${release}" test 1234
|
||||
fi
|
||||
if [[ -z $(aptly repo list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-desktop") ]]; then
|
||||
aptly repo create -config="${CONFIG}" -distribution="${release}" \
|
||||
-component="${release}-desktop" -comment="Armbian ${release} desktop" "${release}-desktop" > /dev/null 2>&1
|
||||
fake_package "${release}" test 1234
|
||||
fi
|
||||
# adding main
|
||||
if find "$1"/ -maxdepth 1 -type f -name "*.deb" 2> /dev/null | grep -q .; then
|
||||
[[ "${ADDING_PACKAGES}" == true ]] && adding_packages "$release" "" "main" "$1"
|
||||
fi
|
||||
# make it for all that exists. It costs little extra time
|
||||
local distributions=($(grep -rw config/distributions/*/support -ve '' | cut -d"/" -f3))
|
||||
for release in "${distributions[@]}"; do
|
||||
|
||||
local COMPONENTS="main"
|
||||
# adding release-specific main
|
||||
if find "${1}/${release}" -maxdepth 1 -type f -name "*.deb" 2> /dev/null | grep -q .; then
|
||||
[[ "${ADDING_PACKAGES}" == true ]] && adding_packages "${release}" "/${release}" "release packages" "$1"
|
||||
fi
|
||||
# create for each one
|
||||
if [[ -z $(aptly repo list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}") ]]; then
|
||||
aptly repo create -config="${CONFIG}" -component="${release}" -distribution="${release}" -comment="Armbian ${release} repository" "${release}" | sudo tee -a "${DEBUGFILE}" >/dev/null
|
||||
fi
|
||||
if [[ -z $(aptly repo list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-utils") ]]; then
|
||||
aptly repo create -config="${CONFIG}" -component="${release}-utils" -distribution="${release}" -comment="Armbian ${release}-utils repository" "${release}-utils" | sudo tee -a "${DEBUGFILE}" >/dev/null
|
||||
fi
|
||||
if [[ -z $(aptly repo list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-desktop") ]]; then
|
||||
aptly repo create -config="${CONFIG}" -component="${release}-desktop" -distribution="${release}" -comment="Armbian ma${release}-desktop repository" "${release}-desktop" | sudo tee -a "${DEBUGFILE}" >/dev/null
|
||||
fi
|
||||
|
||||
# adding release-specific utils
|
||||
if find "${1}/extra/${release}-utils" -maxdepth 1 -type f -name "*.deb" 2> /dev/null | grep -q .; then
|
||||
[[ "${ADDING_PACKAGES}" == true ]] && adding_packages "${release}-utils" "/extra/${release}-utils" "release utils" "$1"
|
||||
fi
|
||||
COMPONENTS="${COMPONENTS} ${release}-utils"
|
||||
# adding release-specific desktop
|
||||
if find "${1}/extra/${release}-desktop" -maxdepth 1 -type f -name "*.deb" 2> /dev/null | grep -q .; then
|
||||
[[ "${ADDING_PACKAGES}" == true ]] && adding_packages "${release}-desktop" "/extra/${release}-desktop" "desktop" "$1"
|
||||
fi
|
||||
adding_packages "${release}" "/${release}" "release packages" "$1"
|
||||
adding_packages "${release}-utils" "/extra/${release}-utils" "release utils" "$1"
|
||||
adding_packages "${release}-desktop" "/extra/${release}-desktop" "release desktop" "$1"
|
||||
|
||||
# drop release snapshot
|
||||
if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}") ]]; then
|
||||
aptly -config="${CONFIG}" snapshot drop ${release} | sudo tee -a "${DEBUGFILE}" 2>/dev/null
|
||||
fi
|
||||
# drop release utils snapshot
|
||||
if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-utils") ]]; then
|
||||
aptly -config="${CONFIG}" snapshot drop ${release}-utils | sudo tee -a "${DEBUGFILE}" 2>/dev/null
|
||||
fi
|
||||
# drop release desktop snapshot
|
||||
if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-desktop") ]]; then
|
||||
aptly -config="${CONFIG}" snapshot drop ${release}-desktop | sudo tee -a "${DEBUGFILE}" 2>/dev/null
|
||||
fi
|
||||
|
||||
aptly -config="${CONFIG}" snapshot create ${release} from repo ${release} | sudo tee -a "${DEBUGFILE}" >/dev/null
|
||||
aptly -config="${CONFIG}" snapshot create ${release}-utils from repo ${release}-utils | sudo tee -a "${DEBUGFILE}" >/dev/null
|
||||
aptly -config="${CONFIG}" snapshot create ${release}-desktop from repo ${release}-desktop | sudo tee -a "${DEBUGFILE}" >/dev/null
|
||||
|
||||
echo "Publishing $release"
|
||||
|
||||
COMPONENTS="${COMPONENTS} ${release}-desktop"
|
||||
local mainnum utilnum desknum
|
||||
mainnum=$(aptly repo show -with-packages -config="${CONFIG}" "${release}" | grep "Number of packages" | awk '{print $NF}')
|
||||
utilnum=$(aptly repo show -with-packages -config="${CONFIG}" "${release}-desktop" | grep "Number of packages" | awk '{print $NF}')
|
||||
desknum=$(aptly repo show -with-packages -config="${CONFIG}" "${release}-utils" | grep "Number of packages" | awk '{print $NF}')
|
||||
#if [ $mainnum -gt 0 ] && [ $utilnum -gt 0 ] && [ $desknum -gt 0 ]; then
|
||||
# write repo sync control file
|
||||
mkdir -p ${2}/public/
|
||||
sudo date +%s > ${2}/public/control
|
||||
# publish
|
||||
echo "Publishing: ${release}"
|
||||
aptly publish \
|
||||
-acquire-by-hash \
|
||||
-architectures="armhf,arm64,amd64,riscv64,i386,all" \
|
||||
|
|
@ -167,32 +133,18 @@ for release in "${distributions[@]}"; do
|
|||
-origin="Armbian" \
|
||||
-label="Armbian" \
|
||||
-config="${CONFIG}" \
|
||||
-component="${COMPONENTS// /,}" \
|
||||
-distribution="${release}" repo "${release}" ${COMPONENTS//main/} > /dev/null
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "Publishing failed ${release}"
|
||||
errors=$((errors + 1))
|
||||
exit 0
|
||||
fi
|
||||
#else
|
||||
# errors=$((errors + 1))
|
||||
# local err_txt=": All components must be present: main, utils and desktop for first build"
|
||||
#fi
|
||||
-component=main,${release},${release}-utils,${release}-desktop \
|
||||
-distribution="${release}" snapshot common ${release} ${release}-utils ${release}-desktop > /dev/null
|
||||
done
|
||||
# cleanup
|
||||
aptly db cleanup -config="${CONFIG}" > /dev/null
|
||||
aptly db cleanup -config="${CONFIG}"
|
||||
# key
|
||||
mkdir -p "${2}"/public/
|
||||
cp config/armbian.key "${2}"/public/
|
||||
# write repo sync control file
|
||||
sudo date +%s > ${2}/public/control
|
||||
# display what we have
|
||||
(aptly repo list -config="${CONFIG}") | grep -E packages
|
||||
# remove debs if no errors found
|
||||
if [[ $errors -eq 0 ]]; then
|
||||
echo "Purging incoming debs"
|
||||
sudo find "${1}" -name "*.deb" -type f -delete
|
||||
else
|
||||
echo "There were some problems $err_txt - leaving incoming directory intact" "err"
|
||||
fi
|
||||
showall
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -232,29 +184,25 @@ case $3 in
|
|||
return 0
|
||||
;;
|
||||
|
||||
delete)
|
||||
for release in "${DISTROS[@]}"; do
|
||||
echo "Deleting $6 from $release"
|
||||
aptly -config="${CONFIG}" repo remove "${release}" "$6"
|
||||
echo "Deleting $6 from $release-utils"
|
||||
aptly -config="${CONFIG}" repo remove "${release}-utils" "$6"
|
||||
echo "Deleting $6 from $release-desktop"
|
||||
aptly -config="${CONFIG}" repo remove "${release}-desktop" "$6"
|
||||
done
|
||||
return 0
|
||||
;;
|
||||
delete)
|
||||
echo "Deleting $6 from common"
|
||||
aptly -config="${CONFIG}" repo remove common "$6"
|
||||
for release in "${DISTROS[@]}"; do
|
||||
echo "Deleting $6 from $release"
|
||||
aptly -config="${CONFIG}" repo remove "${release}" "$6"
|
||||
echo "Deleting $6 from $release-utils"
|
||||
aptly -config="${CONFIG}" repo remove "${release}-utils" "$6"
|
||||
echo "Deleting $6 from $release-desktop"
|
||||
aptly -config="${CONFIG}" repo remove "${release}-desktop" "$6"
|
||||
done
|
||||
return 0
|
||||
;;
|
||||
|
||||
show)
|
||||
|
||||
for release in "${DISTROS[@]}"; do
|
||||
echo "Displaying repository contents for $release"
|
||||
aptly repo show -with-packages -config="${CONFIG}" "${release}" | tail -n +7
|
||||
echo "Displaying repository contents for $release-utils"
|
||||
aptly repo show -with-packages -config="${CONFIG}" "${release}-utils" | tail -n +7
|
||||
echo "Displaying repository contents for $release-desktop"
|
||||
aptly repo show -with-packages -config="${CONFIG}" "${release}-desktop" | tail -n +7
|
||||
done
|
||||
showall
|
||||
return 0
|
||||
|
||||
;;
|
||||
|
||||
unique)
|
||||
|
|
@ -262,6 +210,7 @@ case $3 in
|
|||
IFS=$'\n'
|
||||
while true; do
|
||||
LIST=()
|
||||
LIST+=($(aptly repo show -with-packages -config="${CONFIG}" common | tail -n +7))
|
||||
for release in "${DISTROS[@]}"; do
|
||||
LIST+=($(aptly repo show -with-packages -config="${CONFIG}" "${release}" | tail -n +7))
|
||||
LIST+=($(aptly repo show -with-packages -config="${CONFIG}" "${release}-utils" | tail -n +7))
|
||||
|
|
@ -282,6 +231,7 @@ case $3 in
|
|||
exitstatus=$?
|
||||
exec 3>&-
|
||||
if [[ $exitstatus -eq 0 ]]; then
|
||||
aptly repo remove -config="${CONFIG}" "common" "$TARGET_VERSION"
|
||||
for release in "${DISTROS[@]}"; do
|
||||
aptly repo remove -config="${CONFIG}" "${release}" "$TARGET_VERSION"
|
||||
aptly repo remove -config="${CONFIG}" "${release}-utils" "$TARGET_VERSION"
|
||||
|
|
@ -298,7 +248,7 @@ case $3 in
|
|||
|
||||
update)
|
||||
# remove old releases from publishing
|
||||
drop_unsupported_releases
|
||||
drop_unsupported_releases "all"
|
||||
publishing "$1" "$2" "$3" "$4" "$5"
|
||||
;;
|
||||
|
||||
|
|
@ -347,6 +297,7 @@ Usage: $0 [ -short | --long ]
|
|||
SHORT=i:,l:,o:,c:,p:,r:,h
|
||||
LONG=input:,list:,output:,command:,password:,releases:,help
|
||||
OPTS=$(getopt -a -n repo --options $SHORT --longoptions $LONG -- "$@")
|
||||
DEBUGFILE="/var/log/repo-management.log"
|
||||
|
||||
VALID_ARGUMENTS=$# # Returns the count of arguments that are in short or long options
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue