qiurui
/
qtbase
mirror of https://github.com/qt/qtbase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

PSL update script: Change to use direct upstream source 

The upstream "Public Suffix List" .dat file has gained a comment line
with a "// VERSION:" field, so the work to extract a git SHA is not
needed, nor is it canonical with their new versioning. Though they also
embed the SHA from which the file originated, which we continue to use
for the PURL.

Furthermore the file now warns that any other source of the PSL other
than the one from publicsuffix.org is "not guaranteed to be supported".
So in this patch we move to use their canonical upstream http download:

https://publicsuffix.org/list/public_suffix_list.dat

Pick-to: 6.10 6.9 6.8 6.5 5.15
Task-number: QTBUG-137782
Change-Id: I50ec8ed6a55738560d9108ad08517c243c1bb518
Reviewed-by: Alexandru Croitor <alexandru.croitor@qt.io>
This commit is contained in:
Mårten Nordheim 2025-06-18 14:45:52 +02:00
parent 231321a91d
commit 1275d96d3c
1 changed files with 23 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
util/update_public_suffix_list.sh
View File

@ -3,11 +3,10 @@
# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
PICK_TO_BRANCHES="6.10 6.9 6.8 6.5 5.15" PICK_TO_BRANCHES="6.10 6.9 6.8 6.5 5.15"
#UPSTREAM=github.com:publicsuffix/list.git # use this if you have a github account UPSTREAM=https://publicsuffix.org/list/public_suffix_list.dat
UPSTREAM=https://github.com/publicsuffix/list.git # and this if you don't
THIS="util/update_public_suffix_list.sh" THIS="util/update_public_suffix_list.sh"
PUBLIC_SUFFIX_LIST_DAT_DIR="$1" PUBLIC_SUFFIX_LIST_DAT_DIR="$(mktemp -d)"
MAKE_DAFSA=src/3rdparty/libpsl/src/psl-make-dafsa MAKE_DAFSA=src/3rdparty/libpsl/src/psl-make-dafsa
PSL_DATA_CPP=src/3rdparty/libpsl/psl_data.cpp PSL_DATA_CPP=src/3rdparty/libpsl/psl_data.cpp
PUBLIC_SUFFIX_LIST_DAFSA=tests/auto/network/access/qnetworkcookiejar/testdata/publicsuffix/public_suffix_list.dafsa PUBLIC_SUFFIX_LIST_DAFSA=tests/auto/network/access/qnetworkcookiejar/testdata/publicsuffix/public_suffix_list.dafsa
@ -26,7 +25,7 @@ function check_or_die() {
TYPE=$1 TYPE=$1
OP=$2 OP=$2
FILE="$3" FILE="$3"
test $OP "$FILE" || die "$TYPE \"$FILE\" not found (test $OP \"$FILE\" failed). Please run $THIS from \$SRCDIR/qtbase and pass the directory containing a checkout of $UPSTEAM on the command line." test $OP "$FILE" || die "$TYPE \"$FILE\" not found (test $OP \"$FILE\" failed). Please run $THIS from \$SRCDIR/qtbase."
} }
function run_or_die() { function run_or_die() {
@ -35,36 +34,33 @@ function run_or_die() {
msg "Done" msg "Done"
} }
INPUT="$PUBLIC_SUFFIX_LIST_DAT_DIR/public_suffix_list.dat" DAT_FILE="$PUBLIC_SUFFIX_LIST_DAT_DIR/public_suffix_list.dat"
check_or_die tool -x "$MAKE_DAFSA" check_or_die tool -x "$MAKE_DAFSA"
if [ ! -d "$PUBLIC_SUFFIX_LIST_DAT_DIR" ]; then check_or_die directory -d "$PUBLIC_SUFFIX_LIST_DAT_DIR"
msg -n "$PUBLIC_SUFFIX_LIST_DAT_DIR does not exist; Clone $UPSTREAM there? [y/N]" run_or_die wget $UPSTREAM -O "$DAT_FILE"
read -N1 -t60
msg check_or_die input -r "$DAT_FILE"
if [ "x$REPLY" = "xy" -o "x$REPLY" = "xY" ]; then
run_or_die git clone "$UPSTREAM" "$PUBLIC_SUFFIX_LIST_DAT_DIR"
else
check_or_die publicsuffix/list.git -d "$PUBLIC_SUFFIX_LIST_DAT_DIR" # reuse error message
fi
fi
check_or_die publicsuffix/list.git -d "$PUBLIC_SUFFIX_LIST_DAT_DIR"
check_or_die input -r "$INPUT"
check_or_die output -w "$PSL_DATA_CPP" check_or_die output -w "$PSL_DATA_CPP"
check_or_die binary-output -w "$PUBLIC_SUFFIX_LIST_DAFSA" check_or_die binary-output -w "$PUBLIC_SUFFIX_LIST_DAFSA"
GITSHA1=$(cd "$PUBLIC_SUFFIX_LIST_DAT_DIR" && git log -1 --format=format:%H) VERSION=$(run_or_die sed -nE 's,^// VERSION: (.*)$,\1,p' "$DAT_FILE")
GITDATE=$(cd "$PUBLIC_SUFFIX_LIST_DAT_DIR" && git log -1 --format=format:%cs) if [[ -z "$VERSION" ]]; then
msg "Using $INPUT @ $GITSHA1, fetched on $GITDATE" die "Something is wrong! Recheck the VERSION line in $DAT_FILE and update the script."
fi
GITSHA=$(run_or_die sed -nE 's,^// COMMIT: (.*)$,\1,p' "$DAT_FILE")
msg "Using $DAT_FILE @ ${VERSION} (commit ${GITSHA})"
run_or_die "$MAKE_DAFSA" "$INPUT" "$PSL_DATA_CPP" run_or_die "$MAKE_DAFSA" "$DAT_FILE" "$PSL_DATA_CPP"
run_or_die "$MAKE_DAFSA" --output-format=binary "$INPUT" "$PUBLIC_SUFFIX_LIST_DAFSA" run_or_die "$MAKE_DAFSA" --output-format=binary "$DAT_FILE" "$PUBLIC_SUFFIX_LIST_DAFSA"
rm "$DAT_FILE"
# update the first Version line in qt_attribution.json with the new SHA1 and date: # update the first Version line in qt_attribution.json with the new SHA1 and date:
run_or_die sed -i -e "1,/\"Version\":/{ /\"Version\":/ { s/[0-9a-fA-F]\{40\}/$GITSHA1/; s/[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}/$GITDATE/ } }" "$ATTRIBUTION_JSON" run_or_die sed -i -E -e "1,/\"Version\":/{ /\"Version\":/ { s/(Version\": )\".*?\"/\\1\"${VERSION}\"/ } }" "$ATTRIBUTION_JSON"
# update the first "PURL" line with the new SHA1: # update the first "PURL" line with the new SHA1:
run_or_die sed -i -e "1,/\"PURL\":/{ /\"PURL\":/ { s/[0-9a-fA-F]\{40\}/$GITSHA1/; } }" "$ATTRIBUTION_JSON" run_or_die sed -i -E -e "1,/\"PURL\":/{ /\"PURL\":/ { s/@.*?\\?/@${GITSHA}\\?/ } }" "$ATTRIBUTION_JSON"
run_or_die git add "$PSL_DATA_CPP" run_or_die git add "$PSL_DATA_CPP"
run_or_die git add "$PUBLIC_SUFFIX_LIST_DAFSA" run_or_die git add "$PUBLIC_SUFFIX_LIST_DAFSA"
@ -72,13 +68,12 @@ run_or_die git add "$ATTRIBUTION_JSON"
run_or_die git commit -m "Update public suffix list run_or_die git commit -m "Update public suffix list
Version $GITSHA1, fetched on Version ${VERSION}.
$GITDATE.
[ChangeLog][Third-Party Code] Updated the public suffix list to upstream [ChangeLog][Third-Party Code] Updated the public suffix list to upstream
SHA $GITSHA1. version ${VERSION}.
Pick-to: $PICK_TO_BRANCHES Pick-to: $PICK_TO_BRANCHES
" --edit " --edit
msg "Please use topic=publicsuffix-list-$GITSHA1 when pushing." msg "Please use topic=publicsuffix-list-${VERSION} when pushing."