[oe-commits] org.oe.dev angstrom feed upload script: reuse file list to lessen load on server, add status output for the upload script

[koen commit]

angstrom feed upload script: reuse file list to lessen load on server, add status output for the upload script

Author: koen at openembedded.org
Branch: org.openembedded.dev
Revision: fbdba8d2fa1b51a743e0287b9d0adaee95687ffe
ViewMTN: http://monotone.openembedded.org/revision/info/fbdba8d2fa1b51a743e0287b9d0adaee95687ffe
Files:
1
contrib/angstrom/sort.sh
contrib/angstrom/upload-packages.sh
Diffs:

#
# mt diff -r934a179b506ce73dc1418c67653101f1c80388ed -rfbdba8d2fa1b51a743e0287b9d0adaee95687ffe
#
#
#
# patch "contrib/angstrom/sort.sh"
#  from [1799a6de9fd2f5d0c3771f7d2e8077651abd68e5]
#    to [dc8fe370555422cd208f5e9e5165380f1e2b8b88]
# 
# patch "contrib/angstrom/upload-packages.sh"
#  from [6a90671d9dc884ff105a99e0cef1d23d5dd59b2d]
#    to [659ddd27ab51b8b723b96e8c01d0622d532b7887]
#
============================================================
--- contrib/angstrom/sort.sh	1799a6de9fd2f5d0c3771f7d2e8077651abd68e5
+++ contrib/angstrom/sort.sh	dc8fe370555422cd208f5e9e5165380f1e2b8b88
@@ -14,11 +14,6 @@ for i in $(find . -name "*.ipk") ; do ba
 # Make a list of ipkg files already present in feeds and in unsorted
 echo "Making a list of unsorted packages"
 for i in $(find . -name "*.ipk") ; do basename $i ; done > files-unsorted
-if [ "$1" != "--skip-sorted-list" ]; then
-    echo "Making a list of sorted packages (takes long)"
-    for i in $(find ../ -name "*.ipk"| grep -v unsorted) ; do basename $i ; done > files-sorted
-fi
-
 # Make a list of duplicates and delete those
 echo "Finding duplicate packages in unsorted"
 cat files-sorted files-unsorted | sort | uniq -d > files-duplicate
@@ -28,8 +23,9 @@ for newfile in $(cat new-files.txt | sed
 # Log remaining packages to a file 
 find . -name "*.ipk" |grep -v dbg | grep -v -- -dev | grep -v -- -doc | grep -v angstrom-version | grep -v locale > new-files.txt
 for newfile in $(cat new-files.txt | sed s:./::g) ; do
-    echo "$(date -u +%s) $newfile $(basename ${PWD})" >> ../upload.txt
+    echo "$(date -u +%s) $newfile $(basename ${PWD})" >> ../upload-full.txt
 done    
+tail -n 100 ../upload-full.txt > ../upload.txt
 
 do_sort() {
 archdir=$arch
@@ -142,4 +138,9 @@ done
 	do_sort
 done
 
+if [ "$1" != "--skip-sorted-list" ]; then
+    echo "Updating list of sorted packages (takes long)"
+    for i in $(find ../ -name "*.ipk"| grep -v unsorted) ; do basename $i ; done > files-sorted
+fi
+
 (cd ~/website/repo ; php update.php)
============================================================
--- contrib/angstrom/upload-packages.sh	6a90671d9dc884ff105a99e0cef1d23d5dd59b2d
+++ contrib/angstrom/upload-packages.sh	659ddd27ab51b8b723b96e8c01d0622d532b7887
@@ -14,22 +14,28 @@ mkdir -p upload-queue || true
 mkdir -p upload-queue || true
 
 # Find and delete morgue dirs, we don't need them
+echo "Deleting morgue directories"
 find ipk/ -name "morgue" -exec rm -rf \{\} \;
 
 # Copy all packages to an upload queue
+echo "Copying packages to upload queue"
 find ipk/ -name "*.ipk" -exec cp \{\} upload-queue/ \;
 
 # Find file already present on webserver
-ssh $REMOTEM "find $REMOTED/ -name "*.ipk" -exec basename \{\} \;" > files-remote
+echo "Getting file list from server"
+scp $REMOTEM:$REMOTED/unsorted/files-sorted files-remote
 ls upload-queue/ | grep -v morgue > files-local
 
 # Check for files already present on webserver
+echo "Checking for duplicates"
 cat files-remote files-local | sort | uniq -u >files-uniq
 cat files-uniq files-local | sort | uniq -d > files-trans
 
 # Copy over non-duplicate files
+echo "Starting rsync..."
 rsync -vz --files-from=files-trans upload-queue/ $REMOTEM:$REMOTED/unsorted/
 
 # Clean up temporary files
+echo "Removing upload queue"
 rm -rf files-remote files-local files-uniq files-trans upload-queue