From a446405f2f24e953b002fd615cf066c905316d8f Mon Sep 17 00:00:00 2001 From: Markus Neteler Date: Sun, 29 Sep 2024 22:57:31 +0200 Subject: [PATCH] cronjobs: expand script to run stand-alone With this PR instructions to set up the source code repo have been turned into executable script chunks. Hence, if the scripts are executed on a new server, the GitHub repo cloning is done as well as needed directories are created. Furthermore, some minor script tuning. --- ...ron_grass_current_stable_build_binaries.sh | 76 ++++++++++++++----- .../cron_grass_current_stable_src_snapshot.sh | 36 ++++++--- .../cron_grass_legacy_build_binaries.sh | 69 ++++++++++++----- .../cron_grass_legacy_src_snapshot.sh | 36 ++++++--- .../cron_grass_old_build_binaries.sh | 69 ++++++++++++----- .../cron_grass_old_src_snapshot.sh | 37 ++++++--- .../cron_grass_preview_build_binaries.sh | 67 ++++++++++++---- .../cron_grass_preview_src_snapshot.sh | 37 ++++++--- 8 files changed, 314 insertions(+), 113 deletions(-) diff --git a/utils/cronjobs_osgeo_lxd/cron_grass_current_stable_build_binaries.sh b/utils/cronjobs_osgeo_lxd/cron_grass_current_stable_build_binaries.sh index 731fe0038b..9f43c869f5 100755 --- a/utils/cronjobs_osgeo_lxd/cron_grass_current_stable_build_binaries.sh +++ b/utils/cronjobs_osgeo_lxd/cron_grass_current_stable_build_binaries.sh @@ -8,7 +8,7 @@ ################################################################### # how it works: # - it updates locally the GRASS source code from github server -# - configures, compiles +# - configures source code and then compiles it # - packages the binaries # - generated the install scripts # - generates the pyGRASS 8 HTML manual @@ -20,24 +20,20 @@ # - Install GDAL # - Install apt-get install texlive-latex-extra python3-sphinxcontrib.apidoc # - Clone source from github: -# mkdir -p ~/src ; cd ~/src -# git clone https://github.com/OSGeo/grass.git releasebranch_8_4 -# cd releasebranch_8_4 -# git checkout releasebranch_8_4 -# - Prepare target directories: -# cd /var/www/code_and_data/ -# mkdir grass84 +# - cross-link code into web space on grasslxd server: # cd /var/www/html/ # ln -s /var/www/code_and_data/grass84 . # ################################# # variables for build environment (grass.osgeo.org specific) -MAINDIR=/home/neteler +MAINDIR=/home/$USER PATH=$MAINDIR/bin:/bin:/usr/bin:/usr/local/bin +# https://github.com/OSGeo/grass/tags GMAJOR=8 GMINOR=4 GPATCH="0dev" # required by grass-addons-index.sh +BRANCH=releasebranch_${GMAJOR}_${GMINOR} DOTVERSION=$GMAJOR.$GMINOR VERSION=$GMAJOR$GMINOR GVERSION=$GMAJOR @@ -53,13 +49,12 @@ LDFLAGSSTRING='-s' # define GRASS GIS build related paths: # where to find the GRASS sources (git clone): SOURCE=$MAINDIR/src/ -BRANCH=releasebranch_${GMAJOR}_$GMINOR GRASSBUILDDIR=$SOURCE/$BRANCH TARGETMAIN=/var/www/code_and_data TARGETDIR=$TARGETMAIN/grass${VERSION}/binary/linux/snapshot TARGETHTMLDIR=$TARGETMAIN/grass${VERSION}/manuals/ -# progman not built for older dev versions or old stable, only for preview +# progman not built for older dev versions or old stable, only for preview version #TARGETPROGMAN=$TARGETMAIN/programming${GVERSION} MYBIN=$MAINDIR/binaries @@ -80,6 +75,31 @@ halt_on_error() # function to configure for compilation configure_grass() { +# setup source code repo + +mkdir -p $SOURCE $TARGETDIR + +# fetch repo if needed +cd "$SOURCE/" +# Check if the repository is already cloned +if [ -d "$BRANCH" ]; then + echo "The GRASS GIS repository <$BRANCH> has already been cloned. Continuing..." +else + echo "Cloning the GRASS GIS repository <$BRANCH> first..." + git clone https://github.com/OSGeo/grass.git $BRANCH + if [ $? -eq 0 ]; then + echo "Repository successfully cloned." + else + echo "Error: Failed to clone the repository." + exit 1 + fi +fi + +cd $SOURCE/$BRANCH/ +date + +# be sure to be on the right branch +git checkout $BRANCH # cleanup from previous run rm -f config_$GMAJOR.$GMINOR.git_log.txt @@ -135,6 +155,7 @@ git fetch --all --prune && git checkout $BRANCH && git pull --rebase || halt_on_ git status # for the "contributors list" in old CMSMS (still needed for hugo?) +mkdir -p $TARGETMAIN/uploads/grass/ cp -f *.csv $TARGETMAIN/uploads/grass/ # configure for compilation @@ -164,7 +185,7 @@ $MYMAKE sphinxdoclib echo "Copy over the manual + pygrass HTML pages:" mkdir -p $TARGETHTMLDIR mkdir -p $TARGETHTMLDIR/addons # indeed only relevant the very first compile time -# don't destroy the addons +# don't destroy the addons during update \mv $TARGETHTMLDIR/addons /tmp rm -f $TARGETHTMLDIR/*.* (cd $TARGETHTMLDIR ; rm -rf barscales colortables icons northarrows) @@ -212,9 +233,8 @@ cd $GRASSBUILDDIR/ #(cd $TARGETPROGMAN/ ; ln -s index.html main.html) #### end unused -# note: from G82+ onwards the gettext POT files are managed in git and OSGeo Weblate - -##### generate i18N stats for HTML page path (WebSVN): +##### generate i18N stats for HTML page path: +# note: the gettext POT files are managed in git and OSGeo Weblate ## Structure: grasslibs_ar.po 144 translated messages 326 fuzzy translations 463 untranslated messages. cd $GRASSBUILDDIR (cd locale/ ; @@ -274,15 +294,31 @@ cd $GRASSBUILDDIR # update addon repo (addon repo has been cloned twice on the server to # have separate grass7 and grass8 addon compilation) -(cd ~/src/grass$GMAJOR-addons/; git checkout grass$GMAJOR; git pull origin grass$GMAJOR) +# fetch addon repo if needed +cd "$SOURCE/" +# Check if the addon repository is already cloned +if [ -d "grass${GMAJOR}-addons" ]; then + echo "The GRASS GIS repository has already been cloned. Continuing..." +else + echo "Cloning the GRASS GIS repository first..." + git clone https://github.com/OSGeo/grass-addons.git grass${GMAJOR}-addons + if [ $? -eq 0 ]; then + echo "Repository successfully cloned." + else + echo "Error: Failed to clone the repository." + exit 1 + fi +fi +# setup source code repo +(cd $SOURCE/grass$GMAJOR-addons/; git checkout grass$GMAJOR; git pull origin grass$GMAJOR) # compile addons cd $GRASSBUILDDIR sh ~/cronjobs/compile_addons_git.sh $GMAJOR \ $GMINOR \ - ~/src/grass$GMAJOR-addons/src/ \ - ~/src/$BRANCH/dist.$ARCH/ \ + $SOURCE/grass$GMAJOR-addons/src/ \ + $SOURCE/$BRANCH/dist.$ARCH/ \ ~/.grass$GMAJOR/addons \ - ~/src/$BRANCH/bin.$ARCH/grass \ + $SOURCE/$BRANCH/bin.$ARCH/grass \ 1 mkdir -p $TARGETHTMLDIR/addons/ # copy individual addon html files into one target dir if compiled addon @@ -311,7 +347,7 @@ mkdir -p $TARGETMAIN/addons/grass$GMAJOR/logs/ cp -p ~/.grass$GMAJOR/addons/logs/* $TARGETMAIN/addons/grass$GMAJOR/logs/ # generate addons modules.xml file (required for g.extension module) -~/src/$BRANCH/bin.$ARCH/grass --tmp-project EPSG:4326 --exec ~/cronjobs/build-xml.py --build ~/.grass$GMAJOR/addons +$SOURCE/$BRANCH/bin.$ARCH/grass --tmp-project EPSG:4326 --exec ~/cronjobs/build-xml.py --build ~/.grass$GMAJOR/addons cp ~/.grass$GMAJOR/addons/modules.xml $TARGETMAIN/addons/grass$GMAJOR/modules.xml # regenerate keywords.html file with addons modules keywords diff --git a/utils/cronjobs_osgeo_lxd/cron_grass_current_stable_src_snapshot.sh b/utils/cronjobs_osgeo_lxd/cron_grass_current_stable_src_snapshot.sh index 45101b6b0f..b52b4a6f46 100755 --- a/utils/cronjobs_osgeo_lxd/cron_grass_current_stable_src_snapshot.sh +++ b/utils/cronjobs_osgeo_lxd/cron_grass_current_stable_src_snapshot.sh @@ -1,6 +1,6 @@ #!/bin/sh -# script to build GRASS GIS new current sources package from the main branch +# script to build GRASS GIS new current sources package from the `release_branch_8_4` branch # (c) 2002-2024, GPL 2+ Markus Neteler # # GRASS GIS github, https://github.com/OSGeo/grass @@ -10,19 +10,17 @@ # - it updates locally the GRASS source code from github server # - packages the source code tarball # -# Preparations, on server (neteler@grasslxd:$): -# mkdir -p ~/src -# cd ~/src -# git clone https://github.com/OSGeo/grass.git release_branch_8_4 +# To be executed on server (neteler@grasslxd:$) # ################################################################### # variables for packaging environment (grass.osgeo.org specific) -MAINDIR=/home/neteler +MAINDIR=/home/$USER PATH=$MAINDIR/bin:/bin:/usr/bin:/usr/local/bin # https://github.com/OSGeo/grass/tags GMAJOR=8 GMINOR=4 +BRANCH=releasebranch_${GMAJOR}_${GMINOR} GVERSION=$GMAJOR.$GMINOR.git DOTVERSION=$GMAJOR.$GMINOR GSHORTGVERSION=$GMAJOR$GMINOR @@ -33,7 +31,6 @@ set -e ################### # where to find the GRASS sources (git clone): SOURCE=$MAINDIR/src/ -BRANCH=main # where to put the resulting .tar.gz file: TARGETMAIN=/var/www/code_and_data/ TARGETDIR=$TARGETMAIN/grass${GSHORTGVERSION}/source/snapshot @@ -56,10 +53,30 @@ halt_on_error() # create a source code snapshot: CWD=`pwd` -mkdir -p $TARGETDIR +# setup source code repo + +mkdir -p $SOURCE $TARGETDIR + +# fetch repo if needed +cd "$SOURCE/" +# Check if the repository is already cloned +if [ -d "$BRANCH" ]; then + echo "The GRASS GIS repository <$BRANCH> has already been cloned. Continuing..." +else + echo "Cloning the GRASS GIS repository <$BRANCH> first..." + git clone https://github.com/OSGeo/grass.git $BRANCH + if [ $? -eq 0 ]; then + echo "Repository successfully cloned." + else + echo "Error: Failed to clone the repository." + exit 1 + fi +fi cd $SOURCE/$BRANCH/ date +# be sure to be on the right branch +git checkout $BRANCH # clean up from previous run touch include/Make/Platform.make $MYMAKE distclean > /dev/null 2>&1 @@ -73,9 +90,6 @@ rm -f config_*.git_log.txt ChangeLog # reset i18N POT files to git, just to be sure git checkout locale/templates/*.pot -## hard reset local git repo (just in case) -#git checkout main && git reset --hard HEAD~1 && git reset --hard origin - echo "git update..." git fetch --all --prune || halt_on_error "git fetch error!" # we dont have upstream in this cronjob repo diff --git a/utils/cronjobs_osgeo_lxd/cron_grass_legacy_build_binaries.sh b/utils/cronjobs_osgeo_lxd/cron_grass_legacy_build_binaries.sh index ccc6015fc7..5bc20931f0 100755 --- a/utils/cronjobs_osgeo_lxd/cron_grass_legacy_build_binaries.sh +++ b/utils/cronjobs_osgeo_lxd/cron_grass_legacy_build_binaries.sh @@ -8,7 +8,7 @@ ################################################################### # how it works: # - it updates locally the GRASS source code from github server -# - configures, compiles +# - configures source code and then compiles it # - packages the binaries # - generated the install scripts # - generates the pyGRASS 7 HTML manual @@ -22,25 +22,20 @@ # - Install GDAL # - Install apt-get install texlive-latex-extra python3-sphinxcontrib.apidoc # - Clone source from github: -# mkdir -p ~/src ; cd ~/src -# git clone https://github.com/OSGeo/grass.git releasebranch_7_8 -# cd releasebranch_7_8 -# git checkout releasebranch_7_8 -# - Prepare target directories: -# cd /var/www/code_and_data/ -# mkdir grass78 +# - cross-link code into web space on grasslxd server: # cd /var/www/html/ # ln -s /var/www/code_and_data/grass78 . # ################################# # variables for build environment (grass.osgeo.org specific) -MAINDIR=/home/neteler +MAINDIR=/home/$USER PATH=$MAINDIR/bin:/bin:/usr/bin:/usr/local/bin # https://github.com/OSGeo/grass/tags GMAJOR=7 GMINOR=8 GPATCH=7 # required by grass-addons-index.sh +BRANCH=releasebranch_${GMAJOR}_$GMINOR # NEW_CURRENT: set to same value as in cron_grass_old_build_binaries.sh NEW_CURRENT=84 @@ -60,13 +55,12 @@ LDFLAGSSTRING='-s' # define GRASS GIS build related paths: # where to find the GRASS sources (git clone): SOURCE=$MAINDIR/src/ -BRANCH=releasebranch_${GMAJOR}_$GMINOR GRASSBUILDDIR=$SOURCE/$BRANCH TARGETMAIN=/var/www/code_and_data TARGETDIR=$TARGETMAIN/grass${VERSION}/binary/linux/snapshot TARGETHTMLDIR=$TARGETMAIN/grass${VERSION}/manuals/ -# progman not built for older dev versions or old stable, only for preview +# progman not built for older dev versions or old stable, only for preview version #TARGETPROGMAN=$TARGETMAIN/programming${GVERSION} MYBIN=$MAINDIR/binaries @@ -87,6 +81,31 @@ halt_on_error() # function to configure for compilation configure_grass() { +# setup source code repo + +mkdir -p $SOURCE $TARGETDIR + +# fetch repo if needed +cd "$SOURCE/" +# Check if the repository is already cloned +if [ -d "$BRANCH" ]; then + echo "The GRASS GIS repository <$BRANCH> has already been cloned. Continuing..." +else + echo "Cloning the GRASS GIS repository <$BRANCH> first..." + git clone https://github.com/OSGeo/grass.git $BRANCH + if [ $? -eq 0 ]; then + echo "Repository successfully cloned." + else + echo "Error: Failed to clone the repository." + exit 1 + fi +fi + +cd $SOURCE/$BRANCH/ +date + +# be sure to be on the right branch +git checkout $BRANCH # cleanup from previous run rm -f config_$GMAJOR.$GMINOR.git_log.txt @@ -138,6 +157,7 @@ git fetch --all --prune && git checkout $BRANCH && git pull --rebase || halt_on_ git status # for the "contributors list" in old CMSMS (still needed for hugo?) +mkdir -p $TARGETMAIN/uploads/grass/ cp -f *.csv $TARGETMAIN/uploads/grass/ # configure for compilation @@ -251,15 +271,31 @@ cd $GRASSBUILDDIR # update addon repo (addon repo has been cloned twice on the server to # have separate grass7 and grass8 addon compilation) -(cd ~/src/grass$GMAJOR-addons/; git checkout grass$GMAJOR; git pull origin grass$GMAJOR) +# fetch addon repo if needed +cd "$SOURCE/" +# Check if the addon repository is already cloned +if [ -d "grass${GMAJOR}-addons" ]; then + echo "The GRASS GIS repository has already been cloned. Continuing..." +else + echo "Cloning the GRASS GIS repository first..." + git clone https://github.com/OSGeo/grass-addons.git grass${GMAJOR}-addons + if [ $? -eq 0 ]; then + echo "Repository successfully cloned." + else + echo "Error: Failed to clone the repository." + exit 1 + fi +fi +# setup source code repo +(cd $SOURCE/grass$GMAJOR-addons/; git checkout grass$GMAJOR; git pull origin grass$GMAJOR) # compile addons cd $GRASSBUILDDIR sh ~/cronjobs/compile_addons_git.sh $GMAJOR \ $GMINOR \ - ~/src/grass$GMAJOR-addons/src/ \ - ~/src/$BRANCH/dist.$ARCH/ \ + $SOURCE/grass$GMAJOR-addons/src/ \ + $SOURCE/$BRANCH/dist.$ARCH/ \ ~/.grass$GMAJOR/addons \ - ~/src/$BRANCH/bin.$ARCH/grass$VERSION \ + $SOURCE/$BRANCH/bin.$ARCH/grass$VERSION \ 1 mkdir -p $TARGETHTMLDIR/addons/ # copy individual addon html files into one target dir if compiled addon @@ -282,7 +318,7 @@ mkdir -p $TARGETMAIN/addons/grass$GMAJOR/logs/ cp -p ~/.grass$GMAJOR/addons/logs/* $TARGETMAIN/addons/grass$GMAJOR/logs/ # generate addons modules.xml file (required for g.extension module) -~/src/$BRANCH/bin.$ARCH/grass$VERSION --tmp-location EPSG:4326 --exec ~/cronjobs/build-xml.py --build ~/.grass$GMAJOR/addons +$SOURCE/$BRANCH/bin.$ARCH/grass$VERSION --tmp-location EPSG:4326 --exec ~/cronjobs/build-xml.py --build ~/.grass$GMAJOR/addons cp ~/.grass$GMAJOR/addons/modules.xml $TARGETMAIN/addons/grass$GMAJOR/modules.xml # regenerate keywords.html file with addons modules keywords @@ -316,7 +352,6 @@ echo "Injecting G8.x new current version hint in a red box into MAN pages..." # - run sed to replace an existing HTML header string in the upper part of the HTML file # with itself + canonical link of stable version # --> do this for core manual pages, addons, libpython -## (cd $TARGETHTMLDIR/ ; for myfile in `grep -L 'link rel="canonical"' *.html` ; do sed -i -e "s::\n:g" $myfile ; done) (cd $TARGETHTMLDIR/addons/ ; for myfile in `grep -L 'link rel="canonical"' *.html` ; do sed -i -e "s::\n:g" $myfile ; done) (cd $TARGETHTMLDIR/libpython/ ; for myfile in `grep -L 'link rel="canonical"' *.html` ; do sed -i -e "s::\n:g" $myfile ; done) diff --git a/utils/cronjobs_osgeo_lxd/cron_grass_legacy_src_snapshot.sh b/utils/cronjobs_osgeo_lxd/cron_grass_legacy_src_snapshot.sh index b8d0c715ee..4df1ab68bc 100755 --- a/utils/cronjobs_osgeo_lxd/cron_grass_legacy_src_snapshot.sh +++ b/utils/cronjobs_osgeo_lxd/cron_grass_legacy_src_snapshot.sh @@ -10,20 +10,17 @@ # - it updates locally the GRASS source code from github server # - packages the source code tarball # -# Preparations, on server (neteler@grasslxd:$): -# mkdir -p ~/src -# cd ~/src -# for i in 2 4 6 ; do git clone ​https://github.com/OSGeo/grass.git releasebranch_7_$i ; done -# for i in 2 4 6 ; do (cd releasebranch_7_$i ; git checkout releasebranch_7_$i ) ; done +# To be executed on server (neteler@grasslxd:$) # ################################################################### # variables for packaging environment (grass.osgeo.org specific) -MAINDIR=/home/neteler +MAINDIR=/home/$USER PATH=$MAINDIR/bin:/bin:/usr/bin:/usr/local/bin # https://github.com/OSGeo/grass/tags GMAJOR=7 GMINOR=8 +BRANCH=releasebranch_${GMAJOR}_${GMINOR} GVERSION=$GMAJOR.$GMINOR.git DOTVERSION=$GMAJOR.$GMINOR GSHORTGVERSION=$GMAJOR$GMINOR @@ -34,7 +31,6 @@ set -e ################### # where to find the GRASS sources (git clone): SOURCE=$MAINDIR/src/ -BRANCH=releasebranch_${GMAJOR}_${GMINOR} # where to put the resulting .tar.gz file: TARGETMAIN=/var/www/code_and_data/ TARGETDIR=$TARGETMAIN/grass${GSHORTGVERSION}/source/snapshot @@ -57,10 +53,31 @@ halt_on_error() # create a source code snapshot: CWD=`pwd` -mkdir -p $TARGETDIR +# setup source code repo +mkdir -p $SOURCE $TARGETDIR + +# fetch repo if needed +cd "$SOURCE/" +# Check if the repository is already cloned +if [ -d "$BRANCH" ]; then + echo "The GRASS GIS repository <$BRANCH> has already been cloned. Continuing..." +else + echo "Cloning the GRASS GIS repository <$BRANCH> first..." + git clone https://github.com/OSGeo/grass.git $BRANCH + if [ $? -eq 0 ]; then + echo "Repository successfully cloned." + else + echo "Error: Failed to clone the repository." + exit 1 + fi +fi + cd $SOURCE/$BRANCH/ date +# be sure to be on the right branch +git checkout $BRANCH + # clean up from previous run touch include/Make/Platform.make $MYMAKE distclean > /dev/null 2>&1 @@ -71,9 +88,6 @@ git status | grep '.rst' | xargs rm -f rm -rf lib/python/docs/_build/ lib/python/docs/_templates/layout.html rm -f config_*.git_log.txt ChangeLog -# be sure to be on the right branch -git checkout $BRANCH - echo "git update..." git fetch --all --prune || halt_on_error "git fetch error!" # we dont have upstream in this cronjob repo diff --git a/utils/cronjobs_osgeo_lxd/cron_grass_old_build_binaries.sh b/utils/cronjobs_osgeo_lxd/cron_grass_old_build_binaries.sh index 4b0a6fe687..e56adcdaf5 100755 --- a/utils/cronjobs_osgeo_lxd/cron_grass_old_build_binaries.sh +++ b/utils/cronjobs_osgeo_lxd/cron_grass_old_build_binaries.sh @@ -8,7 +8,7 @@ ################################################################### # how it works: # - it updates locally the GRASS source code from github server -# - configures, compiles +# - configures source code and then compiles it # - packages the binaries # - generated the install scripts # - generates the pyGRASS 8 HTML manual @@ -22,25 +22,20 @@ # - Install GDAL # - Install apt-get install texlive-latex-extra python3-sphinxcontrib.apidoc # - Clone source from github: -# mkdir -p ~/src ; cd ~/src -# git clone https://github.com/OSGeo/grass.git releasebranch_8_3 -# cd releasebranch_8_3 -# git checkout releasebranch_8_3 -# - Prepare target directories: -# cd /var/www/code_and_data/ -# mkdir grass83 +# - cross-link code into web space on grasslxd server: # cd /var/www/html/ # ln -s /var/www/code_and_data/grass83 . # ################################# # variables for build environment (grass.osgeo.org specific) -MAINDIR=/home/neteler +MAINDIR=/home/$USER PATH=$MAINDIR/bin:/bin:/usr/bin:/usr/local/bin # https://github.com/OSGeo/grass/tags GMAJOR=8 GMINOR=3 GPATCH=2 # required by grass-addons-index.sh +BRANCH=releasebranch_${GMAJOR}_$GMINOR # NEW_CURRENT: set to GMINOR from above + 1: NEW_CURRENT=84 @@ -60,13 +55,12 @@ LDFLAGSSTRING='-s' # define GRASS GIS build related paths: # where to find the GRASS sources (git clone): SOURCE=$MAINDIR/src/ -BRANCH=releasebranch_${GMAJOR}_$GMINOR GRASSBUILDDIR=$SOURCE/$BRANCH TARGETMAIN=/var/www/code_and_data TARGETDIR=$TARGETMAIN/grass${VERSION}/binary/linux/snapshot TARGETHTMLDIR=$TARGETMAIN/grass${VERSION}/manuals/ -# progman not built for older dev versions or old stable, only for preview +# progman not built for older dev versions or old stable, only for preview version #TARGETPROGMAN=$TARGETMAIN/programming${GVERSION} MYBIN=$MAINDIR/binaries @@ -87,6 +81,31 @@ halt_on_error() # function to configure for compilation configure_grass() { +# setup source code repo + +mkdir -p $SOURCE $TARGETDIR + +# fetch repo if needed +cd "$SOURCE/" +# Check if the repository is already cloned +if [ -d "$BRANCH" ]; then + echo "The GRASS GIS repository <$BRANCH> has already been cloned. Continuing..." +else + echo "Cloning the GRASS GIS repository <$BRANCH> first..." + git clone https://github.com/OSGeo/grass.git $BRANCH + if [ $? -eq 0 ]; then + echo "Repository successfully cloned." + else + echo "Error: Failed to clone the repository." + exit 1 + fi +fi + +cd $SOURCE/$BRANCH/ +date + +# be sure to be on the right branch +git checkout $BRANCH # cleanup from previous run rm -f config_$GMAJOR.$GMINOR.git_log.txt @@ -140,6 +159,7 @@ git fetch --all --prune && git checkout $BRANCH && git pull --rebase || halt_on_ git status # for the "contributors list" in old CMSMS (still needed for hugo?) +mkdir -p $TARGETMAIN/uploads/grass/ cp -f *.csv $TARGETMAIN/uploads/grass/ # configure for compilation @@ -219,7 +239,6 @@ cd $GRASSBUILDDIR/ ##### generate i18N stats for HTML page path: # note: the gettext POT files are managed in git and OSGeo Weblate - ## Structure: grasslibs_ar.po 144 translated messages 326 fuzzy translations 463 untranslated messages. cd $GRASSBUILDDIR (cd locale/ ; @@ -279,15 +298,31 @@ cd $GRASSBUILDDIR # update addon repo (addon repo has been cloned twice on the server to # have separate grass7 and grass8 addon compilation) -(cd ~/src/grass$GMAJOR-addons/; git checkout grass$GMAJOR; git pull origin grass$GMAJOR) +# fetch addon repo if needed +cd "$SOURCE/" +# Check if the addon repository is already cloned +if [ -d "grass${GMAJOR}-addons" ]; then + echo "The GRASS GIS repository has already been cloned. Continuing..." +else + echo "Cloning the GRASS GIS repository first..." + git clone https://github.com/OSGeo/grass-addons.git grass${GMAJOR}-addons + if [ $? -eq 0 ]; then + echo "Repository successfully cloned." + else + echo "Error: Failed to clone the repository." + exit 1 + fi +fi +# setup source code repo +(cd $SOURCE/grass$GMAJOR-addons/; git checkout grass$GMAJOR; git pull origin grass$GMAJOR) # compile addons cd $GRASSBUILDDIR sh ~/cronjobs/compile_addons_git.sh $GMAJOR \ $GMINOR \ - ~/src/grass$GMAJOR-addons/src/ \ - ~/src/$BRANCH/dist.$ARCH/ \ + $SOURCE/grass$GMAJOR-addons/src/ \ + $SOURCE/$BRANCH/dist.$ARCH/ \ ~/.grass$GMAJOR/addons \ - ~/src/$BRANCH/bin.$ARCH/grass \ + $SOURCE/$BRANCH/bin.$ARCH/grass \ 1 mkdir -p $TARGETHTMLDIR/addons/ # copy individual addon html files into one target dir if compiled addon @@ -316,7 +351,7 @@ mkdir -p $TARGETMAIN/addons/grass$GMAJOR/logs/ cp -p ~/.grass$GMAJOR/addons/logs/* $TARGETMAIN/addons/grass$GMAJOR/logs/ # generate addons modules.xml file (required for g.extension module) -~/src/$BRANCH/bin.$ARCH/grass --tmp-location EPSG:4326 --exec ~/cronjobs/build-xml.py --build ~/.grass$GMAJOR/addons +$SOURCE/$BRANCH/bin.$ARCH/grass --tmp-location EPSG:4326 --exec ~/cronjobs/build-xml.py --build ~/.grass$GMAJOR/addons cp ~/.grass$GMAJOR/addons/modules.xml $TARGETMAIN/addons/grass$GMAJOR/modules.xml # regenerate keywords.html file with addons modules keywords diff --git a/utils/cronjobs_osgeo_lxd/cron_grass_old_src_snapshot.sh b/utils/cronjobs_osgeo_lxd/cron_grass_old_src_snapshot.sh index 9b8ee3df9e..01a9ba8e6d 100755 --- a/utils/cronjobs_osgeo_lxd/cron_grass_old_src_snapshot.sh +++ b/utils/cronjobs_osgeo_lxd/cron_grass_old_src_snapshot.sh @@ -1,6 +1,6 @@ #!/bin/sh -# script to build GRASS GIS old current source package from the release branch +# script to build GRASS GIS old current sources package from the `release_branch_8_3` branch # (c) 2002-2024, GPL 2+ Markus Neteler # # GRASS GIS github, https://github.com/OSGeo/grass @@ -10,19 +10,17 @@ # - it updates locally the GRASS source code from github server # - packages the source code tarball # -# Preparations, on server (neteler@grasslxd:$): -# mkdir -p ~/src -# cd ~/src -# git clone https://github.com/OSGeo/grass.git release_branch_8_3 +# To be executed on server (neteler@grasslxd:$) # ################################################################### # variables for packaging environment (grass.osgeo.org specific) -MAINDIR=/home/neteler +MAINDIR=/home/$USER PATH=$MAINDIR/bin:/bin:/usr/bin:/usr/local/bin # https://github.com/OSGeo/grass/tags GMAJOR=8 GMINOR=3 +BRANCH=releasebranch_${GMAJOR}_${GMINOR} GVERSION=$GMAJOR.$GMINOR.git DOTVERSION=$GMAJOR.$GMINOR GSHORTGVERSION=$GMAJOR$GMINOR @@ -33,7 +31,6 @@ set -e ################### # where to find the GRASS sources (git clone): SOURCE=$MAINDIR/src/ -BRANCH=releasebranch_${GMAJOR}_${GMINOR} # where to put the resulting .tar.gz file: TARGETMAIN=/var/www/code_and_data/ TARGETDIR=$TARGETMAIN/grass${GSHORTGVERSION}/source/snapshot @@ -56,10 +53,31 @@ halt_on_error() # create a source code snapshot: CWD=`pwd` -mkdir -p $TARGETDIR +# setup source code repo + +mkdir -p $SOURCE $TARGETDIR + +# fetch repo if needed +cd "$SOURCE/" +# Check if the repository is already cloned +if [ -d "$BRANCH" ]; then + echo "The GRASS GIS repository <$BRANCH> has already been cloned. Continuing..." +else + echo "Cloning the GRASS GIS repository <$BRANCH> first..." + git clone https://github.com/OSGeo/grass.git $BRANCH + if [ $? -eq 0 ]; then + echo "Repository successfully cloned." + else + echo "Error: Failed to clone the repository." + exit 1 + fi +fi + cd $SOURCE/$BRANCH/ date +# be sure to be on the right branch +git checkout $BRANCH # clean up from previous run touch include/Make/Platform.make $MYMAKE distclean > /dev/null 2>&1 @@ -73,9 +91,6 @@ rm -f config_*.git_log.txt ChangeLog # reset i18N POT files to git, just to be sure git checkout locale/templates/*.pot -## hard reset local git repo (just in case) -#git checkout main && git reset --hard HEAD~1 && git reset --hard origin - echo "git update..." git fetch --all --prune || halt_on_error "git fetch error!" # we dont have upstream in this cronjob repo diff --git a/utils/cronjobs_osgeo_lxd/cron_grass_preview_build_binaries.sh b/utils/cronjobs_osgeo_lxd/cron_grass_preview_build_binaries.sh index dfd2fc9fe3..c86b9b194a 100755 --- a/utils/cronjobs_osgeo_lxd/cron_grass_preview_build_binaries.sh +++ b/utils/cronjobs_osgeo_lxd/cron_grass_preview_build_binaries.sh @@ -8,7 +8,7 @@ ################################################################### # how it works: # - it updates locally the GRASS source code from github server -# - configures, compiles +# - configures source code and then compiles it # - packages the binaries # - generated the install scripts # - generates the programmer's 8 HTML manual @@ -21,22 +21,20 @@ # - Install GDAL # - Install apt-get install texlive-latex-extra python3-sphinxcontrib.apidoc # - Clone source from github: -# mkdir -p ~/src ; cd ~/src -# git clone https://github.com/OSGeo/grass.git main -# - Prepare target directories: -# cd /var/www/code_and_data/ -# mkdir grass84 +# - cross-link code into web space on grasslxd server: # cd /var/www/html/ -# ln -s /var/www/code_and_data/grass84 . +# ln -s /var/www/code_and_data/grass85 . # ################################# # variables for build environment (grass.osgeo.org specific) -MAINDIR=/home/neteler +MAINDIR=/home/$USER PATH=$MAINDIR/bin:/bin:/usr/bin:/usr/local/bin +# https://github.com/OSGeo/grass/tags GMAJOR=8 GMINOR=5 GPATCH="0dev" # required by grass-addons-index.sh +BRANCH=main DOTVERSION=$GMAJOR.$GMINOR VERSION=$GMAJOR$GMINOR GVERSION=$GMAJOR @@ -52,13 +50,12 @@ LDFLAGSSTRING='-s' # define GRASS GIS build related paths: # where to find the GRASS sources (git clone): SOURCE=$MAINDIR/src/ -BRANCH=main GRASSBUILDDIR=$SOURCE/$BRANCH TARGETMAIN=/var/www/code_and_data TARGETDIR=$TARGETMAIN/grass${VERSION}/binary/linux/snapshot TARGETHTMLDIR=$TARGETMAIN/grass${VERSION}/manuals/ -# progman compiled below (i.e., only for preview) +# progman compiled below (i.e., only for preview version) # progman not built for older dev versions or old stable TARGETPROGMAN=$TARGETMAIN/programming${GVERSION} @@ -80,6 +77,29 @@ halt_on_error() # function to configure for compilation configure_grass() { +# setup source code repo +mkdir -p $SOURCE $TARGETDIR +# fetch repo if needed +cd "$SOURCE/" +# Check if the repository is already cloned +if [ -d "$BRANCH" ]; then + echo "The GRASS GIS repository <$BRANCH> has already been cloned. Continuing..." +else + echo "Cloning the GRASS GIS repository <$BRANCH> first..." + git clone https://github.com/OSGeo/grass.git $BRANCH + if [ $? -eq 0 ]; then + echo "Repository successfully cloned." + else + echo "Error: Failed to clone the repository." + exit 1 + fi +fi + +cd $SOURCE/$BRANCH/ +date + +# be sure to be on the right branch +git checkout $BRANCH # cleanup from previous run rm -f config_$GMAJOR.$GMINOR.git_log.txt @@ -134,6 +154,7 @@ git fetch --all --prune && git checkout $BRANCH && git pull --rebase || halt_on_ git status # for the "contributors list" in old CMSMS (still needed for hugo?) +mkdir -p $TARGETMAIN/uploads/grass/ cp -f *.csv $TARGETMAIN/uploads/grass/ # configure for compilation @@ -272,15 +293,31 @@ cd $GRASSBUILDDIR # update addon repo (addon repo has been cloned twice on the server to # have separate grass7 and grass8 addon compilation) -(cd ~/src/grass$GMAJOR-addons/; git checkout grass$GMAJOR; git pull origin grass$GMAJOR) +# fetch addon repo if needed +cd "$SOURCE/" +# Check if the addon repository is already cloned +if [ -d "grass${GMAJOR}-addons" ]; then + echo "The GRASS GIS repository has already been cloned. Continuing..." +else + echo "Cloning the GRASS GIS repository first..." + git clone https://github.com/OSGeo/grass-addons.git grass${GMAJOR}-addons + if [ $? -eq 0 ]; then + echo "Repository successfully cloned." + else + echo "Error: Failed to clone the repository." + exit 1 + fi +fi +# setup source code repo +(cd $SOURCE/grass$GMAJOR-addons/; git checkout grass$GMAJOR; git pull origin grass$GMAJOR) # compile addons cd $GRASSBUILDDIR sh ~/cronjobs/compile_addons_git.sh $GMAJOR \ $GMINOR \ - ~/src/grass$GMAJOR-addons/src/ \ - ~/src/$BRANCH/dist.$ARCH/ \ + $SOURCE/grass$GMAJOR-addons/src/ \ + $SOURCE/$BRANCH/dist.$ARCH/ \ ~/.grass$GMAJOR/addons \ - ~/src/$BRANCH/bin.$ARCH/grass \ + $SOURCE/$BRANCH/bin.$ARCH/grass \ 1 mkdir -p $TARGETHTMLDIR/addons/ # copy individual addon html files into one target dir if compiled addon @@ -309,7 +346,7 @@ mkdir -p $TARGETMAIN/addons/grass$GMAJOR/logs/ cp -p ~/.grass$GMAJOR/addons/logs/* $TARGETMAIN/addons/grass$GMAJOR/logs/ # generate addons modules.xml file (required for g.extension module) -~/src/$BRANCH/bin.$ARCH/grass --tmp-project EPSG:4326 --exec ~/cronjobs/build-xml.py --build ~/.grass$GMAJOR/addons +$SOURCE/$BRANCH/bin.$ARCH/grass --tmp-project EPSG:4326 --exec ~/cronjobs/build-xml.py --build ~/.grass$GMAJOR/addons cp ~/.grass$GMAJOR/addons/modules.xml $TARGETMAIN/addons/grass$GMAJOR/modules.xml # regenerate keywords.html file with addons modules keywords diff --git a/utils/cronjobs_osgeo_lxd/cron_grass_preview_src_snapshot.sh b/utils/cronjobs_osgeo_lxd/cron_grass_preview_src_snapshot.sh index 465017c9e3..dfb44276a2 100755 --- a/utils/cronjobs_osgeo_lxd/cron_grass_preview_src_snapshot.sh +++ b/utils/cronjobs_osgeo_lxd/cron_grass_preview_src_snapshot.sh @@ -1,6 +1,6 @@ #!/bin/sh -# script to build GRASS GIS preview source package from the main branch +# script to build GRASS GIS preview source package from the `main` branch # (c) 2002-2024, GPL 2+ Markus Neteler # # GRASS GIS github, https://github.com/OSGeo/grass @@ -10,19 +10,17 @@ # - it updates locally the GRASS source code from github server # - packages the source code tarball # -# Preparations, on server (neteler@grasslxd:$): -# mkdir -p ~/src -# cd ~/src -# git clone https://github.com/OSGeo/grass.git main +# To be executed on server (neteler@grasslxd:$) # ################################################################### # variables for packaging environment (grass.osgeo.org specific) -MAINDIR=/home/neteler +MAINDIR=/home/$USER PATH=$MAINDIR/bin:/bin:/usr/bin:/usr/local/bin # https://github.com/OSGeo/grass/tags GMAJOR=8 GMINOR=5 +BRANCH=main GVERSION=$GMAJOR.$GMINOR.git DOTVERSION=$GMAJOR.$GMINOR GSHORTGVERSION=$GMAJOR$GMINOR @@ -33,7 +31,6 @@ set -e ################### # where to find the GRASS sources (git clone): SOURCE=$MAINDIR/src/ -BRANCH=main # where to put the resulting .tar.gz file: TARGETMAIN=/var/www/code_and_data/ TARGETDIR=$TARGETMAIN/grass${GSHORTGVERSION}/source/snapshot @@ -56,10 +53,31 @@ halt_on_error() # create a source code snapshot: CWD=`pwd` -mkdir -p $TARGETDIR +# setup source code repo +mkdir -p $SOURCE $TARGETDIR + +# fetch repo if needed +cd "$SOURCE/" +# Check if the repository is already cloned +if [ -d "$BRANCH" ]; then + echo "The GRASS GIS repository <$BRANCH> has already been cloned. Continuing..." +else + echo "Cloning the GRASS GIS repository <$BRANCH> first..." + git clone https://github.com/OSGeo/grass.git $BRANCH + if [ $? -eq 0 ]; then + echo "Repository successfully cloned." + else + echo "Error: Failed to clone the repository." + exit 1 + fi +fi + cd $SOURCE/$BRANCH/ date +# be sure to be on the right branch +git checkout $BRANCH + # clean up from previous run touch include/Make/Platform.make $MYMAKE distclean > /dev/null 2>&1 @@ -73,9 +91,6 @@ rm -f config_*.git_log.txt ChangeLog # reset i18N POT files to git, just to be sure git checkout locale/templates/*.pot -## hard reset local git repo (just in case) -#git checkout main && git reset --hard HEAD~1 && git reset --hard origin - echo "git update..." git fetch --all --prune || halt_on_error "git fetch error!" # we dont have upstream in this cronjob repo