#!/bin/bash
#========================================================================
# Package and publish thumbspage: the main, top-level script [2.1].
# Part of the thumbspage program (learning-python.com/thumbspage.html).
#
# USAGE
# -----
# Run me anywhere: automatically cds to target folders along the way.
# Also runs examples generator, user-guide fixer, analytics inserter.
#
# To publish thumbspage, simply do this on a Unix platform:
#     bash build/_PUBLISH.sh
#
# Which in turn runs:
#     python3 build/generate-examples.py
#     python3 build/userguide-online-links.py
#     python3 build/insert-analytics.py
#
# The first of which runs:
#     _generate.sh in examples/* subfolders, plus make-narrow-labels.py
#
# Then, if galleries have changed:
#     Update external clients in website: see x-publish-external-demos*
#
# All told, this script:
#     - Generates example galleries
#     - Makes release zips
#     - Copies zips and content to the local site dir
#     - Builds the site
#     - Zips the upload package
#     - Uploads and unzips on the server
#     - Downloads and verifies its results
#
# And runs the other .py scripts listed above in the first three steps
# (e.g., examples' _generate.sh use 'here documents', sed, and more).
# The generate-examples.py run verifies thumbspage before uploading it.
#
# This script also uses these Python scripts: ziptools create/extract
# (learning-python.com/ziptools.html); diffall.py, a part of Mergeall
# (learning-python.com/mergeall.html), and the larger website's main
# _PUBLISH.py script (learning-python.com/genhtml/__docs__/PUBLISH.py).
#
# NOTES
# -----
# This could be coded in Python, but it's largely shell commands which 
# are marginally easier in Bash.  OTOH, Bash syntax is arguably odd, 
# and Python may be better for portability to Windows/etc.  For more 
# on Bash syntax and usage, see also the "Bash Rant" at end of file.
#
# Config edits, zip uploads, unzips on the server, and downloads are 
# coded with Unix shell tools sed, scp, ssh, and curl, respectively.
# There are equivalents on Windows (but see also its Linux subsystem),
# and Python options exists for all these steps too (e.g., urllib). 
#
# In the end, this script fully automates manual work that used to 
# take hours (or days) every time thumbspage was changed.  Its run
# output must still be inspected for diffs and errors, and its new
# combined zip makes for a long upload (5~10 minutes).  But there are
# no manual steps left after starting this script; this is a huge win.
#
# OTHER
# -----
# See also ./x-publish-external-demos.sh for a script that automates
# builds of other site packages that depend on thumbspage.  Naturally,
# this is specific to the hosting site, but it may serve as an example.
# Folder x-publish-external-demos-prod/ has bits of the live version.
#
# Update: build logs are no longer updated for each release, and the
# sizes of zipfiles are current only in the main web page, not in
# UserGuide.html.  Else, each release must be run multiple times.
#========================================================================


#------------------------------------------------------------------------
# Setup: globals, work dir
#------------------------------------------------------------------------

# verify temp results with long-running cp+diffall?
Verify=y

# download and validate content/zips with diffall at end?
Validate=y

# stop for enter or ctrl-c between steps (y or not)?
Pause=y

# a conditional 'read -p', () optional
function announce() { 
    echo
    echo STARTING: $1
    if [ $Pause == 'y' ]; then 
        read -p 'Press enter/return to continue' 
    fi
}

# all exits happen here
function shutdown() {
    echo 
    echo 'Build stopping at:' $(date)
    echo 'thumbspage' $1
    echo 'Bye.'
    exit
}

echo 'Build starting at:' $(date)

# testing
#Z=~MY-STUFF/Code/ziptools/link
#C=$temp/test-tp-build/C
#W=$temp/test-tp-build/W
#P=$W/P

# live
C=~/MY-STUFF/Code
Z=$C/ziptools/link
W=~/MY-STUFF/Websites
P=$W/Programs/Current/Complete

# ensure scratch folder ({}; optional, ; or \n, -d=nonexist||nondir)
temp=~/Desktop/temp
if [ ! -d $temp ]; then { rm -f $temp; mkdir $temp; } fi


#------------------------------------------------------------------------
# Interact: make galleries?, verify run
#------------------------------------------------------------------------

read -p 'Generate examples galleries first (y or n)? ' userreply
if [[ -n $userreply && $userreply == 'y' ]]
then
    ##bash  $C/thumbspage/build/generate-examples.sh
    python3 $C/thumbspage/build/generate-examples.py
    echo 
    echo '***All galleries built***'
fi

read -p 'About to bundle thumbspage; continue (y or n)? ' userreply
if [[ -z $userreply || $userreply != 'y' ]]; then shutdown 'bundle aborted'; fi


#------------------------------------------------------------------------
# Make three release zips in Code/
#------------------------------------------------------------------------

cd $C
announce 'ZIPS IN CODE'

# save original to verify $C/tp reset later?
if [ $Verify == 'y' ]
then
    rm -rf $temp/thumbspage
    cp -Rp $C/thumbspage $temp
fi

# make a backup copy of configs for users
cp -p thumbspage/user_configs.py thumbspage/docetc/user_configs.py.bkp


#------
# Zip 1: program + ug (local links) + examples [also for online copy]

mv thumbspage/_private    $temp
mv thumbspage/__pycache__ $temp

echo '...Making full-package zip...'
python3 $Z/zip-create.py thumbspage-full-package.zip thumbspage \
           -skipcruft | tail -n 25


#------
# Zip 2: program + ug (online links)

mv thumbspage/.htaccess $temp
mv thumbspage/examples  $temp

echo '...Changing examples/ links in user guide...'
python3 thumbspage/build/userguide-online-links.py
mv thumbspage/UserGuide.html     $temp
mv thumbspage/UserGuideLive.html thumbspage/UserGuide.html

echo '...Making program-only zip...'
python3 $Z/zip-create.py thumbspage-program-only.zip thumbspage \
           -skipcruft | tail -n 25

rm thumbspage/UserGuide.html
mv $temp/UserGuide.html thumbspage


#------
# Zip 3: examples + ug (local links)

mv $temp/examples thumbspage

# unzips to folder; don't include user guide: requires other top-level items
echo '...Making examples-only zip...'
python3 $Z/zip-create.py thumbspage-examples-only.zip thumbspage/examples \
           -skipcruft | tail -n 25

mv $temp/_private    thumbspage
mv $temp/__pycache__ thumbspage
mv $temp/.htaccess   thumbspage


# $C/tp is back to original, 3 new zips are in $C itself
if [ $Verify == 'y' ]
then 
    echo '** Diffall: Code/ to original: there should be no diffs (but docetc/user_configs if changed)'
    python3 $C/mergeall/diffall.py $C/thumbspage $temp/thumbspage -skipcruft | tail -n 10
    rm -rf $temp/thumbspage
fi


#------------------------------------------------------------------------
# Copy zips+folder to Websites/ subdir
#------------------------------------------------------------------------

cd $P
announce 'COPY TO SITE FOLDER'

# the zip here makes ./thumbspage/ (finder may make macos subfolder)

# remake ./thumbspage from $C zip
rm -rf thumbspage/
cp -p $C/thumbspage-full-package.zip .
python3 $Z/zip-extract.py thumbspage-full-package.zip . | tail -n 25

# embed zips in ./thumbspage
mv thumbspage-full-package.zip        thumbspage/
cp -p $C/thumbspage-program-only.zip  thumbspage/
cp -p $C/thumbspage-examples-only.zip thumbspage/

# move zips to private copies with date suffix
private=$C/thumbspage/_private
stamp=$(date +%b-%d-%y)
mv $C/thumbspage-full-package.zip  $private/thumbspage-full-package--${stamp}.zip
mv $C/thumbspage-program-only.zip  $private/thumbspage-program-only--${stamp}.zip
mv $C/thumbspage-examples-only.zip $private/thumbspage-examples-only--${stamp}.zip

# $P/tp is same as $C/tp, except for 3 added zips and no _private/__pycache
if [ $Verify == 'y' ]
then 
    echo '** Diffall: Websites/ to Code/: there should be +3 zips and -1 _file unique'
    python3 $C/mergeall/diffall.py $P/thumbspage $C/thumbspage -skipcruft > $temp/diffall
    grep --context=8 '*UNIQUE' $temp/diffall
    tail -n 10 $temp/diffall
fi


#------------------------------------------------------------------------
# Zip folder with nested zips for upload 
#------------------------------------------------------------------------

cd $W
announce 'ZIP FOR UPLOAD'

# add analytics to online resources
echo '...Adding analytics...'
python3 $C/thumbspage/build/insert-analytics.py

# build/copy site to local UNION; runs ip-anon, fix-readmes, genhtml
echo '...Building site...'
python3 _PUBLISH.py | tail -n 25
cd UNION

# combined upload zip
echo '...Making upload zip...'
python3 $Z/zip-create.py thumbspage.zip thumbspage -skipcruft | tail -n 25

# Zip embeds folder + 3 nested zips
# Nested zips' content has no analytics
# Folder has user guide (local links) + examples/ + .htaccess with analytics/ip-anon
# Folder also has top-level _README.txt, displayed by .htaccess


#------------------------------------------------------------------------
# Upload, unzip on website server
#------------------------------------------------------------------------

# still in UNION
announce 'UPLOAD TO SITE'

# site credentials
sshusr=bitnami@44.232.137.139
sshkey=$W/_admin/SSH-private-key-download/LightsailDefaultKey-us-west-2.pem

# upload combined zip (now automated, was FileZilla, see also curl scp://)
echo '...Uploading to server root (long)...'
scp -i $sshkey thumbspage.zip $sshusr:htdocs 

# unzip on web server (now automated, was copy/paste, run remote commands)
echo '...Unzipping on server...'
ssh -i $sshkey $sshusr <<-EOF
	cd htdocs
	rm -rf thumbspage
	unzip -d . thumbspage.zip | tail -n 25
	rm thumbspage.zip
	exit
	EOF

# from local UNION
rm thumbspage.zip


#------------------------------------------------------------------------
# Fetch and validate online results
#------------------------------------------------------------------------

if [ $Validate != 'y' ]; then shutdown 'validate skipped'; fi

# _really_ make sure - this is a very long task
if [ $Validate == 'y' ]
then 
    echo
    read -p 'Upload complete; fetch and validate results (y or n)? ' Validate
    if [ -z $Validate ] || [ $Validate != 'y' ]; then 
        shutdown 'validate aborted'
    fi
fi

cd $temp


#------
announce 'VALIDATE ONLINE CONTENT'

curl -O https://learning-python.com/thumbspage/UserGuide.html
echo '** Diff: there should be just 1 diff for UG: added analytics'
diff UserGuide.html $C/thumbspage/UserGuide.html

# site: rewrites .py to a display script, rawmode sends just text
# bash: quote for '&', need hack to break the string across lines
# curl: need '-o file' or '> file' else -O names with query param

url='https://learning-python.com/cgi/showcode.py'
url+='?name=thumbspage/thumbspage.py&rawmode=view'

curl $url -o thumbspage.py
echo '** Diff: there should be no diffs for code'
diff thumbspage.py $C/thumbspage/thumbspage.py
rm UserGuide.html thumbspage.py


#------
announce 'VALIDATE ZIPS'

curl -O https://learning-python.com/thumbspage/thumbspage-full-package.zip
curl -O https://learning-python.com/thumbspage/thumbspage-program-only.zip
curl -O https://learning-python.com/thumbspage/thumbspage-examples-only.zip

# bash: {}s optional here
echo '** Cmps: there should be no cmp output for zips'
cmp -bl thumbspage-full-package.zip  $private/thumbspage-full-package--${stamp}.zip
cmp -bl thumbspage-program-only.zip  $private/thumbspage-program-only--${stamp}.zip
cmp -bl thumbspage-examples-only.zip $private/thumbspage-examples-only--${stamp}.zip


# each of the following makes a ./thumbspage folder

#------
announce 'VALIDATE UNZIP ALL'

python3 $Z/zip-extract.py thumbspage-full-package.zip . | tail -n 25
python3 $C/mergeall/diffall.py thumbspage $C/thumbspage -skipcruft > $temp/diffall

echo '** Diffall: fullpkg download to Code/: there should be -1: _file'
grep --context=8 '*UNIQUE' $temp/diffall
tail -n 10 $temp/diffall
rm -rf thumbspage/

#------
announce 'VALIDATE UNZIP PROGRAM'

python3 $Z/zip-extract.py thumbspage-program-only.zip . | tail -n 25
python3 $C/mergeall/diffall.py thumbspage $C/thumbspage -skipcruft > $temp/diffall

echo '** Diffall: program download to Code/: there should be UG + -3=[1 _file, examples/, .htaccess']
grep --context=8 '*UNIQUE' $temp/diffall
tail -n 10 $temp/diffall
# >200 links differ: tail this
diff thumbspage/UserGuide.html $C/thumbspage/UserGuide.html | tail -n 25
rm -rf thumbspage/

#------
announce 'VALIDATE UNZIP EXAMPLES'

python3 $Z/zip-extract.py thumbspage-examples-only.zip . | tail -n 25
python3 $C/mergeall/diffall.py thumbspage $C/thumbspage -skipcruft > $temp/diffall

echo '** Diffall: examples download to Code/: there should be -N: all other top-level items'
grep --context=20 '*UNIQUE' $temp/diffall
tail -n 10 $temp/diffall
rm -rf thumbspage/

rm thumbspage-full-package.zip thumbspage-program-only.zip thumbspage-examples-only.zip
shutdown '_PUBLISH complete'



#=====================================================================
# The Bash Rant - a few notes on the wildly arcane syntax used above:
#
# - '{ ; }' around a statement block is optional
#
# - $x is required to eval vars, even in expression contexts
#
# - Linebreaks can be used instead of--or in addition to--';'
#   statement delimiters
#
# - The quotes around 'y' strings above are optional (well, often),
#   and " " evals nested $ vars but ' ' does not
#
# - Spaces are required around operators and within [ ] tests
#   and { } compounds, but cannot be used around '=' in assignments
#
# - A nested ' within a ' ' requires either $' \' ' or " ' ",
#   but " " has extra semantics (see above)
#
# - There seems no good way to make strings span lines in '=': see
#   the url= and url+= work-around above ('a''b' and $a$b iff no space)
#
# - 'a b c' auto-splits to a list, but ('a' 'b' 'c') is an array
#   which requires bizarre reference syntax like ${examples[@]}
#
# - Function defs can code 'function', '()', or both - so there
#   are already 3 wtdi; and params are '$N', not useful names
#
# - Tests must be coded in '[]' or '[[]]' depending on content - 
#   '[[ a && b ]]' is just '[ a ] && [ b ]', iff '[[]]' is supported
#
# - Plus more bracket hell: '()', '{}', '(())',...  
#
# And all these work the same in macOS's Bash (but why?):
#
#   if test 's'; then echo 'yes'; fi
#   if [ 's' ]; then echo yes; fi
# 
#   if [ 's' ] 
#   then
#     echo yes
#   fi
#
#   if [[ 's' ]]; then
#     echo yes; fi
#
#   if { test 's'; } then { echo yes; } fi
#   if { test 's'; }; then { echo yes; }; fi;
#
# Which is to subjectively say: yuck.  Bash seems ad hock (and 
# plausibly odd hack).  Python is a lot more coherent, but Bash is 
# okay for command-line centric scripts like this one.  But when 
# build/generate-examples.sh was no longer in that category, it 
# was converted to Python, the better option for program logic.
#
# And the closer: editing a Bash script's source file like this one 
# while it is running can cause the script to fail on nonsensical 
# syntax errors, because Bash reads the script's code as it goes. 
# No, really.  So don't do that.  And OMG...
#=====================================================================