wrdk/scripts/Run

#!/bin/bash
# script to set up individual rendering server

LicensesDirectory=XML-Licenses

ERROR()
{
  echo error: $*
  exit 1
}

USAGE()
{
  [ -z "$1" ] || echo error: "$*"
  echo usage: $(basename "$0") '<options>'
  echo '       --help          -h         this message'
  echo '       --verbose       -v         more messages'
  echo '       --get-index=n   -g <n>     where to rsync the indices from {1 => render1}'
  echo '       --index-only    -i         only do the index'
  echo '       --no-run        -n         do not run final make'
  echo '       --clear         -c         clear work and dest dirs'
  echo '       --re-parse      -R         clear parse and render stamps'
  echo '       --re-render     -r         clear render stamps'
  echo '       --machines=m    -m <m>     set machine count [${MACHINE_COUNT} or 9]'
  echo '       --parallel=n    -j <n>     set make -j value [${PARALLEL_BUILD} or 3]'
  echo '       --language=xx   -l <xx>    set language [en]'
  echo '       --suffix=<s>    -s <s>     set the directory suffix [pedia]'
  echo '       --prefix=name   -p <name>  set data file prefix [wiki]'
  echo '       --work=dir      -w <dir>   workdir [work]'
  echo '       --dest=dir      -d <dir>   destdir [image]'
  echo '       --temp=dir      -t <dir>   tempdir [/tmp]'
  echo '       --farm=n        -f <n>     manual farm number [from hostname numeric suffix]'
  echo '       --compress=<c>  -C <c>     set compression {low/high} [low]'
  echo '       --debug         -D         print but do not execute make commands'
  exit 1
}

# run a command or just print it
RUN()
{
  local command log

  log=yes
  case "$1" in
    -n|--no-log)
      log=no
      shift
      ;;
    *)
      ;;
  esac
  command="$*"

  case "${debug}" in
    [yY]|[yY][eE][sS])
      echo DEBUG: ${command}
      return
      ;;
    *)
      ;;
  esac
  case "${log}" in
    [yY]|[yY][eE][sS])
      echo | tee -a "${LogFile}"
      echo '===>' $(date '+%Y-%m-%d %H:%M:%S'):  ${command} | tee -a "${LogFile}"
      eval "time ${command}" 2>&1 | tee -a "${LogFile}"
      ;;
    *)
      eval "${command}"
  esac
}


verbose=no
IndexHost=
clear=no
work=work
dest=image
temp=/tmp
run=yes
IndexOnly=no
farm=
debug=no
language=en
suffix=pedia
FilePrefix=wiki
MaximumThreads=64
export MACHINE_COUNT=${MACHINE_COUNT:-9}
export PARALLEL_BUILD=${PARALLEL_BUILD:-3}
compression=low

getopt=/usr/local/bin/getopt
[ -x "${getopt}" ] || getopt=getopt
args=$(${getopt} -o hvg:p:incrRl:s:w:d:t:f:p:j:m:CD: --long=help,verbose,get-index:,index-only,no-run,clear,re-render,re-parse,language:,suffix:,suffix:,work:,dest:,temp:,farm:,prefix:,parallel:,machines:,compress:,debug -- "$@") ||exit 1

# replace the arguments with the parsed values
eval set -- "${args}"

while :
do
  case "$1" in
    -v|--verbose)
      verbose=yes
      shift
      ;;

    -g|--get-index)
      IndexHost=$2
      shift 2
      ;;

    -i|--index-only)
      IndexOnly=yes
      clear=index
      shift
      ;;

    -n|--no-run)
      run=no
      shift
      ;;

    -c|--clear)
      clear=yes
      shift
      ;;

    -r|--re-render)
      clear=render
      shift
      ;;

    -R|--re-parse)
      clear=parse
      shift
      ;;

    -l|--language)
      language=$2
      shift 2
      ;;

    -s|--suffix)
      suffix=$2
      shift 2
      ;;

    -w|--work)
      work=$2
      shift 2
      ;;

    -d|--dest)
      dest=$2
      shift 2
      ;;

    -t|--temp)
      temp=$2
      shift 2
      ;;

    -f|--farm)
      farm=$2
      shift 2
      ;;

    -p|--prefix)
      FilePrefix=$2
      shift 2
      ;;

    -j|--parallel)
      PARALLEL_BUILD=$2
      shift 2
      ;;

    -m|--machines)
      MACHINE_COUNT=$2
      shift 2
      ;;

    -C|--compress)
      compression=$2
      shift 2
      ;;

    -D|--debug)
      debug=yes
      shift
      ;;

    --)
      shift
      break
      ;;

    -h|--help)
      USAGE
      ;;

    *)
      USAGE invalid option: $1
      ;;
  esac
done


if [[ $((${MACHINE_COUNT} * ${PARALLEL_BUILD})) > ${MaximumThreads} ]]
then
  echo MACHINE_COUNT=${MACHINE_COUNT}
  echo PARALLEL_BUILD=${PARALLEL_BUILD}
  USAGE too many threads: "${MACHINE_COUNT} * ${PARALLEL_BUILD} > ${MaximumThreads}"
fi

[ -z "${language}" ] && USAGE language is not set

licenses=$(readlink -m "${LicensesDirectory}")
license="${licenses}/${language}/license.xml"
terms="${licenses}/${language}/terms.xml"
articles_link="${language}wiki-pages-articles.xml"
articles=$(readlink -m "${articles_link}")

LogFile="${language}.log"

[ -f "${articles}" ] || USAGE error articles link: ${articles_link} not set correctly

[ -f "${license}" ] || license="${licenses}/en/license.xml"
[ -f "${terms}" ] || terms="${licenses}/en/terms.xml"

xml="${license} ${terms} ${articles}"

# extract numeric suffix from host name
# expect that the rendering hosts are numbered from zero
this_host=$(hostname --short)
this_host_prefix=${this_host%%[0-9]}
this_id=${this_host##*[^0-9]}
[ -z "${this_id}" ] && this_id=0

[ -z "${farm}" ] && farm="farm${this_id}" || farm="farm${farm##*[a-z]}"

# truncate the log file
RUN -n rm -f "${LogFile}"
RUN -n touch "${LogFile}"

# update
RUN git pull --rebase

# set up the make options
common_opts="DESTDIR='${dest}' WORKDIR='${work}' TEMPDIR='${temp}'"
common_opts="${common_opts} WIKI_FILE_PREFIX='${FilePrefix}'"
common_opts="${common_opts} WIKI_LANGUAGE='${language}'"
common_opts="${common_opts} WIKI_DIR_SUFFIX='${suffix}'"
common_opts="${common_opts} XML_FILES='${xml}'"
common_opts="${common_opts} VERBOSE='${verbose}'"
case "${language}" in
  en)
    case "${compression}" in
      [hH][iI][gG][hH])
        # Best version for 4GB card
        common_opts="${common_opts} ENABLE_LANGUAGES_LINKS=NO"

        ## ok
        #common_opts="${common_opts} ARTICLES_PER_BLOCK=64"
        #common_opts="${common_opts} ARTICLE_BLOCK_SIZE=250000"

        ## ok
        common_opts="${common_opts} ARTICLES_PER_BLOCK=64"
        common_opts="${common_opts} ARTICLE_BLOCK_SIZE=200000"
        ;;

      *)
        # low compression for 8GB card
        common_opts="${common_opts} ENABLE_LANGUAGES_LINKS=YES"
        common_opts="${common_opts} ARTICLES_PER_BLOCK=1"
        common_opts="${common_opts} ARTICLE_BLOCK_SIZE=131072"
        ;;
    esac
    ;;

  *)
    common_opts="${common_opts} ENABLE_LANGUAGES_LINKS=YES"
    common_opts="${common_opts} ARTICLES_PER_BLOCK=1"
    common_opts="${common_opts} ARTICLE_BLOCK_SIZE=131072"
    ;;
esac

# clean up
case "${clear}" in
  [yY]|[yY][eE][sS])
    RUN make clean index-clean "${farm}-clean" cleandirs ${common_opts}
    ;;
  [iI][nN][dD][eE][xX])
    RUN make clean index-clean ${common_opts}
    ;;
  [rR][eE][nN][dD][eE][rR])
    RUN make clean "${farm}-render-clean" ${common_opts}
    RUN make "${farm}-parse-touch" ${common_opts}
    ;;
  [pP][aA][rR][sS][eE])
    RUN make clean "${farm}-clean" ${common_opts}
    ;;
  [nN]|[nN][oO])
    ;;
  *)
    ERROR unknown clear action: ${clear}
    ;;
esac


# create root directories and fonts
RUN make createdirs ${common_opts}
RUN make fonts ${common_opts}

# copy the index from another machine
#=====================================
#!!! THIS HAS HARD CODED SAMO PATH !!!
#!!! NEED TO FIX THIS              !!!
#=====================================
if [ -n "${IndexHost}" ]
then
  echo THIS IS BROKEN
  exit 99
  [ -z "${IndexHost%%[0-9]}" ] && IndexHost="${this_host_prefix}${IndexHost}"

  list='templates.db articles.db offsets.db counts.text'
  items=
  for i in ${list}
  do
    items="${items} ${IndexHost}:samo/${work}/${i}"
  done
  RUN rsync -avHx --progress ${items} "${work}"/
  RUN touch stamp-r-index  # ***THIS IS WRONG need to have a make target for this
fi

# run the build
case "${run}" in
  [yY]|[yY][eE][sS])

    RUN make farm-index ${common_opts}

    case "${IndexOnly}" in
      [yY]|[yY][eE][sS])
        ;;
      *)
        RUN make -j${PARALLEL_BUILD} "${farm}" ${common_opts}
        ;;
    esac
esac