Automatic backup of config using a "pull method"

freetz_getconfigs -?

freetz_getconfigs

freetz_getconfigs -d /opt remote.domain.com

freetz_getconfigs -c  remote.domain.com

freetz_getconfigs -c  mymodem,192.168.178.1,,81,,,80

freetz_getconfigs -c  mymodem,192.168.178.1,,0,,,80

freetz_getconfigs -c  remote.domain.com,,,,,,451

#!/bin/sh

# Define constants
DATESTAMP=`date +%Y-%m-%d.%H-%M`
FOLDER=/var/www/vhosts/domain.com/freetz

USER_DEFAULT=admin
PASS_DEFAULT=secret
PORT_DEFAULT=6080
PORTAVM_DEFAULT=450
MINSIZE=20000
HEADLESS=
tty >/dev/null || HEADLESS=true

while getopts cd: opt
do
  case $opt in
    c)   FOLDER="`pwd`" ;;
    d)   FOLDER="$OPTARG";;
    ?)   printf "Usage: %s [-c] [-d <folder>] [<HOST>],[<IP>],[PASS_FREETZ],[<PORT_FREETZ>],[<USER>],[PASS_EXTERNAL],[PORT_AVM]\n" $0
    exit 2;;
  esac
done
shift `expr ${OPTIND} - 1`


# Subroutines

age_of_file ()
{
  if [ ! -e "$1" ] ; then
    echo 0
  else
    AGE_IN_SECONDS=$((`date +%s` - `date +%s -r "$1"` ))
    echo $((${AGE_IN_SECONDS} / 86400))
  fi
}

httpspeed ()
{
  # Returns no value and  errorlevel 1 if site is unreachable or too slow
  # Returns the speed in milliseconds if it's below 2 seconds
  _SPEED=`httping -c1 -t1 ${1} 2>/dev/null | \
         egrep '.+/.+/.+/.+/.+' | tail -n1 | \
         awk -F/ '{print $4}' | awk -F. '{print $1}'`

  if [ -z "${_SPEED}" ] ; then
    echo "${URL} is unreachable" >&2
  elif [ ${_SPEED} -gt 2000 ] ; then
    echo "${URL} is slow (${_SPEED} ms)" >&2
  else
    echo ${_SPEED}
    return 0
  fi
  return 1
}


get_avm_config ()
{
  [ ${PORTAVM} -eq 0 ] && return

  TMPDIR2=`mktemp -p /dev/shm -d ${0//*\/}.XXXXXXXXXX`
  if ! cd ${TMPDIR2} 2>/dev/null ; then
    echo "Error changing current directory to ${TMPDIR2}. You should never see this message!" >&2
  else

    # It's intended to be called from WAN (https)
    # If PORTAVM begins with 8 it assumes LAN and turns to http
    PROTO=https
    echo ${PORTAVM} | grep -q '^8' && PROTO=http
    URL="${PROTO}://${IP}:${PORTAVM}"

    if httpspeed ${URL} >/dev/null ; then
      # get challenge key from FB

      _CHALLENGE=`curl -s -k \
                       --user ${USER}:${PASSE} \
                       "${URL}/login.lua" | \
                       grep 'challenge' | egrep '[0-9a-f]{8}' | \
                       awk -F= '{print $2}' | tail -n1 | egrep -o '[0-9a-f]{8}'`

      if [ -z "${_CHALLENGE}" ] ; then
        echo "Error getting challenge for ${HOST} using ${URL}" >&2
      else
        # echo "Received Challenge \"${_CHALLENGE}\"" >&2
        # build md5 from challenge key and password
        _MD5=`echo -n ${_CHALLENGE}"-"${PASSE} | \
              iconv -f ISO8859-1 -t UTF-16LE   | \
              md5sum -b | awk '{print substr($0,1,32)}'`

        # assemble challenge key and md5
        _RESPONSE=${_CHALLENGE}"-"${_MD5}

        # get sid for later use
        _SID=`curl -i -s -k \
                    --user ${USER}:${PASSE} \
                    -d 'response='${_RESPONSE} \
                    -d 'page=' \
                    -d 'username='${USER} \
                    ${URL}/login.lua | \
                    grep "Location:" | awk -F'=' {' print $NF '}`

        if ! curl -s -k \
                  --user ${USER}:${PASSE} \
                  --form 'sid='${_SID} \
                  --form 'ImportExportPassword='${PASSE} \
                  --form 'ConfigExport=' \
                  ${URL}/cgi-bin/firmwarecfg >TMP1 ; then

          echo "Error getting .export for ${HOST} (no data)" >&2
        else
          if [ `stat -c%s TMP1` -lt ${MINSIZE} ] ; then
            echo "Error getting .export for ${HOST} using ${URL} (too small)" >&2
            echo "SID: \"${_SID}\" CHALLENGE: \"${_CHALLENGE}\"  MD5: \"${_MD5}\"  RESPONSE: \"${_RESPONSE}\"" >&2
          else
            chown ${FUSER}:${FGROUP} TMP1
            mkdir "${DEST_FOLDER}" 2>/dev/null && chown ${FUSER}:${FGROUP} "${DEST_FOLDER}"
            LATESTCONFIG=`find "${DEST_FOLDER}" -maxdepth 1 -type f -size +10 | \
                          grep "${IP}.*\.export" | xargs -I{} stat -c '%Y %n' {} | \
                          sort -rn | head -n1 | awk '{print $2}'`

            if [ -z "${LATESTCONFIG}" ] ; then                     # No previous .export is found
              echo "Saving \"${DEST_BASE}.export\" because no previous AVM-config has been found" >&2
              mv TMP1 "${DEST_BASE}.export"
            else
              if [ ${TRIED_FREETZ} ] ; then      # The Freetz comparison is better, so make that leading
                if ! diff TMP1 "${LATESTCONFIG}" >/dev/null 2>&1 ; then
                  echo "Saving \"${DEST_BASE}.export\" because Freetz config has changed" >&2
                  mv TMP1 "${DEST_BASE}.export"
                fi
              else
                if [ `age_of_file "${LATESTCONFIG}"` -gt 30 ] ; then      # Compare EXACTLY if age is older than 30 days
                  if ! diff TMP1 "${LATESTCONFIG}" >/dev/null 2>&1 ; then
                    echo "Saving \"${DEST_BASE}.export\" because latest AVM-config was older than 30 days and is different" >&2
                    mv TMP1 "${DEST_BASE}.export"
                  fi
                else
                  # remove BINARY data and timestamps before making a diff
                  sed '/ BINFILE:/,/ END OF FILE /d;/ [0-2][0-9]:[0-5][0-9]:[0-9][0-9]/d;/END OF EXPORT/d' "${LATESTCONFIG}" >TMP2
                  sed '/ BINFILE:/,/ END OF FILE /d;/ [0-2][0-9]:[0-5][0-9]:[0-9][0-9]/d;/END OF EXPORT/d' TMP1              >TMP3
                  if ! diff TMP2 TMP3 >/dev/null 2>&1 ; then                  # Compare the 2 stripped configs
                    echo "Saving \"${DEST_BASE}.export\" because AVM-config has changed" >&2
                    diff TMP2 TMP3
                    mv TMP1 "${DEST_BASE}.export"
                  fi
                fi
              fi
            fi
          fi
        fi
      fi
    fi
  fi
  cd "${FOLDER}"
  rm -rf ${TMPDIR2}
}

patch_files ()
{
  # remove or patch some of the untarred files as they spoil the check for a changed config
  if cd "$1" >/dev/null 2>&1 ; then
    rm -f stat.cfg chrony.drift multid.leases 2>/dev/null
    rm -f fx_cg fonctrl phonebook voipd_call_stat 2>/dev/null

    sed -i '/ [0-2][0-9]:[0-5][0-9]:[0-9][0-9]/d' voip.cfg  # remove datestamp in voip.cfg
    sed -i '/ [0-2][0-9]:[0-5][0-9]:[0-9][0-9]/d' ar7.cfg   # remove datestamp in ar7.cfg
    sed -i 's/ *\/\*AVM\*\///g' ar7.cfg                     # remove the /*AVM*/ tag in ar7.cfg
    sed -i 's/; *$/;/g' ar7.cfg                             # remove trailing spaces after a semicolon
    cd - >/dev/null 2>&1
  fi
}

get_freetz_config ()
{
  # Fetch config-file with wget in background
  # Somehow it doesn't listen to the 6 seconds timeout I'm giving it
  wget -q --timeout=6                              \
       --http-user=${USER} --http-password=${PASS} \
       ${URL}/cgi-bin/backup/do_backup.cgi         \
       -O TMP_CONFIG 2>/dev/null &
  # Little wait loop to give wget the time to fetch the file
  n=1
  while sleep 1 ; do
    [ `stat -c%s TMP_CONFIG` -ge ${MINSIZE} ] && break
    [ $n -gt 7 ]                              && break
    let n+=1
  done

  if [ `stat -c%s TMP_CONFIG` -gt 300 ] ; then
    # If I received at least some data, then wait some more and it might get everything
    sleep 2
  else
    echo "wget didn't get any data from host \"${HOST}\"" >&2
    exec 2>/dev/null
    kill -9 %1 2>/dev/null
    exec 2>&2
    return 1
  fi
  return 0
}

keep_config_if_different ()
{
  LATESTCONFIG=`find "${DEST_FOLDER}" -maxdepth 1 -type f -size +10 | \
                      grep "${IP}.*\.tgz" | xargs -I{} stat -c '%Y %n' {} | \
                      sort -rn | head -n1 | awk '{print $2}'`

  # If no older config is found
  if [ -z "${LATESTCONFIG}" ] ; then
    echo "Saving \"${DEST_BASE}.tgz\" because no previous Freetz config has been found" >&2
    mv TMP_CONFIG "${DEST_BASE}.tgz"
    get_avm_config
  else

    TRIED_FREETZ=1  # A flag for get_avm_config to know if the freetz config has been tried or not
    mkdir f1   # folder to untar the current config
    mkdir f2   # folder to untar the previous config
    if ! tar xzf TMP_CONFIG -C f1 2>/dev/null ; then
      echo "I just downloaded an invalid gzipped tar-file" >&2
      rm -f TMP_CONFIG
    else
      if ! tar xzf ${LATESTCONFIG} -C f2 ; then
        echo "A previously downloaded config turns out to be invalid... very strange... (${LATESTCONFIG})" >&2
      else
        # Remove files that are likely to be changed, but don't contain important config
        patch_files f1/var_flash
        patch_files f2/var_flash

        if ! diff f1/var_flash f2/var_flash >/dev/null 2>&1 ; then
          echo "Saving \"${DEST_BASE}.tgz\" because Freetz config changed" >&2
          mv TMP_CONFIG "${DEST_BASE}.tgz"
          # Now get the AVM config too
          get_avm_config
        elif [ `find "${DEST_FOLDER}" -maxdepth 1 -type f -name \*.export | wc -l` -lt 1 ] ; then
          # No previous AVM config exists, so try and fetch one anyhow
          get_avm_config
       fi
     fi
   fi
 fi
}

get_both_configs ()
{
  PROTO=http
  echo ${PORT} | grep -q '^4[456][0-9]' && PROTO=https
  URL="${PROTO}://${IP}:${PORT}"

  if ! httpspeed ${URL} >/dev/null ; then
    # Freetz is not reacting, maybe it is a plain jane AVM Fritzbox
    # I will not be able to see if a config has changed.......
    echo "No reaction from Freetz.... will try AVM now.."
    get_avm_config
  else
    TMPDIR1=`mktemp -p /dev/shm -d ${0//*\/}.XXXXXXXXXX`

    if ! cd ${TMPDIR1} ; then
      echo "Error changing folder to ${TMPDIR1}" >&2
    else
      touch TMP_CONFIG

      get_freetz_config

      if [ `stat -c%s TMP_CONFIG` -lt ${MINSIZE} ] ; then
        echo "Error getting config from ${HOST} on ${IP}" >&2
      else

        # I think we're getting successful here, let's create a folder already
        chown ${FUSER}:${FGROUP} TMP_CONFIG
        mkdir "${DEST_FOLDER}" 2>/dev/null && chown ${FUSER}:${FGROUP} "${DEST_FOLDER}"

        keep_config_if_different
      fi
    fi
    cd "${FOLDER}"
    rm -rf ${TMPDIR1}
  fi
}


# Main Program starts here

if ! cd "${FOLDER}" ; then
  echo "${FOLDER} does not exist" >&2
  exit 1
fi

BASENAME="${0//*\/}"                           # Strip Path of script name
BASENAME="${BASENAME%.*}"                      # Strip extension
CONFIGSOURCE="${FOLDER}/${BASENAME}.list"      # Construct modemlist from script name
CONFIGFILE=`mktemp`                            # temp file for sanitized modemlist

if [ -n "$*" ] ; then
  echo "$*" >"${CONFIGFILE}"                     # push the commandline parameters in the configfile
elif [ -e "${CONFIGSOURCE}" ] ; then
  awk -F# '{print $1}' "${CONFIGSOURCE}" | \
      sed '/^ *$/d' >"${CONFIGFILE}"             # Get all the data before # and remove empty lines
else
  echo "Configfile: \"${CONFIGSOURCE}\" does not exist" >&2
  exit 1
fi

if [ ! -s "${CONFIGFILE}" ] ; then
  echo "No workable data in \"${CONFIGSOURCE}\"" >&2
  exit 1
fi

# Main Loop
FGROUP=`stat -c%G .`
FUSER=`stat -c%U .`

while read MODEMLINE ; do
  HOST="`echo "${MODEMLINE}"    | awk -F, '{print $1}'`"
  IP="`echo "${MODEMLINE}"      | awk -F, '{print $2}'`"
  PASS="`echo "${MODEMLINE}"    | awk -F, '{print $3}'`"
  PORT="`echo "${MODEMLINE}"    | awk -F, '{print $4}' | tr -cd '0-9'`"
  USER="`echo "${MODEMLINE}"    | awk -F, '{print $5}'`"
  PASSE="`echo "${MODEMLINE}"   | awk -F, '{print $6}'`"
  PORTAVM="`echo "${MODEMLINE}" | awk -F, '{print $7}' | tr -cd '0-9'`"

  HOSTORG="${HOST}"
  TRIED_FREETZ=

  if [ -z "${IP}" ] ; then
    HOST=`echo "${HOST}" | tr 'A-Z' 'a-z' | \
          egrep -o '[a-z0-9.-]+\.[a-z0-9]+'`     # Loosely Sanitize domain if no IP is given
  fi
  if [ -z "${HOST}" ] ; then
    echo "${HOSTORG} is not a valid host" >&2
  else
    [ ${HEADLESS} ] || echo ${HOST}
    if [ -z "${IP}" ] ; then
      # If HOST is so clearly an IP-address then use that instead....
      if echo "${HOST}" | egrep -q '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' ; then
        IP=${HOST}
      else
        IP=`host -t A ${HOST} 2>/dev/null | grep -o 'has address .*' | awk '{print $3}' | head -n1`
      fi
    fi
    if [ -z "${IP}" ] ; then
      echo "Unable to resolve ${HOST}" >&2
    else
      [ -z "${PORT}" ]    &&    PORT=${PORT_DEFAULT}      # port of Freetz interface
      [ -z "${PASS}" ]    &&    PASS=${PASS_DEFAULT}      # internal password
      [ -z "${USER}" ]    &&    USER=${USER_DEFAULT}      # external username
      [ -z "${PASSE}" ]   &&   PASSE=${PASS}              # external password
      [ -z "${PORTAVM}" ] && PORTAVM=${PORTAVM_DEFAULT}   # port of AVM interface (https)

      DEST_FOLDER="${FOLDER}/${HOST}"
      DEST_BASE="${DEST_FOLDER}/${IP}.${DATESTAMP}"

      if [ ${PORT} -eq 0 ] ; then
        get_avm_config     # Fetch only AVM-config when the Freetz port is defined as 0
      else
        get_both_configs
      fi
    fi
  fi
done<"${CONFIGFILE}"
rm -f "${CONFIGFILE}"

#!/bin/sh

# Define constants
DATESTAMP=`date +%Y-%m-%d.%H-%M`
FOLDER=/var/www/vhosts/freetz.yourdomain.com/httpdocs

USER_DEFAULT=admin
PASS_DEFAULT="defaultpass"
PORT_DEFAULT=61080
PORTAVM_DEFAULT=450
MINSIZE=20000

HTTP_OPTIONS=
HTTPS_OPTIONS='--cipher RC4-SHA'

HEADLESS=
tty >/dev/null || HEADLESS=true

while getopts cd: opt
do
  case $opt in
    c)   FOLDER="`pwd`" ;;
    d)   FOLDER="$OPTARG";;
    ?)   printf "Usage: %s [-c] [-d <folder>] [<HOST>],[<IP>],[PASS_FREETZ],[<PORT_FREETZ>],[<USER>],[PASS_EXTERNAL],[PORT_AVM]\n" $0
    exit 2;;
  esac
done
shift `expr ${OPTIND} - 1`


# Subroutines

age_of_file ()
{
  if [ ! -e "$1" ] ; then
    echo 0
  else
    AGE_IN_SECONDS=$((`date +%s` - `date +%s -r "$1"` ))
    echo $((${AGE_IN_SECONDS} / 86400))
  fi
}

httpspeed ()
{
  # Returns no value and  errorlevel 1 if site is unreachable or too slow
  # Returns the speed in milliseconds if it's below 2 seconds
  _SPEED=`httping -c1 -t1 ${1} 2>/dev/null | \
         egrep '.+/.+/.+/.+/.+' | tail -n1 | \
         awk -F/ '{print $4}' | awk -F. '{print $1}'`

  if [ -z "${_SPEED}" ] ; then
    echo -e "${URL} is unreachable\n" >&2
  elif [ ${_SPEED} -gt 2000 ] ; then
    echo "${URL} is slow (${_SPEED} ms)" >&2
  else
    echo ${_SPEED}
    return 0
  fi
  return 1
}


get_avm_config ()
{
  [ ${PORTAVM} -eq 0 ] && return

  TMPDIR2=`mktemp -p /dev/shm -d ${0//*\/}.XXXXXXXXXX`
  if ! cd ${TMPDIR2} 2>/dev/null ; then
    echo "Error changing current directory to ${TMPDIR2}. You should never see this message!" >&2
  else

    # It's intended to be called from WAN (https)
    # If PORTAVM begins with 8 it assumes LAN and turns to http
    PROTO=https
    echo ${PORTAVM} | grep -q '^80$' && PROTO=http

    OPTIONS="${HTTP_OPTIONS}"
    [ "${PROTO}" = 'https' ] && OPTIONS="${HTTPS_OPTIONS}"

    URL="${PROTO}://${IP}:${PORTAVM}"

    if httpspeed ${URL} >/dev/null ; then
      # get challenge key from FB

      _CHALLENGE=`curl -s -k ${OPTIONS} \
                       --user ${USER}:${PASSE} \
                       "${URL}/login.lua" | \
                       grep 'challenge' | egrep '[0-9a-f]{8}' | \
                       awk -F= '{print $2}' | tail -n1 | egrep -o '[0-9a-f]{8}'`

      if [ -z "${_CHALLENGE}" ] ; then
        echo "Error getting challenge for ${HOST} using ${URL}" >&2
      else
        # echo "Received Challenge \"${_CHALLENGE}\"" >&2
        # build md5 from challenge key and password
        _MD5=`echo -n ${_CHALLENGE}"-"${PASSE} | \
              iconv -f ISO8859-1 -t UTF-16LE   | \
              md5sum -b | awk '{print substr($0,1,32)}'`

        # assemble challenge key and md5
        _RESPONSE=${_CHALLENGE}"-"${_MD5}

        # get sid for later use
        _SID=`curl -i -s -k ${OPTIONS} \
                    --user ${USER}:${PASSE} \
                    -d 'response='${_RESPONSE} \
                    -d 'page=' \
                    -d 'username='${USER} \
                    ${URL}/login.lua | \
                    grep "Location:" | awk -F'=' {' print $NF '}`

        if ! curl -s -k ${OPTIONS} \
                  --user ${USER}:${PASSE} \
                  --form 'sid='${_SID} \
                  --form 'ImportExportPassword='${PASSE} \
                  --form 'ConfigExport=' \
                  ${URL}/cgi-bin/firmwarecfg >TMP1 ; then

          echo "Error getting .export for ${HOST} (no data)" >&2
        else
          if [ `stat -c%s TMP1` -lt ${MINSIZE} ] ; then
            echo "Error getting .export for ${HOST} using ${URL} (too small)" >&2
            echo "SID: \"${_SID}\" CHALLENGE: \"${_CHALLENGE}\"  MD5: \"${_MD5}\"  RESPONSE: \"${_RESPONSE}\"" >&2
          else
            chown ${FUSER}:${FGROUP} TMP1
            mkdir "${DEST_FOLDER}" 2>/dev/null && chown ${FUSER}:${FGROUP} "${DEST_FOLDER}"
            LATESTCONFIG=`find "${DEST_FOLDER}" -maxdepth 1 -type f -size +10 | \
                          grep "${IP}.*\.export" | xargs -I{} stat -c '%Y %n' {} | \
                          sort -rn | head -n1 | awk '{print $2}'`

            if [ -z "${LATESTCONFIG}" ] ; then                     # No previous .export is found
              echo "Saving \"${DEST_BASE}.export\" because no previous AVM-config has been found" >&2
              mv TMP1 "${DEST_BASE}.export"
            else
              if [ ${TRIED_FREETZ} ] ; then      # The Freetz comparison is better, so make that leading
                if ! diff TMP1 "${LATESTCONFIG}" >/dev/null 2>&1 ; then
                  echo -e "Saving \"${DEST_BASE}.export\" because Freetz config has changed\n" >&2
                  mv TMP1 "${DEST_BASE}.export"
                fi
              else
                if [ `age_of_file "${LATESTCONFIG}"` -gt 30 ] ; then      # Compare EXACTLY if age is older than 30 days
                  if ! diff TMP1 "${LATESTCONFIG}" >/dev/null 2>&1 ; then
                    echo "Saving \"${DEST_BASE}.export\" because latest AVM-config was older than 30 days and is not EXACTLY the same" >&2
                    mv TMP1 "${DEST_BASE}.export"
                  fi
                else

                  # remove BINARY data and timestamps before making a diff
                  sed    '/ BINFILE:/,/ END OF FILE /d' "${LATESTCONFIG}" >TMP2
                  sed -i '/ [0-2][0-9]:[0-5][0-9]:[0-9][0-9]/d'            TMP2
                  sed -i '/END OF EXPORT/d;/update_found/d'                TMP2
                  sed -i '/END OF EXPORT/d;/running_version/d'             TMP2

                  sed    '/ BINFILE:/,/ END OF FILE /d'             TMP1  >TMP3
                  sed -i '/ [0-2][0-9]:[0-5][0-9]:[0-9][0-9]/d'            TMP3
                  sed -i '/END OF EXPORT/d;/update_found/d'                TMP3
                  sed -i '/END OF EXPORT/d;/running_version/d'             TMP3

                  if ! diff TMP2 TMP3 >/dev/null 2>&1 ; then                  # Compare the 2 stripped configs
                    echo -e "Saving \"${DEST_BASE}.export\" because AVM-config has changed\n" >&2
                    diff TMP2 TMP3 >&2
                    echo -e '\n---\n\n' >&2
                    mv TMP1 "${DEST_BASE}.export"
                  fi
                fi
              fi
            fi
          fi
        fi
      fi
    fi
  fi
  cd "${FOLDER}"
  rm -rf ${TMPDIR2}
}

patch_files ()
{
  # remove or patch some of the untarred files as they spoil the check for a changed config
  if cd "$1" >/dev/null 2>&1 ; then
    rm -f stat.cfg chrony.drift multid.leases 2>/dev/null
    rm -f fx_cg fonctrl phonebook voipd_call_stat 2>/dev/null

    sed -i '/ [0-2][0-9]:[0-5][0-9]:[0-9][0-9]/d' voip.cfg  # remove datestamp in voip.cfg
    sed -i '/ [0-2][0-9]:[0-5][0-9]:[0-9][0-9]/d' ar7.cfg   # remove datestamp in ar7.cfg
    sed -i 's/ *\/\*AVM\*\///g' ar7.cfg                     # remove the /*AVM*/ tag in ar7.cfg
    sed -i 's/; *$/;/g' ar7.cfg                             # remove trailing spaces after a semicolon
    cd - >/dev/null 2>&1
  fi
}

get_freetz_config ()
{
  # Fetch config-file with wget in background
  # Somehow it doesn't listen to the 6 seconds timeout I'm giving it
  wget -q --timeout=6                              \
       --http-user=${USER} --http-password=${PASS} \
       ${URL}/cgi-bin/backup/do_backup.cgi         \
       -O TMP_CONFIG 2>/dev/null &
  # Little wait loop to give wget the time to fetch the file
  n=1
  while sleep 1 ; do
    [ `stat -c%s TMP_CONFIG` -ge ${MINSIZE} ] && break
    [ $n -gt 7 ]                              && break
    let n+=1
  done

  if [ `stat -c%s TMP_CONFIG` -gt 300 ] ; then
    # If I received at least some data, then wait some more and it might get everything
    sleep 2
  else
    echo "wget didn't get any data from host \"${HOST}\"" >&2
    exec 2>/dev/null
    kill -9 %1 2>/dev/null
    exec 2>&2
    return 1
  fi
  return 0
}

keep_config_if_different ()
{
  LATESTCONFIG=`find "${DEST_FOLDER}" -maxdepth 1 -type f -size +10 | \
                      grep "${IP}.*\.tgz" | xargs -I{} stat -c '%Y %n' {} | \
                      sort -rn | head -n1 | awk '{print $2}'`

  # If no older config is found
  if [ -z "${LATESTCONFIG}" ] ; then
    echo "Saving \"${DEST_BASE}.tgz\" because no previous Freetz config has been found" >&2
    mv TMP_CONFIG "${DEST_BASE}.tgz"
    get_avm_config
  else

    TRIED_FREETZ=1  # A flag for get_avm_config to know if the freetz config has been tried or not
    mkdir f1   # folder to untar the current config
    mkdir f2   # folder to untar the previous config
    if ! tar xzf TMP_CONFIG -C f1 2>/dev/null ; then
      echo "I just downloaded an invalid gzipped tar-file" >&2
      rm -f TMP_CONFIG
    else
      if ! tar xzf ${LATESTCONFIG} -C f2 ; then
        echo "A previously downloaded config turns out to be invalid... very strange... (${LATESTCONFIG})" >&2
      else
        # Remove files that are likely to be changed, but don't contain important config
        patch_files f1/var_flash
        patch_files f2/var_flash

        if ! diff f1/var_flash f2/var_flash >/dev/null 2>&1 ; then
          echo -e "Saving \"${DEST_BASE}.tgz\" because Freetz config changed\n" >&2
          mv TMP_CONFIG "${DEST_BASE}.tgz"
          # Now get the AVM config too
          get_avm_config
        elif [ `find "${DEST_FOLDER}" -maxdepth 1 -type f -name \*.export | wc -l` -lt 1 ] ; then
          # No previous AVM config exists, so try and fetch one anyhow
          get_avm_config
       fi
     fi
   fi
 fi
}

get_both_configs ()
{
  PROTO=http
  echo ${PORT} | grep -q '^4[456][0-9]' && PROTO=https
  URL="${PROTO}://${IP}:${PORT}"

  if ! httpspeed ${URL} >/dev/null ; then
    # Freetz is not reacting, maybe it is a plain jane AVM Fritzbox
    # I will not be able to see if a config has changed.......
    echo "No reaction from Freetz (\"${HOST}\").... will try AVM now.."
    get_avm_config
  else
    TMPDIR1=`mktemp -p /dev/shm -d ${0//*\/}.XXXXXXXXXX`

    if ! cd ${TMPDIR1} ; then
      echo "Error changing folder to ${TMPDIR1}" >&2
    else
      touch TMP_CONFIG

      get_freetz_config

      if [ `stat -c%s TMP_CONFIG` -lt ${MINSIZE} ] ; then
        echo "Error getting config from ${HOST} on ${IP}" >&2
      else

        # I think we're getting successful here, let's create a folder already
        chown ${FUSER}:${FGROUP} TMP_CONFIG
        mkdir "${DEST_FOLDER}" 2>/dev/null && chown ${FUSER}:${FGROUP} "${DEST_FOLDER}"

        keep_config_if_different
      fi
    fi
    cd "${FOLDER}"
    rm -rf ${TMPDIR1}
  fi
}


# Main Program starts here

if ! cd "${FOLDER}" ; then
  echo "${FOLDER} does not exist" >&2
  exit 1
fi

BASENAME="${0//*\/}"                           # Strip Path of script name
BASENAME="${BASENAME%.*}"                      # Strip extension
CONFIGSOURCE="${FOLDER}/${BASENAME}.list"      # Construct modemlist from script name
CONFIGFILE=`mktemp`                            # temp file for sanitized modemlist

if [ -n "$*" ] ; then
  echo "$*" >"${CONFIGFILE}"                     # push the commandline parameters in the configfile
elif [ -e "${CONFIGSOURCE}" ] ; then
  sed '/^ *$/d;/^ *#/d' "${CONFIGSOURCE}" | \
  awk -F'[\t ]#' '{print $1}' >"${CONFIGFILE}"             # Get all the data before # and remove empty lines
else
  echo "Configfile: \"${CONFIGSOURCE}\" does not exist" >&2
  exit 1
fi

if [ ! -s "${CONFIGFILE}" ] ; then
  echo "No workable data in \"${CONFIGSOURCE}\"" >&2
  exit 1
fi

# Main Loop
FGROUP=`stat -c%G .`
FUSER=`stat -c%U .`

while read MODEMLINE ; do
  HOST="`echo "${MODEMLINE}"    | awk -F, '{print $1}' | awk '{print $1}'`"
  IP="`echo "${MODEMLINE}"      | awk -F, '{print $2}' | awk '{print $1}'`"
  PASS="`echo "${MODEMLINE}"    | awk -F, '{print $3}' | awk '{print $1}'`"
  PORT="`echo "${MODEMLINE}"    | awk -F, '{print $4}' | tr -cd '0-9'`"
  USER="`echo "${MODEMLINE}"    | awk -F, '{print $5}' | awk '{print $1}'`"
  PASSE="`echo "${MODEMLINE}"   | awk -F, '{print $6}' | awk '{print $1}'`"
  PORTAVM="`echo "${MODEMLINE}" | awk -F, '{print $7}' | tr -cd '0-9'`"

  HOSTORG="${HOST}"
  TRIED_FREETZ=

  if [ -z "${IP}" ] ; then
    HOST=`echo "${HOST}" | tr 'A-Z' 'a-z' | \
          egrep -o '[a-z0-9.-]+\.[a-z0-9]+'`     # Loosely Sanitize domain if no IP is given
  fi
  if [ -z "${HOST}" ] ; then
    echo "${HOSTORG} is not a valid host" >&2
  else
    [ ${HEADLESS} ] || echo ${HOST}
    if [ -z "${IP}" ] ; then
      # If HOST is so clearly an IP-address then use that instead....
      if echo "${HOST}" | egrep -q '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' ; then
        IP=${HOST}
      else
        IP=`host -t A ${HOST} 2>/dev/null | grep -o 'has address .*' | awk '{print $3}' | head -n1`
      fi
    fi
    if [ -z "${IP}" ] ; then
      echo "Unable to resolve ${HOST}" >&2
    else
      [ -z "${PORT}" ]    &&    PORT=${PORT_DEFAULT}      # port of Freetz interface
      [ -z "${PASS}" ]    &&    PASS=${PASS_DEFAULT}      # internal password
      [ -z "${USER}" ]    &&    USER=${USER_DEFAULT}      # external username
      [ -z "${PASSE}" ]   &&   PASSE=${PASS}              # external password
      [ -z "${PORTAVM}" ] && PORTAVM=${PORTAVM_DEFAULT}   # port of AVM interface (https)

      DEST_FOLDER="${FOLDER}/${HOST}"
      DEST_BASE="${DEST_FOLDER}/${IP}.${DATESTAMP}"

      if [ ${PORT} -eq 0 ] ; then
        get_avm_config     # Fetch only AVM-config when the Freetz port is defined as 0
      else
        get_both_configs
      fi
    fi
  fi
done<"${CONFIGFILE}"
rm -f "${CONFIGFILE}"

#!/bin/sh


# Define constants
DATESTAMP=`date +%Y-%m-%d.%H-%M`
FOLDER=/var/www/vhosts/freetz.mr-wolf.nl/httpdocs/courant


USER_DEFAULT=admin
PASS_DEFAULT=q1w2e3r4
PORT_DEFAULT=61080
PORTAVM_DEFAULT=450
MINSIZE=20000


HTTP_OPTIONS=''
HTTPS_OPTIONS=''
# HTTPS_OPTIONS='--cipher RC4-SHA'


HEADLESS=
tty >/dev/null || HEADLESS=true


while getopts cd: opt
do
  case $opt in
    c)   FOLDER="`pwd`" ;;
    d)   FOLDER="$OPTARG";;
    ?)   printf "Usage: %s [-c] [-d <folder>] [<HOST>],[<IP>],[PASS_FREETZ],[<PORT_FREETZ>],[<USER>],[PASS_EXTERNAL],[PORT_AVM]\n" $0
    exit 2;;
  esac
done
shift `expr ${OPTIND} - 1`




# Subroutines


age_of_file ()
{
  if [ ! -e "$1" ] ; then
    echo 0
  else
    AGE_IN_SECONDS=$((`date +%s` - `date +%s -r "$1"` ))
    echo $((${AGE_IN_SECONDS} / 86400))
  fi
}


httpspeed ()
{
  # Returns no value and  errorlevel 1 if site is unreachable or too slow
  # Returns the speed in milliseconds if it's below 2 seconds
  _SPEED=`httping -c1 -t1 ${1} 2>/dev/null | \
         egrep '.+/.+/.+/.+/.+' | tail -n1 | \
         awk -F/ '{print $4}' | awk -F. '{print $1}'`


  if [ -z "${_SPEED}" ] ; then
    echo -e "${URL} is unreachable\n" >&2
  elif [ ${_SPEED} -gt 2000 ] ; then
    echo "${URL} is slow (${_SPEED} ms)" >&2
  else
    echo ${_SPEED}
    return 0
  fi
  return 1
}




get_avm_config ()
{
  [ ${PORTAVM} -eq 0 ] && return


  TMPDIR2=`mktemp -p /dev/shm -d ${0//*\/}.XXXXXXXXXX`
  if ! cd ${TMPDIR2} 2>/dev/null ; then
    echo "Error changing current directory to ${TMPDIR2}. You should never see this message!" >&2
  else


    # It's intended to be called from WAN (https)
    # If PORTAVM begins with 8 it assumes LAN and turns to http
    PROTO=https
    echo ${PORTAVM} | grep -q '^80$' && PROTO=http


    CURLOPTS="${HTTP_OPTIONS}"
    [ "${PROTO}" = 'https' ] && CURLOPTS="${HTTPS_OPTIONS}"


    URL="${PROTO}://${IP}:${PORTAVM}"


    if httpspeed ${URL} >/dev/null ; then
      # get challenge key from FB


      _CHALLENGE=`curl ${CURLOPTS} -s -k \
                       --user ${USER}:${PASSE} \
                       "${URL}/login_sid.lua" | \
                       grep 'Challenge' | sed -e 's/.*<Challenge>//' | sed -e 's/<\/Challenge>.*//'`


      if [ -z "${_CHALLENGE}" ] ; then
        echo "Error getting challenge for ${HOST} using ${URL}/login_sid.lua" >&2
      else
        # echo "Received Challenge \"${_CHALLENGE}\"" >&2
        # build md5 from challenge key and password
        _MD5=`echo -n ${_CHALLENGE}"-"${PASSE} | \
              iconv -f ISO8859-1 -t UTF-16LE   | \
              md5sum -b | awk '{print substr($0,1,32)}'`


        # assemble challenge key and md5
        _RESPONSE=${_CHALLENGE}"-"${_MD5}


        # get sid for later use
        _SID=`curl ${CURLOPTS} -i -s -k \
                    --user ${USER}:${PASSE} \
                    -d 'response='${_RESPONSE} \
                    -d 'page=' \
                    -d 'username='${USER} \
                    ${URL}/login_sid.lua | \
                    grep SID | sed -e 's/.*<SID>//' | sed -e 's/<\/SID>.*//'`


        if ! curl ${CURLOPTS} -s -k \
                  --user ${USER}:${PASSE} \
                  --form 'sid='${_SID} \
                  --form 'ImportExportPassword='${PASSE} \
                  --form 'ConfigExport=' \
                  ${URL}/cgi-bin/firmwarecfg >TMP1 ; then


          echo "Error getting .export for ${HOST} (no data)" >&2
        else
          if [ `stat -c%s TMP1` -lt ${MINSIZE} ] ; then
            echo "Error getting .export for ${HOST} using ${URL} (too small)" >&2
            echo "SID: \"${_SID}\" CHALLENGE: \"${_CHALLENGE}\"  MD5: \"${_MD5}\"  RESPONSE: \"${_RESPONSE}\"" >&2
          else
            chown ${FUSER}:${FGROUP} TMP1
            mkdir "${DEST_FOLDER}" 2>/dev/null && chown ${FUSER}:${FGROUP} "${DEST_FOLDER}"
            LATESTCONFIG=`find "${DEST_FOLDER}" -maxdepth 1 -type f -size +10 | \
                          grep "${IP}.*\.export" | xargs -I{} stat -c '%Y %n' {} | \
                          sort -rn | head -n1 | awk '{print $2}'`


            if [ -z "${LATESTCONFIG}" ] ; then                     # No previous .export is found
              echo "Saving \"${DEST_BASE}.export\" because no previous AVM-config has been found" >&2
              mv TMP1 "${DEST_BASE}.export"
            else
              if [ ${TRIED_FREETZ} ] ; then      # The Freetz comparison is better, so make that leading
                if ! diff TMP1 "${LATESTCONFIG}" >/dev/null 2>&1 ; then
                  echo -e "Saving \"${DEST_BASE}.export\" because Freetz config has changed\n" >&2
                  mv TMP1 "${DEST_BASE}.export"
                fi
              else
                if [ `age_of_file "${LATESTCONFIG}"` -gt 30 ] ; then      # Compare EXACTLY if age is older than 30 days
                  if ! diff TMP1 "${LATESTCONFIG}" >/dev/null 2>&1 ; then
                    echo "Saving \"${DEST_BASE}.export\" because latest AVM-config was older than 30 days and is not EXACTLY the same" >&2
                    mv TMP1 "${DEST_BASE}.export"
                  fi
                else


                  # remove BINARY data and timestamps before making a diff
                  sed    '/ BINFILE:/,/ END OF FILE /d' "${LATESTCONFIG}" >TMP2
                  sed -i '/ [0-2][0-9]:[0-5][0-9]:[0-9][0-9]/d'            TMP2
                  sed -i '/END OF EXPORT/d;/update_found/d'                TMP2
                  sed -i '/\$\$\$\$/d;/running_version/d'                  TMP2


                  sed    '/ BINFILE:/,/ END OF FILE /d'             TMP1  >TMP3
                  sed -i '/ [0-2][0-9]:[0-5][0-9]:[0-9][0-9]/d'            TMP3
                  sed -i '/END OF EXPORT/d;/update_found/d'                TMP3
                  sed -i '/\$\$\$\$/d;/running_version/d'                  TMP3


                  if ! diff TMP2 TMP3 >/dev/null 2>&1 ; then                  # Compare the 2 stripped configs
                    echo -e "Saving \"${DEST_BASE}.export\" because AVM-config has changed\n" >&2
                    diff TMP2 TMP3 >&2
                    echo -e '\n---\n\n' >&2
                    mv TMP1 "${DEST_BASE}.export"
                  fi
                fi
              fi
            fi
          fi
        fi
      fi
    fi
  fi
  cd "${FOLDER}"
  rm -rf ${TMPDIR2}
}


patch_files ()
{
  # remove or patch some of the untarred files as they spoil the check for a changed config
  if cd "$1" >/dev/null 2>&1 ; then
    rm -f userstat.cfg stat.cfg chrony.drift multid.leases 2>/dev/null
    rm -f fx_cg fonctrl phonebook voipd_call_stat 2>/dev/null


    sed -i '/ [0-2][0-9]:[0-5][0-9]:[0-9][0-9]/d' voip.cfg  # remove datestamp in voip.cfg
    sed -i '/ [0-2][0-9]:[0-5][0-9]:[0-9][0-9]/d' ar7.cfg   # remove datestamp in ar7.cfg
    sed -i 's/ *\/\*AVM\*\///g' ar7.cfg                     # remove the /*AVM*/ tag in ar7.cfg
    sed -i 's/; *$/;/g' ar7.cfg                             # remove trailing spaces after a semicolon
    cd - >/dev/null 2>&1
  fi
}


get_freetz_config ()
{
  # Fetch config-file with wget in background
  # Somehow it doesn't listen to the 6 seconds timeout I'm giving it
  wget -q --timeout=6                              \
       --http-user=${USER} --http-password=${PASS} \
       ${URL}/cgi-bin/backup/do_backup.cgi         \
       -O TMP_CONFIG 2>/dev/null &
  # Little wait loop to give wget the time to fetch the file
  n=1
  while sleep 1 ; do
    [ `stat -c%s TMP_CONFIG` -ge ${MINSIZE} ] && break
    [ $n -gt 7 ]                              && break
    let n+=1
  done


  if [ `stat -c%s TMP_CONFIG` -gt 300 ] ; then
    # If I received at least some data, then wait some more and it might get everything
    sleep 2
  else
    echo "wget didn't get any data from host \"${HOST}\"" >&2
    exec 2>/dev/null
    kill -9 %1 2>/dev/null
    exec 2>&2
    return 1
  fi
  return 0
}


keep_config_if_different ()
{
  LATESTCONFIG=`find "${DEST_FOLDER}" -maxdepth 1 -type f -size +10 | \
                      grep "${IP}.*\.tgz" | xargs -I{} stat -c '%Y %n' {} | \
                      sort -rn | head -n1 | awk '{print $2}'`


  # If no older config is found
  if [ -z "${LATESTCONFIG}" ] ; then
    echo "Saving \"${DEST_BASE}.tgz\" because no previous Freetz config has been found" >&2
    mv TMP_CONFIG "${DEST_BASE}.tgz"
    get_avm_config
  else


    TRIED_FREETZ=1  # A flag for get_avm_config to know if the freetz config has been tried or not
    mkdir f1   # folder to untar the current config
    mkdir f2   # folder to untar the previous config
    if ! tar xzf TMP_CONFIG -C f1 2>/dev/null ; then
      echo "I just downloaded an invalid gzipped tar-file" >&2
      rm -f TMP_CONFIG
    else
      if ! tar xzf ${LATESTCONFIG} -C f2 ; then
        echo "A previously downloaded config turns out to be invalid... very strange... (${LATESTCONFIG})" >&2
      else
        # Remove files that are likely to be changed, but don't contain important config
        patch_files f1/var_flash
        patch_files f2/var_flash


        if ! diff f1/var_flash f2/var_flash >/dev/null 2>&1 ; then
          [ ${HEADLESS} ] || diff f1/var_flash f2/var_flash >&2
          echo -e "Saving \"${DEST_BASE}.tgz\" because Freetz config changed\n" >&2
          mv TMP_CONFIG "${DEST_BASE}.tgz"
          # Now get the AVM config too
          get_avm_config
        elif [ `find "${DEST_FOLDER}" -maxdepth 1 -type f -name \*.export | wc -l` -lt 1 ] ; then
          # No previous AVM config exists, so try and fetch one anyhow
          get_avm_config
       fi
     fi
   fi
 fi
}


get_both_configs ()
{
  PROTO=http
  echo ${PORT} | grep -q '^4[456][0-9]' && PROTO=https
  URL="${PROTO}://${IP}:${PORT}"


  if ! httpspeed ${URL} >/dev/null ; then
    # Freetz is not reacting, maybe it is a plain jane AVM Fritzbox
    # I will not be able to see if a config has changed.......
    echo "No reaction from Freetz (\"${HOST}\").... will try AVM now.."
    get_avm_config
  else
    TMPDIR1=`mktemp -p /dev/shm -d ${0//*\/}.XXXXXXXXXX`


    if ! cd ${TMPDIR1} ; then
      echo "Error changing folder to ${TMPDIR1}" >&2
    else
      touch TMP_CONFIG


      get_freetz_config


      if [ `stat -c%s TMP_CONFIG` -lt ${MINSIZE} ] ; then
        echo "Error getting config from ${HOST} on ${IP}" >&2
      else


        # I think we're getting successful here, let's create a folder already
        chown ${FUSER}:${FGROUP} TMP_CONFIG
        mkdir "${DEST_FOLDER}" 2>/dev/null && chown ${FUSER}:${FGROUP} "${DEST_FOLDER}"


        keep_config_if_different
      fi
    fi
    cd "${FOLDER}"
    rm -rf ${TMPDIR1}
  fi
}




# Main Program starts here


if ! cd "${FOLDER}" ; then
  echo "${FOLDER} does not exist" >&2
  exit 1
fi


BASENAME="${0//*\/}"                           # Strip Path of script name
BASENAME="${BASENAME%.*}"                      # Strip extension
CONFIGSOURCE="${FOLDER}/${BASENAME}.list"      # Construct modemlist from script name
CONFIGFILE=`mktemp`                            # temp file for sanitized modemlist


if [ -n "$*" ] ; then
  echo "$*" >"${CONFIGFILE}"                     # push the commandline parameters in the configfile
elif [ -e "${CONFIGSOURCE}" ] ; then
  sed '/^ *$/d;/^ *#/d' "${CONFIGSOURCE}" | \
  awk -F'[\t ]#' '{print $1}' >"${CONFIGFILE}"             # Get all the data before # and remove empty lines
else
  echo "Configfile: \"${CONFIGSOURCE}\" does not exist" >&2
  exit 1
fi


if [ ! -s "${CONFIGFILE}" ] ; then
  echo "No workable data in \"${CONFIGSOURCE}\"" >&2
  exit 1
fi


# Main Loop
FGROUP=`stat -c%G .`
FUSER=`stat -c%U .`


while read MODEMLINE ; do
  HOST="`echo "${MODEMLINE}"    | awk -F, '{print $1}' | awk '{print $1}'`"
  IP="`echo "${MODEMLINE}"      | awk -F, '{print $2}' | awk '{print $1}'`"
  PASS="`echo "${MODEMLINE}"    | awk -F, '{print $3}' | awk '{print $1}'`"
  PORT="`echo "${MODEMLINE}"    | awk -F, '{print $4}' | tr -cd '0-9'`"
  USER="`echo "${MODEMLINE}"    | awk -F, '{print $5}' | awk '{print $1}'`"
  PASSE="`echo "${MODEMLINE}"   | awk -F, '{print $6}' | awk '{print $1}'`"
  PORTAVM="`echo "${MODEMLINE}" | awk -F, '{print $7}' | tr -cd '0-9'`"


  HOSTORG="${HOST}"
  TRIED_FREETZ=


  if [ -z "${IP}" ] ; then
    HOST=`echo "${HOST}" | tr 'A-Z' 'a-z' | \
          egrep -o '[a-z0-9.-]+\.[a-z0-9]+'`     # Loosely Sanitize domain if no IP is given
  fi
  if [ -z "${HOST}" ] ; then
    echo "${HOSTORG} is not a valid host" >&2
  else
    [ ${HEADLESS} ] || echo ${HOST}
    if [ -z "${IP}" ] ; then
      # If HOST is so clearly an IP-address then use that instead....
      if echo "${HOST}" | egrep -q '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' ; then
        IP=${HOST}
      else
        IP=`host -t A ${HOST} 2>/dev/null | grep -o 'has address .*' | awk '{print $3}' | head -n1`
      fi
    fi
    if [ -z "${IP}" ] ; then
      echo "Unable to resolve ${HOST}" >&2
    else
      [ -z "${PORT}" ]    &&    PORT=${PORT_DEFAULT}      # port of Freetz interface
      [ -z "${PASS}" ]    &&    PASS=${PASS_DEFAULT}      # internal password
      [ -z "${USER}" ]    &&    USER=${USER_DEFAULT}      # external username
      [ -z "${PASSE}" ]   &&   PASSE=${PASS}              # external password
      [ -z "${PORTAVM}" ] && PORTAVM=${PORTAVM_DEFAULT}   # port of AVM interface (https)


      DEST_FOLDER="${FOLDER}/${HOST}"
      DEST_BASE="${DEST_FOLDER}/${IP}.${DATESTAMP}"


      if [ ${PORT} -eq 0 ] ; then
        get_avm_config     # Fetch only AVM-config when the Freetz port is defined as 0
      else
        get_both_configs
      fi
    fi
  fi
done<"${CONFIGFILE}"
rm -f "${CONFIGFILE}"