#!/usr/bin/sh
#
# uln-yum-mirror
#
# Updated: 2022-06-29

function findMdFile () {
    mdPath=${1}
    mdFile=${mdPath}/repomd.xml
    toFind=$2
    fileName=$(grep $toFind $mdFile | grep repodata | sed -e 's/^.*repodata\///' -e 's/"\/>.*$//')
    [ -z $fileName ] && echo "" || echo ${mdPath}/${fileName}
}

# Source yum-uln-mirror configuration.
. /etc/sysconfig/uln-yum-mirror

# utilities to check for before checking the release
PRERELEASE_UTILS="awk cat comm find grep gunzip id mkdir rpm sed sort uname yum"

# utilities to check for after checking the release
RELEASE_UTILS="reposync createrepo modifyrepo"

# Maximum number of packages to download at once
MAXPKG=1000

export PATH=/bin:/usr/bin:/usr/sbin

# check for basic utilities needed for all releases, we'll check for 
# release-specific utils further down
for cmd in $PRERELEASE_UTILS ; do
    if ! type $cmd >/dev/null 2>&1 ; then 
        echo "$0: Needed command not found: $cmd" >&2
        exit 1
    fi
done

# check for root
if [ $(id -u) -ne 0 ] ; then
    echo "$0: You must be root to run this script" >&2
    exit 1
fi

# The script has the option to be able to download the src rpms alongside
# the binary rpms.  The default, however is to NOT download them.  So, set the
# default and then check if the user asked for the src rpms.
if [ $# -eq 1 ] ; then
    if [ "$1" = "src" ] ; then
        SRC=1
    else
        echo "$0: unknown argument \"$1\"" >&2
        echo "usage: $0 [src]" >&2
        exit 1
    fi
fi


# Identify what version of Oracle Linux the script is running on.
if rpm -q --whatprovides oraclelinux-release >/dev/null 2>&1 ; then
    EL_VER=`rpm -qf /etc/oracle-release --qf '%{EPOCH}' | sed 's/[^0-9]*//g'`
    if [ "$EL_VER" = "" ] ; then
        EL_VER=`rpm -qf /etc/oracle-release --qf '%{VERSION}' | sed 's/[^0-9]*//g'`
    fi
    EL_ARCH=`uname -i`
else
    echo "$0: Script must be run on Oracle Linux" >&2
    exit 1
fi


# release-specific processing
case $EL_VER in
    [5-7])
        # We need 6GB of memory, but check if it's greater than 5000 to allow for reserved memory and kdump
        # Anything less and createrepo --update is very likely to trigger the OOM-killer for large repos
        # This will cause the createrepo process to take longer, but at least it will finish
        if [ "$(free -m | awk '/Mem/{print $2}')" -ge 5000 ]; then
            CREATEREPO_EXTRA_ARGS+="--update"
        fi
        # for OL6, use sha type that's compatible with OL5
        if [ $EL_VER -eq 6 ] || [ $EL_VER -eq 7 ] ; then
            CREATEREPO_EXTRA_ARGS+=" --checksum=sha"
            MODIFYREPO_EXTRA_ARGS+="--checksum=sha"
            REPOSYNC_EXTRA_ARGS+="--norepopath --download-metadata"
        fi

        if [ $EL_VER -eq 5 ] ; then
          RELEASE_UTILS+=" yum-arch"
        fi

        REPOSYNC_ARG_DELETE="-d"
        
        ;;
    [8-9])
        REPOSYNC_ARG_DELETE="--delete"
        ;;
    *)
        echo "$0: unsupported version ($EL_VER)" >&2
        exit 1
        ;;
esac


# By default, if an rpm is removed from a channel on ULN, the rpm will also be
# removed from the local copy of that channel.  This can be overridden in the
# config file
# note: The delete argument $REPOSYNC_ARG_DELETE could be different in distros.
#       Decided in previous step
if [ ! -z $DELETE ] && [ $DELETE -eq 1 ]; then
    REPOSYNC_EXTRA_ARGS+=" $REPOSYNC_ARG_DELETE"
fi


# Only download latest packages and remove old packages if ALL_PKGS=0 in the configuration
if [ ! -z $ALL_PKGS ] && [ $ALL_PKGS -eq 0 ]; then
    REPOSYNC_EXTRA_ARGS+=" -n"
    if [[ ! $REPOSYNC_EXTRA_ARGS == *"$REPOSYNC_ARG_DELETE"* ]]; then
        REPOSYNC_EXTRA_ARGS+=" $REPOSYNC_ARG_DELETE"
    fi
fi


# check for utilities that are release-specific or should be installed
# by the uln-yum-mirror rpm
for cmd in $RELEASE_UTILS ; do
    if ! type $cmd >/dev/null 2>&1 ; then 
        echo "$0: Needed command not found: $cmd" >&2
        exit 1
    fi
done

# create the directory for the generated repo files
#mkdir -p $REPO_FILE_DIR 2>/dev/null 
#if [ $? -ne 0 ] ; then
#    echo "$0: Permieeion denied creating $REPO_FILE_DIR"
#    exit 1
#fi

# Start lock on /var/lock/subsys/uln-yum-mirror. Wait (fd 200) for 10 seconds.
(flock -x -w 10 200
if [ "$?" != "0" ]; then
    echo "ULN Yum Mirror already running."
    exit 1
fi

# easiest way to get the list of channel labels/names that this system is subscribed to
# is direct from ULN via python
python3 -c "
import sys
sys.path.append('/usr/share/rhn/')
from up2date_client import up2dateAuth,rpcServer
sid = up2dateAuth.getSystemId()
if sid is None:
    print('Error: system not registered')
    sys.exit(-1)
try:
    s = rpcServer.getServer()
    for chan in s.up2date.listChannels(sid):
        print('%s::::%s' % (chan['label'], chan['name']))
except:
    print('Error: ', sys.exc_info()[1].faultString)
    sys.exit(-1)
" | sort | while read LINE; do
  # parse out the channel label and name
  LABEL=$(echo $LINE | awk -F '::::' '{print $1}')
  NAME=$(echo $LINE | awk -F '::::' '{print $2}')

  # Enable source parameter
  [ $SRC -eq 1 ] && SRCARG="--source" || SRCARG=""

  echo "## `date +%Y%m%d%H%M%S` BEGIN PROCESSING $LABEL ##"

  # Path conversion 
  ARCHS="i386 x86_64 ia64"
  MAIN_KEEP=("exadata" "pca*")
  REP_KEEP="oracle_addons Dtrace_ gdm_"

  # Find arch and remove it from label
  for i in $ARCHS; do
      if [[ $LABEL =~ ${i} ]]; then
          REP_ARCH=${i}
          LABEL_NOARCH=${LABEL/${i}_/}
      fi
  done

  REP_MAIN=${LABEL_NOARCH%%_*}

  # Make uppercase if not present in MAIN_KEEP
  if ! { for i in "${MAIN_KEEP[@]}";
  do
      [[ ${i} =~ [^a-zA-Z0-9_-] ]] && 
      { [[ $REP_MAIN =~ ^${i} ]] && break; } || 
      { [[ $REP_MAIN = ${i} ]] && break; }
  done };
  then REP_MAIN=$(echo ${REP_MAIN} | tr '[:lower:]' '[:upper:]');
  fi

  # Get the remainder of the label without the main name
  REP_REMAIN=${LABEL_NOARCH#*_}

  # Escape sublabels containing _ with - to prevent substitution
  for l in $REP_KEEP; do
      SUB=${l/_/-}
      REP_REMAIN=${REP_REMAIN/${l}/${SUB}}
  done

  # Replace uX with X for update releases
  REP_REMAIN=`echo $REP_REMAIN | sed -e 's/u\([0-9]\+\)_/\1_/g'`

  # Replace underscores with path seperator
  REP_REMAIN=${REP_REMAIN//_/\/}

  # Unescape sublabels
  for l in $REP_KEEP; do
      SUB=${l/_/-}
      REP_REMAIN=${REP_REMAIN/${SUB}/${l}}
  done
  DIR=$REP_MAIN/$REP_REMAIN/$REP_ARCH

  # determine the correct yum repository directory 
  # set REP_DIR for further use in the repo file creation
  REP_DIR=$REP_UNK/$DIR

  echo $LABEL | grep '^exadata*_' >/dev/null 2>&1 && \
    REP_DIR=$REP_ENG/$DIR

  echo $LABEL | grep '^pca*' >/dev/null 2>&1 && \
    REP_DIR=$REP_ENG/$DIR

  echo $LABEL | grep '^el[0-9]*_' >/dev/null 2>&1 && \
    REP_DIR=$REP_EL/$DIR

  echo $LABEL | grep '^ol[0-9]*_' >/dev/null 2>&1 && \
    REP_DIR=$REP_OL/$DIR

  echo $LABEL | grep '^ovm[0-9]*[a-z]*\?_' >/dev/null 2>&1 && \
    REP_DIR=$REP_OVM/$DIR

  REP_PATH=$REP_BASE/$REP_DIR

  echo "Channel Dir: $REP_PATH"

  # Create the required yum repository directory.
  # If directory creation failed, print error and exit.
  mkdir -p $REP_PATH

  cd $REP_PATH
  if [ $? -ne 0 ] ; then
      echo "$0: Error creating $REP_PATH" >&2
      exit 1
  fi

  # On OL5 we need to run makecache to pull updateinfo.xml.gz into the local cache
  if [ $EL_VER -eq 5 ] ; then
    yum makecache --disablerepo='*' --enablerepo=$LABEL
  fi

  # if the channel is an x86_64 channel and the mirror host is an x86 system
  # then we need to add the -a option to allow for the non-matching-arch rpms
  # to be found and downloaded
  case $EL_VER in
    [5-9])
      if `echo $LABEL | grep -q x86_64` && [ x`uname -i`x == 'xi386x' ] ; then
          SRCARG="${SRCARG} -a x86_64"
      fi
      ;;
  esac

  # -----------
  # Do reposync
  # -----------
  case $EL_VER in
    [5-7])
      reposync -q -g -l -m -p $REP_PATH -r $LABEL $REPOSYNC_EXTRA_ARGS $SRCARG
      ;;
    [8-9])
      reposync -q --download-metadata -p $REP_PATH --repoid=$LABEL --setopt=$LABEL.module_hotfixes=1 $REPOSYNC_EXTRA_ARGS $SRCARG
      continue #In OL8,9, we can skip manually download metadata, createrepo step and modifyrepo step
  esac

  # --------------------------------------------
  # Run createrepo to generate yum repo metadata
  # --------------------------------------------
  case $EL_VER in
    [5-7])
      GRPARG=
      if [ -f ${REP_PATH}/comps.xml ] ; then
        GRPARG="-g ${REP_PATH}/comps.xml"
      else
        # if the comps.xml file wasn't downloaded by reposync, check the
        # global yum cache in /var/cache for the file as well
        CACHED_GROUP_FILE=${YUM_GLOBAL_CACHE}/${LABEL}/comps.xml
        if [ -f ${CACHED_GROUP_FILE} ] ; then
            GRPARG="-g $CACHED_GROUP_FILE"
        fi
      fi
      createrepo -c ${REP_PATH}/.cache $GRPARG $CREATEREPO_EXTRA_ARGS $REP_PATH

      # for OL5, the updateinfo file will not be downloaded by reposync, but it
      # should already be in the global yum cache if it's available for this repo
      if [ ! -f ${REP_PATH}/updateinfo.xml.gz ] && [ -f ${YUM_GLOBAL_CACHE}/${LABEL}/updateinfo.xml.gz ] ; then
        cp ${YUM_GLOBAL_CACHE}/${LABEL}/updateinfo.xml.gz ${REP_PATH}/updateinfo.xml.gz
      fi

      # if there's an updateinfo file for this repo, then patch it into the repo info
      if [ -f ${REP_PATH}/updateinfo.xml.gz ] ; then
        gunzip -fq ${REP_PATH}/updateinfo.xml.gz
        modifyrepo $MODIFYREPO_EXTRA_ARGS ${REP_PATH}/updateinfo.xml ${REP_PATH}/repodata/
      fi

      if [ ! -f ${REP_PATH}/modules.yaml.gz ] && [ -f ${YUM_GLOBAL_CACHE}/${LABEL}/modules.yaml.gz ] ; then
        cp ${YUM_GLOBAL_CACHE}/${LABEL}/modules.yaml.gz ${REP_PATH}/modules.yaml.gz
      fi

      # if there's an modules.yaml file for this repo, then patch it into the repo info
      if [ -f ${REP_PATH}/modules.yaml.gz ] ; then
        gunzip -fq ${REP_PATH}/modules.yaml.gz
        modifyrepo $MODIFYREPO_EXTRA_ARGS ${REP_PATH}/modules.yaml ${REP_PATH}/repodata/
      fi
      ;;

    [8-9])
      # find comps.xml, updateinfo.xml.gz and modules.yaml.gz if available
      REPO_CACHE_DIR=/var/cache/yum/${EL_ARCH}/${EL_VER}Server/${LABEL}

      COMPSFILE=$(findMdFile $REPO_CACHE_DIR comps)

      GRPARG=
      if [ ! -z ${COMPSFILE} ] && [ -f ${COMPSFILE} ] ; then
          GRPARG="-g ${COMPSFILE}"
      fi
      createrepo -c ${REP_PATH}/.cache $GRPARG $CREATEREPO_EXTRA_ARGS $REP_PATH
    
      UPDATEINFOFILE=$(findMdFile $REPO_CACHE_DIR updateinfo)
    
      # if there's an updateinfo file for this repo, then patch it into the repo info
      if [ ! -z ${UPDATEINFOFILE} ] && [ -f ${UPDATEINFOFILE} ] ; then
          UPDATEINFOFILE_SIMPLE=${REPO_CACHE_DIR}/updateinfo.xml.gz
          cp ${UPDATEINFOFILE} ${UPDATEINFOFILE_SIMPLE}
          gunzip -fq ${UPDATEINFOFILE_SIMPLE}
          UPDATEINFOFILE_SIMPLE_UNZIP=${UPDATEINFOFILE_SIMPLE%\.gz}
          modifyrepo $MODIFYREPO_EXTRA_ARGS ${UPDATEINFOFILE_SIMPLE_UNZIP} ${REP_PATH}/repodata/
          rm -f $UPDATEINFOFILE_SIMPLE_UNZIP
      fi
    
      MODULESYAMLFILE=$(findMdFile $REPO_CACHE_DIR modules.yaml)
          
      # if there's a modules.yaml file for this repo, then patch it into the repo info
      if [ ! -z ${MODULESYAMLFILE} ] && [ -f ${MODULESYAMLFILE} ] ; then
          MODULESYAMLFILE_SIMPLE=${REPO_CACHE_DIR}/modules.yaml.gz
          cp ${MODULESYAMLFILE} ${MODULESYAMLFILE_SIMPLE}
          gunzip -fq ${MODULESYAMLFILE_SIMPLE}
          MODULESYAMLFILE_SIMPLE_UNZIP=${MODULESYAMLFILE_SIMPLE%\.gz}
          modifyrepo $MODIFYREPO_EXTRA_ARGS ${MODULESYAMLFILE_SIMPLE_UNZIP} ${REP_PATH}/repodata/
          rm -f $MODULESTAMLFILE_SIMPLE_UNZIP
      fi
    
      # repodata is set, clean up any files not referenced in the repomd.xml file
      ls ${REP_PATH}/repodata | grep -v repomd.xml | while read f ; do
          if ! grep -q "repodata\/$f" ${REP_PATH}/repodata/repomd.xml ; then
              rm -f ${REP_PATH}/repodata/$f
          fi
      done
      ;;
  esac

  # only call yum-arch for repos <= OL5
  if echo $LABEL | grep -q '^ol6\|^ol7\|^ol8\|^ol9' ; then
      # since we don't really need any yum-arch headers for >= OL6, 
      # remove any headers directory from previous script runs
      rm -rf $REP_PATH/headers >/dev/null 2>&1
  else
      # run yum-arch command to generate up2date repo data
      if [ ${SRC} -eq 1 ] ; then
          yum-arch -q -s $REP_PATH 2>/dev/null
      else
          yum-arch -q $REP_PATH 2>/dev/null
      fi
  fi

  echo "## `date +%Y%m%d%H%M%S` END PROCESSING $LABEL ##"
done

if [ $HARDLINK_RPMS -eq 1 ]; then
    echo "## `date +%Y%m%d%H%M%S` START HARDLINK PROCESSING ##"
    # Run the hardlinkpy script over the yum base directory, excluding the metadata
    hardlinkpy -v 0 -f -t -x '.*/\.cache$' -x '.*/repodata$' -x '.*/headers$' ${REP_BASE}
    echo "## `date +%Y%m%d%H%M%S` END HARDLINK PROCESSING ##"
fi

) 200>/var/lock/subsys/uln-yum-mirror
