#!/bin/sh
#job script built by grid-manager and input file for BOINC job

set -x 
export RUNTIME_CONFIG_DIR=`pwd`/
#rename root file
mv ATLAS.root_0 EVNT.49863876._000010.pool.root.1
tar --strip-components=5 -xvf *input.tar.gz
# source with arguments for DASH shells
sourcewithargs() {
script=$1
shift
. $script
}

# Overide umask of execution node (sometime values are really strange)
umask 077
# Record job start timestamp
ACCOUNTING_STARTTIME=`date +"%s"`
# Select accounting method
echo "Detecting resource accounting method available for the job." 1>&2
JOB_ACCOUNTING=""
# Try to use cgroups first
if command -v arc-job-cgroup >/dev/null 2>&1; then
    echo "Found arc-job-cgroup tool: trying to initialize accounting cgroups for the job." 1>&2
    while true; do
        # memory cgroup
        memory_cgroup="$( arc-job-cgroup -m -n 285e9cb0da25 )"
        if [ $? -ne 0 -o -z "$memory_cgroup" ]; then
            echo "Failed to initialize memory cgroup for accounting." 1>&2
            break;
        fi
        # cpuacct cgroup
        cpuacct_cgroup="$( arc-job-cgroup -c -n 285e9cb0da25 )"
        if [ $? -ne 0 -o -z "$cpuacct_cgroup" ]; then
            echo "Failed to initialize cpuacct cgroup for accounting." 1>&2
            break;
        fi
        echo "Using cgroups method for job accounting" 1>&2
        JOB_ACCOUNTING="cgroup"
        break;
    done
fi

# Fallback to GNU_TIME if cgroups are not working
if [ -z "$JOB_ACCOUNTING" ]; then
    GNU_TIME='/usr/bin/time'
    echo "Looking for $GNU_TIME tool for accounting measurements" 1>&2
    if [ ! -z "$GNU_TIME" ] && ! "$GNU_TIME" --version >/dev/null 2>&1; then
        echo "GNU time is not found at: $GNU_TIME" 1>&2
    else
        echo "GNU time found and will be used for job accounting." 1>&2
        JOB_ACCOUNTING="gnutime"
    fi
fi

# Nothing works: rely on LRMS only
if [ -z "$JOB_ACCOUNTING" ]; then
    echo "Failed to use both cgroups and GNU time for resource usage accounting. Accounting relies on LRMS information only." 1>&2
fi

# Setting environment variables as specified by user
export 'PANDA_JSID=harvester-CERN_central_ACTA'
export 'GTAG=http://aipanda403.cern.ch/data/jobs/2026-04-26/BOINC_MCORE/7114723837.out'
export 'PILOT_NOKILL=YES'
export 'GRID_GLOBAL_JOBID=285e9cb0da25'
export 'GRID_GLOBAL_JOBURL=https://arc-boinc-02.cern.ch:443/arex/285e9cb0da25'
export 'GRID_GLOBAL_JOBINTERFACE=org.nordugrid.arcrest'
export 'GRID_GLOBAL_JOBHOST=arc-boinc-02.cern.ch'

RUNTIME_JOB_DIR=`pwd`
RUNTIME_JOB_STDIN=/dev/null
RUNTIME_JOB_STDOUT=`pwd`/log.49863879._000079.job.log.1
RUNTIME_JOB_STDERR=`pwd`/log.49863879._000079.job.log.1
RUNTIME_JOB_DIAG=`pwd`.diag
if [ ! -z "$RUNTIME_GRIDAREA_DIR" ] ; then
  RUNTIME_JOB_DIR=$RUNTIME_GRIDAREA_DIR/`basename $RUNTIME_JOB_DIR`
  RUNTIME_JOB_STDIN=`echo "$RUNTIME_JOB_STDIN" | sed "s#^$RUNTIME_JOB_DIR#$RUNTIME_GRIDAREA_DIR#"`
  RUNTIME_JOB_STDOUT=`echo "$RUNTIME_JOB_STDOUT" | sed "s#^$RUNTIME_JOB_DIR#$RUNTIME_GRIDAREA_DIR#"`
  RUNTIME_JOB_STDERR=`echo "$RUNTIME_JOB_STDERR" | sed "s#^$RUNTIME_JOB_DIR#$RUNTIME_GRIDAREA_DIR#"`
  RUNTIME_JOB_DIAG=`echo "$RUNTIME_JOB_DIAG" | sed "s#^$RUNTIME_JOB_DIR#$RUNTIME_GRIDAREA_DIR#"`
  RUNTIME_CONTROL_DIR=`echo "$RUNTIME_CONTROL_DIR" | sed "s#^$RUNTIME_JOB_DIR#$RUNTIME_GRIDAREA_DIR#"`
fi
RUNTIME_LOCAL_SCRATCH_DIR=${RUNTIME_LOCAL_SCRATCH_DIR:-}
RUNTIME_LOCAL_SCRATCH_MOVE_TOOL=${RUNTIME_LOCAL_SCRATCH_MOVE_TOOL:-mv}
RUNTIME_FRONTEND_SEES_NODE=${RUNTIME_FRONTEND_SEES_NODE:-}
RUNTIME_NODE_SEES_FRONTEND=${RUNTIME_NODE_SEES_FRONTEND:-yes}
  if [ ! -z "$RUNTIME_LOCAL_SCRATCH_DIR" ] && [ ! -z "$RUNTIME_NODE_SEES_FRONTEND" ]; then
    RUNTIME_NODE_JOB_DIR="$RUNTIME_LOCAL_SCRATCH_DIR"/`basename "$RUNTIME_JOB_DIR"`
    rm -rf "$RUNTIME_NODE_JOB_DIR"
    mkdir -p "$RUNTIME_NODE_JOB_DIR"
    # move directory contents
    for f in "$RUNTIME_JOB_DIR"/.* "$RUNTIME_JOB_DIR"/*; do 
      [ "$f" = "$RUNTIME_JOB_DIR/*" ] && continue # glob failed, no files
      [ "$f" = "$RUNTIME_JOB_DIR/." ] && continue
      [ "$f" = "$RUNTIME_JOB_DIR/.." ] && continue
      [ "$f" = "$RUNTIME_JOB_DIR/.diag" ] && continue
      [ "$f" = "$RUNTIME_JOB_DIR/.comment" ] && continue
      [ -f "$f" ] || continue
      if ! $RUNTIME_LOCAL_SCRATCH_MOVE_TOOL "$f" "$RUNTIME_NODE_JOB_DIR"; then
        echo "Failed to '$RUNTIME_LOCAL_SCRATCH_MOVE_TOOL' '$f' to '$RUNTIME_NODE_JOB_DIR'" 1>&2
        exit 1
      fi
    done
    if [ ! -z "$RUNTIME_FRONTEND_SEES_NODE" ] ; then
      # creating link for whole directory
       ln -s "$RUNTIME_FRONTEND_SEES_NODE"/`basename "$RUNTIME_JOB_DIR"` "$RUNTIME_JOB_DIR"
    else
      # keep stdout, stderr and control directory on frontend
      # recreate job directory
      mkdir -p "$RUNTIME_JOB_DIR"
      # make those files
      mkdir -p `dirname "$RUNTIME_JOB_STDOUT"`
      mkdir -p `dirname "$RUNTIME_JOB_STDERR"`
      touch "$RUNTIME_JOB_STDOUT"
      touch "$RUNTIME_JOB_STDERR"
      RUNTIME_JOB_STDOUT__=`echo "$RUNTIME_JOB_STDOUT" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
      RUNTIME_JOB_STDERR__=`echo "$RUNTIME_JOB_STDERR" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
      rm "$RUNTIME_JOB_STDOUT__" 2>/dev/null
      rm "$RUNTIME_JOB_STDERR__" 2>/dev/null
      if [ ! -z "$RUNTIME_JOB_STDOUT__" ] && [ "$RUNTIME_JOB_STDOUT" != "$RUNTIME_JOB_STDOUT__" ]; then
        ln -s "$RUNTIME_JOB_STDOUT" "$RUNTIME_JOB_STDOUT__"
      fi
      if [ "$RUNTIME_JOB_STDOUT__" != "$RUNTIME_JOB_STDERR__" ] ; then
        if [ ! -z "$RUNTIME_JOB_STDERR__" ] && [ "$RUNTIME_JOB_STDERR" != "$RUNTIME_JOB_STDERR__" ]; then
          ln -s "$RUNTIME_JOB_STDERR" "$RUNTIME_JOB_STDERR__"
        fi
      fi
      if [ ! -z "$RUNTIME_CONTROL_DIR" ] ; then
        # move control directory back to frontend
        RUNTIME_CONTROL_DIR__=`echo "$RUNTIME_CONTROL_DIR" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
        mv "$RUNTIME_CONTROL_DIR__" "$RUNTIME_CONTROL_DIR"
      fi
    fi
    # adjust stdin,stdout & stderr pointers
    RUNTIME_JOB_STDIN=`echo "$RUNTIME_JOB_STDIN" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
    RUNTIME_JOB_STDOUT=`echo "$RUNTIME_JOB_STDOUT" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
    RUNTIME_JOB_STDERR=`echo "$RUNTIME_JOB_STDERR" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
    RUNTIME_FRONTEND_JOB_DIR="$RUNTIME_JOB_DIR"
    RUNTIME_JOB_DIR="$RUNTIME_NODE_JOB_DIR"
  fi
  if [ -z "$RUNTIME_NODE_SEES_FRONTEND" ] ; then
    mkdir -p "$RUNTIME_JOB_DIR"
  fi
RUNTIME_JOB_DIAG=`pwd`/285e9cb0da25.diag
# RunTimeEnvironment function for APPS/HEP/ATLAS-SITE:
RTE_function_0 () {
#!/bin/bash
#
#......ATLAS Site configuration (prod)
#  !!! Managed by puppet
#

export PROJECT_ROOT="/boincdata/boinc/project/lhcathome"
#export PROJECT_DOWNLOAD_ROOT="/share/boinc/project/ATLAS"
#export PROJECT_DOWNLOAD_URL="http://atlasathome.cern.ch/ATLAS/download"
export BOINC_APP="ATLAS"
export WU_TEMPLATE="templates/ATLAS_IN_DYNAMIC"
export RESULT_TEMPLATE="templates/ATLAS_OUT_2"

if [ -z $1 ] || [ $1 == 1 ] ; then
  shift
  export SITE_NAME=BOINC
  export ATLAS_LOCAL_AREA=${RUNTIME_CONFIG_DIR}

  # Check for user-specific proxy
  if [ -e ${RUNTIME_CONFIG_DIR}init_data.xml ] && grep "<use_http_proxy/>" ${RUNTIME_CONFIG_DIR}init_data.xml >/dev/null 2>&1; then
    PROXY_HOST="$(sed -e '/<proxy_info>/,/<\/proxy_info>/ s/^.*<http_server_name>\([^<]*\).*/\1/p' -n ${RUNTIME_CONFIG_DIR}init_data.xml)"
    PROXY_PORT="$(sed -e '/<proxy_info>/,/<\/proxy_info>/ s/^.*<http_server_port>\([^<]*\).*/\1/p' -n ${RUNTIME_CONFIG_DIR}init_data.xml)"
    if [ -n "${PROXY_HOST}" ] && [ -n "${PROXY_PORT}" ]; then
      export FRONTIER_PROXY="http://${PROXY_HOST}:${PROXY_PORT}"
      echo "export FRONTIER_PROXY=\"http://${PROXY_HOST}:${PROXY_PORT}\"" > ${ATLAS_LOCAL_AREA}/setup.sh.local
    fi
  fi
fi

}
# Running RTE scripts (stage 1)
runtimeenvironments=
runtimeenvironments="${runtimeenvironments}APPS/HEP/ATLAS-SITE;"
# Calling APPS/HEP/ATLAS-SITE function: 
RTE_function_0 1 
if [ $? -ne 0 ]; then
    echo "Runtime APPS/HEP/ATLAS-SITE stage 1 execution failed." 1>&2
    echo "Runtime APPS/HEP/ATLAS-SITE stage 1 execution failed." 1>>"${RUNTIME_JOB_STDERR}"
    exit 1
fi

echo "runtimeenvironments=$runtimeenvironments" >> "$RUNTIME_JOB_DIAG"
# Detecting WN operating system for accounting purposes
if [ -f "/etc/os-release" ]; then
  SYSTEM_SOFTWARE="$( eval $( cat /etc/os-release ); echo "${NAME} ${VERSION}" )"
elif [ -f "/etc/system-release" ]; then
  SYSTEM_SOFTWARE="$( cat /etc/system-release )"
elif command -v lsb_release >/dev/null 2>&1; then
  SYSTEM_SOFTWARE=$(lsb_release -ds)
elif command -v hostnamectl >/dev/null 2>&1; then
  SYSTEM_SOFTWARE="$( hostnamectl 2>/dev/null | sed -n '/Operating System/s/^\s*Operating System:\s*//p' )"
elif command -v uname >/dev/null 2>&1; then
  SYSTEM_SOFTWARE="Linux $( uname -r)"
fi
[ -n "$SYSTEM_SOFTWARE" ] && echo "systemsoftware=${SYSTEM_SOFTWARE}" >> "$RUNTIME_JOB_DIAG"

  # Changing to session directory
  HOME=$RUNTIME_JOB_DIR
  export HOME
  if ! cd "$RUNTIME_JOB_DIR"; then
    echo "Failed to switch to '$RUNTIME_JOB_DIR'" 1>&2
    RESULT=1
  fi
  if [ ! -z "$RESULT" ] && [ "$RESULT" != 0 ]; then
    exit $RESULT
  fi
# Write nodename if not already written in LRMS-specific way
if [ -z "$NODENAME_WRITTEN" ] ; then
  nodename=`/bin/hostname -f`
  echo "nodename=$nodename" >> "$RUNTIME_JOB_DIAG"
fi
echo "Processors=2" >> "$RUNTIME_JOB_DIAG"
echo "Nodecount=1" >> "$RUNTIME_JOB_DIAG"
echo "Benchmark=HEPSPEC:1.0" >> "$RUNTIME_JOB_DIAG"
executable='./runpilot2-wrapper.sh'
# Check if executable exists
if [ ! -f "$executable" ]; 
then 
  echo "Path \"$executable\" does not seem to exist" 1>$RUNTIME_JOB_STDOUT 2>$RUNTIME_JOB_STDERR 1>&2
  exit 1
fi
# See if executable is a script, and extract the name of the interpreter
line1=$(dd if="$executable" count=1 2>/dev/null | head -n 1 | tr -d '\0')
shebang=`echo $line1 | sed -n 's/^#! *//p'`
interpreter=`echo $shebang | awk '{print $1}'`
if [ "$interpreter" = /usr/bin/env ]; then interpreter=`echo $shebang | awk '{print $2}'`; fi
# If it's a script and the interpreter is not found ...
[ "x$interpreter" = x ] || type "$interpreter" > /dev/null 2>&1 || {

  echo "Cannot run $executable: $interpreter: not found" 1>$RUNTIME_JOB_STDOUT 2>$RUNTIME_JOB_STDERR 1>&2
  exit 1; }
if [ "x$JOB_ACCOUNTING" = "xgnutime" ]; then
  $GNU_TIME -o "$RUNTIME_JOB_DIAG" -a -f 'WallTime=%es\nKernelTime=%Ss\nUserTime=%Us\nCPUUsage=%P\nMaxResidentMemory=%MkB\nAverageResidentMemory=%tkB\nAverageTotalMemory=%KkB\nAverageUnsharedMemory=%DkB\nAverageUnsharedStack=%pkB\nAverageSharedMemory=%XkB\nPageSize=%ZB\nMajorPageFaults=%F\nMinorPageFaults=%R\nSwaps=%W\nForcedSwitches=%c\nWaitSwitches=%w\nInputs=%I\nOutputs=%O\nSocketReceived=%r\nSocketSent=%s\nSignals=%k\n'  "./runpilot2-wrapper.sh" "-q" "BOINC_MCORE" "-j" "managed" "--pilot-user" "ATLAS" "--harvester-submit-mode" "PUSH" "-w" "generic" "--job-type" "managed" "--resource-type" "MCORE" "--pilotversion" "3.13.0.23" "-z" "-t" "--piloturl" "local" "--mute" "--container" <$RUNTIME_JOB_STDIN 1>$RUNTIME_JOB_STDOUT 2>&1
else
   "./runpilot2-wrapper.sh" "-q" "BOINC_MCORE" "-j" "managed" "--pilot-user" "ATLAS" "--harvester-submit-mode" "PUSH" "-w" "generic" "--job-type" "managed" "--resource-type" "MCORE" "--pilotversion" "3.13.0.23" "-z" "-t" "--piloturl" "local" "--mute" "--container" <$RUNTIME_JOB_STDIN 1>$RUNTIME_JOB_STDOUT 2>&1
fi
RESULT=$?

sed -i -e '/nodename=/d' $RUNTIME_JOB_DIAG
hostname=` grep domain_name init_data.xml |awk -F '>' '{print $2}'|awk -F "<" '{print $1}'|sed -e "s# #_#g"`
username=` grep user_name init_data.xml |awk -F '>' '{print $2}'|awk -F "<" '{print $1}'|sed -e "s# #_#g"`
nodename=$username@$hostname
echo "nodename=$nodename" >> "$RUNTIME_JOB_DIAG"
[ -n "$ATHENA_PROC_NUMBER" ] && sed -i -e 's/Processors=1/Processors='"$ATHENA_PROC_NUMBER"'/' $RUNTIME_JOB_DIAG
echo "exitcode=$RESULT" >> "$RUNTIME_JOB_DIAG"

# Running RTE scripts (stage 2)
runtimeenvironments=
runtimeenvironments="${runtimeenvironments}APPS/HEP/ATLAS-SITE;"
# Calling APPS/HEP/ATLAS-SITE function: 
RTE_function_0 2 
if [ $? -ne 0 ]; then
    echo "Runtime APPS/HEP/ATLAS-SITE stage 2 execution failed." 1>&2
    echo "Runtime APPS/HEP/ATLAS-SITE stage 2 execution failed." 1>>"${RUNTIME_JOB_STDERR}"
    exit 1
fi

echo "zip all output files"
flist="*.diag "
for f in ./heartbeat.json ./output.list ./log.49863879._000079.job.log.1 ./log.49863879._000079.job.log.1 ./gmlog/
do
 if [  -e $f ];then
 flist=$flist" "$f
fi
done
if [ -f output.list ];then
ol=$(awk '{print $1}' output.list)
for i in $ol
do
 if [[ -f $i && ! $i =~ HITS.* ]];then
 flist=$flist" "$i
 fi
done
sed -i -e 's/HITS.* /HITS.pool.root.1 /' output.list
fi
tar cvf result.tar.gz $flist
# Handle cgroup measurements
if [ "x$JOB_ACCOUNTING" = "xcgroup" ]; then
    # Max memory used (total)
    maxmemory=$( cat "${memory_cgroup}/memory.memsw.max_usage_in_bytes" )
    maxmemory=$(( (maxmemory + 1023) / 1024 ))
    echo "maxtotalmemory=${maxmemory}kB" >> "$RUNTIME_JOB_DIAG"

    # Max memory used (RAM)
    maxram=$( cat "${memory_cgroup}/memory.max_usage_in_bytes" )
    maxram=$(( (maxram + 1023) / 1024 ))
    echo "maxresidentmemory=${maxram}kB" >> "$RUNTIME_JOB_DIAG"
    # TODO: this is for compatibilty with current A-REX accounting code. Remove when A-REX will use max value instead.
    echo "averageresidentmemory=${maxram}kB" >> "$RUNTIME_JOB_DIAG"

    # User CPU time
    if [ -f "${cpuacct_cgroup}/cpuacct.usage_user" ]; then
        # cgroup values are in nanoseconds
        user_cputime=$( cat "${cpuacct_cgroup}/cpuacct.usage_user" )
        user_cputime=$(( user_cputime / 1000000 ))
    elif [ -f "${cpuacct_cgroup}/cpuacct.stat" ]; then
        # older kernels have only cpuacct.stat that use USER_HZ units
        user_cputime=$( cat "${cpuacct_cgroup}/cpuacct.stat" | sed -n '/^user/s/user //p' )
        user_hz=$( getconf CLK_TCK )
        user_cputime=$(( user_cputime / user_hz ))
    fi
    [ -n "$user_cputime" ] && echo "usertime=${user_cputime}" >> "$RUNTIME_JOB_DIAG"

    # Kernel CPU time
    if [ -f "${cpuacct_cgroup}/cpuacct.usage_sys" ]; then
        # cgroup values are in nanoseconds
        kernel_cputime=$( cat "${cpuacct_cgroup}/cpuacct.usage_sys" )
        kernel_cputime=$(( kernel_cputime / 1000000 ))
    elif [ -f "${cpuacct_cgroup}/cpuacct.stat" ]; then
        # older kernels have only cpuacct.stat that use USER_HZ units
        kernel_cputime=$( cat "${cpuacct_cgroup}/cpuacct.stat" | sed -n '/^system/s/system //p' )
        [ -z "$user_hz" ] && user_hz=$( getconf CLK_TCK )
        kernel_cputime=$(( kernel_cputime / user_hz ))
    fi
    [ -n "$kernel_cputime" ] && echo "kerneltime=${kernel_cputime}" >> "$RUNTIME_JOB_DIAG"

    # Remove nested job accouting cgroups
    arc-job-cgroup -m -d
    arc-job-cgroup -c -d
fi

# Record CPU benchmarking values for WN user by the job
[ -n "${ACCOUNTING_BENCHMARK}" ] && echo "benchmark=${ACCOUNTING_BENCHMARK}" >> "$RUNTIME_JOB_DIAG"
# Record WN instance tag if defined
[ -n "${ACCOUNTING_WN_INSTANCE}" ] && echo "wninstance=${ACCOUNTING_WN_INSTANCE}" >> "$RUNTIME_JOB_DIAG"

# Record execution clock times
ACCOUNTING_ENDTIME=`date +"%s"`
# Mds date format (YYYYMMDDHHMMSSZ)
echo "LRMSStartTime=`date -d "1970-01-01 UTC ${ACCOUNTING_STARTTIME} seconds" +"%Y%m%d%H%M%SZ"`" >> "$RUNTIME_JOB_DIAG"
echo "LRMSEndTime=`date -d "1970-01-01 UTC ${ACCOUNTING_ENDTIME} seconds" +"%Y%m%d%H%M%SZ"`" >> "$RUNTIME_JOB_DIAG"
echo "walltime=$(( ACCOUNTING_ENDTIME - ACCOUNTING_STARTTIME ))" >> "$RUNTIME_JOB_DIAG"

# Add exit code to the accounting information and exit the job script
echo "exitcode=$RESULT" >> "$RUNTIME_JOB_DIAG"
exit $RESULT