#!/bin/csh -f
#+
#OSIRIS library of scripts
#
#NAME
#     osirisRecover - restart detector server without bothering other servers
#
#SYNOPSIS
#     osirisRecover IMAG|SPEC [-t #] [-m] etc...
#
#DESCRIPTION
#     Allows one to restart a detector server and set all the initial 
#     parameters so one can restart observing with that server.  This
#     script does so in a manner that should not affect the other server.
#
#OPTIONS
#     -t #
#          ID number to attach to all script generated errors, warnings, and
#          questions that are passed to the OGS.  The ID number is separated
#          from the rest of the message with a ";".
#
#     -m
#          simulate calls to RPC servers.  during simulated calls, the 
#          command is echoed (instead of being executed).
#
#     others?
#
#EXAMPLES
#     osirisRecover junk1 -t 123
#          What would happen?
#
#ENVIRONMENT VARIABLES
#     list of environment variables used
#
#FILES
#     list of files used
#
#SERVERS & KEYWORDS
#     list of the servers and keywords used, for example:
#
#     service = osiris
#          keywords: sfilter/ifilter, sscale
#
#SCRIPTS CALLED
#     list of the scripts called by this script, for example:
#
#     help, syncheck
#
#EXIT STATUS
#     0 - normal exit, no error
#     1 - script aborted by an interrupt
#     2 - syncheck error
#     3 - error parsing command line input
#     4 - cannot kill server task
#     5 - server did not restart correctly
#     6 - user chose to exit
#     other errors...
#
#SEE ALSO
#     ???
#-
#
# Modification History:
# 2006jun17 - jlyke: Initial version created
# 2017jul19 - jlyke: Updates for new SPEC detector
# 2020jun18 - jlyke: Updates for new IMAG detector, power down asic first
#

# Boiler plate for "-h" support for command autohelp.

if ("$1" == "-h") then
    help $0 | more
    exit $status
endif

# Boilerplate for syncheck.
# Note that the boiler plate should be transparent for all usages,
# but to make correct use of syncheck you will need to specify the
# correct pattern.

set noglob
set CheckStatus = `syncheck -command $0 $* -pattern [SPEC,spec,OSDS,osds,IMAG,imag,OIDS,oids]` 
unset noglob

if ("$CheckStatus" != "OK") then
    help $0 | more
    exit 2
endif

set buf = $0
set cmd = $buf:t

# End of help/syncheck boiler plate.

# Set up to trap interrupts (Ctrl-C, etc.)
onintr abort

set rpcinfo = "usr/sbin/rpcinfo"
set waittime = 0
set waitmax = 5

# Handle input
# Decide if we should use SPEC or IMAG
set mode = $1
switch ($mode)
    case "SPEC":
    case "spec":
    case "OSDS":
    case "osds":
        set M = "s"
        set mode = "spec"
	set computer = "puunoa"
	set service = "osds"
        breaksw
    case "IMAG":
    case "imag":
    case "OIDS":
    case "oids":
        set M = "i"
        set mode = "imag"
	set computer = "kuiaha"
	set service = "oids"
        breaksw
    default:
        osirisScriptMsg -T "${cmd} ${mode}: Specified mode must be SPEC or IMAG- aborting."
        exit 3
        breaksw
endsw

# set default variable values

# get current date
set date=`date -u "+%y%m%d"`


# check server status, shutdown if necessary, use extreme means if necessary
# server alive?
set serverup = `ssh osiris@${computer} ct | grep ${service} | wc -l`
if ( ${serverup} == 1 ) then
    echo "${service} seems to be alive."
    echo -n "Enter <y> to shutdown and restart ${service}. [n] "
    set shutdown = $<
    switch (${shutdown})
	case Y*:
	case y*:
	breaksw
    default:
	echo "User chose to exit"
	exit 6
    endsw
endif

# shutdown server
# 1) Try to power down Sidecar ASIC (det controller)
#    Shutdown via "osiris stop ${service}
# 2) Shutdown via "kill -9"
# 3) Give up  
set running = `ssh osiris@${computer} ct | grep ${service}`
set count = 0
while ( ${#running} != 0 )
  @ count++
  if ( ${count} == 1 ) then
    echo "Attempting to powerdown ${mode} Sidecar ASIC"
    modify -s ${service} powerdown=1
    sleep 1
    set asicpwr = `show -s ${service} -terse asicpwr`
    while ($asicpwr == 1 && $waittime < $waitmax)
      echo "ASIC Power is $asicpwr.  Attempt # $waittime/$waitmax"
      echo "Attempting powerdown again..."
      modify -s ${service} powerdown=1
      sleep 1
      @ waittime++
      set asicpwr = `show -s ${service} -terse asicpwr`
    end
    set asicpwr = `show -s ${service} -terse asicpwr`
    if ($asicpwr == 1 ) then
      echo "Unable to powerdown ${mode} Sidecar ASIC via keyword, but continuing"
    endif 
    echo "Attempting to stop ${service} via osiris stop ${service}."
    osiris stop ${service}
  else if ( ${count} == 2) then
    echo "Attempting to stop ${service} via kill -9."
    set pids = `ssh osiris@${computer} ct | grep osiris | awk '{print $2}'`
    ssh osiris@${computer} kill -9 "$pids"
#    foreach id ($pids)
#	ssh osiris@${computer} kill -9 $id
#    end
  else if ( ${count} == 3 ) then
    echo "Cannot stop ${service} on ${computer} - exiting."
    exit 4
  endif
  # give servers a chance to exit before checking status again
  sleep 5
  set running = `ssh osiris@${computer} ct | grep ${service}`
end

# service in question should not be running
# cannot start another instance of the server if its place in the 
# rpc table is taken.
switch ( ${service} )
  case oids:
    set prognum = $OIDS_PROGNUM
    breaksw
  case osds:
    set prognum = $OSDS_PROGNUM
    breaksw
endsw
set rpc = `ssh osiris@${computer} $rpcinfo | grep $prognum`
if ( $status == 0 ) then
  echo "Removing old ${service} entry in rpc table on ${computer}."
  ssh osiris@${computer} $rpcinfo -d ${rpc[1]} ${rpc[2]}
endif

# now restart the service in question
echo "Restarting ${service} via osiris start ${service}."
osiris start ${service}

# give the server a chance to start
sleep 5

# check if the server is up
set serverup = `ssh osiris@${computer} ct | grep ${service} | wc -l`
    if ( ${serverup} != 1 ) then
	echo "${service} did not start correctly on ${computer} - exiting."
	exit 5
    endif

# set default values for detector server.
if (`osirisCheckServer ${service}` == 1) then
    switch (${service})
	case oids:
	    # assume soutdir was ok if restarting oids
	    set soutdir = `show -s osiris -terse soutdir`
	    set outdir = `echo $soutdir | sed -e 's/SPEC/IMAG/'`
	    modify -s oids outdir=$outdir
	    set testdir = `echo $soutdir | sed -e 's/SPEC\/raw/tmp/'`
	    breaksw
	case osds:
	    # assume ioutdir was ok if restarting osds
	    set ioutdir = `show -s osiris -terse ioutdir`
	    set outdir = `echo $ioutdir | sed -e 's/IMAG/SPEC/'`
	    modify -s osds outdir=$outdir
	    set testdir = `echo $ioutdir | sed -e 's/IMAG\/raw/tmp/'`
	    breaksw
    endsw

    # ensure service is talking to detector
    osirisCheckoutDetector ${mode}

    # set default keywords (same as osirisSetupNight)
    modify -s ${service} testdir=${testdir}
    modify -s ${service} framenum=0
    modify -s ${service} getkw=1
    modify -s ${service} getdcskw=1
    modify -s ${service} getaokw=1
    modify -s ${service} sampmode=3
    modify -s ${service} numreads=1
    modify -s ${service} coadds=1
    modify -s ${service} itime=2000
    modify -s ${service} extension="fits"
    modify -s ${service} writeall=0
    modify -s ${service} calcreads=0
    
    # jlyke 2017 jul 19: added this sleep as osirisGetSetnum was failing
    sleep 1

    set setnum = `osirisGetSetnum ${service}`
    if ( $status == 0 ) then
	modify -s ${service} setnum=${setnum}
    else
	modify -s ${service} setnum=0
    endif

    
    

else
	echo "--- Warning: ${service} not running - default keywords not set."
endif

# finally determine the higher setnum
set ssetnum = `show -s osiris -terse ssetnum`
set isetnum = `show -s osiris -terse isetnum`
if ( ${isetnum} > ${ssetnum} ) then
    modify -s osiris setnum=${isetnum}
else
    modify -s osiris setnum=${ssetnum}
endif


goto done

abort:
# Block of code to handle interrupts.
exit 1

done:
# is there anything that needs to go here?
echo "The recover script is now complete"
echo "If you're still experiencing detector trouble"
echo "...you need to contact your SA."
exit