#!/bin/tcsh -f
#+
#
# koa_translator [directory] [-test]
#
# looks through the specified directory (or the current directory if
# there is no argument) and all subdirectories for file names of the
# form II.dddddddd.xxxxx.fits, where II is a KOA instrument designator
# (see list in the case statement below), dddddddd is an eight-digit
# date number, and xxxxx is a five-digit "KOAID" index number.
#
# Files ending in ".fits.gz" will also be processed. These filenames are
# assumed to be filenames from KOA (Keck Observatory Archive).
#
# The "-test" argument creates a file (translate.csh) that can be executed to 
# perform the translation. Without this argument, the script will be created,
# run, and then deleted.
#
# The translator will look through each such file's FITS header and
# reconstruct the original filename (including a parent directory
# derived from the UT date, and possibly appropriate subdirectories) and 
# write to a file "translate.csh" the appropriate shell commands to
# rename the KOA file to the original filename.
#
# This is meant to reconstruct the original directory and filename
# structure as observed. The reconstruction will not always be perfect.
# Observers may have used nonstandard directory structure, or KOA
# might have recorded multiple versions of the same filename. The 
# latter might occur, for example, if observers recorded some files
# then deleted them and reset the file number to write new files
# into the same filenames.
#
# Note that if there are duplicate original filenames, the program will 
# use the original filename for the first version it finds, then create
# second (and subsequent, if necessary) versions with "_v2", "_v3", etc.
# added to the filename. THIS DOES NOT GUARANTEE THAT FILES ALL USING
# "_v2", FOR EXAMPLE, ARE REALLY FROM THE SAME DATASET. You may have
# to do some checking to determine which files belong in which dataset.
#
# Written by: R. W. Goodrich, 2013-2014.
#
#-

# Loop through the current directory or the directory provided and create
# a list of KOAID-like filenames.

# debug        : when set to 1, script will print additional verbose output
set debug = 0

# service_mode : used when run as part of KOA archive infrastructure; if you are
#                running this by hand it is recommended to leave it set to 0
set service_mode = 0

# test_mode    : set this to 0 to create a translate.csh script without actually running it,
#                in case you want to do some sanity checks before rearranging files
set test_mode = 0


# Default search directory

set koadir = "."

# Check input parameters

while ($#argv != 0)
	if ("$1" == "-test") then
		set test_mode = 1
	else if ("$1" == "-debug") then
		set debug = 1
	else if ("$1" == "-service_mode") then
		shift
		if ("$1" != 0) set service_mode = 1
	else if (-d "$1") then
		set koadir = "$1"
	else
		if ("$service_mode" == 1) then
			echo '[struct stat="ERROR", msg="$1 is not a directory nor a valid flag."]'
		else
			echo "$1 is not a directory nor a valid flag."
		endif
		exit 1
	endif
	shift
end

if ("$debug" == 1) then
	echo "directory : ${koadir}"
	echo "service_mode : ${service_mode}"
	echo "test_mode : ${test_mode}"
	exit
endif

# Disclaimer about "Input error" lines.

if ("$service_mode" == 0) then
	echo ""
	echo Ignore any \"Input error\" lines that you see from this script.
	echo They occur in some cases but are benign.
	echo ""
endif

# KOAID to instrument

set DE = "DEIMOS"
set DF = "DEIMOS"
set ES = "ESI"
set HI = "HIRES"
set LB = "LRIS"
set LR = "LRIS"
set LW = "LWS"
set MF = "MOSFIRE"
set N1 = "NIRC"
set N2 = "NIRC2"
set NC = "NIRSPEC"
set NS = "NIRSPEC"
set NI = "NIRES"
set NR = "NIRES"
set OI = "OSIRIS"
set OS = "OSIRIS"
set KB = "KCWI"
set KR = "KCWI"
set KF = "KCWI"

# Three-letter abbreviations for months.

set month = ("jan" "feb" "mar" "apr" "may" "jun" "jul" "aug" "sep" "oct" "nov" "dec")

# Create list of KOA files.

set format = '[A-Z][A-Z,0-9].[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9].[0-9][0-9][0-9][0-9][0-9]'
set koafiles = (`find $koadir -name "$format.fits" -print` `find $koadir -name "$format.fits.gz" -print`)

# Verify files found

if ($#koafiles == 0) then
	if ("$service_mode" == 1) then
		echo '[struct stat="ERROR", msg="No files found"]'
	else
		echo "No relevant files found to translate."
		echo "KOA files are of the form OS.20111230.05089.fits with a possible"
		echo ".gz extension."
	endif
	exit 1
endif

# Create the translation file.

set transfile = "$koadir/translate.csh"

# If the translate.csh file exists, ask whether to overwrite or delete.

if (-e $transfile) then
	if ("$service_mode"  == 1) then
		cat /dev/null >! $transfile
	else
		echo "The translation file, $transfile, exists."
		echo -n "Do you want to overwrite it [y/n, default is "y"] ? "
		set answer = $<
		set answer = `echo $answer | tr '[:upper:]' '[:lower:]'`
		switch ($answer)
			case "":
			case "y":
			case "yes":
				echo "Overwriting translate.csh..."
				cat /dev/null >! $transfile
				breaksw
			default:
				echo "Updating translate.csh..."
		endsw
	endif
else
	touch $transfile
endif

# Loop through the file list.

set useGrep = "grep"
set redirect = ""
if ($service_mode != 0) then
	set useGrep = "grep -a"
	set redirect = " >& /dev/null"
endif

set dirlist = "";

set blank = "                                 "
set count = 1
set total = 0
foreach file ($koafiles)

	if ("$service_mode" == 0) then
		echo -n "$blank Z" | tr 'Z' '\015'
		echo -n "Processing $count of $#koafiles KOAIDs Z" | tr 'Z' '\015'
	endif

	# If the file ends in "gz", use "gunzip -c" to list the file.
	# Note that "zcat" does not work properly on some Macs.

	set gztest = `echo $file | grep "\.gz"`
	if ("$gztest" == "") then
		set catcmd = cat
		set gzext  = ""
	else
		set catcmd = "gunzip -c"
		set gzext  = ".gz"
	endif

	# Create the directory name from the KOAID's UT date.
	# The month is replaced by a three-letter abbreviation.

	set explodedName = (`echo $file | sed "s|/| |g"`)
	set filename = $explodedName[$#explodedName]
	set koainst = `echo $filename | cut -c1-2`
	set koaUTyear  = `echo $filename | cut -c4-7`
	set koaUTmonth = `echo $filename | cut -c8-9`
	set koaUTday   = `echo $filename | cut -c10-11`

	# Translate the KOAID instrument abbreviation to a longer string.

	set inst = ""
	set inst = `eval echo \$$koainst`
	if ("$inst" == "") then
		if ("$service_mode" == 1) then
			echo '[struct stat="INFO", msg="Bad two-letter KOAID code"]'
		else
			echo "Bad two-letter KOAID code in ${filename}"
		endif
	endif

	# New directory - HIRES/2015jan01
	set outdir = $inst/$koaUTyear$month[$koaUTmonth]$koaUTday/

	# Filename reconstruction from keywords, reduced products directory name

	set raw = "raw"
	set lev1 = ""

	switch ($inst)
		case "OSIRIS":
			set lev1 = "reduced_cubes"
		case "MOSFIRE":
			set origfile = `$catcmd $file | fold | head -100 | $useGrep "^DATAFILE=" | cut -f2 -d\'`.
			breaksw
		case "HIRES":
			set lev1 = "extracted"
		case "DEIMOS":
		case "ESI":
		case "LRIS":
			set prefix = `$catcmd $file | fold | head -350 | $useGrep "^OUTFILE =" | cut -f2 -d\'`
			set frame  = `$catcmd $file | fold | head -350 | $useGrep "^FRAMENO =" | cut -f2 -d=`
			set paddedframe = `printf "%4.4i" $frame[1]`
                        if ($prefix == '') set prefix = ("none" "none")
			set origfile = $prefix[1]$paddedframe.fits
			breaksw
		case "NIRC":
			set prefix = `$catcmd $file | fold | head -100 | $useGrep "^OUTFILE =" | cut -f2 -d\'`
			set frame  = `$catcmd $file | fold | head -100 | $useGrep "^FRAMENO =" | cut -f2 -d=`
			set paddedframe = `printf "%5.5i" $frame[1]`
			set origfile = $prefix[1]$paddedframe.fits
			breaksw
		case "LWS":
			set lev1 = "reduced"
			set prefix = `$catcmd $file | fold | head -100 | $useGrep "^OUTFILE =" | cut -f2 -d\'`
			set frame  = `$catcmd $file | fold | head -100 | $useGrep "^FRAMENO =" | cut -f2 -d\'`
			set paddedframe = `printf "%4.4i" $frame[1]`
			set origfile = $prefix[1]$paddedframe.fits
			breaksw
		case "NIRC2":
			set lev1 = "calibrated"
			set origfile = `$catcmd $file | fold | head -100 | $useGrep "^FILENAME=" | cut -f2 -d\'`
			breaksw
		case "NIRSPEC":
			if ("$koainst" == "NS") set raw = "raw/spec"
        		if ("$koainst" == "NC") set raw = "raw/scam"
			set origfile = `$catcmd $file | fold | head -100 | $useGrep "^FILENAME=" | cut -f2 -d\'`
			if ("$origfile" == "") set origfile = `$catcmd $file | fold | head -400 | $useGrep "^OFNAME  =" | cut -f2 -d\'`
			breaksw
		case "NIRES":
			set origfile = `$catcmd $file | fold | head -450 | $useGrep "^OFNAME  =" | cut -f2 -d\'`
			breaksw
		case "KCWI":
			set origfile = `$catcmd $file | fold | head -450 | $useGrep "^OFNAME  =" | cut -f2 -d\'`
			breaksw
		default:
			if ("$service_mode" == 1) then
				echo '[struct stat="ERROR", msg="Could not determine the instrument $koainst in $file"]'
			else
				echo "Could not determine the instrument $koainst in $file"
			endif
			exit 1
	endsw

	# Filter for double "/" marks in outdir, then write out the translation line.

	set outdir = `echo $outdir | sed "s://:/:g"`
	set test = `echo $dirlist | grep $outdir$raw`
	if ("$test" == "") then
		set dirlist = "$dirlist $outdir$raw"
		echo "mkdir -p $outdir$raw$redirect" >>! $transfile
	endif
	set test = `echo $dirlist | grep $outdir$lev1`
	if ("$test" == "") then
		set dirlist = "$dirlist $outdir$lev1"
		echo "mkdir -p $outdir$lev1$redirect" >>! $transfile
	endif

	# Check to see if the output file is in the translation file yet.

	@ i=1
	set version = ""
	set test_output = `grep $outdir$raw/$origfile$gzext $transfile`
	while ("$test_output" != "")
		@ i++
		set version = "_v$i"
		set origfile2 = `echo $origfile | sed "s/\.fits/$version.fits/"`
		set test_output = `grep $outdir$raw/$origfile2$gzext $transfile`
	end
	if ($i > 1) then
		set origfile = `echo $origfile | sed "s/\.fits/$version.fits/"`
	endif

	# Find and translate any files starting with the same KOAID.

	set koaid = `echo $filename | cut -c1-17`
	set oldname = `echo $origfile | cut -f1 -d\.`
	set explodedName = (`echo $file | sed "s|/| |g"`)
	set newname = `echo $explodedName[$#explodedName] | sed -e "s/$koaid/$oldname/"`
	echo "mv $file $outdir$raw/$newname$redirect" >>! $transfile
	@ total++

	foreach other (`find $koadir -name "${koaid}*" -print | egrep -v "$file"`)
		set oldname = `echo $origfile | cut -f1 -d\.`
		set explodedName = (`echo $other | sed "s|/| |g"`)
		set newname = `echo $explodedName[$#explodedName] | sed -e "s/$koaid/$oldname/"`
        
		# Calibration files
		set caltest = `echo $other | grep "\/cal\/"`
		if ("$caltest" != "") then
			echo "mv $other $outdir$raw/$newname$redirect" >>! $transfile
			@ total++
		else 
			set lev1test = `echo $other | grep "\/$lev1\/"`
			if ("$lev1test" != "") then
				echo "mv $other $outdir$lev1/$newname$redirect" >>! $transfile
				@ total++
			else
				echo "mv $other $outdir$raw/$newname$redirect" >>! $transfile
				@ total++
			endif
		endif
	end
	@ count++
end

# Make the shell script executable.

chmod a+x $transfile

# Run the shell script if this is not in test mode.

if ("$test_mode" == 0) then
	source $transfile
	rm $transfile
endif

if ("$service_mode" == 1) then
	echo '[struct stat="OK"]'
	exit
else
	set str = "$total files";
	if ("$test_mode" == 0) then
		set str = "$str moved"
	else
		set str = "$str to move"
	endif
	echo $str
	echo "Done!"
endif