#!/usr/bin/perl -w

#######################################
## Get a results file or exit
#######################################
if ($#ARGV > 1) {
	print STDERR "Usage: $0 resultsfile -flags\n";
	die "\n";
}
$results_file = $ARGV[0];
$flags = "";
if ($#ARGV == 1){
	$flags = $ARGV[1];
}
if($flags =~ /h/){
	print STDERR "Flag h: help\n";
	print STDERR "Flag d: detailed scores for each 'gold' assignment\n";
	print STDERR "Flag s: print summary\n";
	print STDERR "Flag 5: special handling of team 5 results\n";
	print STDERR "Flag 7: special handling of team 7 results\n";
	die "\n";
}

#######################################
## Load the answer key
#######################################

open (KEY, "< wngloss.key") or die "\nUnable to open the answer key file \"wngloss.key\"\n";

#######################################
## Store the answer key in 3 hashes
##	according to quality of assignment
#######################################
%golds = ();
%silvers = ();
%normals = ();
while($line = <KEY>){
	chomp $line;
	($glossid, $quality, $answer) = split / /, $line;
	if($quality eq "gold"){
		if(exists $golds{$glossid}){
			$answers = $golds{$glossid};
			$answers .= ";" . $answer;
			$golds{$glossid} = $answers;
		} else {
			$golds{$glossid} = $answer;
		}
	} elsif($quality eq "silver"){
		if(exists $silvers{$glossid}){
			$answers = $silvers{$glossid};
			$answers .= ";" . $answer;
			$silvers{$glossid} = $answers;
		} else {
			$silvers{$glossid} = $answer;
		}
	} elsif($quality eq "normal"){
		if(exists $normals{$glossid}){
			$answers = $normals{$glossid};
			$answers .= ";" . $answer;
			$normals{$glossid} = $answers;
		} else {
			$normals{$glossid} = $answer;
		}
	}
}
close KEY;

#######################################
## Read in and store the glosses
## with different degrees of
## interannotator agreement
#######################################
%ann1 = (); # glosses with full interannotator agreement study
%ann2 = (); # glosses with one tagger (no review)
%corr = (); # glosses thought to be improperly annotated
open (ANN, "< synsetId.noun.500.trec") or die "\nUnable to open synsetId.noun.500.trec\n";
while($line = <ANN>){
	chomp $line;
	$line =~ / /;
	$line = "NOUN." . $`;
	$ann1{$line} = 1;
}
close ANN;
open (ANN, "< synsetId.verb.500.trec") or die "\nUnable to open synsetId.verb.500.trec\n";
while($line = <ANN>){
	chomp $line;
	$line =~ / /;
	$line = "VERB." . $`;
	$ann1{$line} = 1;
}
close ANN;
open (ANN, "< synsetId.noun.4000.trec") or die "\nUnable to open synsetId.noun.4000.trec\n";
while($line = <ANN>){
	chomp $line;
	$line =~ / /;
	$line = "NOUN." . $`;
	$ann2{$line} = 1;
}
close ANN;
open (ANN, "< corrid.idx") or die "\nUnable to open corrid.idx\n";
while($line = <ANN>){
	chomp $line;
	$corr{$line} = 1;
}
close ANN;

#######################################
## Read in and store the system's
## answers
#######################################
open (RESULTS, "< $results_file") or die "\nUnable to open the results file: $results_file\n";
%results = ();
while($line = <RESULTS>){
	chomp $line;
	# special processing for system 7
	if($flags =~ /7/){
		$line =~ / /;
		$line = $';
	}
	($glossid, $result) = split / /, $line;
	# special processing for system 5
	if($flags =~ /5/){
		$glossid =~ /\./;
		$glossid = $' . "." . $`;
	}
	if(exists $results{$glossid}){
		$resultlist = $results{$glossid};
		$resultlist .= ";" . $result;
		$results{$glossid} = $resultlist;
	} else {
		$results{$glossid} = $result;
	}
}
close RESULTS;

#######################################
## Score the system's answers
#######################################
@garray = keys(%golds);
@garray = sort(@garray);
for($i=0;$i<6;$i++){
	$numglosses = 0;
	$numgolds = 0;
	$numattempted = 0;
	$numcorrect = 0;
	foreach $key (@garray){
		if($i){
			if($i == 1 && !exists $ann1{$key}){
				next;
			} elsif($i == 2 && !exists $ann2{$key}){
				next;
			} elsif($i == 3 && (exists $ann1{$key} || exists $ann2{$key})){
				next;
			} elsif($i == 4 && exists $corr{$key}){
				next;
			} elsif($i == 5 && !exists $corr{$key}){
				next;
			}
		}
		$numglosses++;
		$goldlist = $golds{$key};
		@golds = split /;/, $goldlist;
		$numgolds += @golds;
		foreach $gold (@golds){
			$gold =~ /%/;
			$lemma = $`;
			$gold =~ /\|(.*?)%/;
			$wform = $1;
			if(exists $results{$key}){
				$resultlist = $results{$key};
				if($resultlist =~ / /){
					print "Answer for $key contains multiple senses\n";
				}
				$found = 0;
				@results = split /;/, $resultlist;
				foreach $result (@results){
					if($result =~ /\(/){
						print STDERR "$result contains parenthesis\n";
						next;
					}
					if($gold =~ /$result/){
						$found = 1;
						last;
					}
				}
				if($found){
					$numcorrect++;
				}
				@results = split /;/, $resultlist;
				foreach $result (@results){
					$result =~ /%/;
					$rlemma = $`;
					if(($rlemma ne "" && $rlemma eq $lemma) || ($wform ne "" && $rlemma eq $wform)){
						$numattempted++;
						last;
					}
				}
				if($flags =~ /d/){
					if($found){
						print "score for \"$key:$gold\": 1.000\n";
					} else {
						print "score for \"$key:$gold\": 0.000\n";
					}
					print " answers for $key = $resultlist\n";
				}
			} else {
				# system did not try this gloss
				if($flags =~ /d/){
					print "score for \"$key:$gold\": 0.000\n";
					print " answers for $key = (null)\n";
				}
			}
		}
		if($flags =~ /d/){
			print "\n";
		}
	}

	if($flags =~ /s/){
		if(!$i){
			print "\n\nUsing full answer key:\n";
		} elsif($i == 1){
			print "\n\nUsing answer key for interannotator only items:\n";
		} elsif($i == 2){
			print "\n\nUsing answer key for single tagger items:\n";
		} elsif($i == 3){
			print "\n\nUsing answer key for irregularly tagged items:\n";
		} elsif($i == 4){
			print "\n\nUsing answer key with corrections removed:\n";
		} elsif($i == 5){
			print "\n\nUsing answer key for corrections only:\n";
		}
		print "Summary scores for $results_file\n";
		print "Number of glosses: $numglosses\n";
		print "Number of gold assignments: $numgolds\n";
		print "Score for $results_file using key \"wngloss.key\"\n";
		$precision = $numcorrect/$numattempted;
		printf " precision: %5.3f ($numcorrect correct of $numattempted attempted)\n", $precision;
		$recall = $numcorrect/$numgolds;
		printf " recall: %5.3f ($numcorrect correct of $numgolds in total)\n", $recall;
		$attempted = ($numattempted/$numgolds) * 100.0;
		printf " attempted: %4.1f percent ($numattempted attempted of $numgolds in total)\n", $attempted;
	}
}

