from boto.mturk.connection import MTurkConnection
from boto.mturk.connection import MTurkRequestError
import csv
import time
import sys
import pdb

from settings import settings

#Settings for when to approve/reject workers
FREEBIES = 10 #Approve every turker's first 10 HITs for free
APPROVE_THRESHOLD = 0.70 #After that, if their accuracy goes above 70%, whitelist them and approve everything
REJECT_THRESHOLD = 0.4 #If their accuracy goes below 40%, blacklist them and reject everything
PENDING_REJECT_THRESHOLD= 0.5 #If their accuracy is between 40% and 70%, approve/reject each HIT based on the controls in that HIT

#Provide detailed feedback about the control questions they get wrong/right
PREAPPROVE_MSG = "Thank you for doing our HIT! You averaged %.02f accuracy on our control questions. You can see your responses and our expected answers to these control questions below. We approve your first %s HITs, after that, if your accuracy falls below %.02f we will no longer approve them. We use your labels for our research, and we appreciate your care in doing our task well. We understand that some of these are ambiguous, which is why we allow some discrepency between your answers and our own. However, if your answers are consistently disagreeing with our answers, please try to adjust the way you are thinking about our questions, or consider working on different HITs. It is important for us to have consistent labels from our workers. Thank you so much for your work and have a great day! \nControl : %s, Expected response: %s, Your response: %s"

PENDING_APP_MSG = "Thank you for doing our HIT! Your accuracy on the controls for this HIT was %.02f. Over the %d HITs you have done for us, you have averaged %.02f accuracy on our control questions. If your overall accuracy falls below %.02f we will no longer approve your work. We use your labels for our research, and we appreciate your care in doing our task well. Have a great day!"

PENDING_REJ_MSG = "I am sorry, your accuracy on the controls for this HIT was %.02f, and we require above 0.5 in order to approve. Over the %d HITs you have done for us, you have averaged %.02f accuracy on our control questions. You can see your responses and our expected answers to these control questions below. If your overall accuracy falls below %.02f we will no longer approve your work. We use your labels for our research, and we appreciate your care in doing our task well. Make sure you reread the instructions and read each pair carefully. Have a great day!  \nControl pair: %s, Expected response: %s, Your response: %s"

TRUSTED_MSG = "Thank you so much for your continued great work! You are awesome!"

BLOCK_MSG = "I am sorry. You averaged %.02f accuracy on our control questions. We use this data for research, and we have to make sure that all of our noun pairs are labeled thoughtfully and accurately, so we require an accuracy of at least %.02f on our controls to approve your HITs. We appreciate your time. If you choose to do more of our HITs in the future, please make sure you read all the instructions and each paraphrase carefully."

conn = MTurkConnection(aws_access_key_id=settings['ACCESS_ID'], aws_secret_access_key=settings['SECRET_KEY'], host=settings['HOST'])

workers = {}
statuses = {}

new_rows = []
rows = [row for row in csv.DictReader(open(sys.argv[1]))]
headers = rows[0].keys() 

#Keep track of how many assignments you approve/reject, just FYI
approves = 0
pending_rejects = 0
blocked_rejects = 0
#Keep track of how many workers you approve/reject, just FYI
wapproves = set()
wpending_rejects = set()
wblocked_rejects = set()

#Utility function for hiliting approvals and rejections on the terminal (good for catching bugs, e.g. if you are rejecting too many HITs)
def hilite(string, color):
	attr = []
	if color == "green" :  attr.append('32')
	elif color == "red" : attr.append('31')
	return '\x1b[%sm%s\x1b[0m' % (';'.join(attr), string)

for row in rows : 
	worker = row['WorkerId']
	assignment = row['AssignmentId']
	if worker not in workers : workers[worker] = [0.0, 0.0, 0]; statuses[worker] = 'PREAPPROVE'
	workers[worker][2] += 1
	this_correct = 0.0; this_total = 0.0
	ctrls = []
	for i in range(11) : 
		kp = row['Input.known_pos%d'%i] #list of positive controls- paraphrases that you expect the worker to have checked
		kn = row['Input.known_neg%d'%i] #list of negative controls- paraphrases that you expect the worker to not have checked
		answers = row['Answer.paraphrases%d'%i].split('|')
		if not(kp == 'unk' and kn == 'unk') : #if true, this is a control question 
			kps = kp.split('|')
			kns = kn.split('|')
			ncontrols = len(kps) + len(kns) #total number of controls 
			workers[worker][0] += ncontrols
			workers[worker][1] += ncontrols
			this_correct = ncontrols
			this_total = ncontrols
			#subtract 1 for each negative control that was checked
			for a in answers : 
				if a in kns : 
					workers[worker][0] -= 1
					this_correct -= 1
			#subtract 1 for each positive control that was not checked
			for a in kps : 
				if not(a in answers) : 
					workers[worker][0] -= 1	
					this_correct -= 1
			ctrls.append((row['Input.sent%d'%i], ', '.join(kps),', '.join(answers)))
	#update the worker's accuracy
	acc = workers[worker][0] / workers[worker][1]
	total = workers[worker][2]
	status = statuses[worker]

	if status == 'PREAPPROVE' : 
		print 'preapproving worker %s with acc %.03f out of %d hits'%(worker, acc, total)
		new_rows.append([row[k] for k in headers] + ['PREAPPROVE'])
		msg = PREAPPROVE_MSG%(acc, FREEBIES, REJECT_THRESHOLD, ctrls[0][0], ctrls[0][1], ctrls[0][2])
		approves += 1
		wapproves.add(worker)
		#If you have already approved/rejected, MTurk will through an error, so surround in try/catch
		try : conn.approve_assignment(assignment, feedback=msg)
		except MTurkRequestError : pass
		#this is their last free approval
		if total == FREEBIES : 
			print 'final preapprove, worker %s acc is %.03f out of %d hits'%(worker, acc, total)
			if acc >= APPROVE_THRESHOLD : statuses[worker] = 'TRUSTED'
			elif acc <= REJECT_THRESHOLD : statuses[worker] = 'BLOCKED'
			else : statuses[worker] = 'PENDING'

	elif status == 'TRUSTED' : 
		print hilite('approving trusted worker %s with acc %.03f out of %d hits'%(worker, acc, total), "green")
		new_rows.append([row[k] for k in headers] + ['TRUSTED'])
		msg = TRUSTED_MSG
		approves += 1
		wapproves.add(worker)
		try : conn.approve_assignment(assignment, feedback=msg)
		except MTurkRequestError : pass

	elif status == 'BLOCKED' : 
		print hilite('rejecting blocked worker %s with acc %.03f out of %d hits'%(worker, acc, total), "red")
		new_rows.append([row[k] for k in headers] + ['BLOCKED'])
		msg = BLOCK_MSG%(acc, REJECT_THRESHOLD)
		blocked_rejects += 1
		wblocked_rejects.add(worker)
		try : conn.reject_assignment(assignment, feedback=msg)
		except MTurkRequestError : pass

	elif status == 'PENDING' : 
		this_acc = this_correct / this_total
		
		if this_acc > PENDING_REJECT_THRESHOLD : #approve just this hit 
			print hilite('approving pending worker %s with acc %.03f out of %d hits'%(worker, acc, total), "green")
			new_rows.append([row[k] for k in headers] + ['PENDING-APP'])
			msg = PENDING_APP_MSG%(this_acc, total, acc, REJECT_THRESHOLD)
			approves += 1
			wapproves.add(worker)
			try : conn.approve_assignment(assignment, feedback=msg)
			except MTurkRequestError : pass

		else : # reject just this hit 
			print hilite('rejecting pending worker %s with acc %.03f out of %d hits'%(worker, acc, total), "red")
			new_rows.append([row[k] for k in headers] + ['PENDING-REJ'])
			msg = PENDING_REJ_MSG%(this_acc, total, acc, REJECT_THRESHOLD, ctrls[0][0], ctrls[0][1], ctrls[0][2])
			pending_rejects += 1
			wpending_rejects.add(worker)
			try : conn.reject_assignment(assignment, feedback=msg)
			except MTurkRequestError : pass

		#update their status if necessary
		if acc > APPROVE_THRESHOLD : statuses[worker] = 'TRUSTED'
		elif acc < REJECT_THRESHOLD : statuses[worker] = 'BLOCKED'

print 'Approved %d (%d)'%(approves, len(wapproves))
print 'Pending rejects %d (%d)'%(pending_rejects, len(wpending_rejects))
print 'Blocked %d (%d)'%(blocked_rejects, len(wblocked_rejects))

date = time.strftime("%m-%d-%YT%H:%M")
outfile = open('graded.%s.csv'%(date), 'a')
csvwriter = csv.writer(outfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
csvwriter.writerow(headers + ['status'])
for row in new_rows: csvwriter.writerow(row)
