"""
Crowdsourcing for NLP Tutorial
NAACL 2015

Script for downloading the judgements for the HIT 
"""


import csv
import sys
import time
import codecs
from boto.mturk.connection import MTurkConnection

#Parameters used for posting the HIT
ACCESS_ID= 'YOUR_ACCESS_KEY' #found on Amazon Web Services
SECRET_KEY = 'YOUR_SECRET_KEY' #found on Amazon Web Services
HOST = 'mechanicalturk.sandbox.amazonaws.com'

conn = MTurkConnection(aws_access_key_id=ACCESS_ID, aws_secret_access_key=SECRET_KEY, host=HOST)

hits = conn.get_all_hits()

headers = set()
rows = []

for hit in hits:
    assignments = conn.get_assignments(hit.HITId, response_groups=['Minimal'])
    #Iterate through each assignment, and collect the desired metadata
    for assignment in assignments:
	results = {}
	h = conn.get_hit(assignment.HITId, response_groups=['HITDetail', 'HITQuestion'])[0]
	results['HITId'] = assignment.HITId
	results['HITGroupId'] = h.HITGroupId
	results['AssignmentId'] = assignment.AssignmentId
	results['WorkerId'] = assignment.WorkerId
	results['AcceptTime'] = assignment.AcceptTime
	results['SubmitTime'] = assignment.SubmitTime
	#Iterate through each question in that assignment and collect the worker's responses
	for i,question_form_answer in enumerate(assignment.answers[0]):
		qid = question_form_answer.qid
		id, pos, w, sent = qid.split(' ||| ')
		answer = '|'.join([p.encode('utf') for p in question_form_answer.fields])
		results['Input.id%d'%i] = id 
		results['Input.pos%d'%i] = pos.encode('utf-8')
		results['Input.w%d'%i] = w.encode('utf-8')
		results['Input.sent%d'%i] = sent.encode('utf-8')
		results['Answer.paraphrases%d'%i] = answer
	headers.update(set(results.keys()))
	rows.append(results)

#Dump all of the data to a csv file
headers = list(headers)
date = time.strftime("%d-%m-%YT%H:%M")
outfile = csv.DictWriter(codecs.open('output.%s.csv'%date, mode='w'), delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL, fieldnames=headers)
outfile.writeheader()
for row in rows : 
	outfile.writerow(row)

