#!/usr/bin/python
# -*- coding: utf-8 -*-

from nijmetrics import Application, Error, Database
import pg
import json
import urllib, urllib2
import re
import itertools
import cPickle as pickle
import zlib
import nijmetrics
import csv
import types


class Summary(Application):
	
	def __init__(self, db_url, exfilter=None, userfilter=None):
		Application.__init__(self)
		self.db_url = db_url
		self.exfilter = exfilter
		self.userfilter = userfilter

	def run(self):
		try:
				
			dbargs = (self.db_url)
			self.db = Database(self.db_url)
			self.db.connect()
			
			self.loadResults()
			self.analyzeResults()
			self.generateSheets()
			
			self.writeOutput()
			
		except (pg.InternalError,pg.ProgrammingError),err:
			print err
			return False
			
		except Error,err:
			print "ERROR:",err
			return False
		
		return True
	

	def loadResults(self):
		self.users = {}
		self.exids = {}

		assignments = self.collectAssignments(self.userfilter, self.exfilter)
		
		qids = {
			'analysis_step': str(self.db.query("SELECT id FROM resultstep WHERE short = 'ANALYSIS';")[0]['id']),
			'comparison_step': str(self.db.query("SELECT id FROM resultstep WHERE short = 'COMPARISON';")[0]['id']),
			'restype_last': str(self.db.query("SELECT id FROM resulttype WHERE short = 'CURRENT';")[0]['id']),
			'restype_submit': str(self.db.query("SELECT id FROM resulttype WHERE short = 'VALIDATED';")[0]['id']),
		}
		
		# Sheets in the final Excel workbook:
		# - survey questions
		# - users, with completed flag and survey results
		# - assignments, with status and all observations (2 parts: analysis and conclusion)
		print "Loading data ..."
		for a in assignments:
			
			if a['userid'] not in self.users:
				user = self.db.query("SELECT person.*,md5(person.username) AS code FROM person WHERE id = "+str(a['userid'])+";")[0]
				survey = self.db.query("SELECT * FROM survey_result WHERE person = "+str(a['userid'])+";")
				s = {}
				for q in survey:
					s[q['question']] = (q['answer'],q['comment'])
				self.users[a['userid']] = {
					'info': user,
					'survey': s,
					'exs': {}
				}
			
			exs = self.users[a['userid']]['exs']
			assert a['exid'] not in exs
			if not a['exid'] in self.exids:
				self.exids[a['exid']] = True
				
			steps = []
			if a['status'] >= 1:
				steps.append('analysis_step')
			if a['status'] >= 3:
				steps.append('comparison_step')
			
			for step in steps:
			
				results = self.db.query("SELECT * FROM resultv WHERE assignment = "+str(a['id'])+" AND stepid = "+qids[step]+" ORDER BY updatedon DESC;");
				if len(results) > 1:
					# Check the data integrity: the latest result must be the submitted one, and the previous
					# must be the 'last saved' because the 'autosave' feature was not active during the study.
					assert results[0]['type'] == 'Submitted' and results[1]['type'] == 'Last saved'
				result = results[0]

				obsv = self.db.query("SELECT type,value,name FROM observationv WHERE assignment = "+str(a['id'])+" AND resstep = "+str(result['stepid'])+" AND restype = "+str(result['typeid'])+";")
				obs = {}
				for o in obsv:
					o['value'] = json.loads(o['value'])
					obs[o['name']] = o
				
				a[step.split('_')[0]] = obs
				
			exs[a['exid']] = a
			
			#break
	

	def analyzeResults(self):
		print "Analyzing results ..."
		users_awardable = {}
		users_idle = {}
		users_nofinish = {}
		for user in self.users.itervalues():
			user['tags'] = set()
			statuses = {0:0,1:0,2:0,3:0,4:0}
			for ex in user['exs'].itervalues():
				statuses[ex['status']] += 1
			if statuses[4] == len(self.exids):
				users_awardable[user['info']['id']] = user['info']
				user['tags'].update('completed')
			elif statuses[0] == len(self.exids):
				users_idle[user['info']['id']] = user['info']
				user['tags'].update('idle')
			elif statuses[4] == 0:
				users_nofinish[user['info']['id']] = user['info']
				user['tags'].update('nofinish')
			user['num_complete'] = statuses[4]
			user['num_analyzed'] = statuses[3]+statuses[4]
			user['num_started'] = statuses[1]+statuses[3]+statuses[4]

		print " ",len(users_awardable),"users finished all trials"
		print " ",len(users_idle),"users did nothing"
		print " ",len(users_nofinish),"users did not complete a single exercise"
		

	
	def parseSurvey(self):
		survey = self.db.query("SELECT * FROM survey WHERE name = 'survey1';")[0]
		questions = {}
		state = 0
		hdrre = re.compile(r'^\[header\](.+)\[/header\]$', re.UNICODE)
		ftrre = re.compile(r'^\[footer\](.+)\[/footer\]$', re.UNICODE)
		for rawline in survey['definition'].splitlines():
			line = rawline.strip()
			if state == 0:
				if len(line) == 0:
					continue
				num = int(line)
				questions[num] = {'answers':[]}
				state = 1
			elif state == 1:
				questions[num]['type'] = line
				state = 2
			elif state == 2:
				questions[num]['title'] = line
				state = 3
			elif state == 3:
				if len(line) == 0:
					state = 0
				else:
					r = hdrre.findall(line)
					if len(r) == 0:
						r = ftrre.findall(line)
						if len(r) == 0:
							questions[num]['answers'].append(line)
		
		return questions
		
		
	def generateSheets(self):
		print "Generating data sheets ..."
		self.sheets = {}
		self.sheets['survey'] = self.getSurveySheet()
		self.sheets['users'] = self.getUsersSheet()
		self.sheets['exercises'] = self.getExercisesSheet()
		self.sheets['constants'] = self.getConstantsSheet()
		
		
	def getSurveySheet(self):
		
		self.survey = self.parseSurvey()
		
		lines = [[
			'ID',
			'Question',
			'Answers'
		]]
		
		for id,question in self.survey.iteritems():
			line = [
				id,
				question['title']]
			line.extend(question['answers'])
			lines.append(line)

		return lines
		
		
	def getUsersSheet(self):
	
		tagsdef = {
			'completed':'C',
			'idle':'I',
			'nofinish':'A',
		}
		
		header = [
			'ID',
			'Username',
			'Tags',
			'Num started',
			'Num analyzed',
			'Num complete',
		]
		for question in self.survey:
			header.append(question)
		header.append('Prize code')
		
		lines = [header]
		
		for uid,data in self.users.iteritems():
			line = [
				data['info']['id'],
				data['info']['username'],
			]
			tags = ''
			for tag,symbol in tagsdef.items():
				if tag in data['tags']:
					tags += symbol
			line.append(tags)
			
			line.extend([
				data['num_started'],
				data['num_analyzed'],
				data['num_complete'],
			])
			
			for question in self.survey:
				assert question in data['survey']
				if question == 8: # The only question that has freetext answer: put the 'comment' in the output file
					line.append(data['survey'][question][1])
				else: # otherwise just put the numerical value of the answer
					line.append(data['survey'][question][0])
				
			line.append(data['info']['code'])
			
			lines.append(line)
			
		return lines
		
	
	def getExercisesSheet(self):
		r = self.db.query("SELECT * FROM observationtype WHERE name <> 'conclusion' ORDER BY name;")
		obstypes = {'analysis':['conclusion'],'comparison':['conclusion']}
		for ot in r:
			prefix,name = ot['name'].split('-',1)
			step = 'analysis' if prefix == 'mark' else 'comparison'
			obstypes[step].append(name)
		
		header2 = ['Exercise ID','Exercise name','User ID','User name','Status']
		header1 = ['General info']+(len(header2)-1)*['']+['Analysis']+(len(obstypes['analysis'])-1)*['']+['Comparison']+(len(obstypes['comparison'])-1)*['']
		for step in ('analysis','comparison'):
			for ot in obstypes[step]:
				header2 += [ot]
				
		lines = []
		
		statusstr = {
			0: 'Started',
			1: 'In analysis',
			3: 'Analysis submitted',
			4: 'Done'
		}

		for uid,data in self.users.iteritems():
			for exid,ex in data['exs'].iteritems():
					
				line = [
					exid,
					ex['exname'],
					uid,
					data['info']['username'],
					statusstr[ex['status']],
				]
				
				for step in ('analysis','comparison'):
					if step in ex:
						#used = set()
						#missing = set()
						#present = set()
						for ot in obstypes[step]:
							key = ot if ot == 'conclusion' else ('mark' if step == 'analysis' else step) + '-' + ot
							if key in ex[step]:
								line += [str(ex[step][key]['value'])]
								#used.add(key)
							else:
								line += ['N/A']
								#missing.add(key)
						#for key in ex[step]:
						#	present.add(key)
						#print statusstr[ex['status']],present,used,missing
					else:
						line += len(obstypes[step])*['N/A']
						
				assert(len(line) == len(header1) == len(header2))

				lines.append(line)

		
		def sorter(a,b):
			# Sorting operator for lines, where columns 1 and 3 are exercise name and user name, respectively
			c = cmp(a[1],b[1])
			if c == 0:
				return cmp(a[3],b[3])
			else:
				return c
		
		lines.sort(cmp=sorter)
		
		return [header1,header2]+lines
		
	
	def getConstants(self):
		print "Reading constants from PiAnoS sources ..."
		f = open('../../js/pianos4_constants.js')
		lines = f.readlines()
		f.close()
		
		read = []
		reading = False
		for l in lines:
			line = l.strip()
			if 'BEGIN-JSON-PARSABLE-DATA' in line:
				reading = True
				continue
			elif 'END-JSON-PARSABLE-DATA' in line:
				reading = False
				continue
			
			if reading:
				read.append(line)
	
		js = ' '.join(['{']+read+['}'])
		
		# - stringify dictionary keys
		# - double quotes instead of single quotes
		# - no trailing commas
		sdk = re.compile(r'({|,)\s*([a-zA-Z_0-9]+)\s*:')
		dqisq = re.compile(r'(\'([^\']+)\')')
		tc = re.compile(r'(,)\s*(}|]|$)')
		
		js = sdk.sub(r'\1"\2":', js)
		js = dqisq.sub(r'"\2"', js)
		js = tc.sub(r'\2', js)
		
		data = json.loads(js)
		
		return data
		
	def getConstantsSheet(self):
		constants = self.getConstants()
		lines = []
		def writeIndented(indent, data):
			if type(data) is type({}):
				if len(data) == 2 and 'id' in data and 'label' in data and type(data['id']) is types.IntType and type(data['label']) is types.UnicodeType:
					#print indent*2*' '+str(data)
					lines.append(['']*indent+[data['id'],data['label']])
				else:
					for x,y in data.iteritems():
						#print indent*2*' '+x
						lines.append(['']*indent+[x])
						writeIndented(indent+1, y)
			elif type(data) is type([]):
				for x in data:
					writeIndented(indent, x)
			else:
				#print indent*2*' '+str(data)
				lines.append(['']*indent+[str(data)])
		writeIndented(0, constants['conclusions'])
		
		return lines
		
		
	def writeOutput(self):
		print "Writing output files ..."
		
		def stripnl(cell):
			if type(cell) is types.StringType:
				str = cell.decode('utf-8')
			elif type(cell) is types.UnicodeType:
				str = cell
			else:
				return cell
			return cell.replace(u'\n',' ').replace(u'\r',' ').replace(u'\t','    ')
			
		for sheet,lines in self.sheets.iteritems():
			filename = 'nij_summary_'+sheet+'.csv'
			with open(filename,'wb') as csvfile:
				csvfile.write("\xEF\xBB\xBF") # UTF-8 BOM
				writer = csv.writer(csvfile, delimiter=';', quoting=csv.QUOTE_ALL)
				for line in lines:
					ln = [stripnl(cell) for cell in line]
					writer.writerow(ln)
			print "  Wrote",filename
				
			