#!/usr/bin/python
# -*- coding: utf-8 -*-

from nijmetrics import Application, sanitizefn
import cPickle as pickle
import sys
import itertools
import zlib
import copy
import types
import Ellipses
import os

'''
*************************************************************************************
NOTE: The metrics are also described in a more comprehensive manner in the word
document 'data_extraction_log.doc'. There are thus 3 different things, created
in this order:
- what the code does (use the source, Luke)
- what is described here (that should reflect the code, hopefully)
- what is described in the word document (that should reflect both the code and this description)
Care has been taken to keep everything in sync, but in case of doubt, the only reference
that can be trusted is the code.
*************************************************************************************

This script calculate the metrics, named M1<x> where x is a,b,c,d,e, M4, M2<a,b>, M3 and M5<a,b>
The description of each is noted below. 
-> comment: denotes a modification to the specification that this script 
   implements (additions only, so they do not hurt)

M4: # of minutiae of each type (ridge endings, bifurcations, type unknowns, position unknowns)

M1a: # of minutiae placed outside of all quality zones
M1b: (#re+#bif)/(#total_in_zone) for the high quality (q=1) zone and low quality (q=3) zones
	-> added orange zone as well
M1c: # of minutiae in green area / surface of green area
	-> calculated for all 3 areas
M1d: (#re+#bif)/#total
M1e: surface of green area / total work surface (=total surface with quality areas, != total surface)
	-> calculated for all 3 areas
	
M2a: (# of green pixels) / (surface of mask)
M2b: (a.(# of green pixels) + b.(# of orange pixels)) / (surface of mask) with a+b=1 and a>b (e.g. a=0.7, b=0.3)
    -> M2a and M2b are calculated for all values of mask thresholds
	
M3: user's quality maps vs the consensus (more details to come)

M5: user's minutiae vs the consensus (more details to come)
M5a: User minutiae maps vs total minutiae maps
M5b: precision/accuracy (?) = sum of densities (weighted by minutia type) for the user's minutiae
M6: # of paired minutiae
	-> only present during comparison
'''

class MxMetrics(Application):

	def __init__(self, filename, dumpdir=None, cache=None, with_metrics = []):
		Application.__init__(self, dumpdir, cache)
		self.filename = filename
		self.metric_desc = {
			'M1a': '# of minutiae placed outside of all quality zones',
			'M1b': ('', 'quality'),
			'M1c': ('', 'quality'),
			'M1d': '',
			'M1e': ('', 'quality'),
			'M2a': ('', 'threshold'),
			'M2b': ('', 'threshold'),
			'M3': ('', 'threshold'),
			'M4': ('', 'mintype'),
			'M5a': ('', 'ellipses'),
			'M5b': ('', 'ellipses'),
			'M6': '',
		}
		if with_metrics:
			all = set(self.metric_desc.keys())
			keep = set(with_metrics)
			do = keep & all
			if not do:
				print "ERROR: no metrics to calculate (asked for '%s', where available are '%s')" % (','.join(with_metrics), ','.join(all))
				sys.exit(1)
			self.metric_desc = dict(((x,self.metric_desc[x]) for x in do))

		self.metric_names = sorted(self.metric_desc.keys())
		self.situations = [
			'analysis_mark',
			'comparison_mark',
			'comparison_print',
		]
		self.standardTable = {
			1: {'id': 1, 'color': '#2ca02c', 'label': 'High'},
			2: {'id': 2, 'color': '#ff7f0e', 'label': 'Medium'},
			3: {'id': 3, 'color': '#d62728', 'label': 'Low'},
		}
		
		# User value on X axis
		# Consensus value on Y axis
		# -> indexed by (consensus, user) tuples
		self.m3matrix = {
			(1,1): 0, (1,2): 1, (1,3): 2, (1,0): 3, 
			(2,1): 1, (2,2): 0, (2,3): 1, (2,0): 2, 
			(3,1): 2, (3,2): 1, (3,3): 0, (3,0): 1, 
			(0,1): 3, (0,2): 2, (0,3): 1, (0,0): 0, 
		}
		
		self.mergedMapColorScale = {
			1:(0,255,0,255), # high (q=1)      -> green
			2:(0,0,255,255), # medium (q=2)    -> blue (!)
			3:(255,0,0,255), # low (q=3)       -> red
			0:(0,0,0,255),   # nothing         -> black
			-1:(128,128,128,255), # masked out -> gray
			-2:(255,0,255,255), # error        -> shitty color (happens when a single pixel is covered by multiple quality level areas)
		}

		self.all_minutia_types = self.getMinutiaTypes(noext=True)
		self.all_qualities = [1,2,3]

		self.mmap_setups = {
			'20_10': {'bifurcation':(20,10), 'ridge_ending':(20,10), 'unknown':(40,20), 'unknown2':(50,40)}
		}
		self.all_mmap_setups = self.mmap_setups.keys()
		self.bivnorms = {}
	
		# User value on X axis
		# Consensus value on Y axis
		# -> indexed by (consensus, user) tuples
		self.m5matrix = {
			('ridge_ending', 'ridge_ending'): 2.0 , ('ridge_ending', 'bifurcation' ): 1.0 , ('ridge_ending', 'unknown'     ): 0.5, ('ridge_ending', 'unknown2'    ): 0.25, 
			('bifurcation' , 'ridge_ending'): 1.0 , ('bifurcation' , 'bifurcation' ): 2.0 , ('bifurcation' , 'unknown'     ): 0.5, ('bifurcation' , 'unknown2'    ): 0.25, 
			('unknown'     , 'ridge_ending'): 0.5 , ('unknown'     , 'bifurcation' ): 0.5 , ('unknown'     , 'unknown'     ): 2.0, ('unknown'     , 'unknown2'    ): 1.0, 
			('unknown2'    , 'ridge_ending'): 0.25, ('unknown2'    , 'bifurcation' ): 0.25, ('unknown2'    , 'unknown'     ): 1.0, ('unknown2'    , 'unknown2'    ): 2.0,
		}
		
	def prepare_output_format(self, thresholds):
		self.thresholds = sorted(thresholds)
		n = {}
		for mn,m in self.metric_desc.items():
			if type(m) is type(''):
				m = (m,'single')
			if m[1] == 'single':
				m = [x for x in m] + [['']]
			elif m[1] == 'quality':
				m = [x for x in m] + [self.all_qualities]
			elif m[1] == 'threshold':
				m = [x for x in m] + [self.thresholds]
			elif m[1] == 'mintype':
				m = [x for x in m] + [self.all_minutia_types]
			elif m[1] == 'ellipses':
				m = [x for x in m] + [self.all_mmap_setups]
			n[mn] = m
		self.metric_desc = n

		self.header1 = ['General info','','','','']
		self.header2 = ['Exercise','User','Username','Status','Situation']
		for mn in self.metric_names:
			self.header1 += [mn]
			m = self.metric_desc[mn]
			assert m[2]
			first = True
			for extcol in m[2]:
				if not first:
					self.header1 += ['']
				self.header2 += [extcol]
				
				first = False
				
		# Add denormalized values at the end of the line. These are values that really belong
		# another entity (than the result line) but are put here for convenience for data
		# analysis tools that will process the output file. (the row header is actually already
		# denormalized, since it repeats trial name, username, id, and exercise status; but this
		# suffix is even more denormalized as it can be used for pretty much anything)
		# Elements that are recorded here are:
		# - overall surface of the **consensus** mask (for each threshold), not the user's
		self.header1 += ['Surface of the mask'] + [''] * len(self.thresholds[1:])
		self.header2 += [x for x in self.thresholds]
		
		assert len(self.header1) == len(self.header2)
		
			
	def run(self):
	
		with open(self.filename,'r') as f:
			data = pickle.load(f)
		
			
		
		for exid in data:
			outname = sanitizefn('%s'%(data[exid]['exercise']['name']))+'.csv'
		
			self.prepare_output_format(data[exid]['consensus']['masks'].keys())
			
			with open(outname, 'w') as f:
				# Write the column headers
				f.write(';'.join(['"'+str(x)+'"' if type(x) is types.StringType else str(x) for x in self.header1])+'\n')
				f.write(';'.join(['"'+str(x)+'"' if type(x) is types.StringType else str(x) for x in self.header2])+'\n')
				f.flush()
				
				#print self.metric_names
				self.do_run(exid, data, f)
			
		return True
	
	def readCached(self, name):
		assert self.cache
		try:
			with file(os.path.join(self.cache, name+'.pickle')) as f:
				return pickle.load(f)
		except (IOError,EOFError):
			return None

	def do_run(self, exid, data, outputf):
	
		
		print "Exercise %s (ID=%d)" % (data[exid]['exercise']['name'],exid)
		
		image = data[exid]['mark']
		#print "  Image: " % ()
		
		ex_diagnostics = {
			'warn_consensus_sum_differs_from_numcontributors': 0,
		}
		
		
		need_qconsensus = len(set(self.metric_desc.keys()) - set(['M5a','M5b']))
		need_mconsensus = 'M5a' in self.metric_desc or 'M5b' in self.metric_desc
		
		if need_qconsensus:
			
			need_reload = True

			if self.cache:
				print "  Loading quality consensus data from cache ..."
				qmap_masks = self.readCached('qmap_masks')
				qmaps_orig = self.readCached('qmaps_orig')
				qmaps = self.readCached('qmaps')
				need_reload = not (qmap_masks and qmaps_orig and qmaps)
			
			if need_reload:
				print "  Gathering quality consensus data ..."
				
				qmap_masks = {}
				qmaps_orig = {}
				qmaps = {}
							
							
				# Read the quality maps
				print "    Reading quality maps",
				for quality in self.all_qualities:
					print quality,
					if quality in data[exid]['consensus']['qmaps']:
						qmap = data[exid]['consensus']['qmaps'][quality]
						if data[exid]['compressed']:
							qmap = eval(zlib.decompress(qmap))
					else:
						qmap = [0]*(image['width']*image['height'])
					qmaps_orig[quality] = qmap
				print "Done"
				
				# Merge them into a single buffer, where each pixel contains 4 elements:
				# 0 the number of users that did *not* cover the pixel
				# 1,2 and 3: the number of users that covered the pixel in the said quality level
				# Initially, all indices 0 contain the number of contributors. This is subtracted by
				# the number of users who covered in green, orange and red. What remains is the number
				# of users that did not put any quality zone here.
				print "    Merging quality maps",
				qmaps_all = [[copy.copy(data[exid]['consensus']['numcontributors']),0,0,0] for x in range(image['width']*image['height'])]
				for quality in self.all_qualities:
					print quality,
					qmap = qmaps_orig[quality]
					for i in range(len(qmaps_all)):
						num = qmap[i]
						qmaps_all[i][quality] = num
						qmaps_all[i][0] -= num
				for i in range(len(qmaps_all)):
					if qmaps_all[i][0] < 0:
						self.warn(ex_diagnostics, 'warn_consensus_sum_differs_from_numcontributors', "WARNING: at index %d, values are %s (numc=%d)" % (i, str(qmaps_all[i]), data[exid]['consensus']['numcontributors']))
				#self.saveQualityMap(qmaps_all, (image['width'],image['height']), 'consensus_all.png', self.mergedMapColorScale)
				print "Done"
						
				# Read the masks and apply them to the merged qmaps
				print "    Reading and applying masks",
				for threshold,mask in data[exid]['consensus']['masks'].iteritems():
					print threshold,
					if data[exid]['compressed']:
						mask = eval(zlib.decompress(mask))
					qmap_masks[threshold] = {'data':mask, 'surface':sum(mask)}

					qm = [0]*(image['width']*image['height'])
					assert len(mask) == len(qm)
					for i in range(len(qm)):
						qm[i] = qmaps_all[i] if mask[i] else -1
					qmaps[threshold] = qm
					#self.saveQualityMap(qm, (image['width'],image['height']), 'consensus_all_masked_%d.png'%(int(threshold*100)), self.mergedMapColorScale)
				print "Done"
				
				if self.cache:
					print "    Saving to cache"
					for name,obj in [(name,eval(name)) for name in ['qmap_masks', 'qmaps_orig', 'qmaps']]:
						with file(os.path.join(self.cache, name+'.pickle'), 'w') as f:
							pickle.dump(obj, f)
							
							
		if need_mconsensus:
			need_reload = True

			if self.cache:
				print "  Loading minutiae consensus data from cache ..."
				mmaps = self.readCached('mmaps')
				need_reload = not (mmaps)
			
			if need_reload:
				print "  Gathering minutiae consensus data ..."
				mtypes = ['bifurcation','ridge_ending','unknown','unknown2']
				
				mmaps = {}
				
				for setup in self.mmap_setups:
					print "    Setup '%s'" % setup
					mmaps[setup] = {}
					
					for situation in self.situations:
						
						if situation == 'comparison_mark':
							# Discard the 'comparison_mark' results - this allow faster runs and since we need
							# lots of investigation before using these results, I leave them out for the time being.
							continue
							
						mmaps[setup][situation] = {'all':None, 'by_type':{}}
					
						phase,imgtype = situation.split('_')

						# Get a reference to the corresponding image
						assert imgtype in data[exid]
						image = data[exid][imgtype]
						mapdims = (image['width'],image['height'])
						
						# Initialize an empty surface for this situation and all minutiae types
						mmaps[setup][situation]['all'] = Ellipses.Surface(mapdims)
						mmaps[setup][situation]['by_type'] = dict([(mtype,Ellipses.Surface(mapdims)) for mtype in mtypes])

						print "      %s " % situation,
						
						# Run through all users
						for userid in data[exid]['users']:	
							userdata = data[exid]['users'][userid]
							if phase in userdata and imgtype in userdata[phase]:
								annotations = userdata[phase][imgtype]
								if 'minutiae' in annotations and annotations['minutiae']:
									# They have minutiae for this image in this phase
									for mid,m in annotations['minutiae'].iteritems():
										# Skip minutiae that do not participate in the consensus we're building
										if m['type'] not in self.mmap_setups[setup]:
											continue
										bn = self.getBivNorm(setup, m)
										mmaps[setup][situation]['by_type'][m['type']].draw(bn, m['pos'])
							sys.stdout.write(".")

						# Create the collated, normalized mmap using all minutiae types and dividing by the
						# number of contributors
						print "collating ...",
						allbuf = mmaps[setup][situation]['all'].buffer
						for mtype in mmaps[setup][situation]['by_type']:
							mtypebuf = mmaps[setup][situation]['by_type'][mtype].buffer
							allbuf[:] = ((x+y) for x,y in itertools.izip(allbuf,mtypebuf))
						numcontributors = len(data[exid]['users'])
						map(lambda x: x/numcontributors, allbuf)
								
						# Save the surfaces if needed
						if self.dumpdir:
							print "saving images ...",
							surf = mmaps[setup][situation]['all']
							filename = sanitizefn(data[exid]['exercise']['name'])+('_mmap_%s_%s' % (setup,situation))+'.png'
							self.debugSaveSurface(surf, filename)
							for mtype in mmaps[setup][situation]['by_type']:
								surf = mmaps[setup][situation]['by_type'][mtype]
								filename = sanitizefn(data[exid]['exercise']['name'])+('_mmap_%s_%s_%s' % (setup,situation,mtype))+'.png'
								self.debugSaveSurface(surf, filename)
							
						print " Done"
				
				if self.cache:
					print "    Saving to cache"
					for name,obj in [(name,eval(name)) for name in ['mmaps']]:
						with file(os.path.join(self.cache, name+'.pickle'), 'w') as f:
							pickle.dump(obj, f)
			
		count = 0
		for userid in data[exid]['users']:
			print "  User %s (ID=%d)  (%.2f%%)" % (userdata['username'],userid,float(count)/float(len(data[exid]['users']))*100.0)
			count += 1
			
			userdata = data[exid]['users'][userid]
			
			if need_qconsensus:
				maps = {}
				print "    Reading quality maps ...",
				for quality in userdata['qmaps']:
					print quality,
					qmap = userdata['qmaps'][quality]
					if data[exid]['compressed']:
						qmap = eval(zlib.decompress(qmap))
					maps[quality] = qmap
				print "OK"
			
				print "    Masking quality maps ...",
				maps_masked = {}
				for threshold in qmap_masks:
					sys.stdout.write(str(threshold))
					maps_masked[threshold] = {}
					for quality in maps:
						sys.stdout.write('.')
						full = maps[quality]
						mask = qmap_masks[threshold]['data']
						assert len(full) == len(mask)
						res = list((full[i]*mask[i] for i in range(len(mask))))
						maps_masked[threshold][quality] = res
				print " OK"
			
			print "    Calculating metrics"
			
				
			# Calculate all metrics for the 3 situations (if posssible): during analysis (mark only)
			# and during comparison (mark and print)
			for situation in self.situations:
				phase,imgtype = situation.split('_')

				# Get a reference to the corresponding image
				assert imgtype in data[exid]
				image = data[exid][imgtype]
				mapdims = (image['width'],image['height'])
				
				# Prepare the storage for the metric values and troubleshooting indicators
				metrics = {}
				diagnostics = {
					'warn_minutiae_in_multiple_areas': 0,
					'warn_union_of_surfaces_differ_from_sum_by': 0,
					'warn_pixel_covered_multiple_times': 0,
				}
				
				# The calculation only happens if there is something to work on (i.e. if the user
				# did submit data for that situation=phase/image)
				if phase in userdata and imgtype in userdata[phase]:
					
					print "      In",phase,"/",imgtype,"...",
					annotations = userdata[phase][imgtype]
					
					# Strip off centers and differences - as these are not actual minutiae in the sense
					# of our analysis right now
					minutiae = {}
					try:
						for id,m in annotations['minutiae'].iteritems():
							if m['type'] not in ('center','difference'):
								minutiae[id] = m
					except AttributeError,err:
						print "WARNING:",err
						print "NOTE: the offending thing that should be a dict of minutiae and isn't is:"
						print repr(annotations['minutiae'])
					
					if 'M4' in self.metric_desc:
						# M4: # of minutiae of each type (ridge endings, bifurcations, type unknowns, position unknowns)
						print "M4",
						M4 = dict(zip(self.all_minutia_types, [0]*len(self.all_minutia_types)))
						for id,m in minutiae.iteritems():
							M4[m['type']] += 1
						metrics['M4'] = M4
					
					if 'M6' in self.metric_desc and phase == 'comparison':
						# M6: # of paired minutiae
						print "M6",
						M6 = 0
						for id, m in minutiae.iteritems():
							if 'pairing' in m:
								M6 += 1
						metrics['M6'] = M6
					# M1x metrics are calculated on the mark only (because they depend on the quality zones, which
					# are only processed on the mark even though the user could have annotated the quality on the print)
					if imgtype != 'print':
						
						if 'M1a' in self.metric_desc:
							# M1a: # of minutiae placed outside of all quality zones. 
							# Keep a record of the zone the minutia belongs to by the way
							print "M1a",
							mzones = {}
							M1a = 0
							for id,m in minutiae.iteritems():
								inside = 0
								for quality,qmap in maps.iteritems():
									if self.isPointInMap(m['pos'], qmap, mapdims):
										inside += 1
										mzones[id] = quality
								if inside == 0:
									M1a += 1
								elif inside != 1:
									self.warn(diagnostics, 'warn_minutiae_in_multiple_areas', "WARNING: in %s minutia %s is inside %d maps" % (situation,id,inside))
							for id,m in minutiae.iteritems():
								if id not in mzones:
									mzones[id] = -1
							metrics['M1a'] = M1a
								
						if 'M1b' in self.metric_desc:
							# M1b: (#re+#bif)/(#total_in_zone) for the high quality (q=1) zone and low quality (q=3) zones
							print "M1b",
							M1b = {}
							for quality in self.all_qualities:
								numre = 0
								numbif = 0
								numtotal = 0
								for id,m in minutiae.iteritems():
									if mzones[id] == quality:
										numtotal += 1
										if m['type'] == 'bifurcation':
											numbif += 1
										elif m['type'] == 'ridge_ending':
											numre += 1
								if numtotal:
									M1b[quality] = float(numre+numbif) / float(numtotal)
								else:
									M1b[quality] = None
								metrics['M1b'] = M1b
	
						if 'M1c' in self.metric_desc:
							# M1c: # of minutiae in green area / surface of green area
							print "M1c",
							m1_surfaces = {}
							M1c = {}
							for quality in self.all_qualities:
								if quality in maps:
									m1_surfaces[quality] = self.getMapSurface(maps[quality])
									if m1_surfaces[quality] > 0:
										nummin = 0
										for id,m in minutiae.iteritems():
											if mzones[id] == quality:
												nummin += 1
										M1c[quality] = float(nummin) / float(m1_surfaces[quality])
									else:
										m1_surfaces[quality] = None
										M1c[quality] = None
								else:
									m1_surfaces[quality] = None
									M1c[quality] = None
							metrics['M1c'] = M1c
								
								
						if 'M1d' in self.metric_desc:
							# M1d: (#re+#bif)/#total
							print "M1d",
							numtotalmin = sum(M4.values())
							if numtotalmin:
								M1d = float(M4['ridge_ending']+M4['bifurcation'])/float(numtotalmin)
							else:
								M1d = None
							metrics['M1d'] = M1d
						
						
						if 'M1e' in self.metric_desc:
							# M1e: surface of green area / total work surface (=total surface with quality areas, != total surface)
							print "M1e",
							# if total area is considered (as before fix of 2012-10-08): totalsurf=float(image['width']*image['height'])
							# now (after fix): calculate union of the 3 quality maps and get its surface
							# Note that it equals the sum of m1_surfaces[x] if and only if all quality layers have no common pixels - which
							# should be the case at this point (i.e. after extraction, and provided it did its job correctly)
							# So ... the sum of surfaces and the surface of the union should in fact be the same. Otherwise there
							# is something to investigate.
							worksurf = [0]*(image['width']*image['height'])
							for quality in self.all_qualities:
								if quality in maps:
									qmap = maps[quality]
									assert len(qmap) == len(worksurf)
									for i in range(len(qmap)):
										worksurf[i] |= qmap[i]
								assert sum([0 if x in (0,1) else 1 for x in worksurf]) == 0 # There should be only 0s or 1s in the union
							totalsurf = self.getMapSurface(worksurf)
							thesum = sum([m1_surfaces[x] if m1_surfaces[x] is not None else 0.0 for x in self.all_qualities])
							if totalsurf != thesum:
								print "WARNING: surface of union of quality maps (%d) differs from sum of surfaces (%d). It means %d pixels are covered by more than one quality area." % (totalsurf, thesum, thesum-totalsurf) # If the difference is negative you live in a parallel universe
								diagnostics['warn_union_of_surfaces_differ_from_sum_by'] = thesum-totalsurf
							M1e = {}
							for quality in self.all_qualities:
								if totalsurf > 0.0 and quality in m1_surfaces and m1_surfaces[quality] is not None:
									M1e[quality] = float(m1_surfaces[quality]) / totalsurf
								else:
									M1e[quality] = None
							metrics['M1e'] = M1e
						
						
						if 'M2a' in self.metric_desc:
							# M2a: (# of green pixels) / (surface of mask)
							print "M2a",
							M2a = {}
							for threshold,mask in qmap_masks.iteritems():
								green_surf = self.getMapSurface(maps_masked[threshold][1]) if 1 in maps_masked[threshold] else 0.0
								M2a[threshold] = green_surf / float(mask['surface'])
							metrics['M2a'] = M2a
								
						if 'M2b' in self.metric_desc:
							# M2b: (a.(# of green pixels) + b.(# of orange pixels)) / (surface of mask) with a+b=1 and a>b (e.g. a=0.7, b=0.3)
							print "M2b",
							M2b = {}
							a,b = 1.0, 0.5
							for threshold,mask in qmap_masks.iteritems():
								green_surf = self.getMapSurface(maps_masked[threshold][1]) if 1 in maps_masked[threshold] else 0.0
								orange_surf = self.getMapSurface(maps_masked[threshold][2]) if 2 in maps_masked[threshold] else 0.0
								M2b[threshold] = (a*green_surf + b*orange_surf) / float(mask['surface'])
							metrics['M2b'] = M2b
						
					print "Done"
					
					if 'M5a' in self.metric_desc:
						print "        Minutiae maps (M5a)",
						# M5: user's minutiae maps vs the consensus
						metrics['M5a'] = {}
						for setup in self.mmap_setups:
							print setup,
							
							if setup not in mmaps or situation not in mmaps[setup]:
								metrics['M5a'][setup] = None
								continue
								
							# Draw the user's minutia maps, first by minutia type
							layers = {}
							for mtype in mmaps[setup][situation]['by_type']:

								layer = Ellipses.Surface(mapdims)
								sys.stdout.write('.')
							
								if 'minutiae' in annotations and annotations['minutiae']:
									for mid,m in annotations['minutiae'].iteritems():
										if m['type'] != mtype:
											continue
										bn = self.getBivNorm(setup, m)
										layer.draw(bn, m['pos'])
								
								layers[mtype] = layer
								
								filename = sanitizefn(data[exid]['exercise']['name'])+('_user_%d_mmap_%s_%s_%s' % (userid,setup,situation,mtype))+'.png'
								self.debugSaveSurface(layer, filename)


							# and then for all types
							all = Ellipses.Surface(mapdims)
							allbuf = all.buffer
							for mtype in layers:
								mtypebuf = layers[mtype].buffer
								allbuf[:] = ((x+y) for x,y in itertools.izip(allbuf,mtypebuf))
							
							filename = sanitizefn(data[exid]['exercise']['name'])+('_user_%d_mmap_%s_%s' % (userid,setup,situation))+'.png'
							self.debugSaveSurface(all, filename)
								
							# Calculate the "simple" score - the one using all layers at once
							consensus = mmaps[setup][situation]['all']
							assurance, frilosity = 0.0, 0.0
							for x,y in itertools.izip(allbuf, consensus.buffer):
								res = x-y
								if res >= 0.0: assurance += res # User marked something stronger than the consensus
								if res < 0.0: frilosity += res # other way around
							M5a = abs(assurance) + abs(frilosity)
							
							# Calculate the convoluted score using the m5matrix
							# TODO!!!
									
							metrics['M5a'][setup] = M5a
							
						print " Done"
				
					if 'M5b' in self.metric_desc:
						print "        Precision (M5b)",
						# M5b: user's minutiae vs the consensus
						metrics['M5b'] = {}
						for setup in self.mmap_setups:
							print setup,
							
							if setup not in mmaps or situation not in mmaps[setup]:
								metrics['M5b'][setup] = None
								continue
				
							M5b = 0.0
							if 'minutiae' in annotations and annotations['minutiae']:
								for mid,m in annotations['minutiae'].iteritems():
									for mtype,surface in mmaps[setup][situation]['by_type'].iteritems():
										density = surface.sample(m['pos'])
										index = (m['type'], mtype)
										if index not in self.m5matrix:
											continue
										coeff = self.m5matrix[index]
										val = density * coeff
										#print "at",m['pos'],"density=",density,",coeff=",coeff,",val=",val
										M5b += val
							
							metrics['M5b'][setup] = M5b
							
						print " Done"
				
				if situation == 'analysis_mark' and 'M3' in self.metric_desc:
					print "      QMap vs. consensus",
					# M3: user's quality maps vs the consensus
					M3 = {}
					# Get the user's full map (union of all three layers)
					fullmap = [0]*(image['width']*image['height'])
					for quality in self.all_qualities:
						if quality in maps:
							qmap = maps[quality]
							#self.saveQualityMap(qmap, (image['width'],image['height']), 'qmap_q%d.png'%quality)
							assert len(qmap) == len(fullmap)
							for i in range(len(qmap)):
								if fullmap[i] == 0:
									fullmap[i] = quality if qmap[i] else 0
								elif qmap[i] != 0:
									fullmap[i] = -2
									self.warn(diagnostics, 'warn_pixel_covered_multiple_times', "WARNING: pixel %d (x=%d,y=%d) was assigned quality %d, but %d applies as well - keeping old" % (i, i%image['width'], i/image['width'], fullmap[i], quality))
							self.debugSaveQualityMap(fullmap, (image['width'],image['height']), sanitizefn(data[exid]['exercise']['name'])+'_user_%d_qmap_all'%(userid)+'.png', self.mergedMapColorScale)
					
					for threshold,maskdef in qmap_masks.iteritems():
						# Calculate the score m3 = f(x,y) ^ g(x,y)
						# f is the user's qmap function
						# g is the consensus' qmap function
						# ^ is the convolution score using the matrix self.m3matrix
						
						print threshold,
						# The score must be calculated only in the area that is not masked out. The consensus qmaps are already
						# masked, but not the user's. However, we do not need to apply the mask to the latter - having either one
						# or the other operand masked out is the same. (score=-1 if masked(left) or masked(right) else f^g)
						fm = fullmap
						if self.dumpdir:
							# Still, if the debug output image is needed, we apply the mask to the user's qmap :)
							fm = [0]*(image['width']*image['height'])
							mask = maskdef['data']
							assert len(mask) == len(fm)
							for i in range(len(fm)):
								fm[i] = fullmap[i] if mask[i] else -1
							self.debugSaveQualityMap(fm, (image['width'],image['height']), sanitizefn(data[exid]['exercise']['name'])+'_user_%d_qmap_all_masked_%d'%(userid,int(threshold*100))+'.png',self.mergedMapColorScale)
					
						# Calculate f^g=sum(fm,
						fnumc = float(data[exid]['consensus']['numcontributors'])
						badvalues = (-1,-2)
						goodvalues = (0,1,2,3)
						qm = qmaps[threshold]
						assert len(qm) == len(fm)
						m3 = 0
						for i in range(len(qm)):
							cons = qm[i]
							if cons in badvalues or fm[i] in badvalues:
								continue
							#print "px",i
							for qcoef in goodvalues:
								coef = self.m3matrix[(qcoef, fm[i])]
								prop = min(max(cons[qcoef] / fnumc, 0.0), 1.0)
								#print qcoef,fm[i],coef,prop,coef*prop,cons
								m3 += coef * prop
						
						M3[threshold] = m3
					
					metrics['M3'] = M3
					
					print "Done"
			
				
				# Prepare the CSV output row
				line = [data[exid]['exercise']['name'], userid, userdata['username'], userdata['status'], situation]
				for mname in self.metric_names:
					m = self.metric_desc[mname]
					# Fetch the python value of the metric
					if mname in metrics:
						mv = metrics[mname]
					else:
						mv = None
					#print '%s=%s'%(mname,str(mv))
					# Write this in the csv, using multiple columns if required
					if m[1] == 'single':
						line.append(mv)
					elif mv is None:
						for item in m[2]:
							line.append(None)
					else:
						#print m[2],mv
						for item in m[2]:
							assert item in mv
							line.append(mv[item])

				# Append the denormalized (repeated) values
				line += [qmap_masks[x]['surface'] for x in self.thresholds]
				
				# Write the line to the file
				assert len(line) == len(self.header2)
				outputf.write(';'.join(map(MxMetrics.csv_output_var_formatter, line))+'\n')
				outputf.flush()
	
	@staticmethod
	def csv_output_var_formatter(x):
		if x is None:
			return 'NA' # change this to have 'NoneType' translate to the value that best suits the next data analysis tool (R in our case)
		elif type(x) is types.StringType:
			return '"'+x+'"'
		else:
			return str(x)
	
	# Returns True if the provided point=(x,y) is in the specified map whose dimensions are mapdims=(w,h)
	# The coordinates of the point are rounded to the closest integer
	def isPointInMap(self, point, map, mapdims):
		ipoint = [int(round(x)) for x in point]
		if ipoint[0] < 0 or ipoint[0] >= mapdims[0] or ipoint[1] < 0 or ipoint[1] >= mapdims[1]:
			return False
		index = mapdims[0] * ipoint[1] + ipoint[0]
		# if index >= len(map):
			# print "assert index < len(map) is bound to fail, dumping objects"
			# print "point",point
			# print "mapdims",mapdims
			# print "index",index
			# print "length of map",len(map)
		assert index < len(map)
		assert map[index] == 1 or map[index] == 0
		return map[index] != 0
		
	# Get the surface, in number of pixels, of the provided map
	def getMapSurface(self, map):
		return float(sum(map))
		
	# Get the (cached) Ellipses.BivNorm object corresponding to this minutia
	def getBivNorm(self, setup, minutia):
		ref_angle = int(minutia['angle']) % 360
		try:
			# Try to hit the cache (cached by setup AND minutia type AND rounded angle)
			ret = self.bivnorms[setup][minutia['type']][ref_angle]
			#print "[CACHE HIT ] setup=%s, type=%s, angle=%s, ref_angle=%s" % (setup, minutia['type'], minutia['angle'], ref_angle)
			return ret
		except KeyError:
			# Cache miss, build the BivNorm object
			if not setup in self.bivnorms:
				self.bivnorms[setup] = {}
			bivnorms = self.bivnorms[setup]
			if not minutia['type'] in bivnorms:
				bivnorms[minutia['type']] = {}
			bivnorms = bivnorms[minutia['type']]
			#print "[CACHE MISS] setup=%s, type=%s, angle=%s, ref_angle=%s" % (setup, minutia['type'], minutia['angle'], ref_angle)
			assert setup in self.mmap_setups and minutia['type'] in self.mmap_setups[setup]
			ratio = self.mmap_setups[setup][minutia['type']]
			# The actual angle passed to the BivNorm constructor is the opposite of the PiAnoS angle
			# since BivNorm is CCW and PiAnoS is CW (but we still keep the index on the PiAnoS value)
			bn = Ellipses.BivNorm(ratio[0], ratio[1], -ref_angle) 
			bn.render()
			bivnorms[ref_angle] = bn
			return bn
	
	def debugSaveQualityMap(self, *args, **kwargs):
		if self.dumpdir:
			if 'filename' in kwargs:
				newname = os.path.join(self.dumpdir, kwargs['filename'])
				kwargs['filename'] = newname
			else:
				newname = os.path.join(self.dumpdir, args[2])
				args = list(args)
				args[2] = newname
			#print "Saving QMAP",os.path.join(self.dumpdir, newname)
			self.saveQualityMap(*args, **kwargs)
			return True
		else:
			return False

	def debugSaveSurface(self, surface, filename):
		if self.dumpdir:
			try:
				#print "Saving SURF",os.path.join(self.dumpdir, filename)
				surface.saveImage(os.path.join(self.dumpdir, filename))
				return True
			except ZeroDivisionError,err:
				# The surface is empty (max value is 0, so we can't normalize, thus can't save using surf.saveImage)
				#print err
				pass
		return False

	def warn(self, diags, key, message):
		diags[key] += 1
		if diags[key] <= 100:
			if diags[key] == 100:
				print "NOTE: skipping warnings of the same type following next one"
			print message
	
	
	
	
class MinutiaeExport(Application):

	def __init__(self, filename, dumpdir=None, cache=None):
		Application.__init__(self, dumpdir, cache)
		self.filename = filename
		
	def run(self):
		print "Loading",self.filename,"...",
		with open(self.filename,'r') as f:
			data = pickle.load(f)
			print "Done!"
			
		for exid in data:

			print "Exercise %s (ID=%d)" % (data[exid]['exercise']['name'], exid)
			
			for phase, imgindices in {'analysis':['mark'], 'comparison':['mark','print']}.items():

				outname = sanitizefn(data[exid]['exercise']['name'])+'_%s_minutiae.csv' % phase
				
				print "  In %s, writing '%s'" % (phase,outname),
				
				
				with file(outname,'w') as f:
				
					f.write('image;user;id;x;y;angle;type;pairing\n')
				
					for userid, userdata in sorted(data[exid]['users'].iteritems(), key=lambda x:x[1]['username']):
						#userdata = data[exid]['users'][userid]

						if phase not in userdata:
							continue
							
						#print "    User %s (ID=%d)" % (userdata['username'],userid)
						for imgindex in imgindices:

							if imgindex not in userdata[phase]:
								continue
							
							annotations = userdata[phase][imgindex]
							if 'minutiae' not in annotations:
								continue
							
							minutiae = annotations['minutiae']
							if minutiae:
								sys.stdout.write(".")
								#for mid, m in sorted(minutiae.iteritems(), key=lambda x:(x[0],x[1]['username'])):
								for mid, m in sorted(minutiae.iteritems(), key=lambda x:int(x[0])):
									if 'pairing' not in m:
										m['pairing'] = -1
									f.write(';'.join((str(x) for x in [imgindex,userdata['username'],mid,m['pos'][0],m['pos'][1],m['angle'],m['type'],m['pairing']]))+'\n')
							
					print " Done !"