#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import time
from collections import defaultdict

from optparse import OptionParser
import myutil

import csv
import simplejson as json
import keys
from keys import KEY_TO_PRINT

SYMBOL = 'symbol';
PRESSTIMESTAMP = 'pressTimeStamp'
DOWNTIMESTAMP = 'downTimeStamp'
UPTIMESTAMP = 'upTimeStamp'
CHARCODE = 'charCode'
ISSHIFT = 'isShift'
TOGETHERWITH = 'togetherWith'
KEYCODE = 'keyCode'
ISCORRECT = 'isCorrect'

BACKSPACE_KEYCODE = 8
SHIFT_KEYCODE = 16
ALT_KEYCODE = 17
CTRL_KEYCODE = 18
ALTGR_KEYCODE = 225

OUTFILE = 'letterpairtiming.csv'

# either one is used to scale the values obtained
# the purpose of this is to make the numbers from fast and slow typist comparable
# replace these values with the ones the script returns and run again
PAIRTIME_AVERAGE_STANDPOINT = 163.0
PAIRTIME_MEDIAN_STANDPOINT = 160.0

# counts the number of particular symbol pairs typed
globalPairCount = defaultdict(int)
# helper variable that helps to determine which symbol pair resides in which excerpt
globalPairExcerptMap = defaultdict(list)
# helper variable to store more information about entries
globalPairRunMap = defaultdict(list)
# stores the average/median duration of all volunteers
globalPairDurations = []

def deCompact(table, sortColName = None):
	strokeList = []
	tableCols = table['cols']
	
	for row in table['rows']:
		stroke = {}
		colIndex = 0
		for colName in tableCols:
			stroke[colName] = row[colIndex]
			colIndex += 1
		strokeList.append(stroke)
		
	if (sortColName != None):
		if (sortColName in tableCols):
			strokeList.sort(lambda a, b: cmp(a[sortColName], b[sortColName]))
		else:
			raise Exception('Given sorting key does not exist')
		
	return strokeList

def median(li):
	listing = sorted(li)
	length = len(listing)

	if length == 0:
		return None

	if length % 2 == 0:
		midpoint = length // 2
		return int(round((listing[midpoint] + listing[midpoint - 1]) / 2.))
	else:
		return listing[length // 2]

	
def getPairTimes(strokeDataLists):
	global globalPairCount	
	global globalPairExcerptMap	
	global globalPairRunMap
	
	global globalPairDurations
	
	ignoreKeyCodes = [
		13, # \r
		8,  # backspace
	]
	
	totalVolunteerPairDurations = []
	totalVolunteerPairCount = 0
	
	pairMapDurations = defaultdict(list)
	pairMap = {}	
	
	for (strokeList, runCode, excerptCode) in strokeDataLists:

		prevStroke = strokeList[0]
		strokeCount = 0	
		for currentStroke in strokeList[1:]:
						
			if (currentStroke[KEYCODE] in [SHIFT_KEYCODE, ALT_KEYCODE, CTRL_KEYCODE, ALTGR_KEYCODE]):
				continue
			
			if (prevStroke[PRESSTIMESTAMP] 
				and prevStroke[CHARCODE] 
				and prevStroke[KEYCODE] not in ignoreKeyCodes
				
				and prevStroke[ISCORRECT] and currentStroke[ISCORRECT]
				
				and currentStroke[PRESSTIMESTAMP] 
				and currentStroke[CHARCODE]  
				and currentStroke[KEYCODE] not in ignoreKeyCodes):
				
				duration = currentStroke[DOWNTIMESTAMP] - prevStroke[DOWNTIMESTAMP]
				
				if (prevStroke[ISSHIFT]):
					previousLetter = prevStroke[SYMBOL]				
				else:
					previousLetter = prevStroke[SYMBOL].lower()
				
				if (currentStroke[ISSHIFT]):
					currentLetter = currentStroke[SYMBOL]
				else:
					currentLetter = currentStroke[SYMBOL].lower()
											
				pairKey = (previousLetter + currentLetter)
												
				if prevStroke[ISCORRECT] and currentStroke[ISCORRECT] and excerptCode not in globalPairExcerptMap[pairKey]:
					globalPairExcerptMap[pairKey].append(excerptCode)
#					globalPairRunMap[pairKey].append((prevStroke, currentStroke, runCode, strokeCount))

				totalVolunteerPairDurations.append(duration)
				totalVolunteerPairCount += 1
					
				pairMapDurations[pairKey].append(duration) 							
				globalPairCount[pairKey] += 1				 
				
			prevStroke = currentStroke
			strokeCount += 1
				
	totalVolunteerPairDurations.sort()
	globalPairDurations.append(median(totalVolunteerPairDurations))
#	globalPairDurations.append(sum(totalVolunteerPairDurations)/totalVolunteerPairCount)
	
	
#	coefficient = PAIRTIME_AVERAGE_STANDPOINT/(sum(totalVolunteerPairDurations)/totalVolunteerPairCount)
	coefficient = PAIRTIME_MEDIAN_STANDPOINT/median(totalVolunteerPairDurations)
				
	for (pairKey, pairDurations) in pairMapDurations.iteritems():
		pairDurations.sort()
		pairMedian = median(pairDurations)
		# exclude outliers
		newPairDurations = [dur for dur in pairDurations if pairMedian / 2. < dur < pairMedian * 2]
		if len(newPairDurations) < 1:
			continue
		
#		store the average or the median, with or without scaling
#		pairMap[pairKey] = int(round(sum(pairDurations)/len(pairDurations) * coefficient))
		pairMap[pairKey] = int(round(median(newPairDurations) * coefficient))
#		pairMap[pairKey] = int(round(median(newPairDurations)))
	
	return pairMap

def aggregatePairTimes(durationsListMap, pairTimes):
	for (pairKey, duration) in pairTimes.iteritems():
		durationsListMap[pairKey].append(duration)
		
	return durationsListMap

def printRunTable(dataRows):	
		for stroke in dataRows:			
			print stroke[5] if stroke[5] else '',

def lackOfPairs(averagePairMap, medianPairMap, durationsListMap):	
	"introspect stored datalists and dictionaries, display the contents of specific data slots that fulfill certain criteria"
	
	excerptPairListMap = defaultdict(list)
	lackPairCounter = 0
	for prev in keys.RELOCATABLES:
		for cur in keys.RELOCATABLES:
			pairKey = (prev + cur).decode('utf-8')
				
			if (pairKey in globalPairCount and (
				pairKey in ['bv', 'jh', 'õü'.decode('utf-8')]
#				globalPairCount[pairKey] < 5
#				or
#				abs(averagePairMap[pairKey] - medianPairMap[pairKey])/float(averagePairMap[pairKey]) > 0.10
				)):

				cor = abs(averagePairMap[pairKey] - medianPairMap[pairKey])/float(averagePairMap[pairKey])
				print '''"%s" : %s/%s/%s/%s''' % (pairKey, averagePairMap[pairKey], medianPairMap[pairKey], globalPairCount[pairKey], cor), sorted(durationsListMap[pairKey]) 
				for excerptCode in globalPairExcerptMap[pairKey]:
					if pairKey not in excerptPairListMap[excerptCode]:
						excerptPairListMap[excerptCode].append(pairKey)
				lackPairCounter += 1
				
	print lackPairCounter
	sortedExcerptPairListMap = sorted(excerptPairListMap.iteritems(), key = lambda a: len(a[1]), reverse = True)
	
	for (excerptCode, pairs) in sortedExcerptPairListMap:		
		print excerptCode, 
		for letterPair in pairs:
			print letterPair,
		print
		
	print len(sortedExcerptPairListMap)
	
#	excerptList = []
#	looper = iter(sortedExcerptPairListMap)
#	first = looper.next()	
#	while len(first[1]) > 0:
#		print first
#		for pairKey in first[1]:
#			for excerptCode in globalPairExcerptMap[pairKey]:
#				print excerptCode, pairKey, excerptPairListMap[excerptCode]				
#				delIndex = excerptPairListMap[excerptCode].index(pairKey)
#				del excerptPairListMap[excerptCode][delIndex]
#				if excerptCode not in excerptList:
#					excerptList.append(excerptCode)
		
		
#		sortedExcerptPairListMap = sorted(excerptPairListMap.iteritems(), key = lambda a: len(a[1]), reverse = True)
#		looper = iter(sortedExcerptPairListMap)
#		first = looper.next()		
	
#	print excerptList, len(excerptList)

def writeToFile(fileName, averageMap, medianMap):
	out = open(fileName, 'w')	
	for l1 in keys.KEYS:
		out.write("\t%s" % KEY_TO_PRINT.get(l1, l1))
	out.write('\n')
			
	for l1 in keys.KEYS:
		out.write(KEY_TO_PRINT.get(l1, l1))
		for l2 in keys.KEYS:
			pairKey = (l1 + l2).decode('utf-8')
			cor = 1
			if (pairKey in averageMap):				
				cor = abs(averageMap[pairKey] - medianMap[pairKey])/float(averageMap[pairKey])
				
			if globalPairCount.get(pairKey, 0) >= 10 or (cor <= 0.1 and 5 <= globalPairCount.get(pairKey, 0) < 10):
#				one or the other
				duration = medianMap.get(pairKey)
#				duration = averageMap.get(pairKey)
			else:
				duration = ''
			out.write("\t%s" % duration)
		out.write('\n')	

def main(argv = None):
	
	if argv is None:
		argv = sys.argv[1:]
		
	parser = OptionParser(usage = "usage: %prog [options] FILE1 ...", version = "%prog 0.2")
	#parser.add_option('-q', '--quiet',  action="store_false", dest = 'verbose', default=True, help="don't print status messages to stdout")
	parser.add_option('-f', '--file', dest = 'out_filename', default = OUTFILE, metavar = "FILE", help = "output file name")

	(options, datafiles) = parser.parse_args(args = argv)
		
	durationsListMap = defaultdict(list)
		
	time_start = time.time()
	fcsv = open(datafiles[0], 'rb')	
	csv.field_size_limit(500000)
	datarows = csv.reader(fcsv, delimiter="\t") 
	#csv.reader(fcsv, delimiter="\t" )
	
	runDataLists = []
	
	row = datarows.next()
	runCode = row[0]
	excerptCode = row[1]	
	previousVolunteer = row[3]
	runStr = row[4]
	runData = json.loads(runStr, 'utf-8')
	
	runDataLists.append((deCompact(runData, DOWNTIMESTAMP), runCode, excerptCode))
	
	for row in datarows:
		
		runCode = row[0]
		excerptCode = row[1]
		currentVolunteer = row[3]
		runStr = row[4]
		runData = json.loads(runStr, 'utf-8')		
			
		if (currentVolunteer == previousVolunteer):
			runDataLists.append((deCompact(runData, DOWNTIMESTAMP), runCode, excerptCode))
		else:
			pairTimes = getPairTimes(runDataLists)
			
			runDataLists = []		
			runDataLists.append((deCompact(runData, DOWNTIMESTAMP), runCode, excerptCode))		
			durationsListMap = aggregatePairTimes(durationsListMap, pairTimes)
		
		previousVolunteer = currentVolunteer
		
	pairTimes = getPairTimes(runDataLists)
	durationsListMap = aggregatePairTimes(durationsListMap, pairTimes)
		
	totalAveragePairMap = {}
	totalMedianPairMap = {}
	for (pairKey, durationsList) in durationsListMap.iteritems():
		durationsList.sort()
		currentMedian = median(durationsList)				
				
		newDurationsList = [dur for dur in durationsList if (currentMedian / 2 < dur < currentMedian * 2)]
#		newDurationsList = durationsList
				
		durationsListMap[pairKey] = newDurationsList			
		totalAveragePairMap[pairKey] = sum(newDurationsList)/len(newDurationsList)		
		totalMedianPairMap[pairKey] = median(newDurationsList)
		globalPairCount[pairKey] = len(newDurationsList)


#	lackOfPairs(totalAveragePairMap, totalMedianPairMap, durationsListMap)
	
	globalPairDurations.sort()
	print 'overall median: %s, overall average: %s, length: %s'% (median(globalPairDurations), sum(globalPairDurations)/len(globalPairDurations), len(globalPairDurations))
	
	writeToFile(options.out_filename, totalAveragePairMap, totalMedianPairMap)

	time_end = time.time()
	print "Took time %s" % (myutil.hrtime(time_end - time_start))	

			
if __name__ == "__main__":
	sys.exit(main())

