#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import time
import math
import random
import unicodedata as un
import scipy as sp

import gv
import pygraphviz as pgv

from optparse import OptionParser
import numexpr as expr

import keymap as km
import modifier as mod
import myutil
import keys


sys.setcheckinterval(10000)

RELOCATABLE_SYMBOLS = [
		'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 'š', 'ž', 'z', 't', 'u', 'v', 'w', 'õ', 'ä', 'ö', 'ü', 'x', 'y', 
		'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'Š', 'Ž', 'Z', 'T', 'U', 'V', 'W', 'Õ', 'Ä', 'Ö', 'Ü', 'X', 'Y', 
	#	'1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 
		'.', ',', '-', ':', ';',  
		'!', '?',  '/', "'", '"', #'_',
	#	'(', ')','{', '}', '[', ']','*', '<', '>', '|', '+', '#', '¤', '%', '&', '\\', '=', '~', '@', '€', '£', '$',
	#	' ', '\n',
	]

OUTFILE = "speed.txt"
STARTMAP = "start"
QUERTYMAP = 'querty'


#~ E = 1/N*sum(e_i)
#~ e_i = (k_b * b_i) + (k_p * p_i) + (k_s * s_i) 

#~ b_i = k_1 * b_i1 * (1 + k_2 * b_i2 * (1 + k_3 * b_i3))
#~ p_i = k_1 * p_i1 * (1 + k_2 * p_i2 * (1 + k_3 * p_i3))
#~ p_ij = w_0 + (w_hand * P_hand_j) + (w_row * P_row_j) + (w_finger * P_finger_j)

#~ s_i = sum_j=hand,row,finger(f_i*p_j)

ANNEALING = 'annealing'
MAIN = 'main'
INERTIA = 'inertia'
PENALTY = 'penalty'
STROKE = 'stroke'
HAND = 'hand'
ROW = 'row'
FINGER = 'finger'
MODE = 'mode'

configuration = {
	'balanced': {
	#t0, k, p0
		ANNEALING: [1000000000, 5, 1],
#		kb, kp, ks
		MAIN: [1, 1, 0],
#		k1, k2, k3
		INERTIA: [1, 0],	
		PENALTY: {
#			 w0, whand, wrow, wfinger, wmode
			MAIN: [0, 1, 1, 1.3539, 1],								
#			 both, left, right
			HAND: [0, 0, 0],
#			 number, top, middle, bottom, space
			ROW: [3, 1, 0, 2, 0],
#			 thumb, pinky, ring, ... , pinky
			FINGER: [0, 2, 1, 0, 0, 0, 0, 1, 2],
#			 normal, shift, altgr, shift+altgr
			MODE: [0, 1, 3, 6]
		},
		STROKE: {
#		     fhand, frow, ffinger		
			MAIN: [1, 0.3, 0.3],
#			different, same
			HAND: [0, 1],
#			same hand, same key, inward, outward, same finger (different rows)
			FINGER: [0, 1, 2, 3, 4],
#			same hand, same row, down, up, two or more rows jump (new value += this)
			ROW: [0, 1, 2, 3, 2]
		}
	},					
	'balanced0': {
	#t0, k, p0
		ANNEALING: [1000000000, 5, 1],
#		kb, kp, ks
#		MAIN: [0.23497096346319005, 9.13318, 0.759085],
		MAIN: [0, 1, 0],
#		k1, k2, k3
#		INERTIA: [0.14285714169882655, 0.1],
		INERTIA: [0.5, 1],

		PENALTY: {
#			 w0, whand, wrow, wfinger, wmode
#			MAIN: [0, 1, 1, 1.325, 0],	
			MAIN: [0, 1, 1, 0, 0],								
#			 both, left, right
			HAND: [0, 0, 0],
#			 number, top, middle, bottom, space
			ROW: [1.5, 0.5, 0, 1, 0],
#			 thumb, pinky, ring, ... , pinky
			FINGER: [0, 1, 0.5, 0, 0, 0, 0, 0.5, 1],
#			 normal, shift, altgr, shift+altgr
			MODE: [0, 1, 5, 10]
		},
		STROKE: {
#		     fhand, frow, ffinger		
			MAIN: [1, 0.3, 0.3],
#			different, same
			HAND: [0, 1],
#			same hand, same key, inward, outward, same finger (different rows)
			FINGER: [0, 1, 2, 3, 4],
#			same hand, same row, down, up, two or more rows jump (new value += this)
			ROW: [0, 1, 2, 3, 2]
		}
	},
	'carpalx': {
	#t0, k, p0
		ANNEALING: [1000000000, 5, 1],
#		kb, kp, ks
		MAIN: [0.3555, 0.6423, 0.4268],
#		k1, k2, k3
		INERTIA: [1, 0.367, 0.235],

		PENALTY: {
#			 w0, whand, wrow, wfinger, wmode
			MAIN: [0, 0, 1.3088, 2.5948, 0],
#			 both, left, right
			HAND: [0, 0, 0],
#			 number, top, middle, bottom, space
			ROW: [1.5, 0.5, 0, 1, 0],
#			 thumb, pinky, ring, ... , pinky
			FINGER: [0, 1, 0.5, 0, 0, 0, 0, 0.5, 1],
#			 normal, shift, altgr, shift+altgr
			MODE: [0, 1, 5, 10]
		},
		STROKE: {
#		     fhand, frow, ffinger		
			MAIN: [1, 0.3, 0.3],
#			different, same
			HAND: [0, 1],
#			same hand, same key, inward, outward, same finger (different rows)
			FINGER: [0, 1, 2, 3, 4],
#			same hand, same row, down, up, two or more rows jump (new value += this)
			ROW: [0, 1, 2, 3, 2]
		}
	}					
}

# [ <plain>, <shift>, <altgr>, <algr+shift>]
KEYINFO_MODES = [0, 1, 2, 3]
KEYINFO_MODE_NORMAL = 0
KEYINFO_MODE_SHIFT = 1
KEYINFO_MODE_ALTGR = 2
KEYINFO_MODE_ALTGRSHIFT = 3

LETTERINFO_INDEX_DISTANCE = 0

CONFIGURATION_SCHEME = 'balanced'
# FINGER_SCHEME = 'left' 
# FINGER_SCHEME = 'right'
FINGER_SCHEME = 'leftright' 

KEYINFO_INDEX_MODE = 0
KEYINFO_INDEX_ROW = 1
KEYINFO_INDEX_FINGER = 2
KEYINFO_INDEX_BASE = 3
KEYINFO_INDEX_PENALTY = 4
KEYINFO_INDEX_IS_RELOCATABLE = 5
KEYINFO_INDEX_IS_SPOT = 6
KEYINFO_INDEX_IS_LETTER = 7
KEYINFO_INDEX_SYMBOL = 8
KEYINFO_SIZE = 9

class LetterDict (dict):

	def __hash__(self):
		hashstring = '_'.join(self.values())
		print hashstring
		return hash(hashstring)

	def flip(self):
		return dict([(v, k) for (k, v) in self.iteritems()])

keysMatrix = None
statsMatrix = None
isLetterList = []
permutationList = []
modeList = []

# hand: 
#0 different
#1 same

#~ finger:
#~ 0 different hands
#~ 1 key repeat
#~ 2 different toward middle
#~ 3 different toward outside
#~ 4 same, not repeating

#row:
#~ 0 different hand
#~ 1 same row
#~ 2 down progressing
#~ 3 up progressing
#~ 4 down progressing > 1
#~ 5 up progressing >1

def stroke_effort(b1, b2):

	f1, f2 = b1[KEYINFO_INDEX_FINGER], b2[KEYINFO_INDEX_FINGER]
	r1, r2 = b1[KEYINFO_INDEX_ROW], b2[KEYINFO_INDEX_ROW]
	
	hand_flag = None
	finger_flag = None
	row_flag = None
	
	if (0 < f1 < 5 and 0 < f2 < 5):
		hand_flag = 1
	elif (4 < f1 < 9 and 4 < f2 < 9):
		hand_flag = 1
		t = f1
		f1 = f2
		f2 = t
	else:
		hand_flag = 0
	
	if hand_flag == 0:
		finger_flag = 0
	elif b1 == b2:
		finger_flag = 1
	else:
		if f1 == f2:
			finger_flag = 4
		elif f1 < f2:
			finger_flag = 2
		else:
			finger_flag = 3

	if hand_flag == 0:
		row_flag = 0
	elif r1 == r2:
		row_flag = 1
	elif r1 < r2:
		t = r1
		r1 = r2
		r2 = t
		row_flag = 2
	else:
		row_flag = 3

	long_row_jump = 0
	if r1 - r2 > 1:
		long_row_jump = configuration[CONFIGURATION_SCHEME][STROKE][ROW][4]

	return configuration[CONFIGURATION_SCHEME][STROKE][MAIN][0] * configuration[CONFIGURATION_SCHEME][STROKE][HAND][hand_flag] +\
		configuration[CONFIGURATION_SCHEME][STROKE][MAIN][1] * (configuration[CONFIGURATION_SCHEME][STROKE][ROW][row_flag] + long_row_jump) +\
		configuration[CONFIGURATION_SCHEME][STROKE][MAIN][2] * configuration[CONFIGURATION_SCHEME][STROKE][FINGER][finger_flag]

def total_effort(key1, key2):
	"""the effort of a keypair"""
	
	effort = (configuration[CONFIGURATION_SCHEME][MAIN][0] * configuration[CONFIGURATION_SCHEME][INERTIA][0] * key1[KEYINFO_INDEX_BASE] * (1 + configuration[CONFIGURATION_SCHEME][INERTIA][1] * key2[KEYINFO_INDEX_BASE]) +
				configuration[CONFIGURATION_SCHEME][MAIN][1] * configuration[CONFIGURATION_SCHEME][INERTIA][0] * key1[KEYINFO_INDEX_PENALTY] * (1 + configuration[CONFIGURATION_SCHEME][INERTIA][1] * key2[KEYINFO_INDEX_PENALTY]) +
				configuration[CONFIGURATION_SCHEME][MAIN][2] * stroke_effort(key1, key2))
	
#	if key1[KEYINFO_INDEX_SYMBOL] == 'd':
#		print "%s%s: %f" % (key1[KEYINFO_INDEX_SYMBOL], key2[KEYINFO_INDEX_SYMBOL], effort)
		
	return effort

def get_layout(mapping):
	"""load the keyboard and create letter to stats mapping"""
	"""returns a tuple: 1) a list of symbols and their properties from the layout 2) a hashtable letter to index number in the list"""

	symbolList = []
	symbolHash = LetterDict()
	counter = 0
	tb1 = [None] * KEYINFO_SIZE

	relocateKeys  = dict.fromkeys(keys.RELOCATABLES)
	for i in range(len(km.KEYBOARD)):
		row = km.KEYBOARD[i]

		for j in range(len(row)):
			button = row[j]
			button[1:] = km.LAYOUT[mapping][i][j]
			tb2 = tb1[:]
			finger = km.BUTTON_TO_FINGER[FINGER_SCHEME][i][j]
			tb2[KEYINFO_INDEX_BASE] = button[LETTERINFO_INDEX_DISTANCE]
			tb2[KEYINFO_INDEX_FINGER] = finger
			tb2[KEYINFO_INDEX_ROW] = i

			if button[-1] != False:
				tb2[KEYINFO_INDEX_IS_SPOT] = True
			else:
				tb2[KEYINFO_INDEX_IS_SPOT] = False

			for k in range(len(button)-1):
				key = button[k+1]
				if key == False: continue
				tb3 = tb2[:]
				tb3[KEYINFO_INDEX_MODE] = k
				tb3[KEYINFO_INDEX_PENALTY] = configuration[CONFIGURATION_SCHEME][PENALTY][MAIN][0] +\
					configuration[CONFIGURATION_SCHEME][PENALTY][MAIN][1] * configuration[CONFIGURATION_SCHEME][PENALTY][HAND][ km.FINGER_TO_HAND[finger] ] +\
					configuration[CONFIGURATION_SCHEME][PENALTY][MAIN][2] * configuration[CONFIGURATION_SCHEME][PENALTY][ROW][i] +\
					configuration[CONFIGURATION_SCHEME][PENALTY][MAIN][3] * configuration[CONFIGURATION_SCHEME][PENALTY][FINGER][finger] +\
					configuration[CONFIGURATION_SCHEME][PENALTY][MAIN][4] * configuration[CONFIGURATION_SCHEME][PENALTY][MODE][k]

				if key != None:
					symbolHash[key] = counter

					if key in relocateKeys:
						tb3[KEYINFO_INDEX_IS_RELOCATABLE] = True
					else:
						tb3[KEYINFO_INDEX_IS_RELOCATABLE] = False

					if len(key) != 1 or un.category(unicode(key, 'utf-8')) in ['Ll', 'Lu']:
						tb3[KEYINFO_INDEX_IS_LETTER] = True
					else:
						tb3[KEYINFO_INDEX_IS_LETTER] = False

				tb3[KEYINFO_INDEX_SYMBOL] = key
				symbolList.append(tb3)
				counter += 1
	return (symbolList, symbolHash)
	
def buttoncmp(a,b):
	return a[KEYINFO_INDEX_BASE] * configuration[CONFIGURATION_SCHEME][MAIN][0] + a[KEYINFO_INDEX_PENALTY] * configuration[CONFIGURATION_SCHEME][MAIN][1] -\
		(b[KEYINFO_INDEX_BASE] * configuration[CONFIGURATION_SCHEME][MAIN][0] + b[KEYINFO_INDEX_PENALTY] * configuration[CONFIGURATION_SCHEME][MAIN][1])
	
def charcmp(b,a):
	return a['count'] - b['count']
	
def buildGvGraph(stats):
	G = gv.digraph("G")
	N = gv.protonode(G)
	E = gv.protoedge(G)

	#~ gv.setv(G, 'rankdir', 'LR')
	#~ gv.setv(G, 'nodesep', '0.05')
	#~ gv.setv(N, 'shape', 'box')
	gv.setv(N, 'width', '0')
	gv.setv(N, 'height', '0')
	gv.setv(N, 'margin', '.03')
	gv.setv(N, 'fontsize', '8')
	gv.setv(N, 'fontname', 'helvetica')
	gv.setv(E, 'arrowhead', 'none')
	#~ gv.setv(E, 'arrowsize', '.4')


	THRESHOLD = 15
	nodehash = {}
	allow = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 's', 't']
	#~ stats = dict()
	for prev, row in stats.iteritems():
		nodehash[prev] = gv.node(G, prev)
	
	visited = dict()
	for prev, row in stats.iteritems():	
		for followup, pairCount in row.iteritems():
			if followup in nodehash :
				#~ if 0 <= weight <= 1:
					#~ weight = 2
				#~ weight = 1./100*math.log(weight,2)
				
				if visited.has_key(followup + prev): continue

				
				if pairCount == 0:
					weight1 = 1
				else:
					weight1 = pairCount
				pairCount2 = stats[followup][prev]
				if pairCount2 == 0:
					weight2 = 1
				else:
					weight2 = pairCount2
					
				totalWeight = (math.log(weight1, 10) + math.log(weight2, 10)) * 2
				visited[prev + followup] = True
					
				print totalWeight
				if totalWeight > THRESHOLD:
					e = gv.edge(nodehash[prev], nodehash[followup])
					w = str(totalWeight)
#					gv.setv(e, 'weight', w)

	gv.layout(G, 'dot')
	gv.render(G, 'xlib')
	
def buildGraphVizGraph(stats):
	G = pgv.AGraph()

	THRESHOLD = 18
	nodehash = {}
	allow = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 's', 't']
	#~ stats = dict()
	for prev, row in stats.iteritems():
#		G.add_node(prev.decode('utf-8'))
		nodehash[prev] = True
		
	visited = dict()
	for prev, row in stats.iteritems():	
		for followup, pairCount in row.iteritems():
			if followup in nodehash :
				#~ if 0 <= weight <= 1:
					#~ weight = 2
				#~ weight = 1./100*math.log(weight,2)
				
				if visited.has_key(followup + prev): continue

				
				if pairCount == 0:
					weight1 = 1
				else:
					weight1 = pairCount
				pairCount2 = stats[followup][prev]
				if pairCount2 == 0:
					weight2 = 1
				else:
					weight2 = pairCount2
					
				totalWeight = (math.log(weight1, 10) + math.log(weight2, 10)) * 2
				visited[prev + followup] = True
					
#				print totalWeight
				if totalWeight > THRESHOLD:
					if not G.has_node(prev.decode('utf-8')):
						G.add_node(prev.decode('utf-8'))
					if not G.has_node(followup.decode('utf-8')):
						G.add_node(followup.decode('utf-8'))
						
#					G.add_edge(prev.decode('utf-8'), followup.decode('utf-8'), weights=int(totalWeight))
#					G.add_edge(prev.decode('utf-8'), followup.decode('utf-8'), weight = int(totalWeight * 20))
					G.add_edge(prev.decode('utf-8'), followup.decode('utf-8'))

	
	G.layout(prog='neato')
	G.draw(path = '../inout/neatonoweigth.png')

	
def load_matrix(datafiles):
	
	stats = dict()
	str_lower = mod.Modifier(True)
	
	#~ str_lower.registerModifier(str.decode, 'utf-8')
	#~ str_lower.registerModifier(unicode.lower)
	#~ str_lower.registerModifier(unicode.encode, 'utf-8')
	
	for filename in datafiles:
		f = file(filename, "r")
		
		lines = []
		for line in f:
			line = line.rstrip()
			lines.append(line.split("\t"))
			
		for i in range(1,len(lines)):
			
			prev = keys.PRINT_TO_KEY.get(lines[i][0], lines[i][0])
			if prev in keys.KEYS:
				
				prev = str_lower(prev)
				if prev not in stats:
					stats[prev] = dict()
					stats[prev]['letter'] = prev
			
				#second = dict()
				for j in range(1,len(lines[i])):
					
					followup = keys.PRINT_TO_KEY.get(lines[0][j], lines[0][j])
					if followup == 'count' or followup in keys.KEYS:
						followup = str_lower(followup)
						stats[prev][followup] = stats[prev].get(followup, 0) + long(lines[i][j])
						#second[followup] = int(lines[i][j])
			#stats[char] = second
		f.close()	
	return stats

def swap_effort2(keyIndex1, keyIndex2, oldEffort):
	"""Swaps two key positions on the keyboard and returns the new effort"""
	"""It calculates the difference of the old and new effort table and uses the result"""
	
	scrambledKeysMatrix = keysMatrix[permutationList,:][:,permutationList]
	krow1, krow2 = scrambledKeysMatrix[keyIndex1, :], scrambledKeysMatrix[keyIndex2, :]
	kcol1, kcol2 = scrambledKeysMatrix[:, keyIndex1], scrambledKeysMatrix[:, keyIndex2]
	srow1, srow2 = statsMatrix[keyIndex1, :], statsMatrix[keyIndex2, :]
	scol1, scol2 = statsMatrix[:, keyIndex1], statsMatrix[:, keyIndex2]
	before = (sp.sum(sp.multiply(krow1, srow1) + sp.multiply(krow2, srow2) + sp.multiply(kcol1, scol1) + sp.multiply(kcol2, scol2))
		- krow1[keyIndex1] * srow1[keyIndex1] - krow1[keyIndex2] * srow1[keyIndex2]
		- krow2[keyIndex1] * srow2[keyIndex1] - krow2[keyIndex2] * srow2[keyIndex2]
	)
	after = (sp.sum(sp.multiply(krow1, srow2)	+ sp.multiply(krow2, srow1) + sp.multiply(kcol1, scol2) + sp.multiply(kcol2, scol1))
		- krow1[keyIndex1] * srow2[keyIndex1] - krow1[keyIndex2] * srow2[keyIndex2]
		- krow2[keyIndex1] * srow1[keyIndex1] - krow2[keyIndex2] * srow1[keyIndex2]
		- kcol1[keyIndex1] * scol2[keyIndex1] - kcol1[keyIndex2] * scol2[keyIndex2]
		- kcol2[keyIndex1] * scol1[keyIndex1] - kcol2[keyIndex2] * scol1[keyIndex2]
		+ krow1[keyIndex1] * srow2[keyIndex2] + krow1[keyIndex2] * srow2[keyIndex1]
		+ krow2[keyIndex1] * srow1[keyIndex2] + krow2[keyIndex2] * srow1[keyIndex1]
	)
	return oldEffort - before + after

def swap_effort22(keyIndex1, keyIndex2, oldEffort):
	"""Swaps two key positions on the keyboard and returns the new effort"""
	"""It calculates the difference of the old and new effort table and uses the result"""

	scrambledKeysMatrix = keysMatrix[permutationList,:][:,permutationList]
	krow1, krow2 = scrambledKeysMatrix[keyIndex1, :], scrambledKeysMatrix[keyIndex2, :]
	kcol1, kcol2 = scrambledKeysMatrix[:, keyIndex1], scrambledKeysMatrix[:, keyIndex2]
	srow1, srow2 = statsMatrix[keyIndex1, :], statsMatrix[keyIndex2, :]
	scol1, scol2 = statsMatrix[:, keyIndex1], statsMatrix[:, keyIndex2]
	
	krow11 = krow1[keyIndex1]
	krow12 = krow1[keyIndex2]
	srow11 = srow1[keyIndex1]
	srow12 = srow1[keyIndex2]
	
	krow21 = krow2[keyIndex1]
	krow22 = krow2[keyIndex2]
	srow21 = srow2[keyIndex1]
	srow22 = srow2[keyIndex2]
	
	kcol11 = kcol1[keyIndex1]
	kcol12 = kcol1[keyIndex2]	
	scol21 = scol2[keyIndex1]
	scol22 = scol2[keyIndex2]
		
	scol11 = scol1[keyIndex1]
	scol12 = scol1[keyIndex2]	
	kcol21 = kcol2[keyIndex1]
	kcol22 = kcol2[keyIndex2]	
	
	before = (sp.sum(expr.evaluate("krow1 * srow1 + krow2 * srow2 + kcol1 * scol1 + kcol2 * scol2"))
		- expr.evaluate("krow11 * srow11 + krow12 * srow12 \
		+ krow21 * srow21 + krow22 * srow22")
	)
	after = (sp.sum(expr.evaluate("krow1 * srow2	+ krow2 * srow1 + kcol1 * scol2 + kcol2 * scol1"))
		- expr.evaluate("krow11 * srow21 + krow12 * srow22 \
		+ krow21 * srow11 + krow22 * srow12 \
		+ kcol11 * scol21 + kcol12 * scol22 \
		+ kcol21 * scol11 + kcol22 * scol12 \
		- krow11 * srow22 - krow12 * srow21 \
		- krow21 * srow12 - krow22 * srow11")
	)
	return oldEffort - before + after


def swap_effort3(keyIndex1, keyIndex2, oldEffort):
	"""Swaps two key positions on the keyboard and returns the new effort"""
	"""It calculates the difference of the old and new effort table as a vector and uses the result"""
	
	scrambledKeysMatrix = keysMatrix[permutationList, :][:, permutationList]
#	scrambledKeysMatrix = keysMatrix[permutationList[:, None], permutationList]
#	scrambledKeysMatrix = keysMatrix[sp.ix_(permutationList, permutationList)]
#	scrambledKeysMatrix = (keysMatrix.ravel()[(permutationList + (permutationList * keysMatrix.shape[1]).reshape((-1,1))).ravel()]).reshape(permutationList.size, permutationList.size)
	rowdiff1 = sp.subtract(scrambledKeysMatrix[keyIndex1, : ], scrambledKeysMatrix[keyIndex2, : ])
	rowdiff1[keyIndex1] = scrambledKeysMatrix[keyIndex1, keyIndex1] - scrambledKeysMatrix[keyIndex2, keyIndex2]
	rowdiff1[keyIndex2] = scrambledKeysMatrix[keyIndex1, keyIndex2] - scrambledKeysMatrix[keyIndex2, keyIndex1]
	rowdiff2 = sp.negative(rowdiff1)
	rowdiff2[[keyIndex1, keyIndex2]] = rowdiff2[[keyIndex2, keyIndex1]]
	coldiff = sp.subtract(scrambledKeysMatrix[:, keyIndex1], scrambledKeysMatrix[:, keyIndex2])
	coldiff[keyIndex1], coldiff[keyIndex2] = 0, 0
	return oldEffort - sp.sum(
		  sp.multiply(rowdiff1, statsMatrix[keyIndex1, :])
		+ sp.multiply(rowdiff2, statsMatrix[keyIndex2, :])
		+ sp.multiply(coldiff, statsMatrix[:, keyIndex1])
		- sp.multiply(coldiff, statsMatrix[:, keyIndex2]))
	
def swap_effort32(keyIndex1, keyIndex2, oldEffort):
	"""Swaps two key positions on the keyboard and returns the new effort"""
	"""It calculates the difference of the old and new effort table as a vector and uses the result"""
	
	scrambledKeysMatrix = keysMatrix[permutationList, :][:, permutationList]
	rowdiff1 = scrambledKeysMatrix[keyIndex1, : ] - scrambledKeysMatrix[keyIndex2, : ]
	rowdiff1[keyIndex1] = scrambledKeysMatrix[keyIndex1, keyIndex1] - scrambledKeysMatrix[keyIndex2, keyIndex2]
	rowdiff1[keyIndex2] = scrambledKeysMatrix[keyIndex1, keyIndex2] - scrambledKeysMatrix[keyIndex2, keyIndex1]
	rowdiff2 = (-1) * rowdiff1
	rowdiff2[[keyIndex1, keyIndex2]] = rowdiff2[[keyIndex2, keyIndex1]]
	coldiff = sp.subtract(scrambledKeysMatrix[:, keyIndex1], scrambledKeysMatrix[:, keyIndex2])
	coldiff[keyIndex1], coldiff[keyIndex2] = 0, 0
	return oldEffort - sp.sum(
		  rowdiff1 * statsMatrix[keyIndex1, :]
		+ rowdiff2 * statsMatrix[keyIndex2, :]
		+ coldiff * statsMatrix[:, keyIndex1]
		- coldiff * statsMatrix[:, keyIndex2])
	
def swap_effort1(keyIndex1, keyIndex2, oldEffort):
	"""Swaps two key positions on the keyboard and returns the new effort"""
	"""It replaces the effort values on place (side effect) and multiplies with counters."""
#	global keysMatrix
	
#	keysMatrix = keysMatrix[permutationList,:][:,permutationList]
	keysMatrix[[keyIndex1, keyIndex2], :] = keysMatrix[[keyIndex2, keyIndex1], :]
	keysMatrix[:, [keyIndex1, keyIndex2]] = keysMatrix[:, [keyIndex2, keyIndex1]]
	return sp.sum(keysMatrix * statsMatrix)

def swap_effort12(keyIndex1, keyIndex2, oldEffort):
	"""Swaps two key positions on the keyboard and returns the new effort"""
	"""It replaces the effort values on place (side effect) and multiplies with counters."""
	
	permList = permutationList[:]
	permList[keyIndex1], permList[keyIndex2] = permList[keyIndex2], permList[keyIndex1]
	
	keyMatrix = keysMatrix[permList,:][:,permList]
	return sp.sum(keyMatrix * statsMatrix)

def swap_effort13(keyIndex1, keyIndex2, oldEffort):
	"""Swaps two key positions on the keyboard and returns the new effort"""
	"""It replaces the effort values on place (side effect) and multiplies with counters."""
#	global keysMatrix
	
	permutationList[keyIndex1], permutationList[keyIndex2] = permutationList[keyIndex2], permutationList[keyIndex1]	
	keyMatrix = keysMatrix[permutationList,:][:,permutationList]
	permutationList[keyIndex1], permutationList[keyIndex2] = permutationList[keyIndex2], permutationList[keyIndex1]
#	return sp.einsum('ij, ij', keyMatrix, statsMatrix)
	return sp.sum(keyMatrix * statsMatrix)

def benchmarkSwapping(initEffort, ROUNDS, permLength):
	global permutationList
	functions = [swap_effort2, swap_effort22, swap_effort3, swap_effort32, swap_effort12, swap_effort13, swap_effort1,]
	permutationList = sp.random.permutation(permLength)
	for func in functions:
		start = time.time()
		for j in range(ROUNDS):
			s1, s2 = pick_keys(permLength)
			func(s1, s2, initEffort)
		end = time.time()
		print "%s Took time %s" % (func.__name__, myutil.hrtime(end - start))
		
def testSwapping(initEffort, permutationList):		
	a, b = 4, 70
	res = check_swapping(initEffort, a, b)
#	permutationList[a], permutationList[b] = permutationList[b], permutationList[a]
	a, b = 12, 32
	res = check_swapping(res, a, b)
#	permutationList[a], permutationList[b] = permutationList[b], permutationList[a]
	a, b = 20, 52
	res = check_swapping(res, a, b)	
#	permutationList[a], permutationList[b] = permutationList[b], permutationList[a]
	a, b = 24, 40
	res = check_swapping(res, a, b)	
#	permutationList[a], permutationList[b] = permutationList[b], permutationList[a]	

def check_swapping(initEffort, s1, s2):

	functionsUnderTest = [ swap_effort12, swap_effort13, swap_effort2, swap_effort22, swap_effort3, swap_effort32, swap_effort1]	
	
	print 'swapping results: initEffort = %s' % initEffort
	for func in functionsUnderTest:
		result = func(s1, s2, initEffort)
		print '%s:\t= %s' % (func.__name__, result)
		
	return result

def calculate_counters(letterHash, keyList, smallLetterHash, smallKeyList, effortMatrix, stats):
	fingerUsage = [0, 0, 0, 0, 0, 0, 0, 0, 0]
	rowUsage = [0, 0, 0, 0, 0]

	keyCount = dict()
	keyCount['rshift'] = 0
	keyCount['lshift'] = 0
	keyCount['altgr'] = 0
	keyCount['sameHand'] = 0
	keyCount['sameFinger'] = 0
	keyCount['neighFinger'] = 0
	keyCount['distance'] = 0

	flippedPermutationList = [None] * len(permutationList)
	for i in range(len(permutationList)):
		flippedPermutationList[permutationList[i]] = i

	for prev, row in stats.iteritems():
		if prev in letterHash:

			smallIndex	= smallLetterHash.get(prev, False)
			if smallIndex != False:
				prev_key = smallKeyList[flippedPermutationList[smallIndex]]
			else:
				prev_key	= keyList[letterHash[prev]]
			prev_row		= prev_key[KEYINFO_INDEX_ROW]
			prev_finger = prev_key[KEYINFO_INDEX_FINGER]
			prev_row		= prev_key[KEYINFO_INDEX_ROW]
			prev_base	= prev_key[KEYINFO_INDEX_BASE]

			fingerUsage[prev_finger] += row['count']
			#~ print prev, prev_row, row['count']
			rowUsage[prev_row] += row['count']

			#~ print prev, keyinfo[prev][KEYINFO_INDEX_MODE]

			if prev_key[KEYINFO_INDEX_MODE] & KEYINFO_MODE_ALTGR:
				#~ print "altgr: ", prev, row['count']
				keyCount['altgr'] += row['count']

			if prev_key[KEYINFO_INDEX_MODE] & KEYINFO_MODE_SHIFT:
				if (0 < prev_finger < 5):
					keyCount['rshift'] += row['count']
					#~ print "rshift: ", prev, row['count']
				if  (4 < prev_finger < 9):
					keyCount['lshift'] += row['count']
					#~ print "lshift: ", prev, row['count']

			for followup, val in row.iteritems():
				if followup in letterHash:
					smallIndex	= smallLetterHash.get(followup, False)
					if smallIndex != False:
						next_key = smallKeyList[flippedPermutationList[smallIndex]]
					else:
						next_key    = keyList[letterHash[followup]]
					next_finger = next_key[KEYINFO_INDEX_FINGER]
					next_base	= next_key[KEYINFO_INDEX_BASE]

#					if prev == prev_check and followup == next_check:
#						print 'manual %s%s: effort: %s, count: %s' % (prev, followup, effort, val)

					if prev != followup and followup != 'count':

						keyCount['distance'] += (prev_base + next_base) * val
						#~ if val > 0:
							#~ print "'%s': %s, '%s': %s > %s" % (prev, keyinfo[prev]['distance'], followup ,keyinfo[followup]['distance'], (keyinfo[prev]['distance'] + keyinfo[followup]['distance']))

						if (0 < prev_finger <= next_finger < 5) or (4 < prev_finger <= next_finger < 9):
							keyCount['sameHand'] += val
							if prev_finger == next_finger:
								keyCount['sameFinger'] += val
							if abs(prev_finger - next_finger) == 1:
								keyCount['neighFinger'] += val

	keyCount['distance'] += 2 * (
		keyCount['rshift'] * keyList[ letterHash['rshift'] ][KEYINFO_INDEX_BASE] +
		keyCount['lshift'] * keyList[ letterHash['lshift'] ][KEYINFO_INDEX_BASE] +
		keyCount['altgr'] * keyList[ letterHash['altgr'] ][KEYINFO_INDEX_BASE]
	)

	print "fingerUsage:\n%s" % fingerUsage
	print "rowUsage:\n%s" % rowUsage
	print "keyCount:\n%s" % keyCount

def get_real_indexes1(a, b):
	"""given two random layout key indexes, returns correct swap pair indexes (list of tuples)"""
	"""that takes account lower- and uppercase letters and symbols"""

	aIsLetter, bIsLetter = isLetterList[a], isLetterList[b]
	if not aIsLetter and not bIsLetter:
		return [(a, b)]

	aMode, bMode = a % 4, b % 4
	twist = False
	if aMode in [KEYINFO_MODE_NORMAL, KEYINFO_MODE_ALTGR] and bMode in [KEYINFO_MODE_SHIFT, KEYINFO_MODE_ALTGRSHIFT]:
		b -= 1
		bMode -= 1
		twist = True
	elif bMode in [KEYINFO_MODE_NORMAL, KEYINFO_MODE_ALTGR] and aMode in [KEYINFO_MODE_SHIFT, KEYINFO_MODE_ALTGRSHIFT]:
		a -= 1
		aMode -= 1
		twist = True

	if twist and not aIsLetter and bIsLetter:
		return [(a, b), (a + 1, b + 1), (b, b + 1)]
	elif twist and aIsLetter and not bIsLetter:
		return [(a, b), (a + 1, b + 1), (a, a + 1)]

	if aMode in [KEYINFO_MODE_NORMAL, KEYINFO_MODE_ALTGR] and \
		bMode in [KEYINFO_MODE_NORMAL, KEYINFO_MODE_ALTGR]:				
		return [(a, b), (a + 1, b + 1)]
	elif aMode in [KEYINFO_MODE_SHIFT, KEYINFO_MODE_ALTGRSHIFT] and \
		bMode in [KEYINFO_MODE_SHIFT, KEYINFO_MODE_ALTGRSHIFT]:
		return  [(a - 1, b - 1), (a, b)]

def get_real_indexes2(a, b):
	"""given two random layout key indexes, returns correct swap pair indexes (list of tuples)"""
	"""that takes account lower- and uppercase letters and symbols"""

	aIsLetter, bIsLetter = isLetterList[a], isLetterList[b]
	if not aIsLetter and not bIsLetter:
		return [(a, b)]

	if modeList[a] and not modeList[b]:
		b -= 1
		if not aIsLetter and bIsLetter:
#			return [(a, b), (a + 1, b + 1), (b, b + 1)]
			return [(a, b), (a + 1, b + 1)]
		elif aIsLetter and not bIsLetter:
			return [(a, b), (a + 1, b + 1), (a, a + 1)]
	elif not modeList[a] and modeList[b]:
		a -= 1
		if not aIsLetter and bIsLetter:
#			return [(a, b), (a + 1, b + 1), (b, b + 1)]
			return [(a, b), (a + 1, b + 1)]
		elif aIsLetter and not bIsLetter:
#			return [(a, b), (a + 1, b + 1), (a, a + 1)]
			return [(a, b), (a + 1, b + 1)]

	if modeList[a] and modeList[b]:
		return [(a, b), (a + 1, b + 1)]
	elif not modeList[a] and not modeList[b]:
		return  [(a - 1, b - 1), (a, b)]

def test_get_real_indexes():
	testPairs = [
		(0, 0), (0, 1), (0, 2), (0, 3), (1, 2), (1, 3), (2, 2), (2, 3),
		(0, 4), (0, 5), (0, 6), (0, 7), (1, 6), (1, 7), (2, 6), (2, 7),
		(4, 0), (5, 0), (6, 0), (7, 0), (6, 1), (7, 1), (6, 2), (7, 2)]

	for i in range(len(testPairs)):
		a, b = testPairs[i]
		print "pair:", a, b
		print "out:", get_real_indexes1(a, b)
		print "-----"

def pick_keys(n):
	a = random.randrange(n)
	b = random.randrange(n)
	while a == b:
		b = random.randrange(n)
	return (a, b)

def simulated_annealing(initEffort, iterations, indexToSymbol):
	global permutationList
	global isLetterList
	t0, k, p0 = configuration[CONFIGURATION_SCHEME][ANNEALING]
	keyListLength = len(permutationList)
	oldEffort = initEffort
	for i in xrange(iterations):
		swaps = get_real_indexes2(*pick_keys(keyListLength))
		newEffort = oldEffort
		for a, b in swaps:
#			print "%s <-> %s | %d <-> %d" % (indexToSymbol.get(permutationList[a],'!'), indexToSymbol.get(permutationList[b], '!'), a, b)
			newEffort = swap_effort3(a, b, newEffort)
			permutationList[a], permutationList[b] = permutationList[b], permutationList[a]
			isLetterList[a], isLetterList[b] = isLetterList[b], isLetterList[a]
		diffEffort = newEffort - oldEffort

		t = t0 * math.exp(-i * float(k)/iterations)
		p = p0 * math.exp(-math.fabs(diffEffort)/t)

		if diffEffort == 0 or (diffEffort > 0 and random.random() > p):
#			print '(failed)'
			for i in range(len(swaps) - 1, -1, -1):
				a, b = swaps[i]
#				print "%s <-> %s | %d <-> %d, i: %d" % (indexToSymbol.get(permutationList[a],'!'), indexToSymbol.get(permutationList[b], '!'), a, b, i)
				permutationList[a], permutationList[b] = permutationList[b], permutationList[a]
				isLetterList[a], isLetterList[b] = isLetterList[b], isLetterList[a]
		else:
#		if diffEffort < 0 or random.random() < p:
#			print 'passed'
#			print 'diff: %d, (old: %d, new: %d), t: %f, p: %f | %d/%d' % (diffEffort, oldEffort, newEffort, t, p, i, iterations)
			oldEffort = newEffort
			pass
#		print '==========='


def simulated_annealing2(initEffort, iterations, indexToSymbol):
	global permutationList
	global isLetterList
	global keysMatrix
	
	permList = permutationList[:]
	kMatrix = keysMatrix.copy()
	
	t0, k, p0 = configuration[CONFIGURATION_SCHEME][ANNEALING]
	keyListLength = len(permList)
	oldEffort = initEffort
	for i in xrange(iterations):
		swaps = get_real_indexes2(*pick_keys(keyListLength))
		newEffort = oldEffort
		for a, b in swaps:
#			print "%s <-> %s | %d <-> %d" % (indexToSymbol.get(permList[a],'!'), indexToSymbol.get(permList[b], '!'), a, b)
			newEffort = swap_effort1(a, b, newEffort)
			permList[a], permList[b] = permList[b], permList[a]
			isLetterList[a], isLetterList[b] = isLetterList[b], isLetterList[a]
		diffEffort = newEffort - oldEffort

		t = t0 * math.exp(-i * float(k)/iterations)
		p = p0 * math.exp(-math.fabs(diffEffort)/t)

		if diffEffort > 0 and random.random() > p:
#			print '(failed)'
			for i in range(len(swaps) - 1, -1, -1):
				a, b = swaps[i]
#				print "%s <-> %s | %d <-> %d, i: %d" % (indexToSymbol.get(permList[a],'!'), indexToSymbol.get(permList[b], '!'), a, b, i)
				kMatrix[[a, b], :] = kMatrix[[b, a], :]
				kMatrix[:, [a, b]] = kMatrix[:, [b, a]]
				permList[a], permList[b] = permList[b], permList[a]
				isLetterList[a], isLetterList[b] = isLetterList[b], isLetterList[a]
		else:
#			print 'diff: %d, (old: %d, new: %d), t: %f, p: %f | %d/%d' % (diffEffort, oldEffort, newEffort, t, p, i, iterations),
#		if diffEffort < 0 or random.random() < p:
#			print '(passed)'
			oldEffort = newEffort
			pass

#	diffArray.dump('diffEffort.trace')

	permutationList = permList
	keysMatrix = kMatrix

def print_keyboard(indexToSymbol):
#	for i in range(0, len(permutationList), 4):
	for i in range(12):
		print "%s%s%s%s " % (
			indexToSymbol.get(permutationList[4 * i], '_'),
			indexToSymbol.get(permutationList[4 * i + 1], '_'),
			indexToSymbol.get(permutationList[4 * i + 2], '_'),
			indexToSymbol.get(permutationList[4 * i + 3], '_')),
	print "\n ",
	for i in range(12, 24):
		print "%s%s%s%s " % (
			indexToSymbol.get(permutationList[4 * i], '_'),
			indexToSymbol.get(permutationList[4 * i + 1], '_'),
			indexToSymbol.get(permutationList[4 * i + 2], '_'),
			indexToSymbol.get(permutationList[4 * i + 3], '_')),
	print
	for i in range(24, 35):
		print "%s%s%s%s " % (
			indexToSymbol.get(permutationList[4 * i], '_'),
			indexToSymbol.get(permutationList[4 * i + 1], '_'),
			indexToSymbol.get(permutationList[4 * i + 2], '_'),
			indexToSymbol.get(permutationList[4 * i + 3], '_')),
	print

def get_small_layout(symbolList, symbolHash):
	smallSymbolList = []
	smallSymbolHash = LetterDict()
	indexToSymbolHash = symbolHash.flip()
	counter = 0
	for kid in range(len(symbolList)):
		key = symbolList[kid]
		if key[KEYINFO_INDEX_IS_SPOT] == True:
			smallSymbolList.append(key)
			if key[KEYINFO_INDEX_IS_RELOCATABLE] == True:
				lowLetter = indexToSymbolHash[kid]
#				lowLetter = keyHash[kid].decode('utf-8').lower().encode('utf-8')
				if lowLetter in symbolHash:
					if lowLetter == indexToSymbolHash[kid] and lowLetter in symbolHash:
						smallSymbolHash[indexToSymbolHash[kid]] = counter
			counter += 1

	return (smallSymbolList, smallSymbolHash)

def set_matrixes(stats, smallSymbolList, smallSymbolHash):
	global keysMatrix
	global statsMatrix
	global isLetterList
	
	symbolCount = len(smallSymbolList)

	keysMatrix = sp.empty([symbolCount, symbolCount], dtype = float)
	for i in range(symbolCount):
		for j in range(symbolCount):
			key1, key2 = smallSymbolList[i], smallSymbolList[j]
			keysMatrix[i, j] = total_effort(key1, key2)

	isLetterList = [False] * symbolCount
	statsMatrix = sp.zeros([symbolCount, symbolCount], dtype = long)
	for prev, row in stats.iteritems():
		if prev in smallSymbolHash:
			i = smallSymbolHash[prev]
#			print i, prev
			for followup, count in row.iteritems():
				if followup in smallSymbolHash:
					j = smallSymbolHash[followup]
					statsMatrix[i, j] = count
			isLetterList[i] = smallSymbolList[i][KEYINFO_INDEX_IS_LETTER]

def writeToFile(outFile, symbolToIndexHash, keyList):	
	
	for symbol1 in keyList:
		outFile.write("\t%s" % keys.KEY_TO_PRINT.get(symbol1, symbol1))
	outFile.write('\n')

	for symbol1 in keyList:
		outFile.write("%s" % keys.KEY_TO_PRINT.get(symbol1, symbol1))
		for symbol2 in keyList:
#			print "%s: %d, %s: %d" % (symbol1, 0, symbol2, 0)
			index1 = symbolToIndexHash[symbol1]
			index2 = symbolToIndexHash[symbol2]
			
			outFile.write("\t%s" % keysMatrix[index1][index2])
		outFile.write('\n')


def main(argv=None):
	
	if argv is None:
		argv = sys.argv[1:]
		
	global permutationList
	global modeList

	parser = OptionParser(usage = "usage: %prog [options] FILE1 ...", version="%prog 0.2")
	#parser.add_option('-q', '--quiet',  action="store_false", dest = 'verbose', default=True, help="don't print status messages to stdout")
	parser.add_option('-f', '--file', dest='out_filename', default=OUTFILE, metavar="FILE", help="output file name")

	(options, datafiles) = parser.parse_args(args = argv)
	stats = load_matrix(datafiles)

	# list of symbols from layout, hash of symbol to index number
	symbolList, symbolHash = get_layout(STARTMAP)
	# same as prevoious, only needed keys/letters
	smallSymbolList, smallSymbolHash = get_small_layout(symbolList, symbolHash)
	symbolCount = len(smallSymbolList)
	set_matrixes(stats, smallSymbolList, smallSymbolHash)
	print symbolCount
	smallIndexToSymbolHash = smallSymbolHash.flip()
#	print [{i:smallIndexToSymbolHash[i]} for i in sorted(smallIndexToSymbolHash)]
#	print smallSymbolList
	permutationList = range(symbolCount)
	calculate_counters(symbolHash, symbolList, smallSymbolHash, smallSymbolList, keysMatrix, stats)

	modeList = [True, False] * (symbolCount/2)
	
	effortMatrix = keysMatrix * statsMatrix
	initEffort = sp.sum(keysMatrix * statsMatrix)
	
#	keysMatrix.dump('keysMatrix.dump')
#	print keysMatrix
	
	effortFile = open('../inout/efforts.csv' , 'w')
	writeToFile(effortFile, smallSymbolHash, keys.RELOCATABLES)			
	effortFile.close()

	print 'Effort: %d/%d = %f' % (initEffort, sp.sum(statsMatrix), float(initEffort)/sp.sum(statsMatrix))
#	print statsMatrix
#	print permutationList
#	print isLetterList

#	benchmarkSwapping(initEffort, 1000, len(permutationList))
#	testSwapping(initEffort, permutationList)
#	test_get_real_indexes()

	startKeysMatrix = keysMatrix.copy()
	print_keyboard(smallIndexToSymbolHash)
	
	print '======================================================='
	start = time.time()
	simulated_annealing2(initEffort, 100, smallIndexToSymbolHash)
	end = time.time()
	print "Took time %s" % (myutil.hrtime(end - start))

	endEffort = sp.sum(statsMatrix * startKeysMatrix[permutationList, :][:, permutationList])
	print 'starting Effort:%15d\nfinished Effort:%15d' % (initEffort, endEffort)

	print_keyboard(smallIndexToSymbolHash)
	calculate_counters(symbolHash, symbolList, smallSymbolHash, smallSymbolList, startKeysMatrix, stats)

#	for prev, row in stats.iteritems():
#		print "\n'%s, %s': {" % (prev,keyinfo[prev])
#		for next, val in row.iteritems():
#			print "'%s': %s," % (next,val),
#		print "},",

#	sorted_buttons = sorted(buttoninfo, buttoncmp)
#	sorted_stats = sorted(stats.values(), charcmp)
#
#	for i in range(len(sorted_buttons)):
#		print "%s -> %s" % (sorted_stats[i]['letter'], sorted_buttons[i][2].decode('utf-8'))
#	for i in range(len(sorted_buttons), len(sorted_stats)):
#		print "%s ->" % (sorted_stats[i]['letter'])

#	buildGvGraph(stats)
	buildGraphVizGraph(stats)

if __name__ == "__main__":
	sys.exit(main())

