#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import time
from optparse import OptionParser

import modifier as mod
import keys
import myutil
		
class Stat:

	def __init__(self):
		self.table = {}

	def __len(self):
		return len(self.table)

	def __getitem__(self, key):
		if not isinstance(key, tuple):
			return self.table.get(key,0)

		leveler = self.table
		for a in key[:-1]:
			if a not in leveler:
				return 0
			leveler = leveler[a]
		return leveler.get(key[-1],0)

	def __setitem__(self, key, value):
		if not isinstance(key, tuple):
			self.table[key] = value
		else:
			leveler = self.table
			for a in key[:-1]:
				leveler.setdefault(a, dict())
				leveler = leveler[a]
			key = key[-1]
			leveler[key]  = value


	def __contains__(self,k):
		return self.table.has_key(k)

	def __iter__(self):
		return iter(self.table)

	def __str__(self):
		return str(self.table)



OUTFILE = 'stats.csv'

def main(argv=None):


	if argv is None:
		argv = sys.argv[1:]

	parser = OptionParser(usage = "usage: %prog [options] FILE1 ...", version="%prog 0.1")
	#parser.add_option('-q', '--quiet',  action="store_false", dest = 'verbose', default=True, help="don't print status messages to stdout")
	parser.add_option('-f', '--file', dest='out_filename', default=OUTFILE, metavar="FILE", help="output file name")

	(options, datafiles) = parser.parse_args(args = argv)
	
	COUNT = 'count'
	stat = dict()
	stat[COUNT] = dict()
	for k1 in keys.KEYS:
		stat[k1] = dict()
		for k2 in keys.KEYS:
			stat[k1][k2] = 0
		stat[COUNT][k1] = 0
		
	linecount = 0
	time_start = time.time()
	keyHash = dict.fromkeys(keys.KEYS)
	
#	line_modify = mod.Modifier(True)
#	line_modify.registerModifier(str.decode, 'utf-8')
	str_modify = mod.Modifier(True)
	#~ str_modify.registerModifier(unicode.lower)
	str_modify.registerModifier(unicode.encode, 'utf-8')

	for filename in datafiles:
		print filename
		f = open(filename, 'r')
		line = f.readline()
		while line:
			try:
				line = line.decode('utf-8')
				pass
			except:
				print line
				continue
			prev = str_modify(line[0])
			if prev in keyHash:
				if '\n' in keyHash:
					stat['\n'][prev] += 1
				stat[COUNT][prev] += 1

			for letter in line[1:]:
				letter = str_modify(letter)
				
				if letter in keyHash: 
					stat[COUNT][letter] += 1
					
					if prev in keyHash:
						stat[prev][letter] += 1
					#	if prev != letter:
					#		stat[letter][prev] += 1
				prev = letter

			linecount += 1

			if linecount % 10000 == 0:
				sys.stdout.write('|')
				sys.stdout.flush()
			
			#if linecount > 300000:
				#break
			line = f.readline()
		sys.stdout.write('\n')
		f.close()

	time_end = time.time()
	print "Took time %s" % (myutil.hrtime(time_end - time_start))
#	keyHash = stat[COUNT].keyHash()
#	keyHash.sort()

	for key in keyHash:
		print "'%s': %s," % (key, stat[COUNT][key]),

	out = open(options.out_filename, 'w')

	for k1 in keys.KEYS:
		out.write("\t%s" % keys.KEY_TO_PRINT.get(k1, k1))
	out.write("\t%s\n" % COUNT)

	for k1 in keys.KEYS:
		out.write("%s" % keys.KEY_TO_PRINT.get(k1, k1))
		for k2 in keys.KEYS:
			out.write("\t%s" % stat[k1][k2])
		#	out.write("\t%.2f" % math.log(stat[k1][k2] | 1,2))
		out.write("\t%s\n" % stat[COUNT][k1])
		
	out.close()


if __name__ == "__main__":
	sys.exit(main())
