#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import time
import os
import re
import myutil
import tempfile
import modifier as mod
from optparse import OptionParser
import keys

BIN_OUTFILE_SUFFIX = '.bin.csv'
PROGRESS_INTERVAL = 5000
MINIMUM_TEXTLINE_LENGTH = 100
MAXIMUM_TEXTLINE_LENGTH = 900
SYMBOLS_ALLOWED_PATTERN = re.compile(u'^[0-9a-zA-ZõäöüšžÕÄÖÜŠŽ,;.:?!"()\[\]/\\%&@€£=+*_<>\$\'\-\s]+$')

def main(argv=None):

	if argv is None:
		argv = sys.argv[1:]

	parser = OptionParser(usage = "usage: %prog [options] FILE1 ...", version = "%prog 0.1")
	parser.add_option('-s', '--suffix', dest = 'bin_suffix', default = BIN_OUTFILE_SUFFIX, help="suffix of the output file")	

	(options, datafiles) = parser.parse_args(args = argv)
	
	stat = {}
	for k1 in keys.KEYS:
		stat[k1] = dict()
		for k2 in keys.KEYS:
			stat[k1][k2] = 0


	time_start = time.time()
	keysMap = dict.fromkeys(keys.KEYS)
	
#	line_modify = mod.Modifier(True)
#	line_modify.registerModifier(str.decode, 'utf-8')
	str_modify = mod.Modifier(True)
	#~ str_modify.registerModifier(unicode.lower)
	str_modify.registerModifier(unicode.encode, 'utf-8')
			
	tickcount = 0
	
	for filename in datafiles:
					
		inputFileName = os.path.basename(filename)
		inputFilePath  = os.path.dirname(filename)
		purenoextFileName = os.path.splitext(inputFileName)[0]		
		outputFile = inputFilePath + '/' + purenoextFileName + options.bin_suffix
		
		tempout = tempfile.TemporaryFile()
											
		f = open(filename, 'r')
		line = f.readline()
		line = line.decode('utf-8')
		inputLineCount = 0
		outputLineCount = 0
		while True:
			
			while line and (len(line) < MINIMUM_TEXTLINE_LENGTH or MAXIMUM_TEXTLINE_LENGTH < len(line) or SYMBOLS_ALLOWED_PATTERN.match(line) == None):	
#				print line
#				print SYMBOLS_ALLOWED_PATTERN.match(line).group(0)
				line = f.readline()
				line = line.decode('utf-8')
				inputLineCount += 1
			if not line: break			
						
			prev = str_modify(line[0])
			
			stat = dict()
			for k1 in keys.KEYS:
				stat[k1] = dict()
				for k2 in keys.KEYS:
					stat[k1][k2] = 0		

			if prev in keysMap:
				if '\n' in keysMap:
					stat['\n'][prev] += 1

			for nex in line[1:]:
				nex = str_modify(nex)
				
				if prev in keysMap and nex in keysMap: 					
					stat[prev][nex] += 1
								
				prev = nex			

			if inputLineCount // PROGRESS_INTERVAL > tickcount :
				tickcount = inputLineCount // PROGRESS_INTERVAL
				sys.stdout.write('|')
				sys.stdout.flush()
									
			tempout.write("%6d" % inputLineCount)

			for k1 in keys.KEYS:		
				for k2 in keys.KEYS:																	
					tempout.write("%d" % bool(stat[k1][k2]))
			
			tempout.write("\n")
			outputLineCount += 1
							
			line = f.readline()
			line = line.decode('utf-8')
			inputLineCount += 1

		
		f.close()			
		print ""		
				
		tempout.seek(0)
		print "Writing to %s" % outputFile
		out = open(outputFile, 'w')
		out.write("%6d%s\n" % (outputLineCount, "0" * (len(keys.KEYS) * len(keys.KEYS)) ))
		for line in tempout:
			out.write(line)		
		out.close()				
		tempout.close()
							
	time_end = time.time()
	print "Took time %s" % (myutil.hrtime(time_end - time_start))
	
	#print pairLineTracker

if __name__ == "__main__":
	sys.exit(main())
