#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import time
from optparse import OptionParser
from operator import itemgetter
import sortedcollection as sortcol
import modifier as mod
import keys

TEMP_OUTFILE = 'stats.temp.csv'
OUTFILE = 'stats.temp.txt'
PROGRESS_INTERVAL = 2000
MINIMUM_TEXTLINE_LENGTH = 100
# MAX_LINES_PER_KEYPAIR = 100
NUMBER_OF_LINES_PER_PASS = 3

def main(argv=None):


	if argv is None:
		argv = sys.argv[1:]

	parser = OptionParser(usage = "usage: %prog [options] FILE1 ...", version = "%prog 0.1")
	#parser.add_option('-t', '--type', dest='out_type', choices = [1,2,3], type = int, default = 1, help = "1 - bitvector, 2 - tabulated, 3 - text")
	#parser.add_option('-q', '--quiet',  action="store_false", dest = 'verbose', default=True, help="don't print status messages to stdout")
	parser.add_option('-f', '--file', dest='tempout_filename', default=TEMP_OUTFILE, metavar="FILE", help="intermediate output file name")
	parser.add_option('-o', '--out', dest='out_filename', default=OUTFILE, metavar="FILE", help="output file name")

	(options, datafiles) = parser.parse_args(args = argv)
	
# 	print "STAGE 1"
# 	pairLineTracker = dict()
	usedKeyPairs = {}
	stat = {}
	for k1 in keys.KEYS:
		stat[k1] = dict()
		usedKeyPairs[k1] = {}
# 		pairLineTracker[k1] = dict()
		for k2 in keys.KEYS:
			stat[k1][k2] = 0
			usedKeyPairs[k1][k2] = False
# 			pairLineTracker[k1][k2] = sortcol.SortedCollection(key=itemgetter(1))

	time_start = time.time()
	keys = dict.fromkeys(keys.KEYS)
	
#	line_modify = mod.Modifier(True)
#	line_modify.registerModifier(str.decode, 'utf-8')
	str_modify = mod.Modifier(True)
	#~ str_modify.registerModifier(unicode.lower)
	str_modify.registerModifier(unicode.encode, 'utf-8')	
		
# 	out = open(options.tempout_filename, 'w')
# 	out.write("No")
# 	for k1 in keys.KEYS:
# 		for k2 in keys.KEYS:
# 			out.write("\t%s+%s" % (keys.KEY_TO_PRINT.get(k1, k1), keys.KEY_TO_PRINT.get(k2, k2)))
# 	out.write("\t%s" % "distinct pairs")
# 	out.write("\n")

	coverLines = []
	
	currentBestKeyPairCounts = sortcol.SortedCollection(key=lambda x : len(x[1]))	
	currentBestKeyPairCounts.insert((None, []))	
	totalCoveredPairsCount = 0
	coveredPairsCount = 1
	
	stageCounter = 0	
#	continueLoop = True
	
	while coveredPairsCount > 0 :
# 	while continueLoop :
#		continueLoop = False									
		stageCounter += 1
		currentBestKeyPairCounts.clear()
		currentBestKeyPairCounts.insert((None, []))
		tickcount = 0
		
		for filename in datafiles[:1]:
						
			f = open(filename, 'r')
			line = f.readline()
			linecount = 0
			while line:
				
				distinctKeyPairList = []
				
				try:
					line = line.decode('utf-8')
					pass
				except:
					print line
					continue
				prev = str_modify(line[0])
				
				stat = dict()
				for k1 in keys.KEYS:
					stat[k1] = dict()
					for k2 in keys.KEYS:
						stat[k1][k2] = 0		

				if prev in keys:
					if '\n' in keys:
						stat['\n'][prev] += 1
	
				for nex in line[1:]:
					nex = str_modify(nex)
					
					if prev in keys and nex in keys: 					
						stat[prev][nex] += 1
									
					prev = nex			
	
				if linecount // PROGRESS_INTERVAL > tickcount :
					tickcount = linecount // PROGRESS_INTERVAL
					sys.stdout.write('|')
					sys.stdout.flush()
				
	# 			if linecount > 3000000:
	# 				break
										
# 				out.write("%6d" % linecount)

				for k1 in keys.KEYS:		
					for k2 in keys.KEYS:
						
						if stat[k1][k2] > 0 and usedKeyPairs[k1][k2] == False:
							distinctKeyPairList.append((k1, k2))						
						
	# 					if stat[k1][k2] > 0:
	# 						pairsFoundSoFar = len(pairLineTracker[k1][k2])				
	# 							
	# 						if pairsFoundSoFar >= MAX_LINES_PER_KEYPAIR :
	# 							if stat[k1][k2] > pairLineTracker[k1][k2][0][1] : 						 
	# 								del pairLineTracker[k1][k2][0]				 
	# 								pairLineTracker[k1][k2].insert((linecount, stat[k1][k2]))							
	# 						else :
	# 							pairLineTracker[k1][k2].insert((linecount, stat[k1][k2]))

# 						out.write("\t%s" % stat[k1][k2])
# 						out.write("%d" % bool(stat[k1][k2]))
# 				out.write("\t%d" % len(distinctKeyPairList))
# 				out.write("\n")
								
				if len(distinctKeyPairList) > len(currentBestKeyPairCounts[0][1]):						
					if len(currentBestKeyPairCounts) >= NUMBER_OF_LINES_PER_PASS :
						del currentBestKeyPairCounts[0]
						currentBestKeyPairCounts.insert((linecount, distinctKeyPairList))							
					else :
						currentBestKeyPairCounts.insert((linecount, distinctKeyPairList))

				line = f.readline()
				linecount += 1
				while line and len(line) < MINIMUM_TEXTLINE_LENGTH:
					line = f.readline()
					linecount += 1	

# 			sys.stdout.write('\n')
			f.close()
		
		coveredPairsCount = 0
		for lineNumber, dpairList in currentBestKeyPairCounts:
			if not lineNumber == None:
				coverLines.append(lineNumber)
			for k1, k2 in dpairList:
				if usedKeyPairs[k1][k2] == False:
					usedKeyPairs[k1][k2] = True
					coveredPairsCount += 1
					
#		print "\nSTAGE %d\n%s additional pairs covered (%s/%s)" % (stageCounter, coveredPairsCount, coveredPairsCount + totalCoveredPairsCount, len(KEYS) * len(KEYS))
		totalCoveredPairsCount += coveredPairsCount
	
# 	out.close()
							
	coverLines.sort()
	print sorted(coverLines)	
	
	solutionout = open(options.out_filename, 'w')
	
	for filename in datafiles[:1]:	
		f = open(filename, 'r')
		
		coverLinesIndex = 0
		lineCounter = 0
		coverLinesLength = len(coverLines)
		textLine = f.readline()
		while coverLinesIndex < coverLinesLength:
			while lineCounter < coverLines[coverLinesIndex]:
				textLine = f.readline()
				lineCounter += 1
			solutionout.write("%s" % textLine)
			coverLinesIndex += 1
			
	solutionout.close()

# 	print "STAGE 2"
# 	neededRows = set()
# 	for k1, vv in pairLineTracker.iteritems():
# 		for k2, pairList in vv.iteritems():
# 			for rowNumber, pairCount in pairList:
# 				neededRows.add(rowNumber)
				
#	print linecount, len(neededRows)
					
	#b = sp.array([pairLineTracker[k1][k2][1] for k1 in keys.KEYS for k2 in keys.KEYS]).reshape(len(keys.KEYS), len(keys.KEYS))

	time_end = time.time()
	print "Took time %.2f minutes" % ((time_end - time_start) /60.)
	
	#print pairLineTracker

if __name__ == "__main__":
	sys.exit(main())
