#!/usr/bin/env python
# -*- coding: utf-8 -*-

import scipy as sp
import os
import sys
import time
import re
import hashlib
import gc
from optparse import OptionParser
import myutil

CONCURRENT_MAX = 3
BIN_OUTFILE_SUFFIX = '.bin.csv'
EXCERPT_SUFFIX = '.excerpt.txt'
CHUNK_LINES = 20000
REALM = 'Estonian keyboard statistics aggregator'


def main(argv=None):
	if argv is None:
		argv = sys.argv[1:]
		
	parser = OptionParser(usage = "usage: %prog [options] FILE1 ...", version = "%prog 0.1")
	parser.add_option('-l', '--linenumber-size', dest = 'line_size', choices = [str(i) for i in range(1, 8)], default = 6, help = "number of characters that the line number takes")
	parser.add_option('-s', '--bin-suffix', dest = 'bin_suffix', default = BIN_OUTFILE_SUFFIX, help = "bitvector file suffix")
	parser.add_option('-e', '--excerpt-suffix', dest = 'excerpt_suffix', default = EXCERPT_SUFFIX, help = "result file suffix")
	parser.add_option('-c', '--concurrency', dest = 'concurrency', choices = [str(i) for i in range(1, 11)], default = CONCURRENT_MAX, help = "consider this many rows at once, value above 1 speeds up processing considerably")

	(options, textDataFiles) = parser.parse_args(args = argv)
	options.concurrency = int(options.concurrency)
	options.line_size = int(options.line_size)
	
		
	time_start = time.time()
	
	print "preparing ..."
	for filename in textDataFiles:
		inputTextFileName = os.path.basename(filename)
		inputTextFilePath = os.path.dirname(filename)
		inputTextFileNoExt = os.path.splitext(inputTextFileName)[0]		
		inputBinFile = inputTextFilePath + '/' + inputTextFileNoExt + options.bin_suffix
		
		bitmapFile = open(inputBinFile, 'r')
		signatureLine = bitmapFile.readline()
		matches = re.match("^([\d ]{%s})([01]*)$" % options.line_size, signatureLine)
		if matches != None:
			numberOfLines = int(matches.group(1))
			vectorLength = len(matches.group(2))
		
		print numberOfLines, vectorLength

		linesProcessed = 1
		chunkCounter = 0
		
		chunkCoversList = sp.array([], dtype = int)
		chunkCoversArray = sp.empty([0, vectorLength], dtype = bool)
		
		while linesProcessed < numberOfLines + 1:
			print "loading ... "
			inputData = sp.genfromtxt(inputBinFile, delimiter = [options.line_size] + [1] * vectorLength, 
											usecols = range(1, vectorLength + 1), dtype = sp.uint8, skip_header = linesProcessed,
											skip_footer = (numberOfLines - linesProcessed - CHUNK_LINES ))
			
			print "processing ..."	
			
			chunkSetCover = calculateCover(inputData, vectorLength, concurrency = options.concurrency)
			chunkCoversList = sp.append(chunkCoversList, chunkSetCover + chunkCounter * CHUNK_LINES, axis = 0)																			
#			print chunkCoversList
			
			for binLine in chunkSetCover:
				chunkCoversArray = sp.append(chunkCoversArray, inputData[binLine, :].reshape(1, vectorLength), axis = 0)			
												
			chunkCounter += 1
			linesProcessed += CHUNK_LINES
			print "%d/%d" % (linesProcessed, numberOfLines + 1)	
			
		inputData = None
		chunkSetCover = None
		del inputData
		del chunkSetCover
		gc.collect()
			
		print "processing file cover"
		fileSetCover = calculateCover(chunkCoversArray, vectorLength, 1)
#		print "compare", (len(chunkCoversList) == len(chunkSetCover)), len(chunkCoversList)
		print ""
		binCover = []
		
		for lineIndex in fileSetCover:
			binCover.append(chunkCoversList[lineIndex])
			
#		for lineIndex in chunkSetCover:
#			binCover.append(lineIndex)
		print "preparing output"
			
		textLineNumbers = sp.genfromtxt(inputBinFile, delimiter = [options.line_size] + [1] * vectorLength, 
								usecols = 0, dtype = int, skip_header = 1)
		
		
		textLinesCover = textLineNumbers[binCover]
		print "len, checksum", len(textLinesCover), sum(textLinesCover)
		textLinesCover.sort()
		print textLinesCover
		
		f = open(filename, 'r')
		coverFileHandle = open(inputTextFilePath + '/' + inputTextFileNoExt + options.excerpt_suffix, 'w')		
		writeCoverLines(textLinesCover, f, coverFileHandle)	
		f.close()	
		coverFileHandle.close()			
				
	time_end = time.time()
	print "Took time %s" % (myutil.hrtime(time_end - time_start))	
	
def progressUpdate():
	sys.stdout.write('|')
	sys.stdout.flush()		
	
def calculateCover(binData, ylen, concurrency = CONCURRENT_MAX, indexAdd = 0, progresser = progressUpdate ):
	remainingColumnsList = sp.array([1] * ylen, dtype = int)
	sums = sp.sum(binData * remainingColumnsList, axis = 1)

# 		print "max table line number", maxTableLineNumber
	chunkSetCover = []
	
	while sp.count_nonzero(sums) > 0:
		currentMax = []
		
		for i in range(concurrency):
			argmax = sp.argmax(sums)
#			print argmax, sp.count_nonzero(sums)			
			if sums[argmax] <= 0: break
			currentMax.append(argmax)
			sums[currentMax[i]] = 0
					
		for maxIndex in currentMax:
			chunkSetCover.append(maxIndex + indexAdd)			
			remainingColumnsList = sp.where(binData[maxIndex, :] > 0, 0, remainingColumnsList)
				
		sums = sp.sum(binData * remainingColumnsList, axis = 1)

		progresser()
		
	return sp.array(chunkSetCover)
		
		
def writeCoverLines(coverList, fromfile, tofile):
	coverLinesIndex = 0
	lineCounter = 0
	coverLinesLength = len(coverList)
	textLine = fromfile.readline()
	while coverLinesIndex < coverLinesLength:
		while lineCounter < coverList[coverLinesIndex]:
			textLine = fromfile.readline()
			lineCounter += 1
		tofile.write("%s\t%s" % (hashlib.md5(textLine + '::' + REALM).hexdigest(), textLine))
		coverLinesIndex += 1		

if __name__ == "__main__":
	sys.exit(main())
