#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import time
#http://pypi.python.org/pypi/regex/
import regex as re
#import htmlentitydefs
import entities as ent
from optparse import OptionParser

OUTFILE = 'text.txt'
citation_count     = 0
quotation_count    = 0

quotation_pattern = re.compile('(?<=\s)"\s+(?![.,!?\(])([^"]+?)\s+"(?=\s)')
loose_qoute_pattern = re.compile(' " ')
post_cleanup_pattern = re.compile('(?<=\s)"\s')
post_replace_pattern = re.compile('“|”')
missing = dict()
debug_out = ''

def unescape(text):
	"""Removes HTML or XML character references
		and entities from a text string.
	@param text The HTML (or XML) source text.
	@return The plain text, as a Unicode string, if necessary.
	from Fredrik Lundh
	2008-01-03: input only unicode characters string.
	http://effbot.org/zone/re-sub.htm#unescape-html
	"""
	def fixup(m):
		global missing
		text = m.group(0)

		if text[:2] == "&#":
			## character reference

			if text[:3] == "&#x":
				return unichr(int(text[3:-1], 16)).encode('utf-8')
			else:
				return unichr(int(text[2:-1])).encode('utf-8')

		else:
			## named entity
			## reescape the reserved characters.??
			if text not in ent.ENTITIES:
				if text not in missing:
					missing[text] = 0
				missing[text] += 1
			return ent.ENTITIES.get(text,text)
			#try:
				#if text[1:-1] == "amp":
					#text = "&amp;amp;"
				#elif text[1:-1] == "gt":
					#text = "&amp;gt;"
				#elif text[1:-1] == "lt":
					#text = "&amp;lt;"
				#else:
					##print text[1:-1]
					#text = unichr(htmlentitydefs.name2codepoint[text[1:-1]])
			#except KeyError:
				#print "keyerror"
				#pass
		#return text # leave as is
	return re.sub("&#?\w+;", fixup, text)


SYMBOLS_LEFT = {
	'(': True,
	'[': True,
	'{': True,
	'“': True
	}

SYMBOLS_RIGHT = {
	')': True,
	']': True,
	'}': True,
	'.': True,
	',': True,
	':': True,
	';': True,
	'?': True,
	'!': True,
	'”': True
	}

def citation(m):
	"regex 'citation_pattern' callback function"
	global citation_count
	global debug_out
	citation_count += 1
	text = m.group(1)
	if text == None:
		text = m.group(2)
	debug_out += "cite: '%s'\n" % text
	text = quotation_pattern.sub(quotation, text)
#	print m.group(0,1,2)
	return '"' + text.strip() + '"'

def quotation(m):
	"regex 'quotation_pattern' callback function"
	global quotation_count
	global debug_out
	quotation_count += 1
	debug_out += "quote: '%s'\n" % m.group(1)
	return '"' + m.group(1) + '"'

def remove_excess_spaces(text):
	"removes unnecessary spaces around punctuation"

	for key in SYMBOLS_LEFT:
		remove_pattern = '\\'+str(key)+'\s*'
		text = re.sub(remove_pattern, key, text)

	for key in SYMBOLS_RIGHT:
		remove_pattern = '\s*\\'+str(key)
		text = re.sub(remove_pattern, key, text)
	return text

def main(argv=None):
	"name says it all"

	global debug_out
	if argv is None:
		argv = sys.argv[1:]

	parser = OptionParser(usage = "usage: %prog [options] FILE1 ...", version="%prog 0.1")
	#parser.add_option('-q', '--quiet',  action="store_false", dest = 'verbose', default=True, help="don't print status messages to stdout")
	parser.add_option('-f', '--file', dest='out_filename', default=OUTFILE, metavar="FILE", help="output file name")
	parser.add_option('--omit-author', action="store_true", dest='omit_author', default = False, help='omit the "author" tag content')
	parser.add_option('--omit-signature', action="store_true", dest='omit_signature', default = False, help='omit the signature paragraph')
	parser.add_option('--strip-http', action="store_true", dest="strip_http", default = False, help='strip away web addresses starting with "http://" (probably copy-pasted)')

	(options, datafiles) = parser.parse_args(args = argv)

	p_attr = "[^>]*"
	if options.omit_signature:
		p_attr = "(?:\s*|(?:\s*(?!rend)\w+='\w+')+|(?:\s*rend='(?!signature)\w+'))"

	paragraph_pattern_string = "<p"+p_attr+">\s*(.*?)\s*</p>"
	if options.omit_author:
		paragraph_pattern_string = "<p"+p_attr+">\s*(<s>.*?</s>)\s*</p>"

	paragraph_pattern = re.compile(paragraph_pattern_string)

	remove_pattern_string = "</?(?:hi|lb|gap)[^>]*>"
	if options.strip_http:
		remove_pattern_string = "</?(?:hi|lb|gap)[^>]*>|https?\s?:\s?//\s?[a-zA-Z0-9_\.%=&-]*(?:\s?[\?\./]\s?[a-zA-Z0-9_\.%=&-]*)*"
	remove_pattern = re.compile(remove_pattern_string)

	citation_pattern = re.compile('''
	(?<=(?:<s>|[.!?])\s)"           #sentence starts
	(\s
	(?![.,!?\(])                    #can not start with punctuation
	[^"]*?
	(?:"[^"]+?"[^"]*?)*?            #nested quotes must be paired
	(?:"[^"]*?[,.!?]\s"|[,.!?])\s+) #nested or main quote must end with punctuation
	"(?=\s)|
	(?<=[:,]\s)"                    #citation continues
	(\s
	(?![.,!?\(])                    #can not start with punctuation
	[^"]*?
	(?:"[^"]+?"[^"]*?)*?
	(?:"[^"]*?[,.!?]\s"|[,.!?])\s+) #nested or main quote must end with punctuation
	"(?=\s)
	''', re.DOTALL | re.VERBOSE)

#	(?<=(?:<s>|[.!?])\s)"(\s(?![.,!?\(])[^"]*?(?:"[^"]+?"[^"]*?)*?(?:"[^"]*?[,.!?]\s"|[,.!?])\s+)"(?=\s)
# |(?<=[:,]\s)"(\s(?![.,!?\(])[^"]*?(?:"[^"]+?"[^"]*?)*?(?:"[^"]*?[,.!?]\s"|[,.!?])\s+)"(?=\s)
	sentence_pattern = re.compile('<s>\s*.*?\s*</s>')
	cleanup_pattern = re.compile('<(?:s|head)> ?|(?<=")</s> | ?</(?:s|head)>')

	outfile = open(options.out_filename, 'w', 5000000)
#	junk = open('junk.txt', 'w')
	loose = open('loosequotes.txt', 'w')

	time_start = time.time()
	clock_start = time.clock()

	count_patterns = []
	section_patterns = []
	loose_quotes_count = 0
	output = ''

	for i in range(5):
		count_patterns.append(re.compile("<tagusage gi=div"+str(i)+" occurs=(\d+)></tagusage>"))
		section_patterns.append(re.compile("<div"+str(i)+"[^>]*>\s*(<head>\s+.*?\s+</head>)?(.*?)</div"+str(i)+">",re.DOTALL))

	for filename in datafiles:
		f = open(filename,"r")
		print filename
		content = f.read()

		content = unescape(content)
		content = remove_pattern.sub("", content)

#Want to identify articles. These are the sections/div's with highest count.
#Case if section count is given
		maximum = -1
		lowdiv = -1
		num = 0
		sections = False
		for i in range(5):
			matches = count_patterns[i].search(content)
			if matches != None:
				num = int(matches.groups()[0])
				if  num > maximum:
					maximum = num
					lowdiv = i
			else:
				break
		#~ print lowdiv

#Case if section count is not given, we'll do the counting here
		if lowdiv < 0:
			for i in range(5):
				matches = section_patterns[i].findall(content)
				num = len(matches)
				if num > maximum:
					maximum = len(matches)
					lowdiv = i
					sections = matches
				elif num == 0:
					break

			#~ print lowdiv
#Find the appropriate article sections if not done already
		if lowdiv >= 0:
			if sections == False:
				sections = section_patterns[lowdiv].findall(content)
		else:
			sections = (content,)


		for sect in sections:
			debug_out = ''
			out_section = ''
			temp_section = sect[0] + sect[1]
			if sect[0] != '':
				out_section += quotation_pattern.sub(quotation, sect[0]) + "\n"
			par_matches = paragraph_pattern.findall(sect[1])
			for par in par_matches:
				par = citation_pattern.sub(citation, par)
				matches = sentence_pattern.findall(par)
#				print "matches:", matches
#				matches = [citation_pattern.sub(citation, s) for s in matches]
				if len(matches) > 0:
					out_par = " ".join(matches) + "\n"
#					out_par = citation_pattern.sub(citation, out_par)
					out_par = quotation_pattern.sub(quotation, out_par)
					out_section += out_par

			out_section = cleanup_pattern.sub('', out_section)
			out_section = citation_pattern.sub(citation, out_section)
			out_section = quotation_pattern.sub(quotation, out_section)
			section_loose_quotes_count = len(loose_qoute_pattern.findall(out_section))

#this gives an exception if the input does not have the proper encoding, otherwise the overall output might be junk.
#~			try:
			out_section = remove_excess_spaces(out_section.decode('utf-8').encode('utf-8'))
#~			except:
#~				print >> junk, out_content

			if (section_loose_quotes_count > 0):
				print >> loose, temp_section
				print >> loose, out_section
				print >> loose, debug_out
				loose_quotes_count += section_loose_quotes_count				
			# remove dangling (not in pair) qoutes
			out_section = post_cleanup_pattern.sub('', out_section)
			
			output += out_section + "\n"
		f.close()	
	

	# replace proper qoutes
	output = post_replace_pattern.sub('"', output)
	print >> outfile, output

	print "citations: %s, quotations: %s, loose quotes: %s" % (citation_count, quotation_count, loose_quotes_count)
	outfile.close()
#	junk.close()
	loose.close()

	print missing
	time_end = time.time()
	clock_end = time.clock()
	print "Took time %.4f minutes, clock %.4f" % ((time_end - time_start) /60., clock_end - clock_start)

if __name__ == "__main__":
	sys.exit(main())
