#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
import re
import os
import tarfile
import shutil
import datetime
import minimize
import argparse
import urllib2
import cStringIO as StringIO
import subprocess
import urlparse

def findVersion(indir):	
	config = file(os.path.join(indir,'backend','Pianos4.php'),'r')
	regexp = r"'version' => '([0-9\-\.alphbetrc]+)'"
	for line in config:
		res = re.search(regexp, line)
		if res is not None and len(res.groups()) == 1:
			return res.groups()[0]
	return None


def copyFilteredFile(src, dst, repl_lines={}):
	sec = None
	r = re.compile(r'(BEGIN|END)-BUILDSCRIPT-SECTION-([\d+?])')
	repl_written = {}
	repl_found = {}
	eol_symbol = None
	lineno = 0
	with file(src,'rb') as srcf:
		with file(dst,'wb') as dstf:
			for line in srcf:
				lineno += 1
				if eol_symbol is None:
					eol_symbol = line[len(line.strip()):]
				m = r.search(line)
				if m:
					if m.group(1) == 'BEGIN':
						if sec is not None:
							print "WARNING: in %s(%d) : found BEGIN build script tag while already in section %d, ignoring tag" % (src,lineno,sec)
						else:
							nsec = int(m.group(2))
							if nsec in repl_found:
								print "WARNING: in %s(%d) : found duplicate BEGIN build script tag for section %d, ignoring tag" % (src,lineno,nsec)
							else:
								repl_found[nsec] = True
								sec = nsec
					elif m.group(1) == 'END':
						if sec is None:
							print "WARNING: in %s(%d) : found END build script tag while not in a section, ignoring tag" % (src,lineno)
						else:
							nsec = int(m.group(2))
							if nsec != sec:
								print "WARNING: in %s(%d) : found END build script tag for section %d while I was in section %d, ignoring tag" % (src,lineno,nsec,sec)
							else:
								sec = None
								continue
					else:
						print "WARNING: in %s(%d) : unknown build script tag: %s" % (src,lineno,m.group(1))
				
				if sec is None:
					dstf.write(line)
				elif sec not in repl_written:
					if sec not in repl_lines:
						print "WARNING: in %s(%d) : found build script section %d, but no replacement lines were given for it" % (src,lineno,sec)
					else:
						dstf.writelines((x+(eol_symbol or '\n') for x in repl_lines[sec]))
					repl_written[sec] = True

					
def resolveDependency(item, url_abs=None, url_rel=None, docroot_abs=None, docroot_rel=None):
	content = None
	resolvers = []
	print "  %s (at %s)" % (item['key'], item['src']),
	if item['isabs']:
		if docroot_abs:
			resolvers.append(('file', os.path.join(docroot_abs, item['src'][1:])))
		if url_abs:
			resolvers.append(('url', url_abs + '/' + item['src']))
		if not resolvers:
			print "WARNING: absolute path '%s' has no resolver" % item['src']
	else:
		if docroot_rel:
			resolvers.append(('file', docroot_rel))
		if url_rel:
			resolvers.append(('url', url_rel))
		if not resolvers:
			print "WARNING: relative path '%s' has no resolver" % item['src']

	trials = []
	for type,name in resolvers:
		if type == 'file':
			#print "trying file",name
			try:
				with file(name,'rb') as f:
					content = f.read()
				print "[from disk]"
			except IOError:
				trials.append('direct file access: %s' % name)
			
		elif type == 'url':
			#print "trying url",name
			try:
				f = urllib2.urlopen(name)
				content = f.read()
				f.close()
				print "[from url]"
			except IOError:
				trials.append('url file access: %s' % name)
					
		if content is not None:
			return content
			
	print "WARNING: no way to resolve %s, tried %s" % (item['src'], ', '.join(trials) if trials else 'nothing')
	return None
		
def gatherDependencies(indir, resolvers):
	
	def getContentType(x):
		if key[:6] == 'jslib_':
			return 'js'
		elif key[:7] == 'csslib_':
			return 'css'
		return None
	
	cssurlre = re.compile(r'url\((.+?)\)')
	deps = {}
	
	main = os.path.join(indir, 'backend','Pianos4.php')
	r = re.compile(r"^'([\w\-]+)'(?:\s*)=>(?:\s*)'(.+)'(?:\s*),") # matches 'some_variable' => 'some_string_value'
	with file(main, 'r') as f:
		for line in (x.strip() for x in f):
			m = r.match(line)
			if m:
				key, value = m.group(1), m.group(2)
				ctype = getContentType(key)
				if ctype:
					deps[key] = {'key': key, 'ctype': ctype, 'dst':value}
	
	local = os.path.join(indir, 'backend', 'settings-local.php')
	r = re.compile(r"^\s*Application::set\(\s*'([\w\-]+)'(?:\s*),\s*(?:\s*)'(.+)'\s*\);$") # matches Application::set( 'some_variable', 'some_string_value' );
	with file(local, 'r') as f:
		for line in (x.strip() for x in f):
			m = r.match(line)
			if m:
				key, value = m.group(1), m.group(2)
				ctype = getContentType(key)
				if ctype:
					if key not in deps:
						print "WARNING: setting '%s' found in local-settings.php is unknown to main.php, ignored" % key
						continue
					deps[key]['src'] = value
	
	solved = []
	sub = {'abs':{},'rel':{}}
	depsv = deps.values()
	while depsv:
		dep = depsv.pop()

		if 'src' not in dep:
			print "WARNING: setting '%s' defined in main.php was not specified in local-settings.php" % dep['key']
			return None

		dep['isabs'] = dep['src'][:1] == '/'

		content = resolveDependency(dep, **resolvers)
		if content is None:
			print "WARNING: could not resolve dependency '%s' (pointing to '%s')" % (dep['key'],dep['src'])
			return None
		dep['content'] = content
		solved.append(dep)

		if dep['ctype'] == 'css':
			m = cssurlre.findall(dep['content'])
			
			for subdep in m:
				#print "INFO: following dep in css '%s' named '%s'" % (dep['key'], subdep)
				url = urlparse.urlparse(subdep)
				if url.scheme or url.netloc:
					print "WARNING: in CSS file '%', external (on remote server) dependency '%s'. This resource will not be added to the package." % (k, subdep)
					continue

				relpath = url.path \
					+ ('?'+url.query if url.query else '') \
					+ ('#'+url.fragment if url.fragment else '')
				
				if not relpath:
					print "WARNING: in CSS file '%', bad url in external dependency '%s'" % (k, subdep)
					continue
					
				fromdir = os.path.split(dep['src'])[0]
				frompath = fromdir + ('/' if relpath[:1] != '/' else '') + relpath
				isabs = frompath[:1] == '/'
				bin = 'abs' if isabs else 'rel'
				if frompath in sub[bin]:
					# already in seen
					continue
				
				todir = os.path.split(dep['dst'])[0]
				topath = os.path.join(todir,relpath)
				
				# print "d/l",relpath
				# print "  load it next to",dep['src'],"in",fromdir
				# print "  store it next to",dep['dst'],"in",todir
				# print "  so it is",frompath
				# print "  written to",topath
				
				item = {'key': relpath+'@'+dep['key'], 'ctype': None, 'isabs': isabs, 'src': frompath, 'dst': topath}
				depsv.append(item)
				sub[bin][frompath] = True
				
	return solved


def get_pianos_sql_udb(path='.'):
	try:
		import udb
	except ImportError,err:
		if not path in sys.path:
			sys.path.append(path)
		try:
			import udb
		except:
			print "ERROR: could not find microDB. Please check the path passed to --with-udb (%s)" % path
			return None
	
	# This is the original file. We want to remove the inclusion of plugins/customizations
	srcfile = os.path.join('..', 'database', 'pianos4.udb')
	print "  compiling", srcfile
	buf = StringIO.StringIO()
	with file(srcfile,'r') as f:
		for line in f:
			# NIJ plugin (note that this will move in its own plugins/nij/ dir afterwards so this
			# string must be changed into a regex that matches all plugins
			idx = line.find('import pianos4-nij.udb')
			if idx >= 0:
				continue
			# Local customizations
			idx = line.find('import local-database.udb')
			if idx >= 0:
				continue
			buf.write(line)
	# Since only the common data model is specified by pianos4.udb, the developer is
	# responsible for including the desired edition's model using either
	# import pianos4-te.udb (for the teaching edition)
	# or
	# import pianos4-le.udb (for the lab edition)
	# This normally happens in the local-database.udb file (along with the database
	# host, port and name) - but we've explicitly skipped this file. Since this script
	# is actually building a Teaching Edition (Lab Edition being in early beta stage)
	# we explicitly add the correct line to the .udb source.
	buf.write('import pianos4-te.udb\n')
	
	buf.seek(0)
	catalog, create, drop = udb.compileSourceStream(srcfile, buf)
	# print catalog
	# print create
	# print drop
	
	buffer = StringIO.StringIO()

	for tpl in catalog['templates']:
		filename = os.path.join(os.path.split(srcfile)[0], tpl['path'])
		with file(filename, 'r') as f:
			if tpl['type'] == 'builtin':
				buffer.write(f.read())
				buffer.write('\n\n')

	for obj in create:
		buffer.write(obj[4])
	buffer.write('\n\n')
		
	for script in catalog['scripts']:
		if script['type'] == 'sql':
			filename = os.path.join(os.path.split(srcfile)[0], script['path'])
			with file(filename, 'r') as f:
				buffer.write(f.read())
				buffer.write('\n\n')
	
	return buffer.getvalue()
	
	
def run(indir, output, temp, sqlgen, version=None, force=False, resolvers={}, php_unil=None, with_src=False, with_db=False, with_doc=False, with_doc_src=False, with_shibboleth=False, write_vername=False, with_demo=False):

	if version is None:
		print "Auto-detecting version"
		version = findVersion(indir)
	
	if version is None:
		print "ERROR: could not find PiAnoS version"
		return False
	
	basename = output
	if output[-1] != '-':
		basename += '-'
	basename += version
	outdir = os.path.join(temp, basename)
	archive = basename+'.tar.gz'
	
	print "-------------------------------------------------------------------"
	print "Building PiAnoS version:   ",version
	print "Sources are located at:    ",indir
	print "Using temporary directory: ",temp
	print "Creating release file:     ",archive
	print "Resolvers:                 ",', '.join((': '.join(x) for x in filter(lambda x:x[1] is not None, resolvers.iteritems())))
	print "PHP UNIL package path:     ",php_unil if php_unil else 'N/A'
	print "Include Javascript sources:",'yes' if with_src else 'no'
	print "Include database sources:  ",'yes' if with_db else 'no'
	print "Include documentation:     ",'yes' if with_doc else 'no'
	print "Include doc. source:       ",'yes' if with_doc_src else 'no'
	print "Include static demo:       ",'yes' if with_demo else 'no'
	print "Include Shibboleth files:  ",'yes' if with_shibboleth else 'no'
	print "Schema generation:         ", ("from SQL file '%s'" if sqlgen['method'] == 'sql' else "using microDB at '%s'") % sqlgen['data']
	print "-------------------------------------------------------------------"

	if os.path.exists(archive) and not force:
		print "ERROR: output filename",outdir,"already exists. Use --force to override."
		return False

	if os.path.lexists(outdir):
		if force:
			shutil.rmtree(outdir)
		else:
			print "ERROR: intermediate directory",outdir,"already exists. Use --force to override."
			return False

	if php_unil is not None:
		filecheck = os.path.join(php_unil,'api.php')
		if not os.path.isfile(filecheck):
			print "ERROR: specified PHP UNIL directory is not valid: the file '%s' could not be found" % filecheck
			return False
			
	print "Loading dependencies ..."
	if not resolvers:
		print "WARNING: no resolvers set, problems to find dependencies are very likely to happen soon. See the help for '--docroot' and '--url'"
	deps = gatherDependencies(indir, resolvers)
	if deps is None:
		print "ERROR: not all dependencies were satisfied, stopping"
		return False

	# print " js:",', '.join(deps['js'])
	# print " css:",', '.join(deps['css'])
	# return False
	minifiedFiles = []#['pianos4.min.js', 'pianos4-backend.min.js']
	useMinified = all([os.path.isfile(x) for x in minifiedFiles])
	
	basePattern = ['.svn', '*.psd', '*.ai', '*.pyc', 'settings-local*', 'local-database*', 'pianos.sql', 'NIJPlugin.php']
	fileFilter = shutil.ignore_patterns(*basePattern)
	fileFilterSrc = shutil.ignore_patterns(*(basePattern+['*.min.js']))
	fileFilterUnil = shutil.ignore_patterns(*(basePattern+['jquery-plugins','parts','SchemaEditor','Crud.php','Notebook.php','Relation.php']))
	fileFilterPHP = shutil.ignore_patterns(*(basePattern+['Pianos4_LE.php','LE']))
	
	print "Creating intermediate directory",outdir
	os.mkdir(outdir)
	
	print "Copying files ..."
	shutil.copy(os.path.join(indir,'LICENSE.txt'), outdir)
	shutil.copy(os.path.join(indir,'LICENSE-3RD-PARTY.txt'), outdir)
	shutil.copy(os.path.join(indir,'README.txt'), outdir)
	os.mkdir(os.path.join(outdir,'js'))
	if with_src:
		jssrc = os.path.join(indir,'js')
		jsdst = os.path.join(outdir,'js')
		for fname in os.listdir(jssrc):
			if os.path.splitext(fname)[1] == '.js':
				shutil.copy(os.path.join(jssrc, fname), os.path.join(outdir,'js',fname))
	shutil.copytree(os.path.join(indir,'css'), os.path.join(outdir,'css'), ignore=fileFilter)
	shutil.copytree(os.path.join(indir,'backend'), os.path.join(outdir,'backend'), ignore=fileFilterPHP)
	if with_db:
		shutil.copytree(os.path.join(indir,'database'), os.path.join(outdir,'database'), ignore=fileFilter)
	if with_shibboleth:
		shutil.copytree(os.path.join(indir,'shibboleth_auth'), os.path.join(outdir,'shibboleth_auth'), ignore=fileFilter)
	shutil.copytree(os.path.join(indir,'install'), os.path.join(outdir,'install'), ignore=fileFilter)
	if php_unil is not None:
		shutil.copytree(os.path.join(php_unil), os.path.join(outdir,'backend','unil'), ignore=fileFilterUnil)
	copyFilteredFile(os.path.join(indir,'main.php'), os.path.join(outdir,'index.php'), {1:["define('_DEBUG', false);","set_include_path(get_include_path().PATH_SEPARATOR.'backend');",'require_once("backend/Pianos4_TE.php");',"$app = new Pianos4_TE();","$app->run();"]})
	copyFilteredFile(os.path.join(indir,'backend','Pianos4.php'), os.path.join(outdir,'backend','Pianos4.php'), {1:["\t\tif (is_dir('install')) {","\t\t\tif (isset($_GET['install_first_login']) && (int)$_GET['install_first_login'] === 1) {","\t\t\t\theader('Location: install/index.php?step=end&err=1');","\t\t\t\texit;","\t\t\t} else {","\t\t\t\tthrow new Exception('Please remove the install/ directory before you attempt to run PiAnoS');","\t\t\t}","\t\t}"]})
	if with_demo:
		copyFilteredFile(os.path.join(indir,'tools','demo-index.html'), os.path.join(outdir,'demo.html'), {1:["\tvar build_settings = { version: '"+version+"'};"], 2:['\t\t<script type="text/javascript" src="js/pianos-'+version+'.min.js"></script>']})
	print "  Done"
	
	print "Writing dependencies ..."
	dirs_ok = {}
	for dep in iter(deps):
		filename = os.path.join(outdir, dep['dst'])
		dirname,fname = os.path.split(filename)
		print "  %s" % fname
		if not dirname in dirs_ok:
			if not os.path.isdir(dirname):
				os.makedirs(dirname)
			dirs_ok[dirname] = True
		with file(filename,'wb') as f:
			f.write(dep['content'])
	
	sqlfile = os.path.join(outdir,'install','pianos.sql')
	print "Generating '%s'" % sqlfile
	if sqlgen['method'] == 'udb':
		#print "ERROR: --with-udb: unimplemented"
		#return False
		content = get_pianos_sql_udb(sqlgen['data'])
		if content is None:
			print "ERROR: could not generate %s" % sqlfile
			return 1
		with file(sqlfile,'w') as f:
			f.write(content)
	else:
		shutil.copy(sqlgen['data'], sqlfile)
		
		
	print "Compiling Javascript ..."
	minified_files = {
		'pianos-'+version+'.min.js': ['pianos4','imageeditor'],
		'pianos-backend-'+version+'.min.js': ['backend'],
	}
	
	def minimizePostProcessList(files):
		files.sort(key=lambda x: 0 if 'shared' in x else 1)
		return files
	
	for minified,components in minified_files.iteritems():
		saveBeforeMin = None
		if '--debug' in sys.argv:
			saveBeforeMin = 'merged-before-minimization-'+minified
			print "Saving merged source to",saveBeforeMin

		print "Creating '%s' ..."%minified
		data = minimize.main(components=components, noPrint=True, saveBeforeMin=saveBeforeMin, postProcess=minimizePostProcessList)
		if not data:
			print "ERROR: errors reported during compilation, aborting. Please inspect output."
			return 1
		print ""
		f = file(os.path.join(outdir,'js',minified),'w')
		f.write(data)
		f.close()
	
	if with_doc:
		print "Including the documentation ..."
		print "  Generating"
		cwd = os.getcwd()
		docpath = os.path.join(indir, 'doc')
		os.chdir(docpath)
		if subprocess.call(['make.bat','html']) != 0:
			print "ERROR: errors reported during Sphinx run"
			return 1
		os.chdir(cwd)
		shutil.copytree(os.path.join(docpath,'build','html'), os.path.join(outdir,'docs'))
	
	if with_doc_src:
		print "Including the documentation source ..."
		docpath = os.path.join(indir, 'doc')
		def ignore(d,f):
			if d == docpath:
				return set(f) - set(['make.bat','Makefile','source'])
			else:	
				return []
		shutil.copytree(docpath, os.path.join(outdir,'doc'), ignore=ignore)
		
	# print "Creating settings file ..."
	# src = os.path.join(indir, 'backend', 'settings-local.php.template')
	# dst = os.path.join(outdir, 'backend', 'settings-local.php')
	# config_lines = [
		# "\tApplication::set('use_minified_pianos4', 'js/pianos-%s.min.js');" % version,
		# "\tApplication::set('use_minified_pianos4_backend', 'js/pianos-backend-%s.min.js');" % version,
	# ]
	# copyFilteredFile(src, dst, {1:config_lines})
	
	print "Tagging release ..."
	vercontent = [
		'This is PiAnoS '+version,
		'',
		'Archive created on '+str(datetime.datetime.now())
	]
	with file(os.path.join(outdir,'VERSION.txt'),'w') as verfile:
		verfile.writelines([x+'\n' for x in vercontent])

	with file(os.path.join(outdir,'install','version'),'w') as f:
		f.write(version)
	
	with file(os.path.join(outdir,'INSTALL.txt'),'w') as instfile:
		instfile.writelines([x+'\n' for x in ['Please refer to install/README.txt']])

	if os.path.lexists(archive):
		if '--force' in sys.argv:
			os.remove(archive)
		else:
			print "ERROR: output file",archive,"already exists. Use --force to override"
			return False

	print "Creating destination archive",archive
	tar = tarfile.open(archive, 'w:gz')
	print "Adding files ..."
	tar.add(outdir, arcname=os.path.split(outdir)[1])
	tar.close()
	
	if sys.platform == 'win32' and False:
		print "Re-packing destination archive with correct permission bits"
		repackname = archive+'.repack'
		itar = tarfile.open(archive, "r:gz")
		otar = tarfile.open(repackname, "w:gz")
		for itarinfo in itar:
			otarinfo = otar.gettarinfo(itarinfo.name)
			print otarinfo
			if itarinfo.isfile():
				otarinfo.mode = 0644
			else:
				otarinfo.mode = 0755
			otarinfo.uid = 0
			otarinfo.gid = 0
			otarinfo.uname = "root"
			otarinfo.gname = "root"
			otar.addfile(otarinfo, itar.extractfile(itarinfo))
		itar.close()
		otar.close()
		
		os.remove(archive)
		os.rename(repackname, archive)


	if write_vername:
		with file(write_vername, 'w') as f:
			f.write(version)
			
	print "Done !"
	
	return True

def main():
	parser = argparse.ArgumentParser(
		description='prepares the PiAnoS 4.x release tarball.',
		epilog='This script allows one to build the .tar.gz release with all dependencies included for production installations. It is only meaningful when executed from a running development (SVN) version (otherwise dependencies will not be resolved).'
	)

	default_output_file = 'pianos'
	default_input_dir = '..'
	default_temp_dir = '.'
	default_version = None
	default_docroot = '..'
	default_url = 'http://127.0.0.1/'
	#default_udb_path = os.path.join('..', '..', '..', 'microdb')
	
	parser.add_argument('-f','--force', action='store_true', help='overwrite output files')
	parser.add_argument('-i','--input', metavar='DIRECTORY', help="the top directory of PiAnoS. Defaults to '%s'" % default_input_dir, default=default_input_dir)
	parser.add_argument('-o','--output', metavar='PREFIX', help="the output filename (path and) prefix. The version number and .tar.gz extension will be added automatically. Defaults to '%s'" % default_output_file, default=default_output_file)
	parser.add_argument('-t','--temp', metavar='DIRECTORY', help="the intermediate directory where temporary files will be created. Defaults to '%s'" % default_temp_dir, default=default_temp_dir)
	parser.add_argument('-v','--version', metavar='VERSION', help="override the automatic detection of version number (e.g. '4.1.3'), using VERSION instead.")
	parser.add_argument('--docroot', metavar='DIRECTORY', help="document root of the web server, used to resolve absolute paths to dependencies. Defaults to '%s'"  % default_docroot, default=default_docroot)
	parser.add_argument('--docroot-rel', metavar='DIRECTORY', help="directory of the web server to be used to resolve relative paths to dependencies")
	parser.add_argument('--url', metavar='URL', help="url to the web server used to resolve absolute paths to dependencies. Defaults to '%s'" % default_url, default=default_url)
	parser.add_argument('--url-rel', metavar='URL', help="url to the web server used to resolve relative paths to dependencies")
	parser.add_argument('--with-unil', metavar='DIRECTORY', help="path to the 'unil' PHP library. Will not be included in the package if not specified")
	parser.add_argument('--inc-src', help="include Javascript source files in the package", action='store_true')
	parser.add_argument('--inc-db', help="include database source files in the package", action='store_true')
	parser.add_argument('--inc-doc', help="include the HTML documentation. You must have Sphinx installed for this to work. See doc/make.bat", action='store_true')
	parser.add_argument('--inc-doc-src', help="include the Sphinx documentation source files (*.rst + Sphinx build scripts)", action='store_true')
	parser.add_argument('--inc-shib', help="include the Shibboleth single-sign on system files.", action='store_true')
	parser.add_argument('--with-udb', metavar='DIRECTORY', help="use microDB (located at DIRECTORY) to generate the 'pianos.sql' file. Mutually exclusive with --with-sql")
	parser.add_argument('--with-sql', metavar='FILENAME', help="use the specified .sql file as 'pianos.sql' file. Mutually exclusive with --with-udb.")
	parser.add_argument('--echo-vername', metavar='FILENAME', help="write the version (e.g. '4.0.1') to FILENAME. Useful for automation.")
	parser.add_argument('--inc-demo', help="include the static HTML demo in the package. Note that this require that the files 'mark.jpg' and 'print.jpg' exist in this directory or the parent.", action='store_true')
	
	args = parser.parse_args()

	sqlgen = { 'method': None, 'data': None }
	if (args.with_udb):
		sqlgen['method'] = 'udb'
		sqlgen['data'] = args.with_udb
	elif (args.with_sql):
		sqlgen['method'] = 'sql'
		sqlgen['data'] = args.with_sql

	if sqlgen['method'] is None:
		print "ERROR: you must specify the schema file generation method using either --with-sql or --with-udb"
		return 1
	elif args.with_udb and args.with_sql:
		print "ERROR: --with-sql and --with-udb are mutually exclusive"
		return 1
		
	resolvers = {
		'docroot_abs': args.docroot,
		'docroot_rel': args.docroot_rel,
		'url_abs': args.url,
		'url_rel': args.url_rel
	}
	
	return 0 if run(args.input, args.output, args.temp, sqlgen, args.version, args.force, resolvers, args.with_unil, args.inc_src, args.inc_db, args.inc_doc, args.inc_doc_src, args.inc_shib, args.echo_vername, args.inc_demo) else 1
	
	
if __name__ == '__main__':
	sys.exit(main())
