Add new genpopo and translation_tool that work with new system.

Signed-off-by: Loui Chang <louipc.ist@gmail.com>

2 files changed, 242 insertions, 379 deletions

diff --git a/web/utils/genpopo b/web/utils/genpopo
index a2a3ac43..66bf9a12 100755
--- a/web/utils/genpopo
+++ b/web/utils/genpopo
@@ -1,10 +1,10 @@
 #! /usr/bin/python -O
 # -*- coding: iso-8859-1 -*-
 
-# this script iterates through the 'html' and 'lib' directories
-# looking for php scripts that contain a include_once("xxx_po.inc")
-# line and __() functions.  It creates/appends to the corresponding
-# "xxx_po.inc" file in the 'lang' subdirectory and places the
+# This script iterates through the script directories
+# looking for php scripts that contain  __() functions.
+# It creates/appends to the corresponding
+# "xxx.po" file in the 'lang' subdirectory and places the
 # i18n strings into the file in the proper format.
 #
 # usage: genpopo [-v] [-f]
@@ -12,77 +12,51 @@
 #        -f: force, overwrite existing translated files, otherwise append
 #
 
+import re
+import os
+import sys
+
 INC_HEADER = """\
 <?php
 # INSTRUCTIONS TO TRANSLATORS
 #
 # This file contains the i18n translations for a subset of the
-# Arch Linux User-community Repository (AUR).  This is a PHP
+# Arch Linux User Community Repository (AUR).  This is a PHP
 # script, and as such, you MUST pay great attention to the syntax.
 # If your text contains any double-quotes ("), you MUST escape
-# them with the backslash character (\).
+# them with a backslash (\).
 #
 
-include_once("translator.inc");
 global $_t;
 """
 
+language = 'en'
+lang = {}
 
-import sys
 print_dupes = '-v' in sys.argv
 force = '-f' in sys.argv
 
-import re, os
 up = re.compile('_\(\s*"(([^"]|(?<=\\\\)["])+)"')
 
-lang = { 'common_po.inc': {} }
+scriptdirs = ['html', 'lib', 'template']
+pofile = '%s.po' % language
 
 current_dir = os.getcwd()
 
-# Find the common_po.inc file.
-#
-common = {}
-for dir in ['../lang', 'lang']:
-	if os.path.exists(dir):
-		os.chdir(dir)
-		if os.path.exists('common_po.list'):
-			f = open('common_po.list','r')
-			lines = f.readlines()
-			f.close()
-			for line in lines:
-				if line[0] != '#':
-					common[line[:-1]] = 0
-					lang['common_po.inc'][line[:-1]] = 1
-			os.chdir(current_dir)
-			break
-		os.chdir(current_dir)
-else:
-	print "Can't find common_po.list file."
-	raise SystemExit
-
-# Find the lang directory.
-#
-for dir in ['../lang', 'lang']:
-	if os.path.exists(dir):
-		lang_dir = dir
-		break
-else:
-	print "Can't find the lang directory."
-	raise SystemExit
-
 # Iterate through various places where the php files might be.
 #
-for dir in ['../html', '../lib', 'html', 'lib']:
+for dir in scriptdirs:
+	dir = "../%s" % dir
 
 	if os.path.exists(dir):
 		# Find all the PHP files in the current directory.
 		#
 		files = [x for x in os.listdir(dir)
-					if (x[-4:] == '.inc' and x[-7:] != '_po.inc')
-					or x[-6:] == '.class'
-					or x[-4:] == '.php'
-					or x[-6:] == '.phtml'
-				]
+				if (x[-4:] == '.inc' and x[-7:] != '.po')
+				or x[-6:] == '.class'
+				or x[-4:] == '.php'
+				or x[-6:] == '.phtml'
+			]
 		os.chdir(dir)
 
 		for file in files:
@@ -90,157 +64,107 @@ for dir in ['../html', '../lib', 'html', 'lib']:
 			lines = f.readlines()
 			f.close()
 
-			# Is this file one we need to parse for internationalized strings?
-			#
-			parse_file = 0
+			# Parse the file
+			print "Parsing %s..." % file
 			for line in lines:
-				match = re.search("include(_once|)\s*\(\s*[\"']([A-Za-z_]+_po.inc)[\"']\s*\);",line)
-				if match and match.group(2) != "common_po.inc":
-					po = match.group(2)
-					if not lang.has_key(po):
-						lang[po] = {}
-					parse_file = 1
-					break
-
-			# If we need to parse the file, do so.
-			#
-			if parse_file:
-				print "Parsing %s..." % file
-				for line in lines:
+				match = up.search(line)
+				while match:
+					term = match.group(1).replace('\\"','"')
+					if print_dupes:
+						if term in lang.keys():
+							print 'Multiple use of "%s"' % term
+
+					lang[term] = 1
+					line = line[match.end(1):]
 					match = up.search(line)
-					while match:
-						term = match.group(1).replace('\\"','"')
-						if common.has_key(term):
-							common[term] += 1
-						else:
-							if print_dupes:
-								for key in lang.keys():
-									if key != po and lang[key].has_key(term):
-										print "...Duplicate term: \"%s\" is also in %s." % (term,key)
-							lang[po][term] = 1
-						line = line[match.end(1):]
-						match = up.search(line)
 
 		os.chdir(current_dir)
 
-# Now generate all the .inc files if they don't already exist.
-# if they do exist, only append new stuff to the end.  If the 'force'
-# option is passed, just overwrite the entire thing.
-#
-os.chdir(lang_dir)
-if not os.path.exists('en'):
-	os.mkdir('en')
+# Generate the .po file if it doesn't already exist.
+# If it does exist, only append new stuff to the end.
+# If the 'force' option is passed, just overwrite.
 
 if force:
-	# just going to overwrite any existing files
-	#
-	for po in lang.keys():
-		print "Generating %s..." % po
+	# Just overwrite any existing files
+	print "Generating %s..." % pofile
+	
+	f = open(pofile,'w')
+	f.write(INC_HEADER)
 
-		f = open(po,'w')
-		f.write(INC_HEADER)
-		f.write('\ninclude_once(\"en/%s\");\n' % po)
-		f.write('\n')
+	for term in lang.keys():
+		f.write("\n")
+		f.write('$_t["%s"]\n  = "%s";\n' % (term, term))
+
+	f.write("\n");
+	f.close()
+else:
+	# Need to leave existing file intact. Only append on new terms.
+	mapre = re.compile('^\$_t\["(.*)"\].*$')
+	got_match = False
+
+	print "Updating %s..." % pofile
+
+	try:
+		f = open(pofile, 'r')
+		new_file = 0
+	except:
+		new_file = 1
+
+	if not new_file:
+		contents = f.readlines()
 		f.close()
-		
-		f = open("en/"+po,'w')
-		f.write(INC_HEADER)
-		for term in lang[po].keys():
-			f.write("\n")
-			f.write('$_t["en"]["%s"] = "%s";\n' % (term, term))
 
+		# Strip beginning/ending empty lines
+		while contents[0] == '':
+			del contents[0]
+		while contents[-1] in ['', "\n", "?>", "?>\n", "\n?>"]:
+			del contents[-1]
+			
+		f = open(pofile,'w')
+		f.write("".join(contents))
 		f.write("\n");
 		f.close()
-else:
-	# need to leave existing file intact, and only append on terms that are new
-	#
-	incre = re.compile('^include_once\("en\/(.*)"\);')
-	mapre = re.compile('^\$_t\["en"\]\["(.*)"\].*$')
-	for po in lang.keys():
-		got_match = False
-		print "Updating %s..." % po
-		try:
-			f = open(po, 'r')
-			new_file = 0
-		except:
-			new_file = 1
-
-		if not new_file:
-			contents = f.readlines()
-			f.close()
-			# strip off beginning/ending empty lines
-			#
-			while contents[0] == '':
-				del contents[0]
-			while contents[-1] in ['', "\n", "?>", "?>\n", "\n?>"]:
-				del contents[-1]
-				
-			for line in contents:
-				match = incre.search(line)
-				if match:
-					got_match = True
-			if not got_match:
-				f = open(po,'w')
-				f.write("".join(contents))
-				f.write('\ninclude_once(\"en/%s\");\n' % po)
-				f.write("\n");
-				f.close()
-		else:
-				f = open(po,'w')
-				f.write(INC_HEADER)
-				f.write('\ninclude_once(\"en/%s\");\n' % po)
-				f.write('\n')
-				f.close()
-	
-		# first read in file contents so we can hash what already exists
-		#
-		try:
-			f = open('en/'+po, 'r')
-			new_file = 0
-		except:
-			new_file = 1
-
-		existing_terms = []
-		if not new_file:
-			contents = f.readlines()
-			f.close()
+	else:
+		f = open(pofile,'w')
+		f.write(INC_HEADER)
+		f.write('\n')
+		f.close()
 
-			# strip off beginning/ending empty lines
-			#
-			while contents[0] == '':
-				del contents[0]
-			while contents[-1] in ['', "\n", "?>", "?>\n", "\n?>"]:
-				del contents[-1]
-
-			# next, collect existing terms
-			#
-			for line in contents:
-				match = mapre.search(line)
-				if match:
-					existing_terms.append(match.group(1))
-
-		# now append any new terms to EOF
-		#
-		f = open('en/'+po, 'w')
-		if not new_file:
-			f.write("".join(contents))
-		else:
-			f.write(INC_HEADER)
-
-		for term in lang[po].keys():
-			if term not in existing_terms:
-				f.write("\n");
-				f.write('$_t["en"]["%s"] = "%s";\n' % (term, term))
-		f.write("\n");
+	# Read file contents so we can hash what already exists
+	try:
+		f = open(pofile, 'r')
+		new_file = 0
+	except:
+		new_file = 1
+
+	existing_terms = []
+	if not new_file:
+		contents = f.readlines()
 		f.close()
 
-# Print out warnings for unused and little-used common entries.
-#
-for key in common.keys():
-	if common[key] == 1:
-		print "Warning: common entry '%s' is only used once." % key
-for key in common.keys():
-	if common[key] == 0:
-		print "Warning: unused common entry '%s'." % key
-
-# vim: ts=2 sw=2 noet ft=python
+		# Strip beginning/ending empty lines
+		while contents[0] == '':
+			del contents[0]
+		while contents[-1] in ['', "\n", "?>", "?>\n", "\n?>"]:
+			del contents[-1]
+
+		# Collect existing terms
+		for line in contents:
+			match = mapre.search(line)
+			if match:
+				existing_terms.append(match.group(1))
+
+	# Append any new terms to EOF
+	f = open(pofile, 'w')
+	if not new_file:
+		f.write("".join(contents))
+	else:
+		f.write(INC_HEADER)
+
+	for term in lang.keys():
+		if term not in existing_terms:
+			f.write("\n");
+			f.write('$_t["%s"]\n  = "%s";\n' % (term, term))
+	f.write("\n");
+	f.close()
+
diff --git a/web/utils/translation_tool b/web/utils/translation_tool
index 6967139a..241f1ee8 100755
--- a/web/utils/translation_tool
+++ b/web/utils/translation_tool
@@ -1,92 +1,79 @@
 #! /usr/bin/python -O
 # -*- coding: iso-8859-1 -*-
 
-# this script iterates through the 'html' and 'lib' directories
-# looking for php scripts that contain a include_once("xxx_po.inc")
-# line and __() functions.  It creates/appends to the corresponding
-# "xxx_po.inc" file in the 'lang' subdirectory and places the
-# i18n strings into the file in the proper format.
+# This script iterates through the html, lib, and template directories
+# looking for php scripts that contain __() functions.
+# It creates/appends to the corresponding 'xxx.po' file in the
+# 'lang' subdirectory and places the i18n strings into the file
+# in the proper format.
 #
-# usage: genpopo [-v] [-f]
-#        -v: verbose, print duplicate terms that could be moved to common_po
-#        -f: force, overwrite existing translated files, otherwise append
+# usage: translation_tool [-v] [-f]
+#        -v: Verbose, print duplicate terms that could be moved to common_po
+#        -f: Force, overwrite existing translated files, otherwise append
 #
 
-translator_name=raw_input("What is your full name? ")
-translator_email=raw_input("What is your email address? ")
-trans_native=raw_input("What is the native name of the language? ")
-trans_eng=raw_input("What is the English name of the language? ")
-trans_abbrv=raw_input("What is the ISO 639-1 Alpha-2 abbreviation for the language? ")
+import getopt
+import os
+import re
+import sys
+
+scriptdirs = ['html', 'lib', 'template']
+
+try:
+	opts, args =getopt.getopt(sys.argv[1:], 'vf')
+except getopt.GetoptError, e:
+	print 'error: %s' % str(e)
+	raise SystemExit
+
+translator_name = raw_input("What is your full name? ")
+translator_email = raw_input("What is your email address? ")
+trans_native = raw_input("What is the native name of the language? ")
+trans_eng = raw_input("What is the English name of the language? ")
+trans_abbrv = raw_input("What is the ISO 639-1 Alpha-2 abbreviation for the language? ")
+
 if len(trans_abbrv) != 2:
 	print "Must use 2 character abbreviation"
 	raise SystemExit
 
-
 INC_HEADER = """\
 <?php
+# INSTRUCTIONS TO TRANSLATORS
+#
+# This file contains the i18n translations for a subset of the
+# Arch Linux User Community Repository (AUR).  This is a PHP
+# script, and as such, you MUST pay great attention to the syntax.
+# If your text contains any double-quotes ("), you MUST escape
+# them with a backslash (\).
+#
 # %s (%s) translation
 # Translator: %s <%s>
 
-include_once("translator.inc");
 global $_t;
+
+
 """ % (trans_eng, trans_native, translator_name, translator_email)
 
 
-import sys
 print_dupes = '-v' in sys.argv
 force = '-f' in sys.argv
 
-import re, os
 up = re.compile('_\(\s*"(([^"]|(?<=\\\\)["])+)"')
 
-lang = { 'common_po.inc': {} }
+lang = {}
 
 current_dir = os.getcwd()
 
-# Find the common_po.inc file.
-#
-common = {}
-for dir in ['../lang', 'lang']:
-	if os.path.exists(dir):
-		os.chdir(dir)
-		if os.path.exists('common_po.list'):
-			f = open('common_po.list','r')
-			lines = f.readlines()
-			f.close()
-			for line in lines:
-				if line[0] != '#':
-					common[line[:-1]] = 0
-					lang['common_po.inc'][line[:-1]] = 1
-			os.chdir(current_dir)
-			break
-		os.chdir(current_dir)
-else:
-	print "Can't find common_po.list file."
-	raise SystemExit
-
-#Find the lang directory
-for dir in ['../lang', 'lang']:
-	if os.path.exists(dir):
-		lang_dir = dir
-		break
-else:
-	print "Can't find the lang directory."
-	raise SystemExit
-
-trans_dir = trans_abbrv
-if not os.path.exists(os.path.join(lang_dir,trans_dir)):
-	os.mkdir(os.path.join(lang_dir,trans_dir))
-
+pofile = '%s.po' % trans_abbrv
 
 # Iterate through various places where the php files might be.
 #
-for dir in ['../html', '../lib', 'html', 'lib']:
+for dir in scriptdirs:
+	dir = '../%s' % dir
 
 	if os.path.exists(dir):
 		# Find all the PHP files in the current directory.
-		#
 		files = [x for x in os.listdir(dir)
-					if (x[-4:] == '.inc' and x[-7:] != '_po.inc')
+					if (x[-4:] == '.inc' and x[-7:] != '.po')
 					or x[-6:] == '.class'
 					or x[-4:] == '.php'
 					or x[-6:] == '.phtml'
@@ -98,47 +85,27 @@ for dir in ['../html', '../lib', 'html', 'lib']:
 			lines = f.readlines()
 			f.close()
 
-			# Is this file one we need to parse for internationalized strings?
-			#
-			parse_file = 0
+			# Parse files.
+			print "Parsing %s..." % file
 			for line in lines:
-				match = re.search("include(_once|)\s*\(\s*[\"']([A-Za-z_]+_po.inc)[\"']\s*\);",line)
-				if match and match.group(2) != "common_po.inc":
-					po = match.group(2)
-					if not lang.has_key(po):
-						lang[po] = {}
-					parse_file = 1
-					break
-
-			# If we need to parse the file, do so.
-			#
-			if parse_file:
-				print "Parsing %s..." % file
-				for line in lines:
+				match = up.search(line)
+				while match:
+					term = match.group(1).replace('\\"','"')
+					if print_dupes:
+						if term in lang.keys():
+							print 'Multiple use of "%s"' % term
+
+					lang[term] = 1
+					line = line[match.end(1):]
 					match = up.search(line)
-					while match:
-						term = match.group(1).replace('\\"','"')
-						if common.has_key(term):
-							common[term] += 1
-						else:
-							if print_dupes:
-								for key in lang.keys():
-									if key != po and lang[key].has_key(term):
-										print "...Duplicate term: \"%s\" is also in %s." % (term,key)
-							lang[po][term] = 1
-						line = line[match.end(1):]
-						match = up.search(line)
 
 		os.chdir(current_dir)
 
-# Now generate all the .inc files if they don't already exist.
-# if they do exist, only append new stuff to the end.  If the 'force'
-# option is passed, just overwrite the entire thing.
-#
+# Now generate the .po file if it doesn't already exist.
+# If it does exist, only append new stuff to the end.
+# If the 'force' option is passed, just overwrite.
 
 print """
-INSTRUCTIONS:
-****************************************************
 You will now be prompted for all needed translations.
 Please translate the requested lines, hitting [enter]
 goes to the next one. You may stop at any time using
@@ -153,126 +120,98 @@ following is a list of escapes and what they do:
 \\" - A double quote (")
 
 When you have finished your translation, make a tarball
-of the lang/ directory and send it to Simo (simo@neotuli.net)
+of the .po file and send it to aur-dev@archlinux.org
 for inclusion in the AUR.
+
 By submitting a translation, you are implying that you
 are also willing to maintain it. When there are
 new strings to be translated, you will be contacted.
 ****************************************************
 """
 
-os.chdir(lang_dir)
+os.chdir(current_dir)
 if force:
-	# just going to overwrite any existing files
+	# Just overwrite any existing files.
 	# NOT RECOMMENDED! OVERWRITES ALL OTHER LANGUAGE SUPPORT
 	#
-	for po in lang.keys():
-		print "Generating %s..." % po
+	print "Generating %s..." % pofile
 
-		f = open(po,'w')
+	f = open(pofile,'w')
+	f.write(INC_HEADER)
+
+	for term in lang:
+		f.write("\n")
+		trans = raw_input("\n%s\n= " % term)
+		f.write('$_t["%s"] = "%s";\n' % (term, trans))
+	f.write("\n");
+	f.close()
+else:
+	# Leave existing file intact. Only append on new terms.
+	mapre = re.compile('^\$_t\["(.*)"\].*$')
+
+	got_match = False
+	print "Updating %s..." % pofile
+	try:
+		f = open(pofile, 'r')
+		new_file = 0
+	except:
+		new_file = 1
+	if not new_file:
+		contents = f.readlines()
+		f.close()
+
+		# Strip beginning/ending empty lines
+		while contents[0] == '':
+			del contents[0]
+		while contents[-1] in ['', "\n", "?>", "?>\n", "\n?>"]:
+			del contents[-1]
+					
+		f = open(pofile,'w')
+		f.write("".join(contents))
+		f.write("\n");
+		f.close()
+	else:
+		f = open(pofile,'w')
 		f.write(INC_HEADER)
-		f.write('\ninclude_once(\"en/%s\");\n' % po)
 		f.write('\n')
 		f.close()
 
-		f = open(trans_dir+"/"+po,'w')
-		f.write(INC_HEADER)
+	# Read in file contents so we can hash what already exists
+	try:
+		f = open(pofile, 'r')
+		new_file = 0
+	except:
+		new_file = 1
 
-		for term in lang[po].keys():
-			f.write("\n")
-			trans = raw_input(term+"  = ")
-			f.write('$_t["%s"]["%s"] = "%s";\n' % (trans_abbrv, term, trans))
-		f.write("\n");
+	existing_terms = []
+	if not new_file:
+		contents = f.readlines()
 		f.close()
-else:
-	# need to leave existing file intact, and only append on terms that are new
-	#
-	incre = re.compile('^include_once\("%s\/(.*)"\);' % trans_abbrv)
-	mapre = re.compile('^\$_t\["%s"\]\["(.*)"\].*$' % trans_abbrv)
-	for po in lang.keys():
-		got_match = False
-		print "Updating %s..." % po
-		try:
-			f = open(po, 'r')
-			new_file = 0
-		except:
-			new_file = 1
-		if not new_file:
-			contents = f.readlines()
-			f.close()
-			# strip off beginning/ending empty lines
-			#
-			while contents[0] == '':
-				del contents[0]
-			while contents[-1] in ['', "\n", "?>", "?>\n", "\n?>"]:
-				del contents[-1]
-						
-			for line in contents:
-				match = incre.search(line)
-				if match:
-					got_match = True
-			if not got_match:
-				f = open(po,'w')
-				f.write("".join(contents))
-				f.write('\ninclude_once(\"%s/%s\");\n' % (trans_abbrv, po))
-				f.write("\n");
-				f.close()
-		else:
-			f = open(po,'w')
-			f.write(INC_HEADER)
-			f.write('\ninclude_once(\"%s/%s\");\n' % (trans_abbrv, po))
-			f.write('\n')
-			f.close()
-		# first read in file contents so we can hash what already exists
-		#
-		try:
-			f = open(trans_dir+"/"+po, 'r')
-			new_file = 0
-		except:
-			new_file = 1
-
-		existing_terms = []
-		if not new_file:
-			contents = f.readlines()
-			f.close()
 
-			# strip off beginning/ending empty lines
-			#
-			while contents[0] == '':
-				del contents[0]
-			while contents[-1] in ['', "\n", "?>", "?>\n", "\n?>"]:
-				del contents[-1]
-
-			# next, collect existing terms
-			#
-			for line in contents:
-				match = mapre.search(line)
-				if match:
-					existing_terms.append(match.group(1))
-
-		# now append any new terms to EOF
-		#
-		f = open(trans_dir+"/"+po, 'w')
-		if not new_file:
-			f.write("".join(contents))
-		else:
-			f.write(INC_HEADER)
-
-		for term in lang[po].keys():
-			if term not in existing_terms:
-				f.write("\n");
-				trans = raw_input(term+"  = ")
-				f.write('$_t["%s"]["%s"] = "%s";\n' % (trans_abbrv, term, trans))
-		f.write("\n");
-		f.close()
+		# Strip beginning/ending empty lines
+		while contents[0] == '':
+			del contents[0]
+		while contents[-1] in ['', "\n", "?>", "?>\n", "\n?>"]:
+			del contents[-1]
+
+		# Collect existing terms
+		for line in contents:
+			match = mapre.search(line)
+			if match:
+				existing_terms.append(match.group(1))
+
+	# Append any new terms to EOF
+	f = open(pofile, 'w')
+	if not new_file:
+		f.write("".join(contents))
+	else:
+		f.write(INC_HEADER)
+
+	for term in lang.keys():
+		if term not in existing_terms:
+			f.write("\n");
+			trans = raw_input("\n%s\n= " % term)
+			f.write('$_t["%s"] = "%s";\n' % (term, trans))
+	f.write("\n");
+	f.close()
 
-# Print out warnings for unused and little-used common entries.
-#
-for key in common.keys():
-	if common[key] == 1:
-		print "Warning: common entry '%s' is only used once." % key
-for key in common.keys():
-	if common[key] == 0:
-		print "Warning: unused common entry '%s'." % key
-
-# vim: ts=2 sw=2 noet ft=python