diff options
Diffstat (limited to 'contrib/gnatsparse/gnatsparse.py')
| -rwxr-xr-x | contrib/gnatsparse/gnatsparse.py | 804 |
1 files changed, 804 insertions, 0 deletions
diff --git a/contrib/gnatsparse/gnatsparse.py b/contrib/gnatsparse/gnatsparse.py new file mode 100755 index 000000000..5f7cde713 --- /dev/null +++ b/contrib/gnatsparse/gnatsparse.py @@ -0,0 +1,804 @@ +try: +# Using Psyco makes it about 25% faster, but there's a bug in psyco in +# handling of eval causing it to use unlimited memory with the magic +# file enabled. +# import psyco +# psyco.full() +# from psyco.classes import * + pass +except: + pass +import re +import base64 +import cStringIO +import specialuu +import array +import email.Utils +import zlib +import magic + +# Comment out if you don't want magic detection +magicf = magic.MagicFile() + +# Open our output file +outfile = open("gnats2bz_data.sql", "w") + +# List of GNATS fields +fieldnames = ("Number", "Category", "Synopsis", "Confidential", "Severity", + "Priority", "Responsible", "State", "Quarter", "Keywords", + "Date-Required", "Class", "Submitter-Id", "Arrival-Date", + "Closed-Date", "Last-Modified", "Originator", "Release", + "Organization", "Environment", "Description", "How-To-Repeat", + "Fix", "Release-Note", "Audit-Trail", "Unformatted") + +# Dictionary telling us which GNATS fields are multiline +multilinefields = {"Organization":1, "Environment":1, "Description":1, + "How-To-Repeat":1, "Fix":1, "Release-Note":1, + "Audit-Trail":1, "Unformatted":1} + +# Mapping of GCC release to version. Our version string is updated every +# so we need to funnel all release's with 3.4 in the string to be version +# 3.4 for bug tracking purposes +# The key is a regex to match, the value is the version it corresponds +# with +releasetovermap = {r"3\.4":"3.4", r"3\.3":"3.3", r"3\.2\.2":"3.2.2", + r"3\.2\.1":"3.2.1", r"3\.2":"3.2", r"3\.1\.2":"3.1.2", + r"3\.1\.1":"3.1.1", r"3\.1":"3.1", r"3\.0\.4":"3.0.4", + r"3\.0\.3":"3.0.3", r"3\.0\.2":"3.0.2", r"3\.0\.1":"3.0.1", + r"3\.0":"3.0", r"2\.95\.4":"2.95.4", r"2\.95\.3":"2.95.3", + r"2\.95\.2":"2.95.2", r"2\.95\.1":"2.95.1", + r"2\.95":"2.95", r"2\.97":"2.97", + r"2\.96.*[rR][eE][dD].*[hH][aA][tT]":"2.96 (redhat)", + r"2\.96":"2.96"} + +# These map the field name to the field id bugzilla assigns. We need +# the id when doing bug activity. +fieldids = {"State":8, "Responsible":15} + +# These are the keywords we use in gcc bug tracking. They are transformed +# into bugzilla keywords. The format here is <keyword>-><bugzilla keyword id> +keywordids = {"wrong-code":1, "ice-on-legal-code":2, "ice-on-illegal-code":3, + "rejects-legal":4, "accepts-illegal":5, "pessimizes-code":6} + +# Map from GNATS states to Bugzilla states. Duplicates and reopened bugs +# are handled when parsing the audit trail, so no need for them here. +state_lookup = {"":"NEW", "open":"ASSIGNED", "analyzed":"ASSIGNED", + "feedback":"WAITING", "closed":"CLOSED", + "suspended":"SUSPENDED"} + +# Table of versions that exist in the bugs, built up as we go along +versions_table = {} + +# Delimiter gnatsweb uses for attachments +attachment_delimiter = "----gnatsweb-attachment----\n" + +# Here starts the various regular expressions we use +# Matches an entire GNATS single line field +gnatfieldre = re.compile(r"""^([>\w\-]+)\s*:\s*(.*)\s*$""") + +# Matches the name of a GNATS field +fieldnamere = re.compile(r"""^>(.*)$""") + +# Matches the useless part of an envelope +uselessre = re.compile(r"""^(\S*?):\s*""", re.MULTILINE) + +# Matches the filename in a content disposition +dispositionre = re.compile("(\\S+);\\s*filename=\"([^\"]+)\"") + +# Matches the last changed date in the entire text of a bug +# If you have other editable fields that get audit trail entries, modify this +# The field names are explicitly listed in order to speed up matching +lastdatere = re.compile(r"""^(?:(?:State|Responsible|Priority|Severity)-Changed-When: )(.+?)$""", re.MULTILINE) + +# Matches the From line of an email or the first line of an audit trail entry +# We use this re to find the begin lines of all the audit trail entries +# The field names are explicitly listed in order to speed up matching +fromtore=re.compile(r"""^(?:(?:State|Responsible|Priority|Severity)-Changed-From-To: |From: )""", re.MULTILINE) + +# These re's match the various parts of an audit trail entry +changedfromtore=re.compile(r"""^(\w+?)-Changed-From-To: (.+?)$""", re.MULTILINE) +changedbyre=re.compile(r"""^\w+?-Changed-By: (.+?)$""", re.MULTILINE) +changedwhenre=re.compile(r"""^\w+?-Changed-When: (.+?)$""", re.MULTILINE) +changedwhyre=re.compile(r"""^\w+?-Changed-Why:\s*(.*?)$""", re.MULTILINE) + +# This re matches audit trail text saying that the current bug is a duplicate of another +duplicatere=re.compile(r"""(?:")?Dup(?:licate)?(?:d)?(?:")? of .*?(\d+)""", re.IGNORECASE | re.MULTILINE) + +# Get the text of a From: line +fromre=re.compile(r"""^From: (.*?)$""", re.MULTILINE) + +# Get the text of a Date: Line +datere=re.compile(r"""^Date: (.*?)$""", re.MULTILINE) + +# Map of the responsible file to email addresses +responsible_map = {} +# List of records in the responsible file +responsible_list = [] +# List of records in the categories file +categories_list = [] +# List of pr's in the index +pr_list = [] +# Map usernames to user ids +usermapping = {} +# Start with this user id +userid_base = 2 + +# Name of gnats user +gnats_username = "gnats@gcc.gnu.org" +# Name of unassigned user +unassigned_username = "unassigned@gcc.gnu.org" + +gnats_db_dir = "." +product = "gcc" +productdesc = "GNU Compiler Connection" +milestoneurl = "http://gcc/gnu.org" +defaultmilestone = "3.4" + +def write_non_bug_tables(): + """ Write out the non-bug related tables, such as products, profiles, etc.""" + # Set all non-unconfirmed bugs's everconfirmed flag + print >>outfile, "update bugs set everconfirmed=1 where bug_status != 'UNCONFIRMED';" + + # Set all bugs assigned to the unassigned user to NEW + print >>outfile, "update bugs set bug_status='NEW',assigned_to='NULL' where bug_status='ASSIGNED' AND assigned_to=3;" + + # Insert the products + print >>outfile, "\ninsert into products (" + print >>outfile, " product, description, milestoneurl, disallownew," + print >>outfile, " defaultmilestone, votestoconfirm) values (" + print >>outfile, " '%s', '%s', '%s', 0, '%s', 1);" % (product, + productdesc, + milestoneurl, + defaultmilestone) + + # Insert the components + for category in categories_list: + component = SqlQuote(category[0]) + productstr = SqlQuote(product) + description = SqlQuote(category[1]) + initialowner = SqlQuote("3") + print >>outfile, "\ninsert into components ("; + print >>outfile, " value, program, initialowner, initialqacontact," + print >>outfile, " description) values (" + print >>outfile, " %s, %s, %s, '', %s);" % (component, productstr, + initialowner, description) + + # Insert the versions + for productstr, version_list in versions_table.items(): + productstr = SqlQuote(productstr) + for version in version_list: + version = SqlQuote(version) + print >>outfile, "\ninsert into versions (value, program) " + print >>outfile, " values (%s, %s);" % (version, productstr) + + # Insert the users + for username, userid in usermapping.items(): + realname = map_username_to_realname(username) + username = SqlQuote(username) + realname = SqlQuote(realname) + print >>outfile, "\ninsert into profiles (" + print >>outfile, " userid, login_name, password, cryptpassword, realname, groupset" + print >>outfile, ") values (" + print >>outfile, "%s,%s,'password',encrypt('password'), %s, 0);" % (userid, username, realname) + print >>outfile, "update profiles set groupset=1 << 32 where login_name like '%\@gcc.gnu.org';" + +def unixdate2datetime(unixdate): + """ Convert a unix date to a datetime value """ + year, month, day, hour, min, sec, x, x, x, x = email.Utils.parsedate_tz(unixdate) + return "%d-%02d-%02d %02d:%02d:%02d" % (year,month,day,hour,min,sec) + +def unixdate2timestamp(unixdate): + """ Convert a unix date to a timestamp value """ + year, month, day, hour, min, sec, x, x, x, x = email.Utils.parsedate_tz(unixdate) + return "%d%02d%02d%02d%02d%02d" % (year,month,day,hour,min,sec) + +def SqlQuote(str): + """ Perform SQL quoting on a string """ + return "'%s'" % str.replace("'", """''""").replace("\\", "\\\\").replace("\0","\\0") + +def convert_gccver_to_ver(gccver): + """ Given a gcc version, convert it to a Bugzilla version. """ + for k in releasetovermap.keys(): + if re.search(".*%s.*" % k, gccver) is not None: + return releasetovermap[k] + result = re.search(r""".*(\d\.\d) \d+ \(experimental\).*""", gccver) + if result is not None: + return result.group(1) + return "unknown" + +def load_index(fname): + """ Load in the GNATS index file """ + global pr_list + ifp = open(fname) + for record in ifp.xreadlines(): + fields = record.split("|") + pr_list.append(fields[0]) + ifp.close() + +def load_categories(fname): + """ Load in the GNATS categories file """ + global categories_list + cfp = open(fname) + for record in cfp.xreadlines(): + if re.search("^#", record) is not None: + continue + categories_list.append(record.split(":")) + cfp.close() + +def map_username_to_realname(username): + """ Given a username, find the real name """ + name = username + name = re.sub("@.*", "", name) + for responsible_record in responsible_list: + if responsible_record[0] == name: + return responsible_record[1] + if len(responsible_record) > 2: + if responsible_record[2] == username: + return responsible_record[1] + return "" + + +def get_userid(responsible): + """ Given an email address, get the user id """ + global responsible_map + global usermapping + global userid_base + if responsible is None: + return -1 + responsible = responsible.lower() + responsible = re.sub("sources.redhat.com", "gcc.gnu.org", responsible) + if responsible_map.has_key(responsible): + responsible = responsible_map[responsible] + if usermapping.has_key(responsible): + return usermapping[responsible] + else: + usermapping[responsible] = userid_base + userid_base += 1 + return usermapping[responsible] + +def load_responsible(fname): + """ Load in the GNATS responsible file """ + global responsible_map + global responsible_list + rfp = open(fname) + for record in rfp.xreadlines(): + if re.search("^#", record) is not None: + continue + split_record = record.split(":") + responsible_map[split_record[0]] = split_record[2].rstrip() + responsible_list.append(record.split(":")) + rfp.close() + +def split_csl(list): + """ Split a comma seperated list """ + newlist = re.split(r"""\s*,\s*""", list) + return newlist + +def fix_email_addrs(addrs): + """ Perform various fixups and cleaning on an e-mail address """ + addrs = split_csl(addrs) + trimmed_addrs = [] + for addr in addrs: + addr = re.sub(r"""\(.*\)""","",addr) + addr = re.sub(r""".*<(.*)>.*""","\\1",addr) + addr = addr.rstrip() + addr = addr.lstrip() + trimmed_addrs.append(addr) + addrs = ", ".join(trimmed_addrs) + return addrs + +class Bugzillabug(object): + """ Class representing a bugzilla bug """ + def __init__(self, gbug): + """ Initialize a bugzilla bug from a GNATS bug. """ + self.bug_id = gbug.bug_id + self.long_descs = [] + self.bug_ccs = [get_userid("gcc-bugs@gcc.gnu.org")] + self.bug_activity = [] + self.attachments = gbug.attachments + self.gnatsfields = gbug.fields + self.need_unformatted = gbug.has_unformatted_attach == 0 + self.need_unformatted &= gbug.fields.has_key("Unformatted") + self.translate_pr() + self.update_versions() + if self.fields.has_key("Audit-Trail"): + self.parse_audit_trail() + self.write_bug() + + def parse_fromto(type, string): + """ Parses the from and to parts of a changed-from-to line """ + fromstr = "" + tostr = "" + + # Some slightly messed up changed lines have unassigned-new, + # instead of unassigned->new. So we make the > optional. + result = re.search(r"""(.*)-(?:>?)(.*)""", string) + + # Only know how to handle parsing of State and Responsible + # changed-from-to right now + if type == "State": + fromstr = state_lookup[result.group(1)] + tostr = state_lookup[result.group(2)] + elif type == "Responsible": + if result.group(1) != "": + fromstr = result.group(1) + if result.group(2) != "": + tostr = result.group(2) + if responsible_map.has_key(fromstr): + fromstr = responsible_map[fromstr] + if responsible_map.has_key(tostr): + tostr = responsible_map[tostr] + return (fromstr, tostr) + parse_fromto = staticmethod(parse_fromto) + + def parse_audit_trail(self): + """ Parse a GNATS audit trail """ + trail = self.fields["Audit-Trail"] + # Begin to split the audit trail into pieces + result = fromtore.finditer(trail) + starts = [] + ends = [] + pieces = [] + # Make a list of the pieces + for x in result: + pieces.append (x) + # Find the start and end of each piece + if len(pieces) > 0: + for x in xrange(len(pieces)-1): + starts.append(pieces[x].start()) + ends.append(pieces[x+1].start()) + starts.append(pieces[-1].start()) + ends.append(len(trail)) + pieces = [] + # Now make the list of actual text of the pieces + for x in xrange(len(starts)): + pieces.append(trail[starts[x]:ends[x]]) + # And parse the actual pieces + for piece in pieces: + result = changedfromtore.search(piece) + # See what things we actually have inside this entry, and + # handle them approriately + if result is not None: + type = result.group(1) + changedfromto = result.group(2) + # If the bug was reopened, mark it as such + if changedfromto.find("closed->analyzed") != -1: + if self.fields["bug_status"] == "'NEW'": + self.fields["bug_status"] = "'REOPENED'" + if type == "State" or type == "Responsible": + oldstate, newstate = self.parse_fromto (type, changedfromto) + result = changedbyre.search(piece) + if result is not None: + changedby = result.group(1) + result = changedwhenre.search(piece) + if result is not None: + changedwhen = result.group(1) + changedwhen = unixdate2datetime(changedwhen) + changedwhen = SqlQuote(changedwhen) + result = changedwhyre.search(piece) + changedwhy = piece[result.start(1):] + #changedwhy = changedwhy.lstrip() + changedwhy = changedwhy.rstrip() + changedby = get_userid(changedby) + # Put us on the cc list if we aren't there already + if changedby != self.fields["userid"] \ + and changedby not in self.bug_ccs: + self.bug_ccs.append(changedby) + # If it's a duplicate, mark it as such + result = duplicatere.search(changedwhy) + if result is not None: + newtext = "*** This bug has been marked as a duplicate of %s ***" % result.group(1) + newtext = SqlQuote(newtext) + self.long_descs.append((self.bug_id, changedby, + changedwhen, newtext)) + self.fields["bug_status"] = "'RESOLVED'" + self.fields["resolution"] = "'DUPLICATE'" + self.fields["userid"] = changedby + else: + newtext = "%s-Changed-From-To: %s\n%s-Changed-Why: %s\n" % (type, changedfromto, type, changedwhy) + newtext = SqlQuote(newtext) + self.long_descs.append((self.bug_id, changedby, + changedwhen, newtext)) + if type == "State" or type == "Responsible": + newstate = SqlQuote("%s" % newstate) + oldstate = SqlQuote("%s" % oldstate) + fieldid = fieldids[type] + self.bug_activity.append((newstate, oldstate, fieldid, changedby, changedwhen)) + + else: + # It's an email + result = fromre.search(piece) + if result is None: + continue + fromstr = result.group(1) + fromstr = fix_email_addrs(fromstr) + fromstr = get_userid(fromstr) + result = datere.search(piece) + if result is None: + continue + datestr = result.group(1) + datestr = SqlQuote(unixdate2timestamp(datestr)) + if fromstr != self.fields["userid"] \ + and fromstr not in self.bug_ccs: + self.bug_ccs.append(fromstr) + self.long_descs.append((self.bug_id, fromstr, datestr, + SqlQuote(piece))) + + + + def write_bug(self): + """ Output a bug to the data file """ + fields = self.fields + print >>outfile, "\ninsert into bugs(" + print >>outfile, " bug_id, assigned_to, bug_severity, priority, bug_status, creation_ts, delta_ts," + print >>outfile, " short_desc," + print >>outfile, " reporter, version," + print >>outfile, " product, component, resolution, target_milestone, qa_contact," + print >>outfile, " gccbuild, gcctarget, gcchost, keywords" + print >>outfile, " ) values (" + print >>outfile, "%s, %s, %s, %s, %s, %s, %s," % (self.bug_id, fields["userid"], fields["bug_severity"], fields["priority"], fields["bug_status"], fields["creation_ts"], fields["delta_ts"]) + print >>outfile, "%s," % (fields["short_desc"]) + print >>outfile, "%s, %s," % (fields["reporter"], fields["version"]) + print >>outfile, "%s, %s, %s, %s, 0," %(fields["product"], fields["component"], fields["resolution"], fields["target_milestone"]) + print >>outfile, "%s, %s, %s, %s" % (fields["gccbuild"], fields["gcctarget"], fields["gcchost"], fields["keywords"]) + print >>outfile, ");" + if self.fields["keywords"] != 0: + print >>outfile, "\ninsert into keywords (bug_id, keywordid) values (" + print >>outfile, " %s, %s);" % (self.bug_id, fields["keywordid"]) + for id, who, when, text in self.long_descs: + print >>outfile, "\ninsert into longdescs (" + print >>outfile, " bug_id, who, bug_when, thetext) values(" + print >>outfile, " %s, %s, %s, %s);" % (id, who, when, text) + for name, data, who in self.attachments: + print >>outfile, "\ninsert into attachments (" + print >>outfile, " bug_id, filename, description, mimetype, ispatch, submitter_id, thedata) values (" + ftype = None + # It's *magic*! + if name.endswith(".ii") == 1: + ftype = "text/x-c++" + elif name.endswith(".i") == 1: + ftype = "text/x-c" + else: + ftype = magicf.detect(cStringIO.StringIO(data)) + if ftype is None: + ftype = "application/octet-stream" + + print >>outfile, "%s,%s,%s, %s,0, %s,%s);" %(self.bug_id, SqlQuote(name), SqlQuote(name), SqlQuote (ftype), who, SqlQuote(zlib.compress(data))) + for newstate, oldstate, fieldid, changedby, changedwhen in self.bug_activity: + print >>outfile, "\ninsert into bugs_activity (" + print >>outfile, " bug_id, who, bug_when, fieldid, added, removed) values (" + print >>outfile, " %s, %s, %s, %s, %s, %s);" % (self.bug_id, + changedby, + changedwhen, + fieldid, + newstate, + oldstate) + for cc in self.bug_ccs: + print >>outfile, "\ninsert into cc(bug_id, who) values (%s, %s);" %(self.bug_id, cc) + def update_versions(self): + """ Update the versions table to account for the version on this bug """ + global versions_table + if self.fields.has_key("Release") == 0 \ + or self.fields.has_key("Category") == 0: + return + curr_product = "gcc" + curr_version = self.fields["Release"] + if curr_version == "": + return + curr_version = convert_gccver_to_ver (curr_version) + if versions_table.has_key(curr_product) == 0: + versions_table[curr_product] = [] + for version in versions_table[curr_product]: + if version == curr_version: + return + versions_table[curr_product].append(curr_version) + def translate_pr(self): + """ Transform a GNATS PR into a Bugzilla bug """ + self.fields = self.gnatsfields + if (self.fields.has_key("Organization") == 0) \ + or self.fields["Organization"].find("GCC"): + self.fields["Originator"] = "" + self.fields["Organization"] = "" + self.fields["Organization"].lstrip() + if (self.fields.has_key("Release") == 0) \ + or self.fields["Release"] == "" \ + or self.fields["Release"].find("unknown-1.0") != -1: + self.fields["Release"]="unknown" + if self.fields.has_key("Responsible"): + result = re.search(r"""\w+""", self.fields["Responsible"]) + self.fields["Responsible"] = "%s%s" % (result.group(0), "@gcc.gnu.org") + self.fields["gcchost"] = "" + self.fields["gcctarget"] = "" + self.fields["gccbuild"] = "" + if self.fields.has_key("Environment"): + result = re.search("^host: (.+?)$", self.fields["Environment"], + re.MULTILINE) + if result is not None: + self.fields["gcchost"] = result.group(1) + result = re.search("^target: (.+?)$", self.fields["Environment"], + re.MULTILINE) + if result is not None: + self.fields["gcctarget"] = result.group(1) + result = re.search("^build: (.+?)$", self.fields["Environment"], + re.MULTILINE) + if result is not None: + self.fields["gccbuild"] = result.group(1) + self.fields["userid"] = get_userid(self.fields["Responsible"]) + self.fields["bug_severity"] = "normal" + if self.fields["Class"] == "change-request": + self.fields["bug_severity"] = "enhancement" + elif self.fields.has_key("Severity"): + if self.fields["Severity"] == "critical": + self.fields["bug_severity"] = "critical" + elif self.fields["Severity"] == "serious": + self.fields["bug_severity"] = "major" + elif self.fields.has_key("Synopsis"): + if re.search("crash|assert", self.fields["Synopsis"]): + self.fields["bug_severity"] = "critical" + elif re.search("wrong|error", self.fields["Synopsis"]): + self.fields["bug_severity"] = "major" + self.fields["bug_severity"] = SqlQuote(self.fields["bug_severity"]) + self.fields["keywords"] = 0 + if keywordids.has_key(self.fields["Class"]): + self.fields["keywords"] = self.fields["Class"] + self.fields["keywordid"] = keywordids[self.fields["Class"]] + self.fields["keywords"] = SqlQuote(self.fields["keywords"]) + self.fields["priority"] = "P1" + if self.fields.has_key("Severity") and self.fields.has_key("Priority"): + severity = self.fields["Severity"] + priority = self.fields["Priority"] + if severity == "critical": + if priority == "high": + self.fields["priority"] = "P1" + else: + self.fields["priority"] = "P2" + elif severity == "serious": + if priority == "low": + self.fields["priority"] = "P4" + else: + self.fields["priority"] = "P3" + else: + if priority == "high": + self.fields["priority"] = "P4" + else: + self.fields["priority"] = "P5" + self.fields["priority"] = SqlQuote(self.fields["priority"]) + state = self.fields["State"] + if (state == "open" or state == "analyzed") and self.fields["userid"] != 3: + self.fields["bug_status"] = "ASSIGNED" + self.fields["resolution"] = "" + elif state == "feedback": + self.fields["bug_status"] = "WAITING" + self.fields["resolution"] = "" + elif state == "closed": + self.fields["bug_status"] = "CLOSED" + if self.fields.has_key("Class"): + theclass = self.fields["Class"] + if theclass.find("duplicate") != -1: + self.fields["resolution"]="DUPLICATE" + elif theclass.find("mistaken") != -1: + self.fields["resolution"]="INVALID" + else: + self.fields["resolution"]="FIXED" + else: + self.fields["resolution"]="FIXED" + elif state == "suspended": + self.fields["bug_status"] = "SUSPENDED" + self.fields["resolution"] = "" + elif state == "analyzed" and self.fields["userid"] == 3: + self.fields["bug_status"] = "NEW" + self.fields["resolution"] = "" + else: + self.fields["bug_status"] = "UNCONFIRMED" + self.fields["resolution"] = "" + self.fields["bug_status"] = SqlQuote(self.fields["bug_status"]) + self.fields["resolution"] = SqlQuote(self.fields["resolution"]) + self.fields["creation_ts"] = "" + if self.fields.has_key("Arrival-Date") and self.fields["Arrival-Date"] != "": + self.fields["creation_ts"] = unixdate2datetime(self.fields["Arrival-Date"]) + self.fields["creation_ts"] = SqlQuote(self.fields["creation_ts"]) + self.fields["delta_ts"] = "" + if self.fields.has_key("Audit-Trail"): + result = lastdatere.findall(self.fields["Audit-Trail"]) + result.reverse() + if len(result) > 0: + self.fields["delta_ts"] = unixdate2timestamp(result[0]) + if self.fields["delta_ts"] == "": + if self.fields.has_key("Arrival-Date") and self.fields["Arrival-Date"] != "": + self.fields["delta_ts"] = unixdate2timestamp(self.fields["Arrival-Date"]) + self.fields["delta_ts"] = SqlQuote(self.fields["delta_ts"]) + self.fields["short_desc"] = SqlQuote(self.fields["Synopsis"]) + if self.fields.has_key("Reply-To") and self.fields["Reply-To"] != "": + self.fields["reporter"] = get_userid(self.fields["Reply-To"]) + elif self.fields.has_key("Mail-Header"): + result = re.search(r"""From .*?([\w.]+@[\w.]+)""", self.fields["Mail-Header"]) + if result: + self.fields["reporter"] = get_userid(result.group(1)) + else: + self.fields["reporter"] = get_userid(gnats_username) + else: + self.fields["reporter"] = get_userid(gnats_username) + long_desc = self.fields["Description"] + long_desc2 = "" + for field in ["Release", "Environment", "How-To-Repeat"]: + if self.fields.has_key(field) and self.fields[field] != "": + long_desc += ("\n\n%s:\n" % field) + self.fields[field] + if self.fields.has_key("Fix") and self.fields["Fix"] != "": + long_desc2 = "Fix:\n" + self.fields["Fix"] + if self.need_unformatted == 1 and self.fields["Unformatted"] != "": + long_desc += "\n\nUnformatted:\n" + self.fields["Unformatted"] + if long_desc != "": + self.long_descs.append((self.bug_id, self.fields["reporter"], + self.fields["creation_ts"], + SqlQuote(long_desc))) + if long_desc2 != "": + self.long_descs.append((self.bug_id, self.fields["reporter"], + self.fields["creation_ts"], + SqlQuote(long_desc2))) + for field in ["gcchost", "gccbuild", "gcctarget"]: + self.fields[field] = SqlQuote(self.fields[field]) + self.fields["version"] = "" + if self.fields["Release"] != "": + self.fields["version"] = convert_gccver_to_ver (self.fields["Release"]) + self.fields["version"] = SqlQuote(self.fields["version"]) + self.fields["product"] = SqlQuote("gcc") + self.fields["component"] = "invalid" + if self.fields.has_key("Category"): + self.fields["component"] = self.fields["Category"] + self.fields["component"] = SqlQuote(self.fields["component"]) + self.fields["target_milestone"] = "---" + if self.fields["version"].find("3.4") != -1: + self.fields["target_milestone"] = "3.4" + self.fields["target_milestone"] = SqlQuote(self.fields["target_milestone"]) + if self.fields["userid"] == 2: + self.fields["userid"] = "\'NULL\'" + +class GNATSbug(object): + """ Represents a single GNATS PR """ + def __init__(self, filename): + self.attachments = [] + self.has_unformatted_attach = 0 + fp = open (filename) + self.fields = self.parse_pr(fp.xreadlines()) + self.bug_id = int(self.fields["Number"]) + if self.fields.has_key("Unformatted"): + self.find_gnatsweb_attachments() + if self.fields.has_key("How-To-Repeat"): + self.find_regular_attachments("How-To-Repeat") + if self.fields.has_key("Fix"): + self.find_regular_attachments("Fix") + + def get_attacher(fields): + if fields.has_key("Reply-To") and fields["Reply-To"] != "": + return get_userid(fields["Reply-To"]) + else: + result = None + if fields.has_key("Mail-Header"): + result = re.search(r"""From .*?([\w.]+\@[\w.]+)""", + fields["Mail-Header"]) + if result is not None: + reporter = get_userid(result.group(1)) + else: + reporter = get_userid(gnats_username) + get_attacher = staticmethod(get_attacher) + def find_regular_attachments(self, which): + fields = self.fields + while re.search("^begin [0-7]{3}", fields[which], + re.DOTALL | re.MULTILINE): + outfp = cStringIO.StringIO() + infp = cStringIO.StringIO(fields[which]) + filename, start, end = specialuu.decode(infp, outfp, quiet=0) + fields[which]=fields[which].replace(fields[which][start:end], + "See attachments for %s\n" % filename) + self.attachments.append((filename, outfp.getvalue(), + self.get_attacher(fields))) + + def decode_gnatsweb_attachment(self, attachment): + result = re.split(r"""\n\n""", attachment, 1) + if len(result) == 1: + return -1 + envelope, body = result + envelope = uselessre.split(envelope) + envelope.pop(0) + # Turn the list of key, value into a dict of key => value + attachinfo = dict([(envelope[i], envelope[i+1]) for i in xrange(0,len(envelope),2)]) + for x in attachinfo.keys(): + attachinfo[x] = attachinfo[x].rstrip() + if (attachinfo.has_key("Content-Type") == 0) or \ + (attachinfo.has_key("Content-Disposition") == 0): + raise ValueError, "Unable to parse file attachment" + result = dispositionre.search(attachinfo["Content-Disposition"]) + filename = result.group(2) + filename = re.sub(".*/","", filename) + filename = re.sub(".*\\\\","", filename) + attachinfo["filename"]=filename + result = re.search("""(\S+);.*""", attachinfo["Content-Type"]) + if result is not None: + attachinfo["Content-Type"] = result.group(1) + if attachinfo.has_key("Content-Transfer-Encoding"): + if attachinfo["Content-Transfer-Encoding"] == "base64": + attachinfo["data"] = base64.decodestring(body) + else: + attachinfo["data"]=body + + return (attachinfo["filename"], attachinfo["data"], + self.get_attacher(self.fields)) + + def find_gnatsweb_attachments(self): + fields = self.fields + attachments = re.split(attachment_delimiter, fields["Unformatted"]) + fields["Unformatted"] = attachments.pop(0) + for attachment in attachments: + result = self.decode_gnatsweb_attachment (attachment) + if result != -1: + self.attachments.append(result) + self.has_unformatted_attach = 1 + def parse_pr(lines): + #fields = {"envelope":[]} + fields = {"envelope":array.array("c")} + hdrmulti = "envelope" + for line in lines: + line = line.rstrip('\n') + line += '\n' + result = gnatfieldre.search(line) + if result is None: + if hdrmulti != "": + if fields.has_key(hdrmulti): + #fields[hdrmulti].append(line) + fields[hdrmulti].fromstring(line) + else: + #fields[hdrmulti] = [line] + fields[hdrmulti] = array.array("c", line) + continue + hdr, arg = result.groups() + ghdr = "*not valid*" + result = fieldnamere.search(hdr) + if result != None: + ghdr = result.groups()[0] + if ghdr in fieldnames: + if multilinefields.has_key(ghdr): + hdrmulti = ghdr + #fields[ghdr] = [""] + fields[ghdr] = array.array("c") + else: + hdrmulti = "" + #fields[ghdr] = [arg] + fields[ghdr] = array.array("c", arg) + elif hdrmulti != "": + #fields[hdrmulti].append(line) + fields[hdrmulti].fromstring(line) + if hdrmulti == "envelope" and \ + (hdr == "Reply-To" or hdr == "From" \ + or hdr == "X-GNATS-Notify"): + arg = fix_email_addrs(arg) + #fields[hdr] = [arg] + fields[hdr] = array.array("c", arg) + if fields.has_key("Reply-To") and len(fields["Reply-To"]) > 0: + fields["Reply-To"] = fields["Reply-To"] + else: + fields["Reply-To"] = fields["From"] + if fields.has_key("From"): + del fields["From"] + if fields.has_key("X-GNATS-Notify") == 0: + fields["X-GNATS-Notify"] = array.array("c") + #fields["X-GNATS-Notify"] = "" + for x in fields.keys(): + fields[x] = fields[x].tostring() + #fields[x] = "".join(fields[x]) + for x in fields.keys(): + if multilinefields.has_key(x): + fields[x] = fields[x].rstrip() + + return fields + parse_pr = staticmethod(parse_pr) +load_index("%s/gnats-adm/index" % gnats_db_dir) +load_categories("%s/gnats-adm/categories" % gnats_db_dir) +load_responsible("%s/gnats-adm/responsible" % gnats_db_dir) +get_userid(gnats_username) +get_userid(unassigned_username) +for x in pr_list: + print "Processing %s..." % x + a = GNATSbug ("%s/%s" % (gnats_db_dir, x)) + b = Bugzillabug(a) +write_non_bug_tables() +outfile.close() |