diff options
Diffstat (limited to 'aurweb')
-rw-r--r-- | aurweb/scripts/__init__.py | 0 | ||||
-rwxr-xr-x | aurweb/scripts/aurblup.py | 55 | ||||
-rwxr-xr-x | aurweb/scripts/mkpkglists.py | 38 | ||||
-rwxr-xr-x | aurweb/scripts/notify.py | 455 | ||||
-rwxr-xr-x | aurweb/scripts/pkgmaint.py | 20 | ||||
-rwxr-xr-x | aurweb/scripts/popupdate.py | 26 | ||||
-rwxr-xr-x | aurweb/scripts/tuvotereminder.py | 28 |
7 files changed, 622 insertions, 0 deletions
diff --git a/aurweb/scripts/__init__.py b/aurweb/scripts/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/aurweb/scripts/__init__.py diff --git a/aurweb/scripts/aurblup.py b/aurweb/scripts/aurblup.py new file mode 100755 index 00000000..1b6de2f0 --- /dev/null +++ b/aurweb/scripts/aurblup.py @@ -0,0 +1,55 @@ +#!/usr/bin/python3 + +import pyalpm +import re + +import aurweb.config +import aurweb.db + +db_path = aurweb.config.get('aurblup', 'db-path') +sync_dbs = aurweb.config.get('aurblup', 'sync-dbs').split(' ') +server = aurweb.config.get('aurblup', 'server') + + +def main(): + blacklist = set() + providers = set() + repomap = dict() + + h = pyalpm.Handle("/", db_path) + for sync_db in sync_dbs: + repo = h.register_syncdb(sync_db, pyalpm.SIG_DATABASE_OPTIONAL) + repo.servers = [server.replace("%s", sync_db)] + t = h.init_transaction() + repo.update(False) + t.release() + + for pkg in repo.pkgcache: + blacklist.add(pkg.name) + [blacklist.add(x) for x in pkg.replaces] + providers.add((pkg.name, pkg.name)) + repomap[(pkg.name, pkg.name)] = repo.name + for provision in pkg.provides: + provisionname = re.sub(r'(<|=|>).*', '', provision) + providers.add((pkg.name, provisionname)) + repomap[(pkg.name, provisionname)] = repo.name + + conn = aurweb.db.Connection() + + cur = conn.execute("SELECT Name, Provides FROM OfficialProviders") + oldproviders = set(cur.fetchall()) + + for pkg, provides in providers.difference(oldproviders): + repo = repomap[(pkg, provides)] + conn.execute("INSERT INTO OfficialProviders (Name, Repo, Provides) " + "VALUES (?, ?, ?)", [pkg, repo, provides]) + for pkg, provides in oldproviders.difference(providers): + conn.execute("DELETE FROM OfficialProviders " + "WHERE Name = ? AND Provides = ?", [pkg, provides]) + + conn.commit() + conn.close() + + +if __name__ == '__main__': + main() diff --git a/aurweb/scripts/mkpkglists.py b/aurweb/scripts/mkpkglists.py new file mode 100755 index 00000000..8a0f2e9d --- /dev/null +++ b/aurweb/scripts/mkpkglists.py @@ -0,0 +1,38 @@ +#!/usr/bin/python3 + +import datetime +import gzip + +import aurweb.config +import aurweb.db + +packagesfile = aurweb.config.get('mkpkglists', 'packagesfile') +pkgbasefile = aurweb.config.get('mkpkglists', 'pkgbasefile') + + +def main(): + conn = aurweb.db.Connection() + + datestr = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT") + pkglist_header = "# AUR package list, generated on " + datestr + pkgbaselist_header = "# AUR package base list, generated on " + datestr + + with gzip.open(packagesfile, "w") as f: + f.write(bytes(pkglist_header + "\n", "UTF-8")) + cur = conn.execute("SELECT Packages.Name FROM Packages " + + "INNER JOIN PackageBases " + + "ON PackageBases.ID = Packages.PackageBaseID " + + "WHERE PackageBases.PackagerUID IS NOT NULL") + f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) + + with gzip.open(pkgbasefile, "w") as f: + f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) + cur = conn.execute("SELECT Name FROM PackageBases " + + "WHERE PackagerUID IS NOT NULL") + f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) + + conn.close() + + +if __name__ == '__main__': + main() diff --git a/aurweb/scripts/notify.py b/aurweb/scripts/notify.py new file mode 100755 index 00000000..ddd6e49b --- /dev/null +++ b/aurweb/scripts/notify.py @@ -0,0 +1,455 @@ +#!/usr/bin/python3 + +import email.mime.text +import subprocess +import sys +import textwrap + +import aurweb.config +import aurweb.db + +aur_location = aurweb.config.get('options', 'aur_location') +aur_request_ml = aurweb.config.get('options', 'aur_request_ml') + +sendmail = aurweb.config.get('notifications', 'sendmail') +sender = aurweb.config.get('notifications', 'sender') +reply_to = aurweb.config.get('notifications', 'reply-to') + + +def headers_cc(cclist): + return {'Cc': str.join(', ', cclist)} + + +def headers_msgid(thread_id): + return {'Message-ID': thread_id} + + +def headers_reply(thread_id): + return {'In-Reply-To': thread_id, 'References': thread_id} + + +def send_notification(to, subject, body, refs, headers={}): + wrapped = '' + for line in body.splitlines(): + wrapped += textwrap.fill(line, break_long_words=False) + '\n' + if refs: + body = wrapped + '\n' + refs + else: + body = wrapped + + for recipient in to: + msg = email.mime.text.MIMEText(body, 'plain', 'utf-8') + msg['Subject'] = subject + msg['From'] = sender + msg['Reply-to'] = reply_to + msg['To'] = recipient + + for key, value in headers.items(): + msg[key] = value + + p = subprocess.Popen([sendmail, '-t', '-oi'], stdin=subprocess.PIPE) + p.communicate(msg.as_bytes()) + + +def username_from_id(conn, uid): + cur = conn.execute('SELECT UserName FROM Users WHERE ID = ?', [uid]) + return cur.fetchone()[0] + + +def pkgbase_from_id(conn, pkgbase_id): + cur = conn.execute('SELECT Name FROM PackageBases WHERE ID = ?', + [pkgbase_id]) + return cur.fetchone()[0] + + +def pkgbase_from_pkgreq(conn, reqid): + cur = conn.execute('SELECT PackageBaseID FROM PackageRequests ' + + 'WHERE ID = ?', [reqid]) + return cur.fetchone()[0] + + +def get_user_email(conn, uid): + cur = conn.execute('SELECT Email FROM Users WHERE ID = ?', [uid]) + return cur.fetchone()[0] + + +def get_maintainer_email(conn, pkgbase_id): + cur = conn.execute('SELECT Users.Email FROM Users ' + + 'INNER JOIN PackageBases ' + + 'ON PackageBases.MaintainerUID = Users.ID WHERE ' + + 'PackageBases.ID = ?', [pkgbase_id]) + return cur.fetchone()[0] + + +def get_recipients(conn, pkgbase_id, uid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + + 'INNER JOIN PackageNotifications ' + + 'ON PackageNotifications.UserID = Users.ID WHERE ' + + 'PackageNotifications.UserID != ? AND ' + + 'PackageNotifications.PackageBaseID = ?', + [uid, pkgbase_id]) + return [row[0] for row in cur.fetchall()] + + +def get_comment_recipients(conn, pkgbase_id, uid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + + 'INNER JOIN PackageNotifications ' + + 'ON PackageNotifications.UserID = Users.ID WHERE ' + + 'Users.CommentNotify = 1 AND ' + + 'PackageNotifications.UserID != ? AND ' + + 'PackageNotifications.PackageBaseID = ?', + [uid, pkgbase_id]) + return [row[0] for row in cur.fetchall()] + + +def get_update_recipients(conn, pkgbase_id, uid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + + 'INNER JOIN PackageNotifications ' + + 'ON PackageNotifications.UserID = Users.ID WHERE ' + + 'Users.UpdateNotify = 1 AND ' + + 'PackageNotifications.UserID != ? AND ' + + 'PackageNotifications.PackageBaseID = ?', + [uid, pkgbase_id]) + return [row[0] for row in cur.fetchall()] + + +def get_ownership_recipients(conn, pkgbase_id, uid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + + 'INNER JOIN PackageNotifications ' + + 'ON PackageNotifications.UserID = Users.ID WHERE ' + + 'Users.OwnershipNotify = 1 AND ' + + 'PackageNotifications.UserID != ? AND ' + + 'PackageNotifications.PackageBaseID = ?', + [uid, pkgbase_id]) + return [row[0] for row in cur.fetchall()] + + +def get_request_recipients(conn, reqid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM PackageRequests ' + + 'INNER JOIN PackageBases ' + + 'ON PackageBases.ID = PackageRequests.PackageBaseID ' + + 'INNER JOIN Users ' + + 'ON Users.ID = PackageRequests.UsersID ' + + 'OR Users.ID = PackageBases.MaintainerUID ' + + 'WHERE PackageRequests.ID = ?', [reqid]) + return [row[0] for row in cur.fetchall()] + + +def get_tu_vote_reminder_recipients(conn, vote_id): + cur = conn.execute('SELECT Users.Email FROM Users ' + + 'WHERE AccountTypeID = 2 ' + + 'EXCEPT SELECT Users.Email FROM Users ' + + 'INNER JOIN TU_Votes ' + + 'ON TU_Votes.UserID = Users.ID ' + + 'WHERE TU_Votes.VoteID = ?', [vote_id]) + return [row[0] for row in cur.fetchall()] + + +def get_comment(conn, comment_id): + cur = conn.execute('SELECT Comments FROM PackageComments WHERE ID = ?', + [comment_id]) + return cur.fetchone()[0] + + +def get_flagger_comment(conn, pkgbase_id): + cur = conn.execute('SELECT FlaggerComment FROM PackageBases WHERE ID = ?', + [pkgbase_id]) + return cur.fetchone()[0] + + +def get_request_comment(conn, reqid): + cur = conn.execute('SELECT Comments FROM PackageRequests WHERE ID = ?', + [reqid]) + return cur.fetchone()[0] + + +def get_request_closure_comment(conn, reqid): + cur = conn.execute('SELECT ClosureComment FROM PackageRequests ' + + 'WHERE ID = ?', [reqid]) + return cur.fetchone()[0] + + +def send_resetkey(conn, uid): + cur = conn.execute('SELECT UserName, Email, ResetKey FROM Users ' + + 'WHERE ID = ?', [uid]) + username, to, resetkey = cur.fetchone() + + subject = 'AUR Password Reset' + body = 'A password reset request was submitted for the account %s ' \ + 'associated with your email address. If you wish to reset your ' \ + 'password follow the link [1] below, otherwise ignore this ' \ + 'message and nothing will happen.' % (username) + refs = '[1] ' + aur_location + '/passreset/?resetkey=' + resetkey + + send_notification([to], subject, body, refs) + + +def welcome(conn, uid): + cur = conn.execute('SELECT UserName, Email, ResetKey FROM Users ' + + 'WHERE ID = ?', [uid]) + username, to, resetkey = cur.fetchone() + + subject = 'Welcome to the Arch User Repository' + body = 'Welcome to the Arch User Repository! In order to set an initial ' \ + 'password for your new account, please click the link [1] below. ' \ + 'If the link does not work, try copying and pasting it into your ' \ + 'browser.' + refs = '[1] ' + aur_location + '/passreset/?resetkey=' + resetkey + + send_notification([to], subject, body, refs) + + +def comment(conn, uid, pkgbase_id, comment_id): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = get_comment_recipients(conn, pkgbase_id, uid) + text = get_comment(conn, comment_id) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Comment for %s' % (pkgbase) + body = '%s [1] added the following comment to %s [2]:' % (user, pkgbase) + body += '\n\n' + text + '\n\n' + body += 'If you no longer wish to receive notifications about this ' \ + 'package, please go to the package page [2] and select "%s".' % \ + ('Disable notifications') + refs = '[1] ' + user_uri + '\n' + refs += '[2] ' + pkgbase_uri + thread_id = '<pkg-notifications-' + pkgbase + '@aur.archlinux.org>' + headers = headers_reply(thread_id) + + send_notification(to, subject, body, refs, headers) + + +def update(conn, uid, pkgbase_id): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = get_update_recipients(conn, pkgbase_id, uid) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Package Update: %s' % (pkgbase) + body = '%s [1] pushed a new commit to %s [2].' % (user, pkgbase) + body += '\n\n' + body += 'If you no longer wish to receive notifications about this ' \ + 'package, please go to the package page [2] and select "%s".' % \ + ('Disable notifications') + refs = '[1] ' + user_uri + '\n' + refs += '[2] ' + pkgbase_uri + thread_id = '<pkg-notifications-' + pkgbase + '@aur.archlinux.org>' + headers = headers_reply(thread_id) + + send_notification(to, subject, body, refs, headers) + + +def flag(conn, uid, pkgbase_id): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = [get_maintainer_email(conn, pkgbase_id)] + text = get_flagger_comment(conn, pkgbase_id) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Out-of-date Notification for %s' % (pkgbase) + body = 'Your package %s [1] has been flagged out-of-date by %s [2]:' % \ + (pkgbase, user) + body += '\n\n' + text + refs = '[1] ' + pkgbase_uri + '\n' + refs += '[2] ' + user_uri + + send_notification(to, subject, body, refs) + + +def adopt(conn, pkgbase_id, uid): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = get_ownership_recipients(conn, pkgbase_id, uid) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Ownership Notification for %s' % (pkgbase) + body = 'The package %s [1] was adopted by %s [2].' % (pkgbase, user) + refs = '[1] ' + pkgbase_uri + '\n' + refs += '[2] ' + user_uri + + send_notification(to, subject, body, refs) + + +def disown(conn, pkgbase_id, uid): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = get_ownership_recipients(conn, pkgbase_id, uid) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Ownership Notification for %s' % (pkgbase) + body = 'The package %s [1] was disowned by %s [2].' % (pkgbase, user) + refs = '[1] ' + pkgbase_uri + '\n' + refs += '[2] ' + user_uri + + send_notification(to, subject, body, refs) + + +def comaintainer_add(conn, pkgbase_id, uid): + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = [get_user_email(conn, uid)] + + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Co-Maintainer Notification for %s' % (pkgbase) + body = 'You were added to the co-maintainer list of %s [1].' % (pkgbase) + refs = '[1] ' + pkgbase_uri + '\n' + + send_notification(to, subject, body, refs) + + +def comaintainer_remove(conn, pkgbase_id, uid): + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = [get_user_email(conn, uid)] + + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = 'AUR Co-Maintainer Notification for %s' % (pkgbase) + body = ('You were removed from the co-maintainer list of %s [1].' % + (pkgbase)) + refs = '[1] ' + pkgbase_uri + '\n' + + send_notification(to, subject, body, refs) + + +def delete(conn, uid, old_pkgbase_id, new_pkgbase_id=None): + user = username_from_id(conn, uid) + old_pkgbase = pkgbase_from_id(conn, old_pkgbase_id) + if new_pkgbase_id: + new_pkgbase = pkgbase_from_id(conn, new_pkgbase_id) + to = get_recipients(conn, old_pkgbase_id, uid) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + old_pkgbase + '/' + + subject = 'AUR Package deleted: %s' % (old_pkgbase) + if new_pkgbase_id: + new_pkgbase_uri = aur_location + '/pkgbase/' + new_pkgbase + '/' + body = '%s [1] merged %s [2] into %s [3].\n\n' \ + 'If you no longer wish receive notifications about the new ' \ + 'package, please go to [3] and click "%s".' %\ + (user, old_pkgbase, new_pkgbase, 'Disable notifications') + refs = '[1] ' + user_uri + '\n' + refs += '[2] ' + pkgbase_uri + '\n' + refs += '[3] ' + new_pkgbase_uri + else: + body = '%s [1] deleted %s [2].\n\n' \ + 'You will no longer receive notifications about this ' \ + 'package.' % (user, old_pkgbase) + refs = '[1] ' + user_uri + '\n' + refs += '[2] ' + pkgbase_uri + + send_notification(to, subject, body, refs) + + +def request_open(conn, uid, reqid, reqtype, pkgbase_id, merge_into=None): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = [aur_request_ml] + cc = get_request_recipients(conn, reqid) + text = get_request_comment(conn, reqid) + + user_uri = aur_location + '/account/' + user + '/' + pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' + + subject = '[PRQ#%d] %s Request for %s' % \ + (int(reqid), reqtype.title(), pkgbase) + if merge_into: + merge_into_uri = aur_location + '/pkgbase/' + merge_into + '/' + body = '%s [1] filed a request to merge %s [2] into %s [3]:' % \ + (user, pkgbase, merge_into) + body += '\n\n' + text + refs = '[1] ' + user_uri + '\n' + refs += '[2] ' + pkgbase_uri + '\n' + refs += '[3] ' + merge_into_uri + else: + body = '%s [1] filed a %s request for %s [2]:' % \ + (user, reqtype, pkgbase) + body += '\n\n' + text + refs = '[1] ' + user_uri + '\n' + refs += '[2] ' + pkgbase_uri + '\n' + thread_id = '<pkg-request-' + reqid + '@aur.archlinux.org>' + # Use a deterministic Message-ID for the first email referencing a request. + headers = headers_msgid(thread_id) + headers.update(headers_cc(cc)) + + send_notification(to, subject, body, refs, headers) + + +def request_close(conn, uid, reqid, reason): + to = [aur_request_ml] + cc = get_request_recipients(conn, reqid) + text = get_request_closure_comment(conn, reqid) + + subject = '[PRQ#%d] Request %s' % (int(reqid), reason.title()) + if int(uid): + user = username_from_id(conn, uid) + user_uri = aur_location + '/account/' + user + '/' + body = 'Request #%d has been %s by %s [1]' % (int(reqid), reason, user) + refs = '[1] ' + user_uri + else: + body = 'Request #%d has been %s automatically by the Arch User ' \ + 'Repository package request system' % (int(reqid), reason) + refs = None + if text.strip() == '': + body += '.' + else: + body += ':\n\n' + text + thread_id = '<pkg-request-' + reqid + '@aur.archlinux.org>' + headers = headers_reply(thread_id) + headers.update(headers_cc(cc)) + + send_notification(to, subject, body, refs, headers) + + +def tu_vote_reminder(conn, vote_id): + to = get_tu_vote_reminder_recipients(conn, vote_id) + + vote_uri = aur_location + '/tu/?id=' + vote_id + + subject = 'TU Vote Reminder: Proposal %d' % (int(vote_id)) + body = 'Please remember to cast your vote on proposal %d [1]. ' \ + 'The voting period ends in less than 48 hours.' % (int(vote_id)) + refs = '[1] ' + vote_uri + + send_notification(to, subject, body, refs) + + +def main(): + action = sys.argv[1] + action_map = { + 'send-resetkey': send_resetkey, + 'welcome': welcome, + 'comment': comment, + 'update': update, + 'flag': flag, + 'adopt': adopt, + 'disown': disown, + 'comaintainer-add': comaintainer_add, + 'comaintainer-remove': comaintainer_remove, + 'delete': delete, + 'request-open': request_open, + 'request-close': request_close, + 'tu-vote-reminder': tu_vote_reminder, + } + + conn = aurweb.db.Connection() + + action_map[action](conn, *sys.argv[2:]) + + conn.commit() + conn.close() + + +if __name__ == '__main__': + main() diff --git a/aurweb/scripts/pkgmaint.py b/aurweb/scripts/pkgmaint.py new file mode 100755 index 00000000..3ad9ed8d --- /dev/null +++ b/aurweb/scripts/pkgmaint.py @@ -0,0 +1,20 @@ +#!/usr/bin/python3 + +import time + +import aurweb.db + + +def main(): + conn = aurweb.db.Connection() + + limit_to = int(time.time()) - 86400 + conn.execute("DELETE FROM PackageBases WHERE " + + "SubmittedTS < ? AND PackagerUID IS NULL", [limit_to]) + + conn.commit() + conn.close() + + +if __name__ == '__main__': + main() diff --git a/aurweb/scripts/popupdate.py b/aurweb/scripts/popupdate.py new file mode 100755 index 00000000..58cd0185 --- /dev/null +++ b/aurweb/scripts/popupdate.py @@ -0,0 +1,26 @@ +#!/usr/bin/python3 + +import time + +import aurweb.db + + +def main(): + conn = aurweb.db.Connection() + + conn.execute("UPDATE PackageBases SET NumVotes = (" + + "SELECT COUNT(*) FROM PackageVotes " + + "WHERE PackageVotes.PackageBaseID = PackageBases.ID)") + + now = int(time.time()) + conn.execute("UPDATE PackageBases SET Popularity = (" + + "SELECT COALESCE(SUM(POWER(0.98, (? - VoteTS) / 86400)), 0.0) " + + "FROM PackageVotes WHERE PackageVotes.PackageBaseID = " + + "PackageBases.ID AND NOT VoteTS IS NULL)", [now]) + + conn.commit() + conn.close() + + +if __name__ == '__main__': + main() diff --git a/aurweb/scripts/tuvotereminder.py b/aurweb/scripts/tuvotereminder.py new file mode 100755 index 00000000..97b1d12e --- /dev/null +++ b/aurweb/scripts/tuvotereminder.py @@ -0,0 +1,28 @@ +#!/usr/bin/python3 + +import subprocess +import time + +import aurweb.config +import aurweb.db + +notify_cmd = aurweb.config.get('notifications', 'notify-cmd') + + +def main(): + conn = aurweb.db.Connection() + + now = int(time.time()) + filter_from = now + 500 + filter_to = now + 172800 + + cur = conn.execute("SELECT ID FROM TU_VoteInfo " + + "WHERE End >= ? AND End <= ?", + [filter_from, filter_to]) + + for vote_id in [row[0] for row in cur.fetchall()]: + subprocess.Popen((notify_cmd, 'tu-vote-reminder', str(vote_id))).wait() + + +if __name__ == '__main__': + main() |