From 10ecd3982decc31bda251ec73d00d6a903a75f9f Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Mon, 1 Jun 2015 23:36:14 +0200 Subject: Restructure scripts * Move scripts/git-integration/ to git-interface/. * Move scripts/aurblup/aurblup.py to scripts/aurblup.py. Signed-off-by: Lukas Fleischer --- INSTALL | 4 +- conf/config.proto | 8 +- git-interface/aurinfo.py | 204 ++++++++++++++++++++++++++ git-interface/git-auth.py | 44 ++++++ git-interface/git-auth.sh | 3 + git-interface/git-serve.py | 162 +++++++++++++++++++++ git-interface/git-update.py | 262 ++++++++++++++++++++++++++++++++++ scripts/aurblup.py | 48 +++++++ scripts/aurblup/aurblup.py | 48 ------- scripts/git-integration/aurinfo.py | 204 -------------------------- scripts/git-integration/git-auth.py | 44 ------ scripts/git-integration/git-auth.sh | 3 - scripts/git-integration/git-serve.py | 162 --------------------- scripts/git-integration/git-update.py | 262 ---------------------------------- 14 files changed, 729 insertions(+), 729 deletions(-) create mode 100644 git-interface/aurinfo.py create mode 100755 git-interface/git-auth.py create mode 100755 git-interface/git-auth.sh create mode 100755 git-interface/git-serve.py create mode 100755 git-interface/git-update.py create mode 100755 scripts/aurblup.py delete mode 100755 scripts/aurblup/aurblup.py delete mode 100644 scripts/git-integration/aurinfo.py delete mode 100755 scripts/git-integration/git-auth.py delete mode 100755 scripts/git-integration/git-auth.sh delete mode 100755 scripts/git-integration/git-serve.py delete mode 100755 scripts/git-integration/git-update.py diff --git a/INSTALL b/INSTALL index 026d4b9a..3aad5156 100644 --- a/INSTALL +++ b/INSTALL @@ -28,12 +28,12 @@ Setup on Arch Linux # mkdir /srv/http/aurweb/aur.git/ # cd /srv/http/aurweb/aur.git/ # git init --bare - # ln -s ../../scripts/git-integration/git-update.py hooks/update + # ln -s ../../git-interface/git-update.py hooks/update # chown -R aur . 7) Install the git-auth wrapper script: - # cd /srv/http/aurweb/scripts/git-integration/ + # cd /srv/http/aurweb/git-interface/ # cp git-auth.sh /usr/local/bin/aur-git-auth # chmod 755 /usr/local/bin/aur-git-auth diff --git a/conf/config.proto b/conf/config.proto index 90b5bd5b..7c56a2b4 100644 --- a/conf/config.proto +++ b/conf/config.proto @@ -30,18 +30,18 @@ auto_delete_age = 86400 [auth] valid-keytypes = ssh-rsa ssh-dss ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521 ssh-ed25519 username-regex = [a-zA-Z0-9]+[.\-_]?[a-zA-Z0-9]+$ -git-serve-cmd = /srv/http/aurweb/scripts/git-integration/git-serve.py +git-serve-cmd = /srv/http/aurweb/git-interface/git-serve.py ssh-options = no-port-forwarding,no-X11-forwarding,no-pty [serve] repo-path = /srv/http/aurweb/aur.git/ repo-regex = [a-z0-9][a-z0-9.+_-]*$ -template-path = /srv/http/aurweb/scripts/git-integration/templates/ -git-update-hook = /srv/http/aurweb/scripts/git-integration/git-update.py +template-path = /srv/http/aurweb/git-interface/templates/ +git-update-hook = /srv/http/aurweb/git-interface/git-update.py git-shell-cmd = /usr/bin/git-shell ssh-cmdline = ssh aur@aur.archlinux.org [aurblup] -db-path = /srv/http/aurweb/scripts/aurblup/ +db-path = /srv/http/aurweb/aurblup/ sync-dbs = core extra community multilib testing community-testing servers = ftp://mirrors.kernel.org/archlinux/%s/os/x86_64 diff --git a/git-interface/aurinfo.py b/git-interface/aurinfo.py new file mode 100644 index 00000000..d9b93729 --- /dev/null +++ b/git-interface/aurinfo.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python + +from copy import copy, deepcopy +import pprint +import sys + +class Attr(object): + def __init__(self, name, is_multivalued=False, allow_arch_extensions=False): + self.name = name + self.is_multivalued = is_multivalued + self.allow_arch_extensions = allow_arch_extensions + +PKGBUILD_ATTRIBUTES = { + 'arch': Attr('arch', True), + 'backup': Attr('backup', True), + 'changelog': Attr('changelog', False), + 'checkdepends': Attr('checkdepends', True), + 'conflicts': Attr('conflicts', True, True), + 'depends': Attr('depends', True, True), + 'epoch': Attr('epoch', False), + 'groups': Attr('groups', True), + 'install': Attr('install', False), + 'license': Attr('license', True), + 'makedepends': Attr('makedepends', True, True), + 'md5sums': Attr('md5sums', True, True), + 'noextract': Attr('noextract', True), + 'optdepends': Attr('optdepends', True, True), + 'options': Attr('options', True), + 'pkgname': Attr('pkgname', False), + 'pkgrel': Attr('pkgrel', False), + 'pkgver': Attr('pkgver', False), + 'provides': Attr('provides', True, True), + 'replaces': Attr('replaces', True, True), + 'sha1sums': Attr('sha1sums', True, True), + 'sha224sums': Attr('sha224sums', True, True), + 'sha256sums': Attr('sha256sums', True, True), + 'sha384sums': Attr('sha384sums', True, True), + 'sha512sums': Attr('sha512sums', True, True), + 'source': Attr('source', True, True), + 'url': Attr('url', False), + 'validpgpkeys': Attr('validpgpkeys', True), +} + +def find_attr(attrname): + # exact match + attr = PKGBUILD_ATTRIBUTES.get(attrname, None) + if attr: + return attr + + # prefix match + # XXX: this could break in the future if PKGBUILD(5) ever + # introduces a key which is a subset of another. + for k in PKGBUILD_ATTRIBUTES.keys(): + if attrname.startswith(k + '_'): + return PKGBUILD_ATTRIBUTES[k] + +def IsMultiValued(attrname): + attr = find_attr(attrname) + return attr and attr.is_multivalued + +class AurInfo(object): + def __init__(self): + self._pkgbase = {} + self._packages = {} + + def GetPackageNames(self): + return self._packages.keys() + + def GetMergedPackage(self, pkgname): + package = deepcopy(self._pkgbase) + package['pkgname'] = pkgname + for k, v in self._packages.get(pkgname).items(): + package[k] = deepcopy(v) + return package + + def AddPackage(self, pkgname): + self._packages[pkgname] = {} + return self._packages[pkgname] + + def SetPkgbase(self, pkgbasename): + self._pkgbase = {'pkgname' : pkgbasename} + return self._pkgbase + + +class StderrECatcher(object): + def Catch(self, lineno, error): + print('ERROR[%d]: %s' % (lineno, error), file=sys.stderr) + + +class CollectionECatcher(object): + def __init__(self): + self._errors = [] + + def Catch(self, lineno, error): + self._errors.append((lineno, error)) + + def HasErrors(self): + return len(self._errors) > 0 + + def Errors(self): + return copy(self._errors) + + +def ParseAurinfoFromIterable(iterable, ecatcher=None): + aurinfo = AurInfo() + + if ecatcher is None: + ecatcher = StderrECatcher() + + current_package = None + lineno = 0 + + for line in iterable: + lineno += 1 + + if line.startswith('#'): + continue + + if not line.strip(): + # end of package + current_package = None + continue + + if not line.startswith('\t'): + # start of new package + try: + key, value = map(str.strip, line.split('=', 1)) + except ValueError: + ecatcher.Catch(lineno, 'unexpected header format in section=%s' % + current_package['pkgname']) + continue + + if key == 'pkgbase': + current_package = aurinfo.SetPkgbase(value) + else: + current_package = aurinfo.AddPackage(value) + else: + # package attribute + if current_package is None: + ecatcher.Catch(lineno, 'package attribute found outside of ' + 'a package section') + continue + + try: + key, value = map(str.strip, line.split('=', 1)) + except ValueError: + ecatcher.Catch(lineno, 'unexpected attribute format in ' + 'section=%s' % current_package['pkgname']) + + if IsMultiValued(key): + if not current_package.get(key): + current_package[key] = [] + if value: + current_package[key].append(value) + else: + if not current_package.get(key): + current_package[key] = value + else: + ecatcher.Catch(lineno, 'overwriting attribute ' + '%s: %s -> %s' % (key, current_package[key], + value)) + + return aurinfo + + +def ParseAurinfo(filename='.AURINFO', ecatcher=None): + with open(filename) as f: + return ParseAurinfoFromIterable(f, ecatcher) + + +def ValidateAurinfo(filename='.AURINFO'): + ecatcher = CollectionECatcher() + ParseAurinfo(filename, ecatcher) + errors = ecatcher.Errors() + for error in errors: + print('error on line %d: %s' % error, file=sys.stderr) + return not errors + + +if __name__ == '__main__': + pp = pprint.PrettyPrinter(indent=4) + + if len(sys.argv) == 1: + print('error: not enough arguments') + sys.exit(1) + elif len(sys.argv) == 2: + action = sys.argv[1] + filename = '.AURINFO' + else: + action, filename = sys.argv[1:3] + + if action == 'parse': + aurinfo = ParseAurinfo() + for pkgname in aurinfo.GetPackageNames(): + print(">>> merged package: %s" % pkgname) + pp.pprint(aurinfo.GetMergedPackage(pkgname)) + print() + elif action == 'validate': + sys.exit(not ValidateAurinfo(filename)) + else: + print('unknown action: %s' % action) + sys.exit(1) + +# vim: set et ts=4 sw=4: diff --git a/git-interface/git-auth.py b/git-interface/git-auth.py new file mode 100755 index 00000000..c9e1f015 --- /dev/null +++ b/git-interface/git-auth.py @@ -0,0 +1,44 @@ +#!/usr/bin/python3 + +import configparser +import mysql.connector +import os +import re +import sys + +config = configparser.RawConfigParser() +config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config") + +aur_db_host = config.get('database', 'host') +aur_db_name = config.get('database', 'name') +aur_db_user = config.get('database', 'user') +aur_db_pass = config.get('database', 'password') +aur_db_socket = config.get('database', 'socket') + +valid_keytypes = config.get('auth', 'valid-keytypes').split() +username_regex = config.get('auth', 'username-regex') +git_serve_cmd = config.get('auth', 'git-serve-cmd') +ssh_opts = config.get('auth', 'ssh-options') + +keytype = sys.argv[1] +keytext = sys.argv[2] +if not keytype in valid_keytypes: + exit(1) + +db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, + passwd=aur_db_pass, db=aur_db_name, + unix_socket=aur_db_socket, buffered=True) + +cur = db.cursor() +cur.execute("SELECT Username FROM Users WHERE SSHPubKey = %s " + + "AND Suspended = 0", (keytype + " " + keytext,)) + +if cur.rowcount != 1: + exit(1) + +user = cur.fetchone()[0] +if not re.match(username_regex, user): + exit(1) + +print('command="%s %s",%s %s' % (git_serve_cmd, user, ssh_opts, + keytype + " " + keytext)) diff --git a/git-interface/git-auth.sh b/git-interface/git-auth.sh new file mode 100755 index 00000000..a5caeec4 --- /dev/null +++ b/git-interface/git-auth.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +/srv/http/aurweb/git-interface/git-auth.py "$1" "$2" diff --git a/git-interface/git-serve.py b/git-interface/git-serve.py new file mode 100755 index 00000000..eebb087c --- /dev/null +++ b/git-interface/git-serve.py @@ -0,0 +1,162 @@ +#!/usr/bin/python3 + +import configparser +import mysql.connector +import os +import pygit2 +import re +import shlex +import sys + +config = configparser.RawConfigParser() +config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config") + +aur_db_host = config.get('database', 'host') +aur_db_name = config.get('database', 'name') +aur_db_user = config.get('database', 'user') +aur_db_pass = config.get('database', 'password') +aur_db_socket = config.get('database', 'socket') + +repo_path = config.get('serve', 'repo-path') +repo_regex = config.get('serve', 'repo-regex') +git_shell_cmd = config.get('serve', 'git-shell-cmd') +ssh_cmdline = config.get('serve', 'ssh-cmdline') +template_path = config.get('serve', 'template-path') + +def pkgbase_exists(pkgbase): + db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, + passwd=aur_db_pass, db=aur_db_name, + unix_socket=aur_db_socket) + cur = db.cursor() + + cur.execute("SELECT COUNT(*) FROM PackageBases WHERE Name = %s ", + [pkgbase]) + + db.close() + return (cur.fetchone()[0] > 0) + +def list_repos(user): + db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, + passwd=aur_db_pass, db=aur_db_name, + unix_socket=aur_db_socket) + cur = db.cursor() + + cur.execute("SELECT ID FROM Users WHERE Username = %s ", [user]) + userid = cur.fetchone()[0] + if userid == 0: + die('%s: unknown user: %s' % (action, user)) + + cur.execute("SELECT Name, PackagerUID FROM PackageBases " + + "WHERE MaintainerUID = %s ", [userid]) + for row in cur: + print((' ' if row[1] else '*') + row[0]) + db.close() + +def setup_repo(pkgbase, user): + if not re.match(repo_regex, pkgbase): + die('%s: invalid repository name: %s' % (action, pkgbase)) + if pkgbase_exists(pkgbase): + die('%s: package base already exists: %s' % (action, pkgbase)) + + db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, + passwd=aur_db_pass, db=aur_db_name, + unix_socket=aur_db_socket) + cur = db.cursor() + + cur.execute("SELECT ID FROM Users WHERE Username = %s ", [user]) + userid = cur.fetchone()[0] + if userid == 0: + die('%s: unknown user: %s' % (action, user)) + + cur.execute("INSERT INTO PackageBases (Name, SubmittedTS, ModifiedTS, " + + "SubmitterUID, MaintainerUID) VALUES (%s, UNIX_TIMESTAMP(), " + + "UNIX_TIMESTAMP(), %s, %s)", [pkgbase, userid, userid]) + pkgbase_id = cur.lastrowid + + cur.execute("INSERT INTO CommentNotify (PackageBaseID, UserID) " + + "VALUES (%s, %s)", [pkgbase_id, userid]) + + db.commit() + db.close() + + repo = pygit2.Repository(repo_path) + repo.create_reference('refs/heads/' + pkgbase, + 'refs/namespaces/' + pkgbase + '/refs/heads/master') + repo.create_reference('refs/namespaces/' + pkgbase + '/HEAD', + 'refs/namespaces/' + pkgbase + '/refs/heads/master') + +def check_permissions(pkgbase, user): + db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, + passwd=aur_db_pass, db=aur_db_name, + unix_socket=aur_db_socket, buffered=True) + cur = db.cursor() + + cur.execute("SELECT AccountTypeID FROM Users WHERE UserName = %s ", [user]) + if cur.fetchone()[0] > 1: + return True + + cur.execute("SELECT COUNT(*) FROM PackageBases " + + "LEFT JOIN PackageComaintainers " + + "ON PackageComaintainers.PackageBaseID = PackageBases.ID " + + "INNER JOIN Users ON Users.ID = PackageBases.MaintainerUID " + + "OR PackageBases.MaintainerUID IS NULL " + + "OR Users.ID = PackageComaintainers.UsersID " + + "WHERE Name = %s AND Username = %s", [pkgbase, user]) + return cur.fetchone()[0] > 0 + +def die(msg): + sys.stderr.write("%s\n" % (msg)) + exit(1) + +def die_with_help(msg): + die(msg + "\nTry `%s help` for a list of commands." % (ssh_cmdline)) + +user = sys.argv[1] +cmd = os.environ.get("SSH_ORIGINAL_COMMAND") +if not cmd: + die_with_help("Interactive shell is disabled.") +cmdargv = shlex.split(cmd) +action = cmdargv[0] + +if action == 'git-upload-pack' or action == 'git-receive-pack': + if len(cmdargv) < 2: + die_with_help("%s: missing path" % (action)) + + path = cmdargv[1].rstrip('/') + if not path.startswith('/') or not path.endswith('.git'): + die('%s: invalid path: %s' % (action, path)) + pkgbase = path[1:-4] + if not re.match(repo_regex, pkgbase): + die('%s: invalid repository name: %s' % (action, repo)) + + if not pkgbase_exists(pkgbase): + setup_repo(pkgbase, user) + + if action == 'git-receive-pack': + if not check_permissions(pkgbase, user): + die('%s: permission denied: %s' % (action, user)) + + os.environ["AUR_USER"] = user + os.environ["AUR_PKGBASE"] = pkgbase + os.environ["GIT_NAMESPACE"] = pkgbase + cmd = action + " '" + repo_path + "'" + os.execl(git_shell_cmd, git_shell_cmd, '-c', cmd) +elif action == 'list-repos': + if len(cmdargv) > 1: + die_with_help("%s: too many arguments" % (action)) + list_repos(user) +elif action == 'setup-repo': + if len(cmdargv) < 2: + die_with_help("%s: missing repository name" % (action)) + if len(cmdargv) > 2: + die_with_help("%s: too many arguments" % (action)) + setup_repo(cmdargv[1], user) +elif action == 'help': + die("Commands:\n" + + " help Show this help message and exit.\n" + + " list-repos List all your repositories.\n" + + " setup-repo Create an empty repository.\n" + + " git-receive-pack Internal command used with Git.\n" + + " git-upload-pack Internal command used with Git.") +else: + die_with_help("invalid command: %s" % (action)) diff --git a/git-interface/git-update.py b/git-interface/git-update.py new file mode 100755 index 00000000..b7eaa14d --- /dev/null +++ b/git-interface/git-update.py @@ -0,0 +1,262 @@ +#!/usr/bin/python3 + +from copy import copy, deepcopy +import configparser +import mysql.connector +import os +import pygit2 +import re +import sys + +import aurinfo + +config = configparser.RawConfigParser() +config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config") + +aur_db_host = config.get('database', 'host') +aur_db_name = config.get('database', 'name') +aur_db_user = config.get('database', 'user') +aur_db_pass = config.get('database', 'password') +aur_db_socket = config.get('database', 'socket') + +repo_path = config.get('serve', 'repo-path') + +def extract_arch_fields(pkginfo, field): + values = [] + + if field in pkginfo: + for val in pkginfo[field]: + values.append({"value": val, "arch": None}) + + for arch in ['i686', 'x86_64']: + if field + '_' + arch in pkginfo: + for val in pkginfo[field + '_' + arch]: + values.append({"value": val, "arch": arch}) + + return values + +def save_srcinfo(srcinfo, db, cur, user): + # Obtain package base ID and previous maintainer. + pkgbase = srcinfo._pkgbase['pkgname'] + cur.execute("SELECT ID, MaintainerUID FROM PackageBases " + "WHERE Name = %s", [pkgbase]) + (pkgbase_id, maintainer_uid) = cur.fetchone() + was_orphan = not maintainer_uid + + # Obtain the user ID of the new maintainer. + cur.execute("SELECT ID FROM Users WHERE Username = %s", [user]) + user_id = int(cur.fetchone()[0]) + + # Update package base details and delete current packages. + cur.execute("UPDATE PackageBases SET ModifiedTS = UNIX_TIMESTAMP(), " + + "PackagerUID = %s, OutOfDateTS = NULL WHERE ID = %s", + [user_id, pkgbase_id]) + cur.execute("UPDATE PackageBases SET MaintainerUID = %s " + + "WHERE ID = %s AND MaintainerUID IS NULL", + [user_id, pkgbase_id]) + cur.execute("DELETE FROM Packages WHERE PackageBaseID = %s", + [pkgbase_id]) + + for pkgname in srcinfo.GetPackageNames(): + pkginfo = srcinfo.GetMergedPackage(pkgname) + + if 'epoch' in pkginfo and int(pkginfo['epoch']) > 0: + ver = '%d:%s-%s' % (int(pkginfo['epoch']), pkginfo['pkgver'], + pkginfo['pkgrel']) + else: + ver = '%s-%s' % (pkginfo['pkgver'], pkginfo['pkgrel']) + + # Create a new package. + cur.execute("INSERT INTO Packages (PackageBaseID, Name, " + + "Version, Description, URL) " + + "VALUES (%s, %s, %s, %s, %s)", + [pkgbase_id, pkginfo['pkgname'], ver, + pkginfo['pkgdesc'], pkginfo['url']]) + db.commit() + pkgid = cur.lastrowid + + # Add package sources. + for source_info in extract_arch_fields(pkginfo, 'source'): + cur.execute("INSERT INTO PackageSources (PackageID, Source, " + + "SourceArch) VALUES (%s, %s, %s)", + [pkgid, source_info['value'], source_info['arch']]) + + # Add package dependencies. + for deptype in ('depends', 'makedepends', + 'checkdepends', 'optdepends'): + cur.execute("SELECT ID FROM DependencyTypes WHERE Name = %s", + [deptype]) + deptypeid = cur.fetchone()[0] + for dep_info in extract_arch_fields(pkginfo, deptype): + depname = re.sub(r'(<|=|>).*', '', dep_info['value']) + depcond = dep_info['value'][len(depname):] + deparch = dep_info['arch'] + cur.execute("INSERT INTO PackageDepends (PackageID, " + + "DepTypeID, DepName, DepCondition, DepArch) " + + "VALUES (%s, %s, %s, %s, %s)", + [pkgid, deptypeid, depname, depcond, deparch]) + + # Add package relations (conflicts, provides, replaces). + for reltype in ('conflicts', 'provides', 'replaces'): + cur.execute("SELECT ID FROM RelationTypes WHERE Name = %s", + [reltype]) + reltypeid = cur.fetchone()[0] + for rel_info in extract_arch_fields(pkginfo, reltype): + relname = re.sub(r'(<|=|>).*', '', rel_info['value']) + relcond = rel_info['value'][len(relname):] + relarch = rel_info['arch'] + cur.execute("INSERT INTO PackageRelations (PackageID, " + + "RelTypeID, RelName, RelCondition, RelArch) " + + "VALUES (%s, %s, %s, %s, %s)", + [pkgid, reltypeid, relname, relcond, relarch]) + + # Add package licenses. + if 'license' in pkginfo: + for license in pkginfo['license']: + cur.execute("SELECT ID FROM Licenses WHERE Name = %s", + [license]) + if cur.rowcount == 1: + licenseid = cur.fetchone()[0] + else: + cur.execute("INSERT INTO Licenses (Name) VALUES (%s)", + [license]) + db.commit() + licenseid = cur.lastrowid + cur.execute("INSERT INTO PackageLicenses (PackageID, " + + "LicenseID) VALUES (%s, %s)", + [pkgid, licenseid]) + + # Add package groups. + if 'groups' in pkginfo: + for group in pkginfo['groups']: + cur.execute("SELECT ID FROM Groups WHERE Name = %s", + [group]) + if cur.rowcount == 1: + groupid = cur.fetchone()[0] + else: + cur.execute("INSERT INTO Groups (Name) VALUES (%s)", + [group]) + db.commit() + groupid = cur.lastrowid + cur.execute("INSERT INTO PackageGroups (PackageID, " + "GroupID) VALUES (%s, %s)", [pkgid, groupid]) + + # Add user to notification list on adoption. + if was_orphan: + cur.execute("INSERT INTO CommentNotify (PackageBaseID, UserID) " + + "VALUES (%s, %s)", [pkgbase_id, user_id]) + + db.commit() + +def die(msg): + sys.stderr.write("error: %s\n" % (msg)) + exit(1) + +def die_commit(msg, commit): + sys.stderr.write("error: The following error " + + "occurred when parsing commit\n") + sys.stderr.write("error: %s:\n" % (commit)) + sys.stderr.write("error: %s\n" % (msg)) + exit(1) + +if len(sys.argv) != 4: + die("invalid arguments") + +refname = sys.argv[1] +sha1_old = sys.argv[2] +sha1_new = sys.argv[3] + +user = os.environ.get("AUR_USER") +pkgbase = os.environ.get("AUR_PKGBASE") + +if refname != "refs/heads/master": + die("pushing to a branch other than master is restricted") + +repo = pygit2.Repository(repo_path) +walker = repo.walk(sha1_new, pygit2.GIT_SORT_TOPOLOGICAL) +if sha1_old != "0000000000000000000000000000000000000000": + walker.hide(sha1_old) + +db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, + passwd=aur_db_pass, db=aur_db_name, + unix_socket=aur_db_socket, buffered=True) +cur = db.cursor() + +cur.execute("SELECT Name FROM PackageBlacklist") +blacklist = [row[0] for row in cur.fetchall()] + +for commit in walker: + if not '.SRCINFO' in commit.tree: + die_commit("missing .SRCINFO", commit.id) + + for treeobj in commit.tree: + if repo[treeobj.id].size > 100000: + die_commit("maximum blob size (100kB) exceeded", commit.id) + + srcinfo_raw = repo[commit.tree['.SRCINFO'].id].data.decode() + srcinfo_raw = srcinfo_raw.split('\n') + ecatcher = aurinfo.CollectionECatcher() + srcinfo = aurinfo.ParseAurinfoFromIterable(srcinfo_raw, ecatcher) + errors = ecatcher.Errors() + if errors: + sys.stderr.write("error: The following errors occurred " + "when parsing .SRCINFO in commit\n") + sys.stderr.write("error: %s:\n" % (commit.id)) + for error in errors: + sys.stderr.write("error: line %d: %s\n" % error) + exit(1) + + srcinfo_pkgbase = srcinfo._pkgbase['pkgname'] + if srcinfo_pkgbase != pkgbase: + die_commit('invalid pkgbase: %s' % (srcinfo_pkgbase), commit.id) + + for pkgname in srcinfo.GetPackageNames(): + pkginfo = srcinfo.GetMergedPackage(pkgname) + + for field in ('pkgver', 'pkgrel', 'pkgname', 'pkgdesc', 'url'): + if not field in pkginfo: + die_commit('missing mandatory field: %s' % (field), commit.id) + + if 'epoch' in pkginfo and not pkginfo['epoch'].isdigit(): + die_commit('invalid epoch: %s' % (pkginfo['epoch']), commit.id) + + if not re.match(r'[a-z0-9][a-z0-9\.+_-]*$', pkginfo['pkgname']): + die_commit('invalid package name: %s' % (pkginfo['pkgname']), + commit.id) + + if pkginfo['pkgname'] in blacklist: + die_commit('package is blacklisted: %s' % (pkginfo['pkgname']), + commit.id) + + if not re.match(r'(?:http|ftp)s?://.*', pkginfo['url']): + die_commit('invalid URL: %s' % (pkginfo['url']), commit.id) + + for field in ('pkgname', 'pkgdesc', 'url'): + if len(pkginfo[field]) > 255: + die_commit('%s field too long: %s' % (field, pkginfo[field]), + commit.id) + + for field in ('install', 'changelog'): + if field in pkginfo and not pkginfo[field] in commit.tree: + die_commit('missing %s file: %s' % (field, pkginfo[field]), + commit.id) + + for fname in pkginfo['source']: + if "://" in fname or "lp:" in fname: + continue + if not fname in commit.tree: + die_commit('missing source file: %s' % (fname), commit.id) + +srcinfo_raw = repo[repo[sha1_new].tree['.SRCINFO'].id].data.decode() +srcinfo_raw = srcinfo_raw.split('\n') +srcinfo = aurinfo.ParseAurinfoFromIterable(srcinfo_raw) + +save_srcinfo(srcinfo, db, cur, user) + +db.close() + +pkglist = list(srcinfo.GetPackageNames()) +if len(pkglist) > 0: + with open(repo_path + '/description', 'w') as f: + pkginfo = srcinfo.GetMergedPackage(pkglist[0]) + f.write(pkginfo['pkgdesc']) diff --git a/scripts/aurblup.py b/scripts/aurblup.py new file mode 100755 index 00000000..e678fee9 --- /dev/null +++ b/scripts/aurblup.py @@ -0,0 +1,48 @@ +#!/usr/bin/python3 + +import configparser +import mysql.connector +import os +import pyalpm + +config = configparser.RawConfigParser() +config.read(os.path.dirname(os.path.realpath(__file__)) + "/../../conf/config") + +aur_db_host = config.get('database', 'host') +aur_db_name = config.get('database', 'name') +aur_db_user = config.get('database', 'user') +aur_db_pass = config.get('database', 'password') +aur_db_socket = config.get('database', 'socket') +db_path = config.get('aurblup', 'db-path') +sync_dbs = config.get('aurblup', 'sync-dbs').split(' ') +servers = config.get('aurblup', 'servers').split(' ') + +blacklist = set() + +h = pyalpm.Handle("/", db_path) +for sync_db in sync_dbs: + repo = h.register_syncdb(sync_db, pyalpm.SIG_DATABASE_OPTIONAL) + repo.servers = [server.replace("%s", sync_db) for server in servers] + t = h.init_transaction() + repo.update(False) + t.release() + + for pkg in repo.pkgcache: + blacklist.add(pkg.name) + [blacklist.add(x) for x in pkg.replaces] + +db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, + passwd=aur_db_pass, db=aur_db_name, + unix_socket=aur_db_socket, buffered=True) +cur = db.cursor() + +cur.execute("SELECT Name FROM PackageBlacklist") +oldblacklist = set([row[0] for row in cur.fetchall()]) + +for pkg in blacklist.difference(oldblacklist): + cur.execute("INSERT INTO PackageBlacklist (Name) VALUES (%s)", [pkg]) +for pkg in oldblacklist.difference(blacklist): + cur.execute("DELETE FROM PackageBlacklist WHERE Name = %s", [pkg]) + +db.commit() +db.close() diff --git a/scripts/aurblup/aurblup.py b/scripts/aurblup/aurblup.py deleted file mode 100755 index e678fee9..00000000 --- a/scripts/aurblup/aurblup.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/python3 - -import configparser -import mysql.connector -import os -import pyalpm - -config = configparser.RawConfigParser() -config.read(os.path.dirname(os.path.realpath(__file__)) + "/../../conf/config") - -aur_db_host = config.get('database', 'host') -aur_db_name = config.get('database', 'name') -aur_db_user = config.get('database', 'user') -aur_db_pass = config.get('database', 'password') -aur_db_socket = config.get('database', 'socket') -db_path = config.get('aurblup', 'db-path') -sync_dbs = config.get('aurblup', 'sync-dbs').split(' ') -servers = config.get('aurblup', 'servers').split(' ') - -blacklist = set() - -h = pyalpm.Handle("/", db_path) -for sync_db in sync_dbs: - repo = h.register_syncdb(sync_db, pyalpm.SIG_DATABASE_OPTIONAL) - repo.servers = [server.replace("%s", sync_db) for server in servers] - t = h.init_transaction() - repo.update(False) - t.release() - - for pkg in repo.pkgcache: - blacklist.add(pkg.name) - [blacklist.add(x) for x in pkg.replaces] - -db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) -cur = db.cursor() - -cur.execute("SELECT Name FROM PackageBlacklist") -oldblacklist = set([row[0] for row in cur.fetchall()]) - -for pkg in blacklist.difference(oldblacklist): - cur.execute("INSERT INTO PackageBlacklist (Name) VALUES (%s)", [pkg]) -for pkg in oldblacklist.difference(blacklist): - cur.execute("DELETE FROM PackageBlacklist WHERE Name = %s", [pkg]) - -db.commit() -db.close() diff --git a/scripts/git-integration/aurinfo.py b/scripts/git-integration/aurinfo.py deleted file mode 100644 index d9b93729..00000000 --- a/scripts/git-integration/aurinfo.py +++ /dev/null @@ -1,204 +0,0 @@ -#!/usr/bin/env python - -from copy import copy, deepcopy -import pprint -import sys - -class Attr(object): - def __init__(self, name, is_multivalued=False, allow_arch_extensions=False): - self.name = name - self.is_multivalued = is_multivalued - self.allow_arch_extensions = allow_arch_extensions - -PKGBUILD_ATTRIBUTES = { - 'arch': Attr('arch', True), - 'backup': Attr('backup', True), - 'changelog': Attr('changelog', False), - 'checkdepends': Attr('checkdepends', True), - 'conflicts': Attr('conflicts', True, True), - 'depends': Attr('depends', True, True), - 'epoch': Attr('epoch', False), - 'groups': Attr('groups', True), - 'install': Attr('install', False), - 'license': Attr('license', True), - 'makedepends': Attr('makedepends', True, True), - 'md5sums': Attr('md5sums', True, True), - 'noextract': Attr('noextract', True), - 'optdepends': Attr('optdepends', True, True), - 'options': Attr('options', True), - 'pkgname': Attr('pkgname', False), - 'pkgrel': Attr('pkgrel', False), - 'pkgver': Attr('pkgver', False), - 'provides': Attr('provides', True, True), - 'replaces': Attr('replaces', True, True), - 'sha1sums': Attr('sha1sums', True, True), - 'sha224sums': Attr('sha224sums', True, True), - 'sha256sums': Attr('sha256sums', True, True), - 'sha384sums': Attr('sha384sums', True, True), - 'sha512sums': Attr('sha512sums', True, True), - 'source': Attr('source', True, True), - 'url': Attr('url', False), - 'validpgpkeys': Attr('validpgpkeys', True), -} - -def find_attr(attrname): - # exact match - attr = PKGBUILD_ATTRIBUTES.get(attrname, None) - if attr: - return attr - - # prefix match - # XXX: this could break in the future if PKGBUILD(5) ever - # introduces a key which is a subset of another. - for k in PKGBUILD_ATTRIBUTES.keys(): - if attrname.startswith(k + '_'): - return PKGBUILD_ATTRIBUTES[k] - -def IsMultiValued(attrname): - attr = find_attr(attrname) - return attr and attr.is_multivalued - -class AurInfo(object): - def __init__(self): - self._pkgbase = {} - self._packages = {} - - def GetPackageNames(self): - return self._packages.keys() - - def GetMergedPackage(self, pkgname): - package = deepcopy(self._pkgbase) - package['pkgname'] = pkgname - for k, v in self._packages.get(pkgname).items(): - package[k] = deepcopy(v) - return package - - def AddPackage(self, pkgname): - self._packages[pkgname] = {} - return self._packages[pkgname] - - def SetPkgbase(self, pkgbasename): - self._pkgbase = {'pkgname' : pkgbasename} - return self._pkgbase - - -class StderrECatcher(object): - def Catch(self, lineno, error): - print('ERROR[%d]: %s' % (lineno, error), file=sys.stderr) - - -class CollectionECatcher(object): - def __init__(self): - self._errors = [] - - def Catch(self, lineno, error): - self._errors.append((lineno, error)) - - def HasErrors(self): - return len(self._errors) > 0 - - def Errors(self): - return copy(self._errors) - - -def ParseAurinfoFromIterable(iterable, ecatcher=None): - aurinfo = AurInfo() - - if ecatcher is None: - ecatcher = StderrECatcher() - - current_package = None - lineno = 0 - - for line in iterable: - lineno += 1 - - if line.startswith('#'): - continue - - if not line.strip(): - # end of package - current_package = None - continue - - if not line.startswith('\t'): - # start of new package - try: - key, value = map(str.strip, line.split('=', 1)) - except ValueError: - ecatcher.Catch(lineno, 'unexpected header format in section=%s' % - current_package['pkgname']) - continue - - if key == 'pkgbase': - current_package = aurinfo.SetPkgbase(value) - else: - current_package = aurinfo.AddPackage(value) - else: - # package attribute - if current_package is None: - ecatcher.Catch(lineno, 'package attribute found outside of ' - 'a package section') - continue - - try: - key, value = map(str.strip, line.split('=', 1)) - except ValueError: - ecatcher.Catch(lineno, 'unexpected attribute format in ' - 'section=%s' % current_package['pkgname']) - - if IsMultiValued(key): - if not current_package.get(key): - current_package[key] = [] - if value: - current_package[key].append(value) - else: - if not current_package.get(key): - current_package[key] = value - else: - ecatcher.Catch(lineno, 'overwriting attribute ' - '%s: %s -> %s' % (key, current_package[key], - value)) - - return aurinfo - - -def ParseAurinfo(filename='.AURINFO', ecatcher=None): - with open(filename) as f: - return ParseAurinfoFromIterable(f, ecatcher) - - -def ValidateAurinfo(filename='.AURINFO'): - ecatcher = CollectionECatcher() - ParseAurinfo(filename, ecatcher) - errors = ecatcher.Errors() - for error in errors: - print('error on line %d: %s' % error, file=sys.stderr) - return not errors - - -if __name__ == '__main__': - pp = pprint.PrettyPrinter(indent=4) - - if len(sys.argv) == 1: - print('error: not enough arguments') - sys.exit(1) - elif len(sys.argv) == 2: - action = sys.argv[1] - filename = '.AURINFO' - else: - action, filename = sys.argv[1:3] - - if action == 'parse': - aurinfo = ParseAurinfo() - for pkgname in aurinfo.GetPackageNames(): - print(">>> merged package: %s" % pkgname) - pp.pprint(aurinfo.GetMergedPackage(pkgname)) - print() - elif action == 'validate': - sys.exit(not ValidateAurinfo(filename)) - else: - print('unknown action: %s' % action) - sys.exit(1) - -# vim: set et ts=4 sw=4: diff --git a/scripts/git-integration/git-auth.py b/scripts/git-integration/git-auth.py deleted file mode 100755 index 09dadecf..00000000 --- a/scripts/git-integration/git-auth.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/python3 - -import configparser -import mysql.connector -import os -import re -import sys - -config = configparser.RawConfigParser() -config.read(os.path.dirname(os.path.realpath(__file__)) + "/../../conf/config") - -aur_db_host = config.get('database', 'host') -aur_db_name = config.get('database', 'name') -aur_db_user = config.get('database', 'user') -aur_db_pass = config.get('database', 'password') -aur_db_socket = config.get('database', 'socket') - -valid_keytypes = config.get('auth', 'valid-keytypes').split() -username_regex = config.get('auth', 'username-regex') -git_serve_cmd = config.get('auth', 'git-serve-cmd') -ssh_opts = config.get('auth', 'ssh-options') - -keytype = sys.argv[1] -keytext = sys.argv[2] -if not keytype in valid_keytypes: - exit(1) - -db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) - -cur = db.cursor() -cur.execute("SELECT Username FROM Users WHERE SSHPubKey = %s " + - "AND Suspended = 0", (keytype + " " + keytext,)) - -if cur.rowcount != 1: - exit(1) - -user = cur.fetchone()[0] -if not re.match(username_regex, user): - exit(1) - -print('command="%s %s",%s %s' % (git_serve_cmd, user, ssh_opts, - keytype + " " + keytext)) diff --git a/scripts/git-integration/git-auth.sh b/scripts/git-integration/git-auth.sh deleted file mode 100755 index c6a54012..00000000 --- a/scripts/git-integration/git-auth.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -/srv/http/aurweb/scripts/git-integration/git-auth.py "$1" "$2" diff --git a/scripts/git-integration/git-serve.py b/scripts/git-integration/git-serve.py deleted file mode 100755 index 81222e8a..00000000 --- a/scripts/git-integration/git-serve.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/python3 - -import configparser -import mysql.connector -import os -import pygit2 -import re -import shlex -import sys - -config = configparser.RawConfigParser() -config.read(os.path.dirname(os.path.realpath(__file__)) + "/../../conf/config") - -aur_db_host = config.get('database', 'host') -aur_db_name = config.get('database', 'name') -aur_db_user = config.get('database', 'user') -aur_db_pass = config.get('database', 'password') -aur_db_socket = config.get('database', 'socket') - -repo_path = config.get('serve', 'repo-path') -repo_regex = config.get('serve', 'repo-regex') -git_shell_cmd = config.get('serve', 'git-shell-cmd') -ssh_cmdline = config.get('serve', 'ssh-cmdline') -template_path = config.get('serve', 'template-path') - -def pkgbase_exists(pkgbase): - db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket) - cur = db.cursor() - - cur.execute("SELECT COUNT(*) FROM PackageBases WHERE Name = %s ", - [pkgbase]) - - db.close() - return (cur.fetchone()[0] > 0) - -def list_repos(user): - db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket) - cur = db.cursor() - - cur.execute("SELECT ID FROM Users WHERE Username = %s ", [user]) - userid = cur.fetchone()[0] - if userid == 0: - die('%s: unknown user: %s' % (action, user)) - - cur.execute("SELECT Name, PackagerUID FROM PackageBases " + - "WHERE MaintainerUID = %s ", [userid]) - for row in cur: - print((' ' if row[1] else '*') + row[0]) - db.close() - -def setup_repo(pkgbase, user): - if not re.match(repo_regex, pkgbase): - die('%s: invalid repository name: %s' % (action, pkgbase)) - if pkgbase_exists(pkgbase): - die('%s: package base already exists: %s' % (action, pkgbase)) - - db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket) - cur = db.cursor() - - cur.execute("SELECT ID FROM Users WHERE Username = %s ", [user]) - userid = cur.fetchone()[0] - if userid == 0: - die('%s: unknown user: %s' % (action, user)) - - cur.execute("INSERT INTO PackageBases (Name, SubmittedTS, ModifiedTS, " + - "SubmitterUID, MaintainerUID) VALUES (%s, UNIX_TIMESTAMP(), " + - "UNIX_TIMESTAMP(), %s, %s)", [pkgbase, userid, userid]) - pkgbase_id = cur.lastrowid - - cur.execute("INSERT INTO CommentNotify (PackageBaseID, UserID) " + - "VALUES (%s, %s)", [pkgbase_id, userid]) - - db.commit() - db.close() - - repo = pygit2.Repository(repo_path) - repo.create_reference('refs/heads/' + pkgbase, - 'refs/namespaces/' + pkgbase + '/refs/heads/master') - repo.create_reference('refs/namespaces/' + pkgbase + '/HEAD', - 'refs/namespaces/' + pkgbase + '/refs/heads/master') - -def check_permissions(pkgbase, user): - db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) - cur = db.cursor() - - cur.execute("SELECT AccountTypeID FROM Users WHERE UserName = %s ", [user]) - if cur.fetchone()[0] > 1: - return True - - cur.execute("SELECT COUNT(*) FROM PackageBases " + - "LEFT JOIN PackageComaintainers " + - "ON PackageComaintainers.PackageBaseID = PackageBases.ID " + - "INNER JOIN Users ON Users.ID = PackageBases.MaintainerUID " + - "OR PackageBases.MaintainerUID IS NULL " + - "OR Users.ID = PackageComaintainers.UsersID " + - "WHERE Name = %s AND Username = %s", [pkgbase, user]) - return cur.fetchone()[0] > 0 - -def die(msg): - sys.stderr.write("%s\n" % (msg)) - exit(1) - -def die_with_help(msg): - die(msg + "\nTry `%s help` for a list of commands." % (ssh_cmdline)) - -user = sys.argv[1] -cmd = os.environ.get("SSH_ORIGINAL_COMMAND") -if not cmd: - die_with_help("Interactive shell is disabled.") -cmdargv = shlex.split(cmd) -action = cmdargv[0] - -if action == 'git-upload-pack' or action == 'git-receive-pack': - if len(cmdargv) < 2: - die_with_help("%s: missing path" % (action)) - - path = cmdargv[1].rstrip('/') - if not path.startswith('/') or not path.endswith('.git'): - die('%s: invalid path: %s' % (action, path)) - pkgbase = path[1:-4] - if not re.match(repo_regex, pkgbase): - die('%s: invalid repository name: %s' % (action, repo)) - - if not pkgbase_exists(pkgbase): - setup_repo(pkgbase, user) - - if action == 'git-receive-pack': - if not check_permissions(pkgbase, user): - die('%s: permission denied: %s' % (action, user)) - - os.environ["AUR_USER"] = user - os.environ["AUR_PKGBASE"] = pkgbase - os.environ["GIT_NAMESPACE"] = pkgbase - cmd = action + " '" + repo_path + "'" - os.execl(git_shell_cmd, git_shell_cmd, '-c', cmd) -elif action == 'list-repos': - if len(cmdargv) > 1: - die_with_help("%s: too many arguments" % (action)) - list_repos(user) -elif action == 'setup-repo': - if len(cmdargv) < 2: - die_with_help("%s: missing repository name" % (action)) - if len(cmdargv) > 2: - die_with_help("%s: too many arguments" % (action)) - setup_repo(cmdargv[1], user) -elif action == 'help': - die("Commands:\n" + - " help Show this help message and exit.\n" + - " list-repos List all your repositories.\n" + - " setup-repo Create an empty repository.\n" + - " git-receive-pack Internal command used with Git.\n" + - " git-upload-pack Internal command used with Git.") -else: - die_with_help("invalid command: %s" % (action)) diff --git a/scripts/git-integration/git-update.py b/scripts/git-integration/git-update.py deleted file mode 100755 index c7d64dff..00000000 --- a/scripts/git-integration/git-update.py +++ /dev/null @@ -1,262 +0,0 @@ -#!/usr/bin/python3 - -from copy import copy, deepcopy -import configparser -import mysql.connector -import os -import pygit2 -import re -import sys - -import aurinfo - -config = configparser.RawConfigParser() -config.read(os.path.dirname(os.path.realpath(__file__)) + "/../../conf/config") - -aur_db_host = config.get('database', 'host') -aur_db_name = config.get('database', 'name') -aur_db_user = config.get('database', 'user') -aur_db_pass = config.get('database', 'password') -aur_db_socket = config.get('database', 'socket') - -repo_path = config.get('serve', 'repo-path') - -def extract_arch_fields(pkginfo, field): - values = [] - - if field in pkginfo: - for val in pkginfo[field]: - values.append({"value": val, "arch": None}) - - for arch in ['i686', 'x86_64']: - if field + '_' + arch in pkginfo: - for val in pkginfo[field + '_' + arch]: - values.append({"value": val, "arch": arch}) - - return values - -def save_srcinfo(srcinfo, db, cur, user): - # Obtain package base ID and previous maintainer. - pkgbase = srcinfo._pkgbase['pkgname'] - cur.execute("SELECT ID, MaintainerUID FROM PackageBases " - "WHERE Name = %s", [pkgbase]) - (pkgbase_id, maintainer_uid) = cur.fetchone() - was_orphan = not maintainer_uid - - # Obtain the user ID of the new maintainer. - cur.execute("SELECT ID FROM Users WHERE Username = %s", [user]) - user_id = int(cur.fetchone()[0]) - - # Update package base details and delete current packages. - cur.execute("UPDATE PackageBases SET ModifiedTS = UNIX_TIMESTAMP(), " + - "PackagerUID = %s, OutOfDateTS = NULL WHERE ID = %s", - [user_id, pkgbase_id]) - cur.execute("UPDATE PackageBases SET MaintainerUID = %s " + - "WHERE ID = %s AND MaintainerUID IS NULL", - [user_id, pkgbase_id]) - cur.execute("DELETE FROM Packages WHERE PackageBaseID = %s", - [pkgbase_id]) - - for pkgname in srcinfo.GetPackageNames(): - pkginfo = srcinfo.GetMergedPackage(pkgname) - - if 'epoch' in pkginfo and int(pkginfo['epoch']) > 0: - ver = '%d:%s-%s' % (int(pkginfo['epoch']), pkginfo['pkgver'], - pkginfo['pkgrel']) - else: - ver = '%s-%s' % (pkginfo['pkgver'], pkginfo['pkgrel']) - - # Create a new package. - cur.execute("INSERT INTO Packages (PackageBaseID, Name, " + - "Version, Description, URL) " + - "VALUES (%s, %s, %s, %s, %s)", - [pkgbase_id, pkginfo['pkgname'], ver, - pkginfo['pkgdesc'], pkginfo['url']]) - db.commit() - pkgid = cur.lastrowid - - # Add package sources. - for source_info in extract_arch_fields(pkginfo, 'source'): - cur.execute("INSERT INTO PackageSources (PackageID, Source, " + - "SourceArch) VALUES (%s, %s, %s)", - [pkgid, source_info['value'], source_info['arch']]) - - # Add package dependencies. - for deptype in ('depends', 'makedepends', - 'checkdepends', 'optdepends'): - cur.execute("SELECT ID FROM DependencyTypes WHERE Name = %s", - [deptype]) - deptypeid = cur.fetchone()[0] - for dep_info in extract_arch_fields(pkginfo, deptype): - depname = re.sub(r'(<|=|>).*', '', dep_info['value']) - depcond = dep_info['value'][len(depname):] - deparch = dep_info['arch'] - cur.execute("INSERT INTO PackageDepends (PackageID, " + - "DepTypeID, DepName, DepCondition, DepArch) " + - "VALUES (%s, %s, %s, %s, %s)", - [pkgid, deptypeid, depname, depcond, deparch]) - - # Add package relations (conflicts, provides, replaces). - for reltype in ('conflicts', 'provides', 'replaces'): - cur.execute("SELECT ID FROM RelationTypes WHERE Name = %s", - [reltype]) - reltypeid = cur.fetchone()[0] - for rel_info in extract_arch_fields(pkginfo, reltype): - relname = re.sub(r'(<|=|>).*', '', rel_info['value']) - relcond = rel_info['value'][len(relname):] - relarch = rel_info['arch'] - cur.execute("INSERT INTO PackageRelations (PackageID, " + - "RelTypeID, RelName, RelCondition, RelArch) " + - "VALUES (%s, %s, %s, %s, %s)", - [pkgid, reltypeid, relname, relcond, relarch]) - - # Add package licenses. - if 'license' in pkginfo: - for license in pkginfo['license']: - cur.execute("SELECT ID FROM Licenses WHERE Name = %s", - [license]) - if cur.rowcount == 1: - licenseid = cur.fetchone()[0] - else: - cur.execute("INSERT INTO Licenses (Name) VALUES (%s)", - [license]) - db.commit() - licenseid = cur.lastrowid - cur.execute("INSERT INTO PackageLicenses (PackageID, " + - "LicenseID) VALUES (%s, %s)", - [pkgid, licenseid]) - - # Add package groups. - if 'groups' in pkginfo: - for group in pkginfo['groups']: - cur.execute("SELECT ID FROM Groups WHERE Name = %s", - [group]) - if cur.rowcount == 1: - groupid = cur.fetchone()[0] - else: - cur.execute("INSERT INTO Groups (Name) VALUES (%s)", - [group]) - db.commit() - groupid = cur.lastrowid - cur.execute("INSERT INTO PackageGroups (PackageID, " - "GroupID) VALUES (%s, %s)", [pkgid, groupid]) - - # Add user to notification list on adoption. - if was_orphan: - cur.execute("INSERT INTO CommentNotify (PackageBaseID, UserID) " + - "VALUES (%s, %s)", [pkgbase_id, user_id]) - - db.commit() - -def die(msg): - sys.stderr.write("error: %s\n" % (msg)) - exit(1) - -def die_commit(msg, commit): - sys.stderr.write("error: The following error " + - "occurred when parsing commit\n") - sys.stderr.write("error: %s:\n" % (commit)) - sys.stderr.write("error: %s\n" % (msg)) - exit(1) - -if len(sys.argv) != 4: - die("invalid arguments") - -refname = sys.argv[1] -sha1_old = sys.argv[2] -sha1_new = sys.argv[3] - -user = os.environ.get("AUR_USER") -pkgbase = os.environ.get("AUR_PKGBASE") - -if refname != "refs/heads/master": - die("pushing to a branch other than master is restricted") - -repo = pygit2.Repository(repo_path) -walker = repo.walk(sha1_new, pygit2.GIT_SORT_TOPOLOGICAL) -if sha1_old != "0000000000000000000000000000000000000000": - walker.hide(sha1_old) - -db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) -cur = db.cursor() - -cur.execute("SELECT Name FROM PackageBlacklist") -blacklist = [row[0] for row in cur.fetchall()] - -for commit in walker: - if not '.SRCINFO' in commit.tree: - die_commit("missing .SRCINFO", commit.id) - - for treeobj in commit.tree: - if repo[treeobj.id].size > 100000: - die_commit("maximum blob size (100kB) exceeded", commit.id) - - srcinfo_raw = repo[commit.tree['.SRCINFO'].id].data.decode() - srcinfo_raw = srcinfo_raw.split('\n') - ecatcher = aurinfo.CollectionECatcher() - srcinfo = aurinfo.ParseAurinfoFromIterable(srcinfo_raw, ecatcher) - errors = ecatcher.Errors() - if errors: - sys.stderr.write("error: The following errors occurred " - "when parsing .SRCINFO in commit\n") - sys.stderr.write("error: %s:\n" % (commit.id)) - for error in errors: - sys.stderr.write("error: line %d: %s\n" % error) - exit(1) - - srcinfo_pkgbase = srcinfo._pkgbase['pkgname'] - if srcinfo_pkgbase != pkgbase: - die_commit('invalid pkgbase: %s' % (srcinfo_pkgbase), commit.id) - - for pkgname in srcinfo.GetPackageNames(): - pkginfo = srcinfo.GetMergedPackage(pkgname) - - for field in ('pkgver', 'pkgrel', 'pkgname', 'pkgdesc', 'url'): - if not field in pkginfo: - die_commit('missing mandatory field: %s' % (field), commit.id) - - if 'epoch' in pkginfo and not pkginfo['epoch'].isdigit(): - die_commit('invalid epoch: %s' % (pkginfo['epoch']), commit.id) - - if not re.match(r'[a-z0-9][a-z0-9\.+_-]*$', pkginfo['pkgname']): - die_commit('invalid package name: %s' % (pkginfo['pkgname']), - commit.id) - - if pkginfo['pkgname'] in blacklist: - die_commit('package is blacklisted: %s' % (pkginfo['pkgname']), - commit.id) - - if not re.match(r'(?:http|ftp)s?://.*', pkginfo['url']): - die_commit('invalid URL: %s' % (pkginfo['url']), commit.id) - - for field in ('pkgname', 'pkgdesc', 'url'): - if len(pkginfo[field]) > 255: - die_commit('%s field too long: %s' % (field, pkginfo[field]), - commit.id) - - for field in ('install', 'changelog'): - if field in pkginfo and not pkginfo[field] in commit.tree: - die_commit('missing %s file: %s' % (field, pkginfo[field]), - commit.id) - - for fname in pkginfo['source']: - if "://" in fname or "lp:" in fname: - continue - if not fname in commit.tree: - die_commit('missing source file: %s' % (fname), commit.id) - -srcinfo_raw = repo[repo[sha1_new].tree['.SRCINFO'].id].data.decode() -srcinfo_raw = srcinfo_raw.split('\n') -srcinfo = aurinfo.ParseAurinfoFromIterable(srcinfo_raw) - -save_srcinfo(srcinfo, db, cur, user) - -db.close() - -pkglist = list(srcinfo.GetPackageNames()) -if len(pkglist) > 0: - with open(repo_path + '/description', 'w') as f: - pkginfo = srcinfo.GetMergedPackage(pkglist[0]) - f.write(pkginfo['pkgdesc']) -- cgit v1.2.3-24-g4f1b