from collections import defaultdict from itertools import chain from operator import attrgetter, itemgetter import re from django.core.serializers.json import DjangoJSONEncoder from django.db import connection from django.db.models import Count, Max, F from django.db.models.query import QuerySet from django.contrib.auth.models import User from main.models import Package, PackageFile, Arch, Repo from main.utils import (database_vendor, groupby_preserve_order, PackageStandin) from .models import (PackageGroup, PackageRelation, License, Depend, Conflict, Provision, Replacement, SignoffSpecification, Signoff, DEFAULT_SIGNOFF_SPEC) VERSION_RE = re.compile(r'^((\d+):)?(.+)-([^-]+)$') def parse_version(version): match = VERSION_RE.match(version) if not match: return None, None, 0 ver = match.group(3) rel = match.group(4) if match.group(2): epoch = int(match.group(2)) else: epoch = 0 return ver, rel, epoch def get_group_info(include_arches=None): raw_groups = PackageGroup.objects.values_list( 'name', 'pkg__arch__name').order_by('name').annotate( cnt=Count('pkg'), last_update=Max('pkg__last_update')) # now for post_processing. we need to seperate things out and add # the count in for 'any' to all of the other architectures. group_mapping = {} for grp in raw_groups: arch_groups = group_mapping.setdefault(grp[1], {}) arch_groups[grp[0]] = {'name': grp[0], 'arch': grp[1], 'count': grp[2], 'last_update': grp[3]} # we want to promote the count of 'any' packages in groups to the # other architectures, and also add any 'any'-only groups if 'any' in group_mapping: any_groups = group_mapping['any'] del group_mapping['any'] for arch, arch_groups in group_mapping.iteritems(): for grp in any_groups.itervalues(): if grp['name'] in arch_groups: found = arch_groups[grp['name']] found['count'] += grp['count'] if grp['last_update'] > found['last_update']: found['last_update'] = grp['last_update'] else: new_g = grp.copy() # override the arch to not be 'any' new_g['arch'] = arch arch_groups[grp['name']] = new_g # now transform it back into a sorted list, including only the specified # architectures if we got a list groups = [] for key, val in group_mapping.iteritems(): if not include_arches or key in include_arches: groups.extend(val.itervalues()) return sorted(groups, key=itemgetter('name', 'arch')) def get_split_packages_info(): '''Return info on split packages that do not have an actual package name matching the split pkgbase.''' pkgnames = Package.objects.values('pkgname') split_pkgs = Package.objects.exclude(pkgname=F('pkgbase')).exclude( pkgbase__in=pkgnames).values('pkgbase', 'repo', 'arch').annotate( last_update=Max('last_update')).order_by().distinct() all_arches = Arch.objects.in_bulk({s['arch'] for s in split_pkgs}) all_repos = Repo.objects.in_bulk({s['repo'] for s in split_pkgs}) for split in split_pkgs: split['arch'] = all_arches[split['arch']] split['repo'] = all_repos[split['repo']] return split_pkgs class Difference(object): def __init__(self, pkgname, repo, pkg_a, pkg_b): self.pkgname = pkgname self.repo = repo self.pkg_a = pkg_a self.pkg_b = pkg_b def classes(self): '''A list of CSS classes that should be applied to this row in any generated HTML. Useful for sorting, filtering, etc. Contains whether this difference is in both architectures or the sole architecture it belongs to, as well as the repo name.''' css_classes = [self.repo.name.lower()] if self.pkg_a and self.pkg_b: css_classes.append('both') elif self.pkg_a: css_classes.append(self.pkg_a.arch.name) elif self.pkg_b: css_classes.append(self.pkg_b.arch.name) return ' '.join(css_classes) def __key(self): return (self.pkgname, hash(self.repo), hash(self.pkg_a), hash(self.pkg_b)) def __eq__(self, other): return self.__key() == other.__key() def __hash__(self): return hash(self.__key()) def get_differences_info(arch_a, arch_b): # This is a monster. Join packages against itself, looking for packages in # our non-'any' architectures only, and not having a corresponding package # entry in the other table (or having one with a different pkgver). We will # then go and fetch all of these packages from the database and display # them later using normal ORM models. sql = """ SELECT p.id, q.id FROM packages p LEFT JOIN packages q ON ( p.pkgname = q.pkgname AND p.repo_id = q.repo_id AND p.arch_id != q.arch_id AND p.id != q.id ) WHERE p.arch_id IN (%s, %s) AND ( q.id IS NULL OR p.pkgver != q.pkgver OR p.pkgrel != q.pkgrel OR p.epoch != q.epoch ) """ cursor = connection.cursor() cursor.execute(sql, [arch_a.id, arch_b.id]) results = cursor.fetchall() # column A will always have a value, column B might be NULL to_fetch = {row[0] for row in results} # fetch all of the necessary packages pkgs = Package.objects.normal().in_bulk(to_fetch) # now build a set containing differences differences = set() for row in results: pkg_a = pkgs.get(row[0]) pkg_b = pkgs.get(row[1]) # We want arch_a to always appear first # pkg_a should never be None if pkg_a.arch == arch_a: item = Difference(pkg_a.pkgname, pkg_a.repo, pkg_a, pkg_b) else: # pkg_b can be None in this case, so be careful name = pkg_a.pkgname if pkg_a else pkg_b.pkgname repo = pkg_a.repo if pkg_a else pkg_b.repo item = Difference(name, repo, pkg_b, pkg_a) differences.add(item) # now sort our list by repository, package name key_func = attrgetter('repo.name', 'pkgname') differences = sorted(differences, key=key_func) return differences def multilib_differences(): # Query for checking multilib out of date-ness if database_vendor(Package) == 'sqlite': pkgname_sql = """ CASE WHEN ml.pkgname LIKE %s THEN SUBSTR(ml.pkgname, 7) WHEN ml.pkgname LIKE %s THEN SUBSTR(ml.pkgname, 1, LENGTH(ml.pkgname) - 9) ELSE ml.pkgname END """ else: pkgname_sql = """ CASE WHEN ml.pkgname LIKE %s THEN SUBSTRING(ml.pkgname, 7) WHEN ml.pkgname LIKE %s THEN SUBSTRING(ml.pkgname FROM 1 FOR CHAR_LENGTH(ml.pkgname) - 9) ELSE ml.pkgname END """ sql = """ SELECT ml.id, reg.id FROM packages ml JOIN packages reg ON ( reg.pkgname = (""" + pkgname_sql + """) AND reg.pkgver != ml.pkgver ) JOIN repos r ON reg.repo_id = r.id WHERE ml.repo_id = %s AND r.testing = %s AND r.staging = %s AND reg.arch_id = %s ORDER BY ml.last_update """ multilib = Repo.objects.get(name__iexact='multilib') i686 = Arch.objects.get(name='i686') params = ['lib32-%', '%-multilib', multilib.id, False, False, i686.id] cursor = connection.cursor() cursor.execute(sql, params) results = cursor.fetchall() # fetch all of the necessary packages to_fetch = set(chain.from_iterable(results)) pkgs = Package.objects.normal().in_bulk(to_fetch) return [(pkgs[ml], pkgs[reg]) for ml, reg in results] def get_wrong_permissions(): sql = """ SELECT DISTINCT id FROM ( SELECT pr.id, p.repo_id, pr.user_id FROM packages p JOIN packages_packagerelation pr ON p.pkgbase = pr.pkgbase WHERE pr.type = %s ) mp LEFT JOIN ( SELECT user_id, repo_id FROM user_profiles_allowed_repos ar INNER JOIN user_profiles up ON ar.userprofile_id = up.id ) ur ON mp.user_id = ur.user_id AND mp.repo_id = ur.repo_id WHERE ur.user_id IS NULL; """ cursor = connection.cursor() cursor.execute(sql, [PackageRelation.MAINTAINER]) to_fetch = [row[0] for row in cursor.fetchall()] relations = PackageRelation.objects.select_related('user').filter( id__in=to_fetch) return relations def attach_maintainers(packages): '''Given a queryset or something resembling it of package objects, find all the maintainers and attach them to the packages to prevent N+1 query cascading.''' if isinstance(packages, QuerySet): pkgbases = packages.values('pkgbase') else: packages = list(packages) pkgbases = {p.pkgbase for p in packages if p is not None} rels = PackageRelation.objects.filter(type=PackageRelation.MAINTAINER, pkgbase__in=pkgbases).values_list( 'pkgbase', 'user_id').order_by().distinct() # get all the user objects we will need user_ids = {rel[1] for rel in rels} users = User.objects.in_bulk(user_ids) # now build a pkgbase -> [maintainers...] map maintainers = defaultdict(list) for rel in rels: maintainers[rel[0]].append(users[rel[1]]) annotated = [] # and finally, attach the maintainer lists on the original packages for package in packages: if package is None: continue package.maintainers = maintainers[package.pkgbase] annotated.append(package) return annotated def approved_by_signoffs(signoffs, spec): if signoffs: good_signoffs = sum(1 for s in signoffs if not s.revoked) return good_signoffs >= spec.required return False class PackageSignoffGroup(object): '''Encompasses all packages in testing with the same pkgbase.''' def __init__(self, packages): if len(packages) == 0: raise Exception self.packages = packages self.user = None self.target_repo = None self.signoffs = set() self.specification = DEFAULT_SIGNOFF_SPEC self.default_spec = True first = packages[0] self.pkgbase = first.pkgbase self.arch = first.arch self.repo = first.repo self.version = '' self.last_update = first.last_update self.packager = first.packager self.maintainers = first.maintainers version = first.full_version if all(version == pkg.full_version for pkg in packages): self.version = version @property def package(self): '''Try and return a relevant single package object representing this group. Start by seeing if there is only one package, then look for the matching package by name, finally falling back to a standin package object.''' if len(self.packages) == 1: return self.packages[0] same_pkgs = [p for p in self.packages if p.pkgname == p.pkgbase] if same_pkgs: return same_pkgs[0] return PackageStandin(self.packages[0]) def find_signoffs(self, all_signoffs): '''Look through a list of Signoff objects for ones matching this particular group and store them on the object.''' for s in all_signoffs: if s.pkgbase != self.pkgbase: continue if self.version and not s.full_version == self.version: continue if s.arch_id == self.arch.id and s.repo_id == self.repo.id: self.signoffs.add(s) def find_specification(self, specifications): for spec in specifications: if spec.pkgbase != self.pkgbase: continue if self.version and not spec.full_version == self.version: continue if spec.arch_id == self.arch.id and spec.repo_id == self.repo.id: self.specification = spec self.default_spec = False return def approved(self): return approved_by_signoffs(self.signoffs, self.specification) @property def completed(self): return sum(1 for s in self.signoffs if not s.revoked) @property def required(self): return self.specification.required def user_signed_off(self, user=None): '''Did a given user signoff on this package? user can be passed as an argument, or attached to the group object itself so this can be called from a template.''' if user is None: user = self.user return user in (s.user for s in self.signoffs if not s.revoked) def __unicode__(self): return u'%s-%s (%s): %d' % ( self.pkgbase, self.version, self.arch, len(self.signoffs)) def signoffs_id_query(model, repos): sql = """ SELECT DISTINCT s.id FROM %s s JOIN packages p ON ( s.pkgbase = p.pkgbase AND s.pkgver = p.pkgver AND s.pkgrel = p.pkgrel AND s.epoch = p.epoch AND s.arch_id = p.arch_id AND s.repo_id = p.repo_id ) WHERE p.repo_id IN (%s) AND s.repo_id IN (%s) """ cursor = connection.cursor() # query pre-process- fill in table name and placeholders for IN repo_sql = ','.join(['%s' for _ in repos]) sql = sql % (model._meta.db_table, repo_sql, repo_sql) repo_ids = [r.pk for r in repos] # repo_ids are needed twice, so double the array cursor.execute(sql, repo_ids * 2) results = cursor.fetchall() return [row[0] for row in results] def get_current_signoffs(repos): '''Returns a list of signoff objects for the given repos.''' to_fetch = signoffs_id_query(Signoff, repos) return Signoff.objects.select_related('user').in_bulk(to_fetch).values() def get_current_specifications(repos): '''Returns a list of signoff specification objects for the given repos.''' to_fetch = signoffs_id_query(SignoffSpecification, repos) return SignoffSpecification.objects.in_bulk(to_fetch).values() def get_target_repo_map(repos): sql = """ SELECT DISTINCT p1.pkgbase, r.name FROM packages p1 JOIN repos r ON p1.repo_id = r.id JOIN packages p2 ON p1.pkgbase = p2.pkgbase WHERE r.staging = %s AND r.testing = %s AND p2.repo_id IN ( """ sql += ','.join(['%s' for _ in repos]) sql += ")" params = [False, False] params.extend(r.pk for r in repos) cursor = connection.cursor() cursor.execute(sql, params) return dict(cursor.fetchall()) def get_signoff_groups(repos=None, user=None): if repos is None: repos = Repo.objects.filter(testing=True) repo_ids = [r.pk for r in repos] test_pkgs = Package.objects.select_related( 'arch', 'repo', 'packager').filter(repo__in=repo_ids) packages = test_pkgs.order_by('pkgname') packages = attach_maintainers(packages) # Filter by user if asked to do so if user is not None: packages = [p for p in packages if user == p.packager or user in p.maintainers] # Collect all pkgbase values in testing repos pkgtorepo = get_target_repo_map(repos) # Collect all possible signoffs and specifications for these packages signoffs = get_current_signoffs(repos) specs = get_current_specifications(repos) same_pkgbase_key = lambda x: (x.repo.name, x.arch.name, x.pkgbase) grouped = groupby_preserve_order(packages, same_pkgbase_key) signoff_groups = [] for group in grouped: signoff_group = PackageSignoffGroup(group) signoff_group.target_repo = pkgtorepo.get(signoff_group.pkgbase, "Unknown") signoff_group.find_signoffs(signoffs) signoff_group.find_specification(specs) signoff_groups.append(signoff_group) return signoff_groups class PackageJSONEncoder(DjangoJSONEncoder): pkg_attributes = ['pkgname', 'pkgbase', 'repo', 'arch', 'pkgver', 'pkgrel', 'epoch', 'pkgdesc', 'url', 'filename', 'compressed_size', 'installed_size', 'build_date', 'last_update', 'flag_date', 'maintainers', 'packager'] pkg_list_attributes = ['groups', 'licenses', 'conflicts', 'provides', 'replaces', 'depends'] def default(self, obj): if hasattr(obj, '__iter__'): # mainly for queryset serialization return list(obj) if isinstance(obj, Package): data = {attr: getattr(obj, attr) for attr in self.pkg_attributes} for attr in self.pkg_list_attributes: data[attr] = getattr(obj, attr).all() return data if isinstance(obj, PackageFile): filename = obj.filename or '' return obj.directory + filename if isinstance(obj, (Repo, Arch)): return obj.name.lower() if isinstance(obj, (PackageGroup, License)): return obj.name if isinstance(obj, (Depend, Conflict, Provision, Replacement)): return unicode(obj) elif isinstance(obj, User): return obj.username return super(PackageJSONEncoder, self).default(obj) # vim: set ts=4 sw=4 et: