From 55179a4f9e8b80d515bae7032af8aefc33ae0192 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 16 Jan 2013 16:07:26 -0600 Subject: reporead: remove batched_bulk_create Now that Django 1.5 is out and realized SQLite3 only allows for 999 parameters per SQL call, we don't need to manually batch things up anymore and can let the underlying bulk_create code do it for us. This basically reverts commit 88ee61a39ac3. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 32 +++++--------------------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index ab0efee..ccac55f 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -32,7 +32,6 @@ from django.utils.timezone import now from devel.utils import UserFinder from main.models import Arch, Package, PackageFile, Repo -from main.utils import database_vendor from packages.models import Depend, Conflict, Provision, Replacement, Update from packages.utils import parse_version @@ -182,27 +181,6 @@ def create_related(model, package, rel_str, equals_only=False): return related -def batched_bulk_create(model, all_objects): - # for short lists, just bulk_create as we should be fine - if len(all_objects) < 20: - return model.objects.bulk_create(all_objects) - - if database_vendor(model, mode='write') == 'sqlite': - # 999 max variables in each SQL statement - incr = 999 // len(model._meta.fields) - else: - incr = 1000 - - def chunks(): - offset = 0 - while offset < len(all_objects): - yield all_objects[offset:offset + incr] - offset += incr - - for items in chunks(): - model.objects.bulk_create(items) - - def create_multivalued(dbpkg, repopkg, db_attr, repo_attr): '''Populate the simplest of multivalued attributes. These are those that only deal with a 'name' attribute, such as licenses, groups, etc. The input @@ -256,20 +234,20 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): deps += [create_depend(dbpkg, y, 'O') for y in repopkg.optdepends] deps += [create_depend(dbpkg, y, 'M') for y in repopkg.makedepends] deps += [create_depend(dbpkg, y, 'C') for y in repopkg.checkdepends] - batched_bulk_create(Depend, deps) + Depend.objects.bulk_create(deps) dbpkg.conflicts.all().delete() conflicts = [create_related(Conflict, dbpkg, y) for y in repopkg.conflicts] - batched_bulk_create(Conflict, conflicts) + Conflict.objects.bulk_create(conflicts) dbpkg.provides.all().delete() provides = [create_related(Provision, dbpkg, y, equals_only=True) for y in repopkg.provides] - batched_bulk_create(Provision, provides) + Provision.objects.bulk_create(provides) dbpkg.replaces.all().delete() replaces = [create_related(Replacement, dbpkg, y) for y in repopkg.replaces] - batched_bulk_create(Replacement, replaces) + Replacement.objects.bulk_create(replaces) create_multivalued(dbpkg, repopkg, 'groups', 'groups') create_multivalued(dbpkg, repopkg, 'licenses', 'license') @@ -319,7 +297,7 @@ def populate_files(dbpkg, repopkg, force=False): directory=dirname, filename=filename) pkg_files.append(pkgfile) - batched_bulk_create(PackageFile, pkg_files) + PackageFile.objects.bulk_create(pkg_files) dbpkg.files_last_update = now() dbpkg.save() -- cgit v1.2.3-24-g4f1b From 10462425f989dc74653179f0845978a6b5e30045 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 16 Jan 2013 16:10:24 -0600 Subject: Remove {% load url from future %} from templates This is now the default in Django 1.5. Signed-off-by: Dan McGee --- templates/base.html | 2 +- templates/packages/removed.html | 1 - templates/public/download.html | 1 - templates/public/index.html | 1 - templates/public/keys.html | 1 - templates/releng/add.html | 1 - templates/releng/iso_overview.html | 1 - templates/releng/release_detail.html | 1 - templates/releng/release_list.html | 1 - templates/releng/result_list.html | 1 - templates/releng/result_section.html | 1 - templates/releng/results.html | 1 - templates/releng/thanks.html | 1 - templates/visualize/index.html | 1 - 14 files changed, 1 insertion(+), 14 deletions(-) diff --git a/templates/base.html b/templates/base.html index cc507fb..15bb7e2 100644 --- a/templates/base.html +++ b/templates/base.html @@ -1,4 +1,4 @@ -{% load url from future %}{% load static from staticfiles %} +{% load static from staticfiles %} diff --git a/templates/packages/removed.html b/templates/packages/removed.html index ea20ce8..f188b6d 100644 --- a/templates/packages/removed.html +++ b/templates/packages/removed.html @@ -1,5 +1,4 @@ {% extends "base.html" %} -{% load url from future %} {% load package_extras %} {% block title %}Arch Linux - Not Available - {{ name }} {{ version }} ({{ arch.name }}){% endblock %} diff --git a/templates/public/download.html b/templates/public/download.html index c68cf66..7936efb 100644 --- a/templates/public/download.html +++ b/templates/public/download.html @@ -1,6 +1,5 @@ {% extends "base.html" %} {% load cache %} -{% load url from future %} {% load static from staticfiles %} {% load flags %} diff --git a/templates/public/index.html b/templates/public/index.html index 8926a06..3f88c18 100644 --- a/templates/public/index.html +++ b/templates/public/index.html @@ -1,6 +1,5 @@ {% extends "base.html" %} {% load cache %} -{% load url from future %} {% load static from staticfiles %} {% block head %} diff --git a/templates/public/keys.html b/templates/public/keys.html index ad2dd19..91b5307 100644 --- a/templates/public/keys.html +++ b/templates/public/keys.html @@ -1,6 +1,5 @@ {% extends "base.html" %} {% load static from staticfiles %} -{% load url from future %} {% load pgp %} {% block title %}Arch Linux - Master Signing Keys{% endblock %} diff --git a/templates/releng/add.html b/templates/releng/add.html index ed02984..d060395 100644 --- a/templates/releng/add.html +++ b/templates/releng/add.html @@ -1,5 +1,4 @@ {% extends "base.html" %} -{% load url from future %} {% block title %}Arch Linux - Test Result Entry{% endblock %} diff --git a/templates/releng/iso_overview.html b/templates/releng/iso_overview.html index 23ef013..196f0c0 100644 --- a/templates/releng/iso_overview.html +++ b/templates/releng/iso_overview.html @@ -1,6 +1,5 @@ {% extends "base.html" %} {% load static from staticfiles %} -{% load url from future %} {% block content %}
diff --git a/templates/releng/release_detail.html b/templates/releng/release_detail.html index 01c0319..0950753 100644 --- a/templates/releng/release_detail.html +++ b/templates/releng/release_detail.html @@ -1,5 +1,4 @@ {% extends "base.html" %} -{% load url from future %} {% block title %}Arch Linux - Release: {{ release.version }}{% endblock %} diff --git a/templates/releng/release_list.html b/templates/releng/release_list.html index ef53a93..5f24826 100644 --- a/templates/releng/release_list.html +++ b/templates/releng/release_list.html @@ -1,5 +1,4 @@ {% extends "base.html" %} -{% load url from future %} {% load static from staticfiles %} {% block title %}Arch Linux - Releases{% endblock %} diff --git a/templates/releng/result_list.html b/templates/releng/result_list.html index be5783b..12bf39d 100644 --- a/templates/releng/result_list.html +++ b/templates/releng/result_list.html @@ -1,6 +1,5 @@ {% extends "base.html" %} {% load static from staticfiles %} -{% load url from future %} {% block content %}
diff --git a/templates/releng/result_section.html b/templates/releng/result_section.html index ae951eb..5e0b6f6 100644 --- a/templates/releng/result_section.html +++ b/templates/releng/result_section.html @@ -1,4 +1,3 @@ -{% load url from future %} {% if option.is_rollback %}Rollback: {% endif %}{{ option.name|title }} Last Success diff --git a/templates/releng/results.html b/templates/releng/results.html index c19b42a..59d8351 100644 --- a/templates/releng/results.html +++ b/templates/releng/results.html @@ -1,5 +1,4 @@ {% extends "base.html" %} -{% load url from future %} {% block title %}Arch Linux - Release Engineering Testbuild Results{% endblock %} diff --git a/templates/releng/thanks.html b/templates/releng/thanks.html index 66d65a5..2462daf 100644 --- a/templates/releng/thanks.html +++ b/templates/releng/thanks.html @@ -1,5 +1,4 @@ {% extends "base.html" %} -{% load url from future %} {% block title %}Arch Linux - Feedback - Thanks!{% endblock %} diff --git a/templates/visualize/index.html b/templates/visualize/index.html index 1e932d9..2b79d60 100644 --- a/templates/visualize/index.html +++ b/templates/visualize/index.html @@ -1,6 +1,5 @@ {% extends "base.html" %} {% load static from staticfiles %} -{% load url from future %} {% block title %}Arch Linux - Visualizations{% endblock %} -- cgit v1.2.3-24-g4f1b From 271d1babbf8038e17d9dc5cfc3cd659463848400 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 20 Jan 2013 12:43:33 -0600 Subject: Revert "Reduce query count when retrieving satisfiers and providers" This reverts commit 20b64e42672d185821cc584dfa4b133ee259a144. Django 1.5 fixed this issue and now parent objects are automatically attached to their children when queries go through the related manager. See "Caching of related model instances" in the release notes. Signed-off-by: Dan McGee --- main/models.py | 4 ++-- packages/models.py | 20 ++++++++------------ 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/main/models.py b/main/models.py index 46fd3a6..da9d8b6 100644 --- a/main/models.py +++ b/main/models.py @@ -277,10 +277,10 @@ class Package(models.Model): # TODO: we can use list comprehension and an 'in' query to make this # more effective for dep in self.depends.all(): - pkg = dep.get_best_satisfier(self) + pkg = dep.get_best_satisfier() providers = None if not pkg: - providers = dep.get_providers(self) + providers = dep.get_providers() deps.append({'dep': dep, 'pkg': pkg, 'providers': providers}) # sort the list; deptype sorting makes this tricker than expected sort_order = {'D': 0, 'O': 1, 'M': 2, 'C': 3} diff --git a/packages/models.py b/packages/models.py index ff67788..7bcdc00 100644 --- a/packages/models.py +++ b/packages/models.py @@ -357,16 +357,14 @@ class RelatedToBase(models.Model): name = models.CharField(max_length=255, db_index=True) version = models.CharField(max_length=255, default='') - def get_best_satisfier(self, main_pkg=None): + def get_best_satisfier(self): '''Find a satisfier for this related package that best matches the given criteria. It will not search provisions, but will find packages named and matching repo characteristics if possible.''' - if main_pkg is None: - main_pkg = self.pkg pkgs = Package.objects.normal().filter(pkgname=self.name) - if not main_pkg.arch.agnostic: + if not self.pkg.arch.agnostic: # make sure we match architectures if possible - arches = main_pkg.applicable_arches() + arches = self.pkg.applicable_arches() pkgs = pkgs.filter(arch__in=arches) # if we have a comparison operation, make sure the packages we grab # actually satisfy the requirements @@ -386,27 +384,25 @@ class RelatedToBase(models.Model): pkg = pkgs[0] # prevents yet more DB queries, these lists should be short; # after each grab the best available in case we remove all entries - pkgs = [p for p in pkgs if p.repo.staging == main_pkg.repo.staging] + pkgs = [p for p in pkgs if p.repo.staging == self.pkg.repo.staging] if len(pkgs) > 0: pkg = pkgs[0] - pkgs = [p for p in pkgs if p.repo.testing == main_pkg.repo.testing] + pkgs = [p for p in pkgs if p.repo.testing == self.pkg.repo.testing] if len(pkgs) > 0: pkg = pkgs[0] return pkg - def get_providers(self, main_pkg=None): + def get_providers(self): '''Return providers of this related package. Does *not* include exact matches as it checks the Provision names only, use get_best_satisfier() instead for exact matches.''' - if main_pkg is None: - main_pkg = self.pkg pkgs = Package.objects.normal().filter( provides__name=self.name).order_by().distinct() - if not main_pkg.arch.agnostic: + if not self.pkg.arch.agnostic: # make sure we match architectures if possible - arches = main_pkg.applicable_arches() + arches = self.pkg.applicable_arches() pkgs = pkgs.filter(arch__in=arches) # If we have a comparison operation, make sure the packages we grab -- cgit v1.2.3-24-g4f1b From 8524a6057c4b99a620850494a22eb3d1f56bee68 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 20 Jan 2013 14:15:53 -0600 Subject: Change caching strategy on package.applicable_arches Rather than use the Django cache for this (aka memcached), just do it on a per-object instantiation basis. This means no external services calls except the first time to the database, which should be quite a bit faster. Signed-off-by: Dan McGee --- main/models.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/main/models.py b/main/models.py index da9d8b6..ec5c2bd 100644 --- a/main/models.py +++ b/main/models.py @@ -177,12 +177,15 @@ class Package(models.Model): def maintainers(self, maintainers): self._maintainers = maintainers - @cache_function(1800) + _applicable_arches = None + def applicable_arches(self): '''The list of (this arch) + (available agnostic arches).''' - arches = set(Arch.objects.filter(agnostic=True)) - arches.add(self.arch) - return list(arches) + if self._applicable_arches is None: + arches = set(Arch.objects.filter(agnostic=True)) + arches.add(self.arch) + self._applicable_arches = list(arches) + return self._applicable_arches @cache_function(119) def get_requiredby(self): -- cgit v1.2.3-24-g4f1b From 5566d43a7734f6bb2f48d5d511351da12ddc5cc1 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 9 Feb 2013 16:43:40 -0600 Subject: Use 'update_fields' model.save() kwarg This was added in Django 1.5 and allows saving only a subset of a model's fields. It makes sense in a few cases to utilize it. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorresolv.py | 2 +- packages/views/signoff.py | 2 +- releng/management/commands/syncisos.py | 2 +- todolists/views.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mirrors/management/commands/mirrorresolv.py b/mirrors/management/commands/mirrorresolv.py index 0370f8e..a6c2523 100644 --- a/mirrors/management/commands/mirrorresolv.py +++ b/mirrors/management/commands/mirrorresolv.py @@ -53,7 +53,7 @@ def resolve_mirrors(): newvals = (mirrorurl.has_ipv4, mirrorurl.has_ipv6) if newvals != oldvals: logger.debug("values changed for %s", mirrorurl) - mirrorurl.save(force_update=True) + mirrorurl.save(update_fields=('has_ipv4', 'has_ipv6')) except socket.error, e: logger.warn("error resolving %s: %s", mirrorurl.hostname, e) diff --git a/packages/views/signoff.py b/packages/views/signoff.py index 17f3095..c37aa0f 100644 --- a/packages/views/signoff.py +++ b/packages/views/signoff.py @@ -45,7 +45,7 @@ def signoff_package(request, name, repo, arch, revoke=False): except Signoff.DoesNotExist: raise Http404 signoff.revoked = now() - signoff.save() + signoff.save(update_fields=('revoked',)) created = False else: # ensure we should even be accepting signoffs diff --git a/releng/management/commands/syncisos.py b/releng/management/commands/syncisos.py index 223c771..c9f6196 100644 --- a/releng/management/commands/syncisos.py +++ b/releng/management/commands/syncisos.py @@ -53,7 +53,7 @@ class Command(BaseCommand): if not existing.active: existing.active = True existing.removed = None - existing.save() + existing.save(update_fields=('active', 'removed')) # and then mark all other names as no longer active Iso.objects.filter(active=True).exclude(name__in=active_isos).update( active=False, removed=now()) diff --git a/todolists/views.py b/todolists/views.py index f333728..9935987 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -47,7 +47,7 @@ def flag(request, slug, pkg_id): else: tlpkg.status = TodolistPackage.INCOMPLETE tlpkg.user = request.user - tlpkg.save() + tlpkg.save(update_fields=('status', 'user', 'last_modified')) if request.is_ajax(): data = { 'status': tlpkg.get_status_display(), -- cgit v1.2.3-24-g4f1b From a25cc4ff459ab0070ce48730f6c6fdc227860f08 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 26 Feb 2013 18:44:37 -0600 Subject: Bump Django requirement to 1.5 Signed-off-by: Dan McGee --- requirements.txt | 2 +- requirements_prod.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index b8a0621..24f83d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin -Django==1.4.3 +Django==1.5 Markdown==2.2.1 South==0.7.6 bencode==1.0 diff --git a/requirements_prod.txt b/requirements_prod.txt index 705c56b..0913d32 100644 --- a/requirements_prod.txt +++ b/requirements_prod.txt @@ -1,5 +1,5 @@ -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin -Django==1.4.3 +Django==1.5 Markdown==2.2.1 South==0.7.6 bencode==1.0 -- cgit v1.2.3-24-g4f1b