From 5575b68272f1bc773f98ecdfd679e58e8be2ac9b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 23 Mar 2012 17:59:54 -0500 Subject: reporead: use Django 1.4 select_for_update() As per TODO comments in the existing code. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) (limited to 'devel/management/commands/reporead.py') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 8369b6e..ebf1711 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -286,17 +286,6 @@ def populate_files(dbpkg, repopkg, force=False): dbpkg.save() -def select_pkg_for_update(dbpkg): - database = router.db_for_write(Package, instance=dbpkg) - connection = connections[database] - if 'sqlite' in connection.settings_dict['ENGINE'].lower(): - return dbpkg - new_pkg = Package.objects.raw( - 'SELECT * FROM packages WHERE id = %s FOR UPDATE', - [dbpkg.id]) - return list(new_pkg)[0] - - def update_common(archname, reponame, pkgs, sanity_check=True): # If isolation level is repeatable-read, we need to ensure each package # update starts a new transaction and re-queries the database as @@ -396,8 +385,7 @@ def db_update(archname, reponame, pkgs, force=False): # simultaneous updates don't happen on a package, causing # files/depends/all related items to be double-imported. with transaction.commit_on_success(): - # TODO Django 1.4 select_for_update() will work once released - dbpkg = select_pkg_for_update(dbpkg) + dbpkg = Package.objects.select_for_update().get(id=dbpkg.id) if not force and pkg_same_version(pkg, dbpkg): logger.debug("Package %s was already updated", pkg.name) continue @@ -428,8 +416,7 @@ def filesonly_update(archname, reponame, pkgs, force=False): elif not force and dbpkg.files_last_update > dbpkg.last_update: logger.debug("Files for %s are up to date", pkg.name) continue - # TODO Django 1.4 select_for_update() will work once released - dbpkg = select_pkg_for_update(dbpkg) + dbpkg = Package.objects.select_for_update().get(id=dbpkg.id) logger.debug("Checking files for package %s", pkg.name) populate_files(dbpkg, pkg, force=force) -- cgit v1.2.3-24-g4f1b From 95520ae26485e5c62ea1431ca82a7b42d01b923a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 23 Mar 2012 18:03:28 -0500 Subject: reporead: use Django 1.4 bulk_create() for package files Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'devel/management/commands/reporead.py') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index ebf1711..a8c58ba 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -270,18 +270,17 @@ def populate_files(dbpkg, repopkg, force=False): delete_pkg_files(dbpkg) logger.info("adding %d files for package %s", len(repopkg.files), dbpkg.pkgname) + pkg_files = [] for f in repopkg.files: dirname, filename = f.rsplit('/', 1) if filename == '': filename = None - # this is basically like calling dbpkg.packagefile_set.create(), - # but much faster as we can skip a lot of the repeated code paths - # TODO use Django 1.4 bulk_create pkgfile = PackageFile(pkg=dbpkg, is_directory=(filename is None), directory=dirname + '/', filename=filename) - pkgfile.save(force_insert=True) + pkg_files.append(pkgfile) + PackageFile.objects.bulk_create(pkg_files) dbpkg.files_last_update = datetime.utcnow() dbpkg.save() -- cgit v1.2.3-24-g4f1b From 90e08b4863dfaecafee5b151478bda4513b12e85 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 23 Mar 2012 19:29:40 -0500 Subject: Make all datetime objects fully timezone aware This is most of the transition to Django 1.4 `USE_TZ = True`. We need to ensure we don't mix aware and non-aware datetime objects when dealing with datetimes in the code. Add a utc_now() helper method that we can use most places, and ensure there is always a timezone attached when necessary. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'devel/management/commands/reporead.py') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index a8c58ba..60ee6ec 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -22,6 +22,7 @@ import tarfile import logging from datetime import datetime from optparse import make_option +from pytz import utc from django.core.management.base import BaseCommand, CommandError from django.db import connections, router, transaction @@ -29,8 +30,10 @@ from django.db.utils import IntegrityError from devel.utils import UserFinder from main.models import Arch, Package, PackageDepend, PackageFile, Repo +from main.utils import utc_now from packages.models import Conflict, Provision, Replacement + logging.basicConfig( level=logging.WARNING, format='%(asctime)s -> %(levelname)s: %(message)s', @@ -113,7 +116,8 @@ class RepoPackage(object): self.epoch = int(match.group(2)) elif k == 'builddate': try: - self.builddate = datetime.utcfromtimestamp(int(v[0])) + builddate = datetime.utcfromtimestamp(int(v[0])) + self.builddate = builddate.replace(tzinfo=utc) except ValueError: try: self.builddate = datetime.strptime(v[0], @@ -281,7 +285,7 @@ def populate_files(dbpkg, repopkg, force=False): filename=filename) pkg_files.append(pkgfile) PackageFile.objects.bulk_create(pkg_files) - dbpkg.files_last_update = datetime.utcnow() + dbpkg.files_last_update = utc_now() dbpkg.save() @@ -351,7 +355,7 @@ def db_update(archname, reponame, pkgs, force=False): dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository) try: with transaction.commit_on_success(): - populate_pkg(dbpkg, pkg, timestamp=datetime.utcnow()) + populate_pkg(dbpkg, pkg, timestamp=utc_now()) except IntegrityError: logger.warning("Could not add package %s; " "not fatal if another thread beat us to it.", @@ -378,7 +382,7 @@ def db_update(archname, reponame, pkgs, force=False): if not force and pkg_same_version(pkg, dbpkg): continue elif not force: - timestamp = datetime.utcnow() + timestamp = utc_now() # The odd select_for_update song and dance here are to ensure # simultaneous updates don't happen on a package, causing -- cgit v1.2.3-24-g4f1b