From 5575b68272f1bc773f98ecdfd679e58e8be2ac9b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 23 Mar 2012 17:59:54 -0500 Subject: reporead: use Django 1.4 select_for_update() As per TODO comments in the existing code. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 8369b6e..ebf1711 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -286,17 +286,6 @@ def populate_files(dbpkg, repopkg, force=False): dbpkg.save() -def select_pkg_for_update(dbpkg): - database = router.db_for_write(Package, instance=dbpkg) - connection = connections[database] - if 'sqlite' in connection.settings_dict['ENGINE'].lower(): - return dbpkg - new_pkg = Package.objects.raw( - 'SELECT * FROM packages WHERE id = %s FOR UPDATE', - [dbpkg.id]) - return list(new_pkg)[0] - - def update_common(archname, reponame, pkgs, sanity_check=True): # If isolation level is repeatable-read, we need to ensure each package # update starts a new transaction and re-queries the database as @@ -396,8 +385,7 @@ def db_update(archname, reponame, pkgs, force=False): # simultaneous updates don't happen on a package, causing # files/depends/all related items to be double-imported. with transaction.commit_on_success(): - # TODO Django 1.4 select_for_update() will work once released - dbpkg = select_pkg_for_update(dbpkg) + dbpkg = Package.objects.select_for_update().get(id=dbpkg.id) if not force and pkg_same_version(pkg, dbpkg): logger.debug("Package %s was already updated", pkg.name) continue @@ -428,8 +416,7 @@ def filesonly_update(archname, reponame, pkgs, force=False): elif not force and dbpkg.files_last_update > dbpkg.last_update: logger.debug("Files for %s are up to date", pkg.name) continue - # TODO Django 1.4 select_for_update() will work once released - dbpkg = select_pkg_for_update(dbpkg) + dbpkg = Package.objects.select_for_update().get(id=dbpkg.id) logger.debug("Checking files for package %s", pkg.name) populate_files(dbpkg, pkg, force=force) -- cgit v1.2.3-24-g4f1b From 95520ae26485e5c62ea1431ca82a7b42d01b923a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 23 Mar 2012 18:03:28 -0500 Subject: reporead: use Django 1.4 bulk_create() for package files Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index ebf1711..a8c58ba 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -270,18 +270,17 @@ def populate_files(dbpkg, repopkg, force=False): delete_pkg_files(dbpkg) logger.info("adding %d files for package %s", len(repopkg.files), dbpkg.pkgname) + pkg_files = [] for f in repopkg.files: dirname, filename = f.rsplit('/', 1) if filename == '': filename = None - # this is basically like calling dbpkg.packagefile_set.create(), - # but much faster as we can skip a lot of the repeated code paths - # TODO use Django 1.4 bulk_create pkgfile = PackageFile(pkg=dbpkg, is_directory=(filename is None), directory=dirname + '/', filename=filename) - pkgfile.save(force_insert=True) + pkg_files.append(pkgfile) + PackageFile.objects.bulk_create(pkg_files) dbpkg.files_last_update = datetime.utcnow() dbpkg.save() -- cgit v1.2.3-24-g4f1b From b8c20439c091aaa56e772441e6c3a7e57e8ef2d4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 23 Mar 2012 18:29:42 -0500 Subject: Change Django urls.py import Until Django 1.3, the functions include(), patterns() and url() plus handler404, handler500 were located in a django.conf.urls.defaults module. In Django 1.4, they live in django.conf.urls. Signed-off-by: Dan McGee --- devel/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'devel') diff --git a/devel/urls.py b/devel/urls.py index 07cb321..31afc86 100644 --- a/devel/urls.py +++ b/devel/urls.py @@ -1,4 +1,4 @@ -from django.conf.urls.defaults import patterns +from django.conf.urls import patterns urlpatterns = patterns('devel.views', (r'^admin_log/$','admin_log'), -- cgit v1.2.3-24-g4f1b From 90e08b4863dfaecafee5b151478bda4513b12e85 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 23 Mar 2012 19:29:40 -0500 Subject: Make all datetime objects fully timezone aware This is most of the transition to Django 1.4 `USE_TZ = True`. We need to ensure we don't mix aware and non-aware datetime objects when dealing with datetimes in the code. Add a utc_now() helper method that we can use most places, and ensure there is always a timezone attached when necessary. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 12 ++++++++---- devel/views.py | 33 +++++++++++++-------------------- 2 files changed, 21 insertions(+), 24 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index a8c58ba..60ee6ec 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -22,6 +22,7 @@ import tarfile import logging from datetime import datetime from optparse import make_option +from pytz import utc from django.core.management.base import BaseCommand, CommandError from django.db import connections, router, transaction @@ -29,8 +30,10 @@ from django.db.utils import IntegrityError from devel.utils import UserFinder from main.models import Arch, Package, PackageDepend, PackageFile, Repo +from main.utils import utc_now from packages.models import Conflict, Provision, Replacement + logging.basicConfig( level=logging.WARNING, format='%(asctime)s -> %(levelname)s: %(message)s', @@ -113,7 +116,8 @@ class RepoPackage(object): self.epoch = int(match.group(2)) elif k == 'builddate': try: - self.builddate = datetime.utcfromtimestamp(int(v[0])) + builddate = datetime.utcfromtimestamp(int(v[0])) + self.builddate = builddate.replace(tzinfo=utc) except ValueError: try: self.builddate = datetime.strptime(v[0], @@ -281,7 +285,7 @@ def populate_files(dbpkg, repopkg, force=False): filename=filename) pkg_files.append(pkgfile) PackageFile.objects.bulk_create(pkg_files) - dbpkg.files_last_update = datetime.utcnow() + dbpkg.files_last_update = utc_now() dbpkg.save() @@ -351,7 +355,7 @@ def db_update(archname, reponame, pkgs, force=False): dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository) try: with transaction.commit_on_success(): - populate_pkg(dbpkg, pkg, timestamp=datetime.utcnow()) + populate_pkg(dbpkg, pkg, timestamp=utc_now()) except IntegrityError: logger.warning("Could not add package %s; " "not fatal if another thread beat us to it.", @@ -378,7 +382,7 @@ def db_update(archname, reponame, pkgs, force=False): if not force and pkg_same_version(pkg, dbpkg): continue elif not force: - timestamp = datetime.utcnow() + timestamp = utc_now() # The odd select_for_update song and dance here are to ensure # simultaneous updates don't happen on a package, causing diff --git a/devel/views.py b/devel/views.py index 328d52e..3a9be75 100644 --- a/devel/views.py +++ b/devel/views.py @@ -1,3 +1,10 @@ +from datetime import datetime, timedelta +import operator +import pytz +import random +from string import ascii_letters, digits +import time + from django import forms from django.http import HttpResponseRedirect from django.contrib.auth.decorators import \ @@ -16,19 +23,13 @@ from django.views.generic.simple import direct_to_template from django.utils.http import http_date from main.models import Package, PackageDepend, PackageFile, TodolistPkg -from main.models import Arch, Repo -from main.models import UserProfile +from main.models import Arch, Repo, UserProfile +from main.utils import utc_now from packages.models import PackageRelation from packages.utils import get_signoff_groups from todolists.utils import get_annotated_todolists from .utils import get_annotated_maintainers -from datetime import datetime, timedelta -import operator -import pytz -import random -from string import ascii_letters, digits -import time @login_required def index(request): @@ -85,22 +86,14 @@ def clock(request): devs = User.objects.filter(is_active=True).order_by( 'first_name', 'last_name').select_related('userprofile') - now = datetime.now() - utc_now = datetime.utcnow().replace(tzinfo=pytz.utc) - # now annotate each dev object with their current time - for dev in devs: - tz = pytz.timezone(dev.userprofile.time_zone) - dev.current_time = utc_now.astimezone(tz) - + now = utc_now() page_dict = { 'developers': devs, - 'now': now, - 'utc_now': utc_now, + 'utc_now': now, } response = direct_to_template(request, 'devel/clock.html', page_dict) if not response.has_header('Expires'): - # why this works only with the non-UTC date I have no idea... expire_time = now.replace(second=0, microsecond=0) expire_time += timedelta(minutes=1) expire_time = time.mktime(expire_time.timetuple()) @@ -168,12 +161,12 @@ def report(request, report_name, username=None): if report_name == 'old': title = 'Packages last built more than two years ago' - cutoff = datetime.utcnow() - timedelta(days=365 * 2) + cutoff = utc_now() - timedelta(days=365 * 2) packages = packages.filter( build_date__lt=cutoff).order_by('build_date') elif report_name == 'long-out-of-date': title = 'Packages marked out-of-date more than 90 days ago' - cutoff = datetime.utcnow() - timedelta(days=90) + cutoff = utc_now() - timedelta(days=90) packages = packages.filter( flag_date__lt=cutoff).order_by('flag_date') elif report_name == 'big': -- cgit v1.2.3-24-g4f1b