summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDan McGee <dan@archlinux.org>2011-02-17 04:51:23 +0100
committerDan McGee <dan@archlinux.org>2011-02-22 22:43:48 +0100
commitfc7fc1d10a13e9b21d222435ba12affd1d08822f (patch)
tree08ccd24ee854506542b546e184f5423dcde0e331
parentb9fdcd06222c674d5fabcf5a4ab6bc55f268c757 (diff)
downloadarchweb-fc7fc1d10a13e9b21d222435ba12affd1d08822f.tar.gz
archweb-fc7fc1d10a13e9b21d222435ba12affd1d08822f.tar.xz
Slight tweaks to mirror commands
We know we are doing updates when setting IPv4/IPv6 information, so set force_update to True to save the useless select query on each save(). For mirror checks, use a less cumbersome deque for the results since it is also thread-safe, and have all the log entries committed in one go. Signed-off-by: Dan McGee <dan@archlinux.org>
-rw-r--r--mirrors/management/commands/mirrorcheck.py17
-rw-r--r--mirrors/management/commands/mirrorresolv.py2
2 files changed, 9 insertions, 10 deletions
diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py
index 44d2b36..51be71e 100644
--- a/mirrors/management/commands/mirrorcheck.py
+++ b/mirrors/management/commands/mirrorcheck.py
@@ -10,7 +10,9 @@ Usage: ./manage.py mirrorcheck
"""
from django.core.management.base import NoArgsCommand
+from django.db import transaction
+from collections import deque
from datetime import datetime, timedelta
import logging
import re
@@ -130,7 +132,7 @@ def mirror_url_worker(work, output):
item = work.get(block=False)
try:
log = check_mirror_url(item)
- output.put(log)
+ output.append(log)
finally:
work.task_done()
except Empty:
@@ -139,7 +141,7 @@ def mirror_url_worker(work, output):
class MirrorCheckPool(object):
def __init__(self, work, num_threads=10):
self.tasks = Queue()
- self.logs = Queue()
+ self.logs = deque()
for i in list(work):
self.tasks.put(i)
self.threads = []
@@ -149,6 +151,7 @@ class MirrorCheckPool(object):
thread.daemon = True
self.threads.append(thread)
+ @transaction.commit_on_success
def run(self):
logger.debug("starting threads")
for t in self.threads:
@@ -156,13 +159,9 @@ class MirrorCheckPool(object):
logger.debug("joining on all threads")
self.tasks.join()
logger.debug("processing log entries")
- try:
- while True:
- log = self.logs.get(block=False)
- log.save()
- self.logs.task_done()
- except Empty:
- logger.debug("all log items saved to database")
+ for log in self.logs:
+ log.save()
+ logger.debug("log entries saved")
def check_current_mirrors():
urls = MirrorUrl.objects.filter(
diff --git a/mirrors/management/commands/mirrorresolv.py b/mirrors/management/commands/mirrorresolv.py
index 8cbd51a..77e7a22 100644
--- a/mirrors/management/commands/mirrorresolv.py
+++ b/mirrors/management/commands/mirrorresolv.py
@@ -50,7 +50,7 @@ def resolve_mirrors():
mirrorurl.has_ipv6 = socket.AF_INET6 in families
logger.debug("%s: v4: %s v6: %s", hostname,
mirrorurl.has_ipv4, mirrorurl.has_ipv6)
- mirrorurl.save()
+ mirrorurl.save(force_update=True)
except socket.error, e:
logger.warn("error resolving %s: %s", hostname, e)