summaryrefslogtreecommitdiffstats
path: root/devel/management/commands
diff options
context:
space:
mode:
authorDan McGee <dan@archlinux.org>2011-11-16 19:49:17 +0100
committerDan McGee <dan@archlinux.org>2011-11-16 19:49:17 +0100
commitb1e406d73844d5a30344ca8ac855fe850c52bc2f (patch)
tree0fd8ec585574a079de6f0a57fca902189a71223e /devel/management/commands
parent404c4b400b2bd2a14e0363e33d66505c51903fe7 (diff)
downloadarchweb-b1e406d73844d5a30344ca8ac855fe850c52bc2f.tar.gz
archweb-b1e406d73844d5a30344ca8ac855fe850c52bc2f.tar.xz
reporead_inotify: spin up read_repo() in separate thread
This prevents memory usage from ballooning to absolutely huge values, such as when multiple threads kick off at the same time. The bulk of our memory allocation obviously comes in these threads and not the main threads, so being able to isolate them in processes helps a lot. Signed-off-by: Dan McGee <dan@archlinux.org>
Diffstat (limited to 'devel/management/commands')
-rwxr-xr-xdevel/management/commands/reporead_inotify.py11
1 files changed, 9 insertions, 2 deletions
diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py
index 4c865ce..ffd49b8 100755
--- a/devel/management/commands/reporead_inotify.py
+++ b/devel/management/commands/reporead_inotify.py
@@ -15,6 +15,7 @@ correctly.
"""
import logging
+import multiprocessing
import os.path
import pyinotify
import sys
@@ -133,6 +134,7 @@ class Database(object):
return
if self.update_thread:
self.update_thread.cancel()
+ self.update_thread = None
self._start_update_countdown()
def update(self):
@@ -142,8 +144,13 @@ class Database(object):
self.updating = True
try:
- # invoke reporead's primary method
- read_repo(self.arch, self.path, {})
+ # invoke reporead's primary method. we do this in a separate
+ # process for memory conservation purposes; these processes grow
+ # rather large so it is best to free up the memory ASAP.
+ process = multiprocessing.Process(target=read_repo,
+ args=[self.arch, self.path, {}])
+ process.start()
+ process.join()
finally:
logger.debug('Done updating database %s.', self.path)
with self.lock: