Work around bulk_create limitations in sqlite3 in reporead

Given the 999 SQL statement variable limit, we can easily hit it when
updating a package with thousands of files or a few hundred depends.

Signed-off-by: Dan McGee <dan@archlinux.org>

1 files changed, 28 insertions, 6 deletions

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index e50686b..2d9b68b 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -31,7 +31,7 @@ from django.db.utils import IntegrityError
 
 from devel.utils import UserFinder
 from main.models import Arch, Package, PackageFile, Repo
-from main.utils import utc_now
+from main.utils import utc_now, database_vendor
 from packages.models import Depend, Conflict, Provision, Replacement, Update
 
 
@@ -184,6 +184,28 @@ def create_related(model, package, rel_str, equals_only=False):
         return None
     return related
 
+
+def batched_bulk_create(model, all_objects):
+    # for short lists, just bulk_create as we should be fine
+    if len(all_objects) < 20:
+        return model.objects.bulk_create(all_objects)
+
+    if database_vendor(model, mode='write') == 'sqlite':
+        # 999 max variables in each SQL statement
+        incr = 999 // len(model._meta.fields)
+    else:
+        incr = 1000
+
+    def chunks():
+        offset = 0
+        while offset < len(all_objects):
+            yield all_objects[offset:offset + incr]
+            offset += incr
+
+    for items in chunks():
+        model.objects.bulk_create(items)
+
+
 def create_multivalued(dbpkg, repopkg, db_attr, repo_attr):
     '''Populate the simplest of multivalued attributes. These are those that
     only deal with a 'name' attribute, such as licenses, groups, etc. The input
@@ -235,20 +257,20 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None):
     dbpkg.depends.all().delete()
     deps = [create_depend(dbpkg, y) for y in repopkg.depends]
     deps += [create_depend(dbpkg, y, True) for y in repopkg.optdepends]
-    Depend.objects.bulk_create(deps)
+    batched_bulk_create(Depend, deps)
 
     dbpkg.conflicts.all().delete()
     conflicts = [create_related(Conflict, dbpkg, y) for y in repopkg.conflicts]
-    Conflict.objects.bulk_create(conflicts)
+    batched_bulk_create(Conflict, conflicts)
 
     dbpkg.provides.all().delete()
     provides = [create_related(Provision, dbpkg, y, equals_only=True)
             for y in repopkg.provides]
-    Provision.objects.bulk_create(provides)
+    batched_bulk_create(Provision, provides)
 
     dbpkg.replaces.all().delete()
     replaces = [create_related(Replacement, dbpkg, y) for y in repopkg.replaces]
-    Replacement.objects.bulk_create(replaces)
+    batched_bulk_create(Replacement, replaces)
 
     create_multivalued(dbpkg, repopkg, 'groups', 'groups')
     create_multivalued(dbpkg, repopkg, 'licenses', 'license')
@@ -295,7 +317,7 @@ def populate_files(dbpkg, repopkg, force=False):
                     directory=dirname,
                     filename=filename)
             pkg_files.append(pkgfile)
-        PackageFile.objects.bulk_create(pkg_files)
+        batched_bulk_create(PackageFile, pkg_files)
         dbpkg.files_last_update = utc_now()
         dbpkg.save()