createopes use only bulk_create

createopes script:
- more than 6x faster
- only bulk_create is used instead of create or save
- correctly create editions (~5% of created operations)
- ratio of withdrawals go from ~10% to ~5%

kfet/management/commands/createopes.py

@@ -14,7 +14,8 @@ from kfet.models import (Account, Article, OperationGroup, Operation,
 
 
 class Command(BaseCommand):
-    help = "Crée des opérations réparties uniformément sur une période de temps"
+    help = ("Crée des opérations réparties uniformément "
+            "sur une période de temps")
 
     def add_arguments(self, parser):
         # Nombre d'opérations à créer
@@ -29,7 +30,6 @@ class Command(BaseCommand):
         parser.add_argument('--transfers', type=int, default=0,
                             help='Number of transfers to create (default 0)')
 
-
     def handle(self, *args, **options):
 
         self.stdout.write("Génération d'opérations")
@@ -44,6 +44,7 @@ class Command(BaseCommand):
         # Convert to seconds
         time = options['days'] * 24 * 3600
 
+        now = timezone.now()
         checkout = Checkout.objects.first()
         articles = Article.objects.all()
         accounts = Account.objects.exclude(trigramme='LIQ')
@@ -55,6 +56,13 @@ class Command(BaseCommand):
         except Account.DoesNotExist:
             con_account = random.choice(accounts)
 
+        # use to fetch OperationGroup pk created by bulk_create
+        at_list = []
+        # use to lazy set OperationGroup pk on Operation objects
+        ope_by_grp = []
+        # OperationGroup objects to bulk_create
+        opegroup_list = []
+
         for i in range(num_ops):
 
             # Randomly pick account
@@ -64,8 +72,7 @@ class Command(BaseCommand):
                 account = liq_account
 
             # Randomly pick time
-            at = timezone.now() - timedelta(
+            at = now - timedelta(seconds=random.randint(0, time))
-                seconds=random.randint(0, time))
 
             # Majoration sur compte 'concert'
             if random.random() < 0.2:
@@ -78,13 +85,6 @@ class Command(BaseCommand):
             # Initialize opegroup amount
             amount = Decimal('0')
 
-            opegroup = OperationGroup.objects.create(
-                on_acc=account,
-                checkout=checkout,
-                at=at,
-                is_cof=account.cofprofile.is_cof
-            )
-
             # Generating operations
             ope_list = []
             for j in range(random.randint(1, 4)):
@@ -94,25 +94,26 @@ class Command(BaseCommand):
                 # 0.1 probability to have a charge
                 if typevar > 0.9 and account != liq_account:
                     ope = Operation(
-                        group=opegroup,
                         type=Operation.DEPOSIT,
-                        is_checkout=(random.random() > 0.2),
                         amount=Decimal(random.randint(1, 99)/10)
                     )
-                # 0.1 probability to have a withdrawal
+                # 0.05 probability to have a withdrawal
+                elif typevar > 0.85 and account != liq_account:
+                    ope = Operation(
+                        type=Operation.WITHDRAW,
+                        amount=-Decimal(random.randint(1, 99)/10)
+                    )
+                # 0.05 probability to have an edition
                 elif typevar > 0.8 and account != liq_account:
                     ope = Operation(
-                        group=opegroup,
+                        type=Operation.EDIT,
-                        type=Operation.WITHDRAW,
+                        amount=Decimal(random.randint(1, 99)/10)
-                        is_checkout=(random.random() > 0.2),
-                        amount=-Decimal(random.randint(1, 99)/10)
                     )
                 else:
                     article = random.choice(articles)
                     nb = random.randint(1, 5)
 
                     ope = Operation(
-                        group=opegroup,
                         type=Operation.PURCHASE,
                         amount=-article.price*nb,
                         article=article,
@@ -129,17 +130,44 @@ class Command(BaseCommand):
                 ope_list.append(ope)
                 amount += ope.amount
 
-            Operation.objects.bulk_create(ope_list)
+            opegroup_list.append(OperationGroup(
-            opes_created += len(ope_list)
+                on_acc=account,
-            opegroup.amount = amount
+                checkout=checkout,
-            opegroup.save()
+                at=at,
+                is_cof=account.cofprofile.is_cof,
+                amount=amount,
+            ))
+            at_list.append(at)
+            ope_by_grp.append((at, ope_list, ))
+
+        OperationGroup.objects.bulk_create(opegroup_list)
+
+        # Fetch created OperationGroup objects pk by at
+        opegroups = (OperationGroup.objects
+                     .filter(at__in=at_list)
+                     .values('id', 'at'))
+        opegroups_by = {grp['at']: grp['id'] for grp in opegroups}
+
+        all_ope = []
+        for _ in range(num_ops):
+            at, ope_list = ope_by_grp.pop()
+            for ope in ope_list:
+                ope.group_id = opegroups_by[at]
+                all_ope.append(ope)
+
+        Operation.objects.bulk_create(all_ope)
+        opes_created = len(all_ope)
 
         # Transfer generation
+
+        transfer_by_grp = []
+        transfergroup_list = []
+        at_list = []
+
         for i in range(num_transfers):
 
             # Randomly pick time
-            at = timezone.now() - timedelta(
+            at = now - timedelta(seconds=random.randint(0, time))
-                seconds=random.randint(0, time))
 
             # Choose whether to have a comment
             if random.random() > 0.5:
@@ -147,24 +175,40 @@ class Command(BaseCommand):
             else:
                 comment = ""
 
-            transfergroup = TransferGroup.objects.create(
+            transfergroup_list.append(TransferGroup(
                 at=at,
                 comment=comment,
-                valid_by=random.choice(accounts)
+                valid_by=random.choice(accounts),
-            )
+            ))
+            at_list.append(at)
 
             # Randomly generate transfer
             transfer_list = []
             for i in range(random.randint(1, 4)):
                 transfer_list.append(Transfer(
-                    group=transfergroup,
                     from_acc=random.choice(accounts),
                     to_acc=random.choice(accounts),
                     amount=Decimal(random.randint(1, 99)/10)
                 ))
 
-            Transfer.objects.bulk_create(transfer_list)
+            transfer_by_grp.append((at, transfer_list, ))
-            transfers += len(transfer_list)
+
+        TransferGroup.objects.bulk_create(transfergroup_list)
+
+        transfergroups = (TransferGroup.objects
+                          .filter(at__in=at_list)
+                          .values('id', 'at'))
+        transfergroups_by = {grp['at']: grp['id'] for grp in transfergroups}
+
+        all_transfer = []
+        for _ in range(num_transfers):
+            at, transfer_list = transfer_by_grp.pop()
+            for transfer in transfer_list:
+                transfer.group_id = transfergroups_by[at]
+                all_transfer.append(transfer)
+
+        Transfer.objects.bulk_create(all_transfer)
+        transfers += len(all_transfer)
 
         self.stdout.write(
             "- {:d} opérations créées dont {:d} commandes d'articles"