From f57bab8ae9cca35f8da45fc85c92142517f3b951 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Thu, 6 Apr 2017 13:33:40 +0200 Subject: [PATCH] createopes use only bulk_create createopes script: - more than 6x faster - only bulk_create is used instead of create or save - correctly create editions (~5% of created operations) - ratio of withdrawals go from ~10% to ~5% --- kfet/management/commands/createopes.py | 106 +++++++++++++++++-------- 1 file changed, 75 insertions(+), 31 deletions(-) diff --git a/kfet/management/commands/createopes.py b/kfet/management/commands/createopes.py index 77663b2b..5a7699ae 100644 --- a/kfet/management/commands/createopes.py +++ b/kfet/management/commands/createopes.py @@ -14,7 +14,8 @@ from kfet.models import (Account, Article, OperationGroup, Operation, class Command(BaseCommand): - help = "Crée des opérations réparties uniformément sur une période de temps" + help = ("Crée des opérations réparties uniformément " + "sur une période de temps") def add_arguments(self, parser): # Nombre d'opérations à créer @@ -29,7 +30,6 @@ class Command(BaseCommand): parser.add_argument('--transfers', type=int, default=0, help='Number of transfers to create (default 0)') - def handle(self, *args, **options): self.stdout.write("Génération d'opérations") @@ -44,6 +44,7 @@ class Command(BaseCommand): # Convert to seconds time = options['days'] * 24 * 3600 + now = timezone.now() checkout = Checkout.objects.first() articles = Article.objects.all() accounts = Account.objects.exclude(trigramme='LIQ') @@ -55,6 +56,13 @@ class Command(BaseCommand): except Account.DoesNotExist: con_account = random.choice(accounts) + # use to fetch OperationGroup pk created by bulk_create + at_list = [] + # use to lazy set OperationGroup pk on Operation objects + ope_by_grp = [] + # OperationGroup objects to bulk_create + opegroup_list = [] + for i in range(num_ops): # Randomly pick account @@ -64,8 +72,7 @@ class Command(BaseCommand): account = liq_account # Randomly pick time - at = timezone.now() - timedelta( - seconds=random.randint(0, time)) + at = now - timedelta(seconds=random.randint(0, time)) # Majoration sur compte 'concert' if random.random() < 0.2: @@ -78,13 +85,6 @@ class Command(BaseCommand): # Initialize opegroup amount amount = Decimal('0') - opegroup = OperationGroup.objects.create( - on_acc=account, - checkout=checkout, - at=at, - is_cof=account.cofprofile.is_cof - ) - # Generating operations ope_list = [] for j in range(random.randint(1, 4)): @@ -94,25 +94,26 @@ class Command(BaseCommand): # 0.1 probability to have a charge if typevar > 0.9 and account != liq_account: ope = Operation( - group=opegroup, type=Operation.DEPOSIT, - is_checkout=(random.random() > 0.2), amount=Decimal(random.randint(1, 99)/10) ) - # 0.1 probability to have a withdrawal + # 0.05 probability to have a withdrawal + elif typevar > 0.85 and account != liq_account: + ope = Operation( + type=Operation.WITHDRAW, + amount=-Decimal(random.randint(1, 99)/10) + ) + # 0.05 probability to have an edition elif typevar > 0.8 and account != liq_account: ope = Operation( - group=opegroup, - type=Operation.WITHDRAW, - is_checkout=(random.random() > 0.2), - amount=-Decimal(random.randint(1, 99)/10) + type=Operation.EDIT, + amount=Decimal(random.randint(1, 99)/10) ) else: article = random.choice(articles) nb = random.randint(1, 5) ope = Operation( - group=opegroup, type=Operation.PURCHASE, amount=-article.price*nb, article=article, @@ -129,17 +130,44 @@ class Command(BaseCommand): ope_list.append(ope) amount += ope.amount - Operation.objects.bulk_create(ope_list) - opes_created += len(ope_list) - opegroup.amount = amount - opegroup.save() + opegroup_list.append(OperationGroup( + on_acc=account, + checkout=checkout, + at=at, + is_cof=account.cofprofile.is_cof, + amount=amount, + )) + at_list.append(at) + ope_by_grp.append((at, ope_list, )) + + OperationGroup.objects.bulk_create(opegroup_list) + + # Fetch created OperationGroup objects pk by at + opegroups = (OperationGroup.objects + .filter(at__in=at_list) + .values('id', 'at')) + opegroups_by = {grp['at']: grp['id'] for grp in opegroups} + + all_ope = [] + for _ in range(num_ops): + at, ope_list = ope_by_grp.pop() + for ope in ope_list: + ope.group_id = opegroups_by[at] + all_ope.append(ope) + + Operation.objects.bulk_create(all_ope) + opes_created = len(all_ope) # Transfer generation + + transfer_by_grp = [] + transfergroup_list = [] + at_list = [] + for i in range(num_transfers): # Randomly pick time - at = timezone.now() - timedelta( - seconds=random.randint(0, time)) + at = now - timedelta(seconds=random.randint(0, time)) # Choose whether to have a comment if random.random() > 0.5: @@ -147,24 +175,40 @@ class Command(BaseCommand): else: comment = "" - transfergroup = TransferGroup.objects.create( + transfergroup_list.append(TransferGroup( at=at, comment=comment, - valid_by=random.choice(accounts) - ) + valid_by=random.choice(accounts), + )) + at_list.append(at) # Randomly generate transfer transfer_list = [] for i in range(random.randint(1, 4)): transfer_list.append(Transfer( - group=transfergroup, from_acc=random.choice(accounts), to_acc=random.choice(accounts), amount=Decimal(random.randint(1, 99)/10) )) - Transfer.objects.bulk_create(transfer_list) - transfers += len(transfer_list) + transfer_by_grp.append((at, transfer_list, )) + + TransferGroup.objects.bulk_create(transfergroup_list) + + transfergroups = (TransferGroup.objects + .filter(at__in=at_list) + .values('id', 'at')) + transfergroups_by = {grp['at']: grp['id'] for grp in transfergroups} + + all_transfer = [] + for _ in range(num_transfers): + at, transfer_list = transfer_by_grp.pop() + for transfer in transfer_list: + transfer.group_id = transfergroups_by[at] + all_transfer.append(transfer) + + Transfer.objects.bulk_create(all_transfer) + transfers += len(all_transfer) self.stdout.write( "- {:d} opérations créées dont {:d} commandes d'articles"