From f57bab8ae9cca35f8da45fc85c92142517f3b951 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Thu, 6 Apr 2017 13:33:40 +0200 Subject: [PATCH 01/43] createopes use only bulk_create createopes script: - more than 6x faster - only bulk_create is used instead of create or save - correctly create editions (~5% of created operations) - ratio of withdrawals go from ~10% to ~5% --- kfet/management/commands/createopes.py | 106 +++++++++++++++++-------- 1 file changed, 75 insertions(+), 31 deletions(-) diff --git a/kfet/management/commands/createopes.py b/kfet/management/commands/createopes.py index 77663b2b..5a7699ae 100644 --- a/kfet/management/commands/createopes.py +++ b/kfet/management/commands/createopes.py @@ -14,7 +14,8 @@ from kfet.models import (Account, Article, OperationGroup, Operation, class Command(BaseCommand): - help = "Crée des opérations réparties uniformément sur une période de temps" + help = ("Crée des opérations réparties uniformément " + "sur une période de temps") def add_arguments(self, parser): # Nombre d'opérations à créer @@ -29,7 +30,6 @@ class Command(BaseCommand): parser.add_argument('--transfers', type=int, default=0, help='Number of transfers to create (default 0)') - def handle(self, *args, **options): self.stdout.write("Génération d'opérations") @@ -44,6 +44,7 @@ class Command(BaseCommand): # Convert to seconds time = options['days'] * 24 * 3600 + now = timezone.now() checkout = Checkout.objects.first() articles = Article.objects.all() accounts = Account.objects.exclude(trigramme='LIQ') @@ -55,6 +56,13 @@ class Command(BaseCommand): except Account.DoesNotExist: con_account = random.choice(accounts) + # use to fetch OperationGroup pk created by bulk_create + at_list = [] + # use to lazy set OperationGroup pk on Operation objects + ope_by_grp = [] + # OperationGroup objects to bulk_create + opegroup_list = [] + for i in range(num_ops): # Randomly pick account @@ -64,8 +72,7 @@ class Command(BaseCommand): account = liq_account # Randomly pick time - at = timezone.now() - timedelta( - seconds=random.randint(0, time)) + at = now - timedelta(seconds=random.randint(0, time)) # Majoration sur compte 'concert' if random.random() < 0.2: @@ -78,13 +85,6 @@ class Command(BaseCommand): # Initialize opegroup amount amount = Decimal('0') - opegroup = OperationGroup.objects.create( - on_acc=account, - checkout=checkout, - at=at, - is_cof=account.cofprofile.is_cof - ) - # Generating operations ope_list = [] for j in range(random.randint(1, 4)): @@ -94,25 +94,26 @@ class Command(BaseCommand): # 0.1 probability to have a charge if typevar > 0.9 and account != liq_account: ope = Operation( - group=opegroup, type=Operation.DEPOSIT, - is_checkout=(random.random() > 0.2), amount=Decimal(random.randint(1, 99)/10) ) - # 0.1 probability to have a withdrawal + # 0.05 probability to have a withdrawal + elif typevar > 0.85 and account != liq_account: + ope = Operation( + type=Operation.WITHDRAW, + amount=-Decimal(random.randint(1, 99)/10) + ) + # 0.05 probability to have an edition elif typevar > 0.8 and account != liq_account: ope = Operation( - group=opegroup, - type=Operation.WITHDRAW, - is_checkout=(random.random() > 0.2), - amount=-Decimal(random.randint(1, 99)/10) + type=Operation.EDIT, + amount=Decimal(random.randint(1, 99)/10) ) else: article = random.choice(articles) nb = random.randint(1, 5) ope = Operation( - group=opegroup, type=Operation.PURCHASE, amount=-article.price*nb, article=article, @@ -129,17 +130,44 @@ class Command(BaseCommand): ope_list.append(ope) amount += ope.amount - Operation.objects.bulk_create(ope_list) - opes_created += len(ope_list) - opegroup.amount = amount - opegroup.save() + opegroup_list.append(OperationGroup( + on_acc=account, + checkout=checkout, + at=at, + is_cof=account.cofprofile.is_cof, + amount=amount, + )) + at_list.append(at) + ope_by_grp.append((at, ope_list, )) + + OperationGroup.objects.bulk_create(opegroup_list) + + # Fetch created OperationGroup objects pk by at + opegroups = (OperationGroup.objects + .filter(at__in=at_list) + .values('id', 'at')) + opegroups_by = {grp['at']: grp['id'] for grp in opegroups} + + all_ope = [] + for _ in range(num_ops): + at, ope_list = ope_by_grp.pop() + for ope in ope_list: + ope.group_id = opegroups_by[at] + all_ope.append(ope) + + Operation.objects.bulk_create(all_ope) + opes_created = len(all_ope) # Transfer generation + + transfer_by_grp = [] + transfergroup_list = [] + at_list = [] + for i in range(num_transfers): # Randomly pick time - at = timezone.now() - timedelta( - seconds=random.randint(0, time)) + at = now - timedelta(seconds=random.randint(0, time)) # Choose whether to have a comment if random.random() > 0.5: @@ -147,24 +175,40 @@ class Command(BaseCommand): else: comment = "" - transfergroup = TransferGroup.objects.create( + transfergroup_list.append(TransferGroup( at=at, comment=comment, - valid_by=random.choice(accounts) - ) + valid_by=random.choice(accounts), + )) + at_list.append(at) # Randomly generate transfer transfer_list = [] for i in range(random.randint(1, 4)): transfer_list.append(Transfer( - group=transfergroup, from_acc=random.choice(accounts), to_acc=random.choice(accounts), amount=Decimal(random.randint(1, 99)/10) )) - Transfer.objects.bulk_create(transfer_list) - transfers += len(transfer_list) + transfer_by_grp.append((at, transfer_list, )) + + TransferGroup.objects.bulk_create(transfergroup_list) + + transfergroups = (TransferGroup.objects + .filter(at__in=at_list) + .values('id', 'at')) + transfergroups_by = {grp['at']: grp['id'] for grp in transfergroups} + + all_transfer = [] + for _ in range(num_transfers): + at, transfer_list = transfer_by_grp.pop() + for transfer in transfer_list: + transfer.group_id = transfergroups_by[at] + all_transfer.append(transfer) + + Transfer.objects.bulk_create(all_transfer) + transfers += len(all_transfer) self.stdout.write( "- {:d} opérations créées dont {:d} commandes d'articles" From 1302adf1566bd10f4e545da7e91b13393308d887 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Thu, 6 Apr 2017 14:15:03 +0200 Subject: [PATCH 02/43] globally fewer db requests --- kfet/middleware.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/kfet/middleware.py b/kfet/middleware.py index dbb192c6..c61d0233 100644 --- a/kfet/middleware.py +++ b/kfet/middleware.py @@ -1,15 +1,28 @@ # -*- coding: utf-8 -*- -from __future__ import (absolute_import, division, - print_function, unicode_literals) -from builtins import * +from django.contrib.auth.models import User -from django.http import HttpResponseForbidden from kfet.backends import KFetBackend -from kfet.models import Account + class KFetAuthenticationMiddleware(object): + """Authenticate another user for this request if KFetBackend succeeds. + + By the way, if a user is authenticated, we refresh its from db to add + values from CofProfile and Account of this user. + + """ def process_request(self, request): + if request.user.is_authenticated(): + # avoid multiple db accesses in views and templates + user_pk = request.user.pk + print(user_pk) + request.user = ( + User.objects + .select_related('profile__account_kfet') + .get(pk=user_pk) + ) + kfet_backend = KFetBackend() temp_request_user = kfet_backend.authenticate(request) if temp_request_user: From c9019c4eb40cfe97bd0f4db848ccb4f89d08f846 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Thu, 6 Apr 2017 14:36:25 +0200 Subject: [PATCH 03/43] Get debug toolbar with ajax calls - Add django-debug-panel module to pip development dependencies - Enable debug toolbar on ajax calls by using the "Django Debug Panel" extension for... Chrome. For further informations about: - django module, see https://github.com/recamshak/django-debug-panel - Chrome extension, see https://github.com/recamshak/chrome-django-panel --- cof/settings_dev.py | 5 ++--- requirements-devel.txt | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cof/settings_dev.py b/cof/settings_dev.py index 18aadaad..e47fe227 100644 --- a/cof/settings_dev.py +++ b/cof/settings_dev.py @@ -42,6 +42,7 @@ INSTALLED_APPS = ( 'captcha', 'django_cas_ng', 'debug_toolbar', + 'debug_panel', 'bootstrapform', 'kfet', 'channels', @@ -50,7 +51,7 @@ INSTALLED_APPS = ( ) MIDDLEWARE_CLASSES = ( - 'debug_toolbar.middleware.DebugToolbarMiddleware', + 'debug_panel.middleware.DebugPanelMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', @@ -189,8 +190,6 @@ def show_toolbar(request): """ if not DEBUG: return False - if request.is_ajax(): - return False return True DEBUG_TOOLBAR_CONFIG = { diff --git a/requirements-devel.txt b/requirements-devel.txt index 425dfc36..83053f76 100644 --- a/requirements-devel.txt +++ b/requirements-devel.txt @@ -1,3 +1,4 @@ -r requirements.txt django-debug-toolbar +django-debug-panel ipython From 56fa70e495cd0d5a52ecee145fe4f5129371d222 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Thu, 6 Apr 2017 14:38:25 +0200 Subject: [PATCH 04/43] fewer calls on account_read view --- kfet/views.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kfet/views.py b/kfet/views.py index de6c906e..1e2edfbc 100644 --- a/kfet/views.py +++ b/kfet/views.py @@ -351,7 +351,11 @@ def account_create_ajax(request, username=None, login_clipper=None, @login_required def account_read(request, trigramme): try: - account = Account.objects.select_related('negative').get(trigramme=trigramme) + account = ( + Account.objects + .select_related('cofprofile__user', 'negative') + .get(trigramme=trigramme) + ) except Account.DoesNotExist: raise Http404 From 33cee05884204a1cde8e5400b388265cb76eab05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Thu, 6 Apr 2017 16:45:44 +0200 Subject: [PATCH 05/43] initial checkout selected Result is the same result but: - auto-select go to javascript - 1 less request for KPsulCheckoutForm - delete debug messages - some PEP8 --- kfet/forms.py | 16 ++++++++++++---- kfet/middleware.py | 1 - kfet/templates/kfet/kpsul.html | 5 +++++ kfet/views.py | 20 +++++--------------- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/kfet/forms.py b/kfet/forms.py index 7acd0880..27248ae9 100644 --- a/kfet/forms.py +++ b/kfet/forms.py @@ -322,12 +322,20 @@ class KPsulAccountForm(forms.ModelForm): }), } + class KPsulCheckoutForm(forms.Form): checkout = forms.ModelChoiceField( - queryset=Checkout.objects.filter( - is_protected=False, valid_from__lte=timezone.now(), - valid_to__gte=timezone.now()), - widget=forms.Select(attrs={'id':'id_checkout_select'})) + queryset=( + Checkout.objects + .filter( + is_protected=False, + valid_from__lte=timezone.now(), + valid_to__gte=timezone.now(), + ) + ), + widget=forms.Select(attrs={'id': 'id_checkout_select'}), + ) + class KPsulOperationForm(forms.ModelForm): article = forms.ModelChoiceField( diff --git a/kfet/middleware.py b/kfet/middleware.py index c61d0233..9502d393 100644 --- a/kfet/middleware.py +++ b/kfet/middleware.py @@ -16,7 +16,6 @@ class KFetAuthenticationMiddleware(object): if request.user.is_authenticated(): # avoid multiple db accesses in views and templates user_pk = request.user.pk - print(user_pk) request.user = ( User.objects .select_related('profile__account_kfet') diff --git a/kfet/templates/kfet/kpsul.html b/kfet/templates/kfet/kpsul.html index f271a306..45416d06 100644 --- a/kfet/templates/kfet/kpsul.html +++ b/kfet/templates/kfet/kpsul.html @@ -393,6 +393,11 @@ $(document).ready(function() { var last_statement_container = $('#last_statement'); var last_statement_html_default = 'Dernier relevé:
€ le par '; + // If only one checkout is available, select it + var checkout_choices = checkoutInput.find("option[value!='']"); + if (checkout_choices.length == 1) { + $(checkout_choices[0]).prop("selected", true); + } // Display data function displayCheckoutData() { diff --git a/kfet/views.py b/kfet/views.py index 1e2edfbc..7dc599c6 100644 --- a/kfet/views.py +++ b/kfet/views.py @@ -883,7 +883,6 @@ class ArticleUpdate(SuccessMessageMixin, UpdateView): return super(ArticleUpdate, self).form_valid(form) - # ----- # K-Psul # ----- @@ -893,19 +892,13 @@ def kpsul(request): data = {} data['operationgroup_form'] = KPsulOperationGroupForm() data['trigramme_form'] = KPsulAccountForm() - initial = {} - try: - checkout = Checkout.objects.filter( - is_protected=False, valid_from__lte=timezone.now(), - valid_to__gte=timezone.now()).get() - initial['checkout'] = checkout - except (Checkout.DoesNotExist, Checkout.MultipleObjectsReturned): - pass - data['checkout_form'] = KPsulCheckoutForm(initial=initial) - operation_formset = KPsulOperationFormSet(queryset=Operation.objects.none()) - data['operation_formset'] = operation_formset + data['checkout_form'] = KPsulCheckoutForm() + data['operation_formset'] = KPsulOperationFormSet( + queryset=Operation.objects.none(), + ) return render(request, 'kfet/kpsul.html', data) + @teamkfet_required def kpsul_get_settings(request): addcost_for = Settings.ADDCOST_FOR() @@ -1643,9 +1636,6 @@ def cancel_transfers(request): if stop: negative_accounts.append(account.trigramme) - print(required_perms) - print(request.user.get_all_permissions()) - if stop_all or not request.user.has_perms(required_perms): missing_perms = get_missing_perms(required_perms, request.user) if missing_perms: From 709d421b2c459fe270f22d0de3759497d9f94402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Thu, 6 Apr 2017 18:23:27 +0200 Subject: [PATCH 06/43] Add custom Manager for Account model. - When Account model is queried with Account.objects, it always add .select_related('cofprofile_user', 'negative'). - Eg benefits: history doesn't do anymore one request by account to fill the account filter. Important Using this workaround (systemically append select_related) can be dangerous normally, however a certain number of properties in cofprofile and user are frequently used (as firstname or lastname), and the benefits seems greater. --- kfet/backends.py | 6 +++++- kfet/models.py | 13 ++++++++++++- kfet/views.py | 29 +++++++++++++---------------- 3 files changed, 30 insertions(+), 18 deletions(-) diff --git a/kfet/backends.py b/kfet/backends.py index ba3bce9d..c0aec699 100644 --- a/kfet/backends.py +++ b/kfet/backends.py @@ -46,6 +46,10 @@ class GenericTeamBackend(object): def get_user(self, user_id): try: - return User.objects.get(pk=user_id) + return ( + User.objects + .select_related('profile__acount_kfet') + .get(pk=user_id) + ) except User.DoesNotExist: return None diff --git a/kfet/models.py b/kfet/models.py index 2ad1c091..bfa1f360 100644 --- a/kfet/models.py +++ b/kfet/models.py @@ -27,8 +27,19 @@ def default_promo(): now = date.today() return now.month <= 8 and now.year-1 or now.year -@python_2_unicode_compatible + +class AccountManager(models.Manager): + """Manager for Account Model.""" + + def get_queryset(self): + """Always append related data to this Account.""" + return super().get_queryset().select_related('cofprofile__user', + 'negative') + + class Account(models.Model): + objects = AccountManager() + cofprofile = models.OneToOneField( CofProfile, on_delete = models.PROTECT, related_name = "account_kfet") diff --git a/kfet/views.py b/kfet/views.py index 7dc599c6..ebe169ca 100644 --- a/kfet/views.py +++ b/kfet/views.py @@ -346,35 +346,32 @@ def account_create_ajax(request, username=None, login_clipper=None, 'user_form' : forms['user_form'], }) + # Account - Read @login_required def account_read(request, trigramme): - try: - account = ( - Account.objects - .select_related('cofprofile__user', 'negative') - .get(trigramme=trigramme) - ) - except Account.DoesNotExist: - raise Http404 + account = get_object_or_404(Account, trigramme=trigramme) # Checking permissions if not request.user.has_perm('kfet.is_team') \ and request.user != account.user: raise PermissionDenied - addcosts = (OperationGroup.objects - .filter(opes__addcost_for=account,opes__canceled_at=None) - .extra({'date':"date(at)"}) - .values('date') - .annotate(sum_addcosts=Sum('opes__addcost_amount')) - .order_by('-date')) + addcosts = ( + OperationGroup.objects + .filter(opes__addcost_for=account, + opes__canceled_at=None) + .extra({'date': "date(at)"}) + .values('date') + .annotate(sum_addcosts=Sum('opes__addcost_amount')) + .order_by('-date') + ) return render(request, "kfet/account_read.html", { - 'account' : account, + 'account': account, 'addcosts': addcosts, - 'settings': { 'subvention_cof': Settings.SUBVENTION_COF() }, + 'settings': {'subvention_cof': Settings.SUBVENTION_COF()}, }) # Account - Update From f8b71b604c6dfc6ddd779bc3ce0059f6f8f24505 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Thu, 6 Apr 2017 18:42:00 +0200 Subject: [PATCH 07/43] Fewer db accesses on transfers list view. - Add select_related for `from_acc`, `to_acc` and `group.valid_by`. - 2 requests instead of (3 * #transfers) --- kfet/views.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/kfet/views.py b/kfet/views.py index ebe169ca..c296d3b3 100644 --- a/kfet/views.py +++ b/kfet/views.py @@ -1477,19 +1477,32 @@ class SettingsUpdate(SuccessMessageMixin, UpdateView): Settings.empty_cache() return super(SettingsUpdate, self).form_valid(form) + # ----- # Transfer views # ----- @teamkfet_required def transfers(request): - transfergroups = (TransferGroup.objects - .prefetch_related('transfers') - .order_by('-at')) + transfers_pre = Prefetch( + 'transfers', + queryset=( + Transfer.objects + .select_related('from_acc', 'to_acc') + ), + ) + + transfergroups = ( + TransferGroup.objects + .select_related('valid_by') + .prefetch_related(transfers_pre) + .order_by('-at') + ) return render(request, 'kfet/transfers.html', { 'transfergroups': transfergroups, }) + @teamkfet_required def transfers_create(request): transfer_formset = TransferFormSet(queryset=Transfer.objects.none()) From afdb08b424c675d68be7f117a73943bf7fbcaa7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Thu, 6 Apr 2017 19:07:13 +0200 Subject: [PATCH 08/43] Fewer db requests with AccountNegative handling. - AccountNegative use new AccountNegativeManager which select_related account, cofprofile and user for instances accessed via AccountNegative.objects. - Compute sum of negatives with python instead of an SQL statement (since we already got the data with a previous query). - Fix bug on account property `real_balance` (happened if an account has a relative AccountNegative instance but balance_offset to NULL). - More compliant to PEP8 --- kfet/models.py | 41 +++++++++++++++++++++++++++++------------ kfet/views.py | 17 +++++++---------- 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/kfet/models.py b/kfet/models.py index bfa1f360..c13eea9e 100644 --- a/kfet/models.py +++ b/kfet/models.py @@ -96,7 +96,8 @@ class Account(models.Model): # Propriétés supplémentaires @property def real_balance(self): - if (hasattr(self, 'negative')): + if (hasattr(self, 'negative') and + self.negative.balance_offset is not None): return self.balance - self.negative.balance_offset return self.balance @@ -229,27 +230,43 @@ class Account(models.Model): def __init__(self, trigramme): self.trigramme = trigramme + +class AccountNegativeManager(models.Manager): + """Manager for AccountNegative model.""" + + def get_queryset(self): + return ( + super().get_queryset() + .select_related('account__cofprofile__user') + ) + + class AccountNegative(models.Model): + objects = AccountNegativeManager() + account = models.OneToOneField( - Account, on_delete = models.PROTECT, - related_name = "negative") - start = models.DateTimeField( - blank = True, null = True, default = None) + Account, on_delete=models.PROTECT, + related_name="negative", + ) + start = models.DateTimeField(blank=True, null=True, default=None) balance_offset = models.DecimalField( "décalage de balance", help_text="Montant non compris dans l'autorisation de négatif", - max_digits = 6, decimal_places = 2, - blank = True, null = True, default = None) + max_digits=6, decimal_places=2, + blank=True, null=True, default=None, + ) authz_overdraft_amount = models.DecimalField( "négatif autorisé", - max_digits = 6, decimal_places = 2, - blank = True, null = True, default = None) + max_digits=6, decimal_places=2, + blank=True, null=True, default=None, + ) authz_overdraft_until = models.DateTimeField( "expiration du négatif", - blank = True, null = True, default = None) - comment = models.CharField("commentaire", max_length = 255, blank = True) + blank=True, null=True, default=None, + ) + comment = models.CharField("commentaire", max_length=255, blank=True) + -@python_2_unicode_compatible class Checkout(models.Model): created_by = models.ForeignKey( Account, on_delete = models.PROTECT, diff --git a/kfet/views.py b/kfet/views.py index c296d3b3..92cdd734 100644 --- a/kfet/views.py +++ b/kfet/views.py @@ -540,10 +540,13 @@ class AccountGroupUpdate(UpdateView): success_message = 'Groupe modifié : %(name)s' success_url = reverse_lazy('kfet.account.group') + class AccountNegativeList(ListView): - queryset = (AccountNegative.objects + queryset = ( + AccountNegative.objects .select_related('account', 'account__cofprofile__user') - .exclude(account__trigramme='#13')) + .exclude(account__trigramme='#13') + ) template_name = 'kfet/account_negative.html' context_object_name = 'negatives' @@ -553,14 +556,8 @@ class AccountNegativeList(ListView): 'overdraft_amount': Settings.OVERDRAFT_AMOUNT(), 'overdraft_duration': Settings.OVERDRAFT_DURATION(), } - negs_sum = (AccountNegative.objects - .exclude(account__trigramme='#13') - .aggregate( - bal = Coalesce(Sum('account__balance'),0), - offset = Coalesce(Sum('balance_offset'),0), - ) - ) - context['negatives_sum'] = negs_sum['bal'] - negs_sum['offset'] + real_balances = (neg.account.real_balance for neg in self.object_list) + context['negatives_sum'] = sum(real_balances) return context # ----- From 2731d4630fa830103a3fe9879c2e52493c179950 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Thu, 6 Apr 2017 19:25:23 +0200 Subject: [PATCH 09/43] Fewer requests on accounts groups. - Saves two queries. (4 prefetch where done, replaced by 2 prefetchs (whose 1 with 2 selected_related)) --- kfet/views.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/kfet/views.py b/kfet/views.py index 92cdd734..6d8af17d 100644 --- a/kfet/views.py +++ b/kfet/views.py @@ -518,13 +518,22 @@ def account_update(request, trigramme): 'pwd_form': pwd_form, }) + @permission_required('kfet.manage_perms') def account_group(request): - groups = (Group.objects - .filter(name__icontains='K-Fêt') - .prefetch_related('permissions', 'user_set__profile__account_kfet') + user_pre = Prefetch( + 'user_set', + queryset=User.objects.select_related('profile__account_kfet'), ) - return render(request, 'kfet/account_group.html', { 'groups': groups }) + groups = ( + Group.objects + .filter(name__icontains='K-Fêt') + .prefetch_related('permissions', user_pre) + ) + return render(request, 'kfet/account_group.html', { + 'groups': groups, + }) + class AccountGroupCreate(SuccessMessageMixin, CreateView): model = Group From 026fba867dd6b8862be4519e8d9e7a2c54764106 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Thu, 6 Apr 2017 19:44:16 +0200 Subject: [PATCH 10/43] Fewer db requests on home view. - 1 request instead of (2 + #articles) --- kfet/views.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/kfet/views.py b/kfet/views.py index 6d8af17d..db13f403 100644 --- a/kfet/views.py +++ b/kfet/views.py @@ -55,11 +55,21 @@ class Home(TemplateView): template_name = "kfet/home.html" def get_context_data(self, **kwargs): - context = super(TemplateView, self).get_context_data(**kwargs) - articles = Article.objects.all().filter(is_sold=True, hidden=False) - context['pressions'] = articles.filter(category__name='Pression') - context['articles'] = (articles.exclude(category__name='Pression') - .order_by('category')) + context = super().get_context_data(**kwargs) + articles = list( + Article.objects + .filter(is_sold=True, hidden=False) + .select_related('category') + .order_by('category__name') + ) + pressions, others = [], [] + while len(articles) > 0: + article = articles.pop() + if article.category.name == 'Pression': + pressions.append(article) + else: + others.append(article) + context['pressions'], context['articles'] = pressions, others return context @method_decorator(login_required) From 0ed70eb0a7ca5809c74eae10328d0fb328d10f54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Thu, 6 Apr 2017 20:30:23 +0200 Subject: [PATCH 11/43] PEP8 No improvement. --- kfet/views.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/kfet/views.py b/kfet/views.py index db13f403..15454aa5 100644 --- a/kfet/views.py +++ b/kfet/views.py @@ -779,12 +779,18 @@ class CategoryUpdate(SuccessMessageMixin, UpdateView): # Article - General class ArticleList(ListView): - queryset = (Article.objects - .select_related('category') - .prefetch_related(Prefetch('inventories', - queryset=Inventory.objects.order_by('-at'), - to_attr='inventory')) - .order_by('category', '-is_sold', 'name')) + queryset = ( + Article.objects + .select_related('category') + .prefetch_related( + Prefetch( + 'inventories', + queryset=Inventory.objects.order_by('-at'), + to_attr='inventory', + ) + ) + .order_by('category__name', '-is_sold', 'name') + ) template_name = 'kfet/article.html' context_object_name = 'articles' From 3e0bd2e758a8e88fcd8fad4c794d42b0ecfa216d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Fri, 7 Apr 2017 13:25:50 +0200 Subject: [PATCH 12/43] Fewer db requests on bda views. bda.views.etat_places - Use select_related on spectacles_set, avoid query issue for each spectacle. - `slots__sum` is computed in view, instead of a query. - Cleaner code (and avoid useless computations). bda.views.places - Add select_related on places, avoid query issue for each spectacle. bda.views.inscription - 1 query for spectacle field choices, instead of (#forms in formset * #spectacles) - Delete BaseBdaFormSet. The validation was redundant with `unique_together` of ChoixSpectacle model. --- bda/forms.py | 23 -------- bda/views.py | 152 +++++++++++++++++++++++++++++++++------------------ 2 files changed, 98 insertions(+), 77 deletions(-) diff --git a/bda/forms.py b/bda/forms.py index 2029645b..6bb36c32 100644 --- a/bda/forms.py +++ b/bda/forms.py @@ -5,33 +5,10 @@ from __future__ import print_function from __future__ import unicode_literals from django import forms -from django.forms.models import BaseInlineFormSet from django.utils import timezone from bda.models import Attribution, Spectacle -class BaseBdaFormSet(BaseInlineFormSet): - def clean(self): - """Checks that no two articles have the same title.""" - super(BaseBdaFormSet, self).clean() - if any(self.errors): - # Don't bother validating the formset unless each form is valid on - # its own - return - spectacles = [] - for i in range(0, self.total_form_count()): - form = self.forms[i] - if not form.cleaned_data: - continue - spectacle = form.cleaned_data['spectacle'] - delete = form.cleaned_data['DELETE'] - if not delete and spectacle in spectacles: - raise forms.ValidationError( - "Vous ne pouvez pas vous inscrire deux fois pour le " - "même spectacle.") - spectacles.append(spectacle) - - class TokenForm(forms.Form): token = forms.CharField(widget=forms.widgets.Textarea()) diff --git a/bda/views.py b/bda/views.py index 8fda604d..089c1f29 100644 --- a/bda/views.py +++ b/bda/views.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +from functools import partial import random import hashlib import time @@ -11,9 +12,9 @@ from custommail.shortcuts import ( from django.shortcuts import render, get_object_or_404 from django.contrib.auth.decorators import login_required from django.contrib import messages -from django.db import models, transaction +from django.db import transaction from django.core import serializers -from django.db.models import Count, Q, Sum +from django.db.models import Count, Q from django.forms.models import inlineformset_factory from django.http import ( HttpResponseBadRequest, HttpResponseRedirect, JsonResponse @@ -29,8 +30,7 @@ from bda.models import ( ) from bda.algorithm import Algorithm from bda.forms import ( - BaseBdaFormSet, TokenForm, ResellForm, AnnulForm, InscriptionReventeForm, - SoldForm + TokenForm, ResellForm, AnnulForm, InscriptionReventeForm, SoldForm, ) @@ -44,39 +44,44 @@ def etat_places(request, tirage_id): Et le total de toutes les demandes """ tirage = get_object_or_404(Tirage, id=tirage_id) - spectacles1 = ChoixSpectacle.objects \ - .filter(spectacle__tirage=tirage) \ - .filter(double_choice="1") \ - .all() \ - .values('spectacle', 'spectacle__title') \ - .annotate(total=models.Count('spectacle')) - spectacles2 = ChoixSpectacle.objects \ - .filter(spectacle__tirage=tirage) \ - .exclude(double_choice="1") \ - .all() \ - .values('spectacle', 'spectacle__title') \ - .annotate(total=models.Count('spectacle')) - spectacles = tirage.spectacle_set.all() - spectacles_dict = {} - total = 0 + + spectacles = tirage.spectacle_set.select_related('location') + spectacles_dict = {} # index of spectacle by id + for spectacle in spectacles: - spectacle.total = 0 - spectacle.ratio = 0.0 + spectacle.total = 0 # init total requests spectacles_dict[spectacle.id] = spectacle - for spectacle in spectacles1: - spectacles_dict[spectacle["spectacle"]].total += spectacle["total"] - spectacles_dict[spectacle["spectacle"]].ratio = \ - spectacles_dict[spectacle["spectacle"]].total / \ - spectacles_dict[spectacle["spectacle"]].slots - total += spectacle["total"] - for spectacle in spectacles2: - spectacles_dict[spectacle["spectacle"]].total += 2*spectacle["total"] - spectacles_dict[spectacle["spectacle"]].ratio = \ - spectacles_dict[spectacle["spectacle"]].total / \ - spectacles_dict[spectacle["spectacle"]].slots - total += 2*spectacle["total"] + + choices = ( + ChoixSpectacle.objects + .filter(spectacle__in=spectacles) + .values('spectacle') + .annotate(total=Count('spectacle')) + ) + + # choices *by spectacles* whose only 1 place is requested + choices1 = choices.filter(double_choice="1") + # choices *by spectacles* whose 2 places is requested + choices2 = choices.exclude(double_choice="1") + + for spectacle in choices1: + pk = spectacle['spectacle'] + spectacles_dict[pk].total += spectacle['total'] + for spectacle in choices2: + pk = spectacle['spectacle'] + spectacles_dict[pk].total += 2*spectacle['total'] + + # here, each spectacle.total contains the number of requests + + slots = 0 # proposed slots + total = 0 # requests + for spectacle in spectacles: + slots += spectacle.slots + total += spectacle.total + spectacle.ratio = spectacle.total / spectacle.slots + context = { - "proposed": tirage.spectacle_set.aggregate(Sum('slots'))['slots__sum'], + "proposed": slots, "spectacles": spectacles, "total": total, 'tirage': tirage @@ -94,11 +99,16 @@ def _hash_queryset(queryset): @cof_required def places(request, tirage_id): tirage = get_object_or_404(Tirage, id=tirage_id) - participant, created = Participant.objects.get_or_create( - user=request.user, tirage=tirage) - places = participant.attribution_set.order_by( - "spectacle__date", "spectacle").all() - total = sum([place.spectacle.price for place in places]) + participant, _ = ( + Participant.objects + .get_or_create(user=request.user, tirage=tirage) + ) + places = ( + participant.attribution_set + .order_by("spectacle__date", "spectacle") + .select_related("spectacle", "spectacle__location") + ) + total = sum(place.spectacle.price for place in places) filtered_places = [] places_dict = {} spectacles = [] @@ -148,33 +158,62 @@ def inscription(request, tirage_id): return render(request, 'bda/resume-inscription-tirage.html', {}) if timezone.now() > tirage.fermeture: # Le tirage est fermé. - participant, created = Participant.objects.get_or_create( - user=request.user, tirage=tirage) - choices = participant.choixspectacle_set.order_by("priority").all() + participant, _ = ( + Participant.objects + .get_or_create(user=request.user, tirage=tirage) + ) + choices = participant.choixspectacle_set.order_by("priority") messages.error(request, " C'est fini : tirage au sort dans la journée !") return render(request, "bda/resume-inscription-tirage.html", {"choices": choices}) - def formfield_callback(f, **kwargs): + def force_for(f, to_choices=None, **kwargs): + """Overrides choices for ModelChoiceField. + + Args: + f (models.Field): To render as forms.Field + to_choices (dict): If a key `f.name` exists, f._choices is set to + its value. + """ - Fonction utilisée par inlineformset_factory ci dessous. - Restreint les spectacles proposés aux spectacles du bo tirage. - """ - if f.name == "spectacle": - kwargs['queryset'] = tirage.spectacle_set - return f.formfield(**kwargs) + formfield = f.formfield(**kwargs) + if to_choices: + if f.name in to_choices: + choices = [('', '---------')] + to_choices[f.name] + formfield._choices = choices + return formfield + + # Restrict spectacles choices to spectacles for this tirage. + spectacles = ( + tirage.spectacle_set + .select_related('location') + ) + spectacles_field_choices = [(sp.pk, str(sp)) for sp in spectacles] + + # Allow for spectacle choices to be set once for all. + # Form display use 1 request instead of (#forms of formset * #spectacles). + # FIXME: Validation still generates too much requests... + formfield_callback = partial( + force_for, + to_choices={ + 'spectacle': spectacles_field_choices, + }, + ) BdaFormSet = inlineformset_factory( Participant, ChoixSpectacle, fields=("spectacle", "double_choice", "priority"), - formset=BaseBdaFormSet, - formfield_callback=formfield_callback) - participant, created = Participant.objects.get_or_create( - user=request.user, tirage=tirage) + formfield_callback=formfield_callback, + ) + participant, _ = ( + Participant.objects + .get_or_create(user=request.user, tirage=tirage) + ) success = False stateerror = False if request.method == "POST": + # use *this* queryset dbstate = _hash_queryset(participant.choixspectacle_set.all()) if "dbstate" in request.POST and dbstate != request.POST["dbstate"]: stateerror = True @@ -187,9 +226,14 @@ def inscription(request, tirage_id): formset = BdaFormSet(instance=participant) else: formset = BdaFormSet(instance=participant) + # use *this* queryset dbstate = _hash_queryset(participant.choixspectacle_set.all()) total_price = 0 - for choice in participant.choixspectacle_set.all(): + choices = ( + participant.choixspectacle_set + .select_related('spectacle') + ) + for choice in choices: total_price += choice.spectacle.price if choice.double: total_price += choice.spectacle.price From 9f307c1bd00558d5f7e7eb4350661f5b6d022fe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Fri, 7 Apr 2017 16:22:10 +0200 Subject: [PATCH 13/43] Fewer db requests on bda tirage. bda.algorithm - use iterator to find max_groups, instead of a db request bda.views.do_tirage - select_related() are now focused on some relationships (they were taking useless relationships) - bda-revente filling takes 1 request (each save and add was issuing 1 request) --- bda/algorithm.py | 3 +-- bda/views.py | 23 ++++++++++++++++------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/bda/algorithm.py b/bda/algorithm.py index bf9b690f..7f18ce18 100644 --- a/bda/algorithm.py +++ b/bda/algorithm.py @@ -22,8 +22,7 @@ class Algorithm(object): show.requests - on crée des tables de demandes pour chaque personne, afin de pouvoir modifier les rankings""" - self.max_group = \ - 2 * choices.aggregate(Max('priority'))['priority__max'] + self.max_group = 2*max(choice.priority for choice in choices) self.shows = [] showdict = {} for show in shows: diff --git a/bda/views.py b/bda/views.py index 089c1f29..b6dd6aec 100644 --- a/bda/views.py +++ b/bda/views.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +from collections import defaultdict from functools import partial import random import hashlib @@ -262,9 +263,9 @@ def do_tirage(tirage_elt, token): # Initialisation du dictionnaire data qui va contenir les résultats start = time.time() data = { - 'shows': tirage_elt.spectacle_set.select_related().all(), + 'shows': tirage_elt.spectacle_set.select_related('location'), 'token': token, - 'members': tirage_elt.participant_set.all(), + 'members': tirage_elt.participant_set.select_related('user'), 'total_slots': 0, 'total_losers': 0, 'total_sold': 0, @@ -277,7 +278,7 @@ def do_tirage(tirage_elt, token): ChoixSpectacle.objects .filter(spectacle__tirage=tirage_elt) .order_by('participant', 'priority') - .select_related().all() + .select_related('participant', 'participant__user', 'spectacle') ) results = Algorithm(data['shows'], data['members'], choices)(token) @@ -334,10 +335,18 @@ def do_tirage(tirage_elt, token): ]) # On inscrit à BdA-Revente ceux qui n'ont pas eu les places voulues - for (show, _, losers) in results: - for (loser, _, _, _) in losers: - loser.choicesrevente.add(show) - loser.save() + ChoixRevente = Participant.choicesrevente.through + + lost_by = defaultdict(set) + for show, _, losers in results: + for loser, _, _, _ in losers: + lost_by[loser].add(show) + + ChoixRevente.objects.bulk_create( + ChoixRevente(participant=member, spectacle=show) + for member, shows in lost_by.items() + for show in shows + ) data["duration"] = time.time() - start data["results"] = results From d31101aff3190a4493df68dac20f8afcf8b2b466 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Fri, 7 Apr 2017 17:04:06 +0200 Subject: [PATCH 14/43] Empty SpectacleRevente and ChoixRevente before do_tirage. - Usefull if a tirage is launched more than once. - Ensure full reset of reventes for this tirage in this case. --- bda/views.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/bda/views.py b/bda/views.py index b6dd6aec..216fc45b 100644 --- a/bda/views.py +++ b/bda/views.py @@ -337,6 +337,18 @@ def do_tirage(tirage_elt, token): # On inscrit à BdA-Revente ceux qui n'ont pas eu les places voulues ChoixRevente = Participant.choicesrevente.through + # Suppression des reventes demandées/enregistrées (si le tirage est relancé) + ( + ChoixRevente.objects + .filter(spectacle__tirage=tirage_elt) + .delete() + ) + ( + SpectacleRevente.objects + .filter(attribution__spectacle__tirage=tirage_elt) + .delete() + ) + lost_by = defaultdict(set) for show, _, losers in results: for loser, _, _, _ in losers: From 3556e3b1b04c87972d088756aa42ecd0198aefa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Sat, 8 Apr 2017 12:10:23 +0200 Subject: [PATCH 15/43] Fewer requests on bda.views.revente --- bda/forms.py | 31 ++++++++++++++++++++++--------- bda/templates/bda/reventes.html | 13 ++++++++----- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/bda/forms.py b/bda/forms.py index 6bb36c32..3565bedf 100644 --- a/bda/forms.py +++ b/bda/forms.py @@ -15,7 +15,7 @@ class TokenForm(forms.Form): class AttributionModelMultipleChoiceField(forms.ModelMultipleChoiceField): def label_from_instance(self, obj): - return "%s" % obj.spectacle + return "%s" % str(obj.spectacle) class ResellForm(forms.Form): @@ -27,9 +27,13 @@ class ResellForm(forms.Form): def __init__(self, participant, *args, **kwargs): super(ResellForm, self).__init__(*args, **kwargs) - self.fields['attributions'].queryset = participant.attribution_set\ - .filter(spectacle__date__gte=timezone.now())\ + self.fields['attributions'].queryset = ( + participant.attribution_set + .filter(spectacle__date__gte=timezone.now()) .exclude(revente__seller=participant) + .select_related('spectacle', 'spectacle__location', + 'participant__user') + ) class AnnulForm(forms.Form): @@ -41,11 +45,15 @@ class AnnulForm(forms.Form): def __init__(self, participant, *args, **kwargs): super(AnnulForm, self).__init__(*args, **kwargs) - self.fields['attributions'].queryset = participant.attribution_set\ + self.fields['attributions'].queryset = ( + participant.attribution_set .filter(spectacle__date__gte=timezone.now(), revente__isnull=False, revente__notif_sent=False, revente__soldTo__isnull=True) + .select_related('spectacle', 'spectacle__location', + 'participant__user') + ) class InscriptionReventeForm(forms.Form): @@ -56,8 +64,11 @@ class InscriptionReventeForm(forms.Form): def __init__(self, tirage, *args, **kwargs): super(InscriptionReventeForm, self).__init__(*args, **kwargs) - self.fields['spectacles'].queryset = tirage.spectacle_set.filter( - date__gte=timezone.now()) + self.fields['spectacles'].queryset = ( + tirage.spectacle_set + .select_related('location') + .filter(date__gte=timezone.now()) + ) class SoldForm(forms.Form): @@ -70,7 +81,9 @@ class SoldForm(forms.Form): super(SoldForm, self).__init__(*args, **kwargs) self.fields['attributions'].queryset = ( participant.attribution_set - .filter(revente__isnull=False, - revente__soldTo__isnull=False) - .exclude(revente__soldTo=participant) + .filter(revente__isnull=False, + revente__soldTo__isnull=False) + .exclude(revente__soldTo=participant) + .select_related('spectacle', 'spectacle__location', + 'participant__user') ) diff --git a/bda/templates/bda/reventes.html b/bda/templates/bda/reventes.html index e61e7c8d..0912babb 100644 --- a/bda/templates/bda/reventes.html +++ b/bda/templates/bda/reventes.html @@ -4,6 +4,8 @@ {% block realcontent %}

Revente de place

+{% with resell_attributions=resellform.attributions annul_attributions=annulform.attributions sold_attributions=soldform.attributions %} + {% if resellform.attributions %}

Places non revendues

@@ -15,14 +17,14 @@
{% endif %}
-{% if annulform.attributions or overdue %} +{% if annul_attributions or overdue %}

Places en cours de revente

{% csrf_token %}
    - {% for attrib in annulform.attributions %} + {% for attrib in annul_attributions %}
  • {{attrib.tag}} {{attrib.choice_label}}
    • {% endfor %} {% for attrib in overdue %} @@ -31,13 +33,13 @@ {{attrib.spectacle}} {% endfor %} - {% if annulform.attributions %} + {% if annul_attributions %} {% endif %} {% endif %}
    -{% if soldform.attributions %} +{% if sold_attributions %}

    Places revendues

    {% csrf_token %} @@ -46,8 +48,9 @@
    {% endif %} -{% if not resellform.attributions and not soldform.attributions and not overdue and not annulform.attributions %} +{% if not resell_attributions and not annul_attributions and not overdue and not sold_attributions %}

    Plus de reventes possibles !

    {% endif %} +{% endwith %} {% endblock %} From fdc1128bd57a1a4f49b61b80467cb027e78a6f22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Sat, 8 Apr 2017 12:12:56 +0200 Subject: [PATCH 16/43] delete useless save --- bda/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bda/views.py b/bda/views.py index 216fc45b..fc932674 100644 --- a/bda/views.py +++ b/bda/views.py @@ -523,7 +523,6 @@ def list_revente(request, tirage_id): ) if min_resell is not None: min_resell.answered_mail.add(participant) - min_resell.save() inscrit_revente.append(spectacle) success = True else: From 93a3a9af2c92c08c7e128cd0e4ea36b34fd3cbf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Sat, 8 Apr 2017 12:53:37 +0200 Subject: [PATCH 17/43] Fewer requests with shotgunable reventes bda.views.revente_shotgun: - 2 requests instead of ~#spectacles requests --- bda/views.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/bda/views.py b/bda/views.py index fc932674..f99dce26 100644 --- a/bda/views.py +++ b/bda/views.py @@ -15,7 +15,7 @@ from django.contrib.auth.decorators import login_required from django.contrib import messages from django.db import transaction from django.core import serializers -from django.db.models import Count, Q +from django.db.models import Count, Q, Prefetch from django.forms.models import inlineformset_factory from django.http import ( HttpResponseBadRequest, HttpResponseRedirect, JsonResponse @@ -560,13 +560,13 @@ def buy_revente(request, spectacle_id): # Si l'utilisateur veut racheter une place qu'il est en train de revendre, # on supprime la revente en question. - if reventes.filter(seller=participant).exists(): - revente = reventes.filter(seller=participant)[0] - revente.delete() + own_reventes = reventes.filter(seller=participant) + if len(own_reventes) > 0: + own_reventes[0].delete() return HttpResponseRedirect(reverse("bda-shotgun", args=[tirage.id])) - reventes_shotgun = list(reventes.filter(shotgun=True).all()) + reventes_shotgun = reventes.filter(shotgun=True) if not reventes_shotgun: return render(request, "bda-no-revente.html", {}) @@ -598,16 +598,21 @@ def buy_revente(request, spectacle_id): @login_required def revente_shotgun(request, tirage_id): tirage = get_object_or_404(Tirage, id=tirage_id) - spectacles = tirage.spectacle_set.filter( - date__gte=timezone.now()) - shotgun = [] - for spectacle in spectacles: - reventes = SpectacleRevente.objects.filter( - attribution__spectacle=spectacle, - shotgun=True, - soldTo__isnull=True) - if reventes.exists(): - shotgun.append(spectacle) + spectacles = ( + tirage.spectacle_set + .filter(date__gte=timezone.now()) + .select_related('location') + .prefetch_related(Prefetch( + 'attribues', + queryset=( + Attribution.objects + .filter(revente__shotgun=True, + revente__soldTo__isnull=True) + ), + to_attr='shotguns', + )) + ) + shotgun = [sp for sp in spectacles if len(sp.shotguns) > 0] return render(request, "bda-shotgun.html", {"shotgun": shotgun}) From 15d2faf8e15a036dff9b92433573daa43561ad14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Sat, 8 Apr 2017 13:01:05 +0200 Subject: [PATCH 18/43] Fewer requests on spectacles and participants list views --- bda/views.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/bda/views.py b/bda/views.py index f99dce26..a3b28af3 100644 --- a/bda/views.py +++ b/bda/views.py @@ -622,7 +622,10 @@ def revente_shotgun(request, tirage_id): def spectacle(request, tirage_id, spectacle_id): tirage = get_object_or_404(Tirage, id=tirage_id) spectacle = get_object_or_404(Spectacle, id=spectacle_id, tirage=tirage) - attributions = spectacle.attribues.all() + attributions = ( + spectacle.attribues + .select_related('participant', 'participant__user') + ) participants = {} for attrib in attributions: participant = attrib.participant @@ -651,7 +654,10 @@ class SpectacleListView(ListView): def get_queryset(self): self.tirage = get_object_or_404(Tirage, id=self.kwargs['tirage_id']) - categories = self.tirage.spectacle_set.all() + categories = ( + self.tirage.spectacle_set + .select_related('location') + ) return categories def get_context_data(self, **kwargs): From 98f355ed209b655e37ae1c46ca898062f326e07e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Sat, 8 Apr 2017 13:08:53 +0200 Subject: [PATCH 19/43] Fewer requests on unpaid view --- bda/views.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bda/views.py b/bda/views.py index a3b28af3..ca14c08f 100644 --- a/bda/views.py +++ b/bda/views.py @@ -670,9 +670,12 @@ class SpectacleListView(ListView): @buro_required def unpaid(request, tirage_id): tirage = get_object_or_404(Tirage, id=tirage_id) - unpaid = tirage.participant_set \ - .annotate(nb_attributions=Count('attribution')) \ - .filter(paid=False, nb_attributions__gt=0).all() + unpaid = ( + tirage.participant_set + .annotate(nb_attributions=Count('attribution')) + .filter(paid=False, nb_attributions__gt=0) + .select_related('user') + ) return render(request, "bda-unpaid.html", {"unpaid": unpaid}) From bbe6f41962a9fc7e21f174bc3c155640f99e59a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Sat, 8 Apr 2017 13:44:21 +0200 Subject: [PATCH 20/43] Fewer requests on descriptions and catalogue views --- bda/views.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/bda/views.py b/bda/views.py index ca14c08f..6e0e73b6 100644 --- a/bda/views.py +++ b/bda/views.py @@ -710,7 +710,11 @@ def send_rappel(request, spectacle_id): def descriptions_spectacles(request, tirage_id): tirage = get_object_or_404(Tirage, id=tirage_id) - shows_qs = tirage.spectacle_set + shows_qs = ( + tirage.spectacle_set + .select_related('location') + .prefetch_related('quote_set') + ) category_name = request.GET.get('category', '') location_id = request.GET.get('location', '') if category_name: @@ -721,7 +725,7 @@ def descriptions_spectacles(request, tirage_id): except ValueError: return HttpResponseBadRequest( "La variable GET 'location' doit contenir un entier") - return render(request, 'descriptions.html', {'shows': shows_qs.all()}) + return render(request, 'descriptions.html', {'shows': shows_qs}) def catalogue(request, request_type): @@ -794,7 +798,11 @@ def catalogue(request, request_type): ) tirage = get_object_or_404(Tirage, id=tirage_id) - shows_qs = tirage.spectacle_set + shows_qs = ( + tirage.spectacle_set + .select_related('location') + .prefetch_related('quote_set') + ) if categories_id: shows_qs = shows_qs.filter(category__id__in=categories_id) if locations_id: @@ -813,14 +821,15 @@ def catalogue(request, request_type): 'vips': spectacle.vips, 'description': spectacle.description, 'slots_description': spectacle.slots_description, - 'quotes': list(Quote.objects.filter(spectacle=spectacle).values( - 'author', 'text')), + 'quotes': [dict(author=quote.author, + text=quote.text) + for quote in spectacle.quote_set.all()], 'image': spectacle.getImgUrl(), 'ext_link': spectacle.ext_link, 'price': spectacle.price, 'slots': spectacle.slots } - for spectacle in shows_qs.all() + for spectacle in shows_qs ] return JsonResponse(data_return, safe=False) # Si la requête n'est pas de la forme attendue, on quitte avec une erreur From 6451f971bd176d85a6ac2328536b42b437e26bb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Sat, 8 Apr 2017 16:11:42 +0200 Subject: [PATCH 21/43] AttributionForm in bda admin - New attribution form issue less queries - Spectacle and Participant are readonly if updating an attribution. ReadOnlyMixin allows to set readonly fields only while updating an object. --- bda/admin.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/bda/admin.py b/bda/admin.py index fc10c326..8926c554 100644 --- a/bda/admin.py +++ b/bda/admin.py @@ -94,6 +94,20 @@ class ParticipantAdmin(admin.ModelAdmin): class AttributionAdminForm(forms.ModelForm): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if 'spectacle' in self.fields: + self.fields['spectacle'].queryset = ( + Spectacle.objects + .select_related('location') + ) + if 'participant' in self.fields: + self.fields['participant'].queryset = ( + Participant.objects + .select_related('user', 'tirage') + ) + def clean(self): cleaned_data = super(AttributionAdminForm, self).clean() participant = cleaned_data.get("participant") @@ -106,7 +120,18 @@ class AttributionAdminForm(forms.ModelForm): return cleaned_data -class AttributionAdmin(admin.ModelAdmin): +class ReadOnlyMixin(object): + readonly_fields_update = () + + def get_readonly_fields(self, request, obj=None): + readonly_fields = super().get_readonly_fields(request, obj) + if obj is None: + return readonly_fields + else: + return readonly_fields + self.readonly_fields_update + + +class AttributionAdmin(ReadOnlyMixin, admin.ModelAdmin): def paid(self, obj): return obj.participant.paid paid.short_description = 'A payé' @@ -116,6 +141,7 @@ class AttributionAdmin(admin.ModelAdmin): 'participant__user__first_name', 'participant__user__last_name') form = AttributionAdminForm + readonly_fields_update = ('spectacle', 'participant') class ChoixSpectacleAdmin(admin.ModelAdmin): From 0750551d7c40a12fb5eb69f7fafdd77425a8506e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Sat, 8 Apr 2017 17:16:33 +0200 Subject: [PATCH 22/43] Fewer requests on participant admin create and updateviews. - Fewer requests on choicesreventes and tabular inlines (of attributions) - User and tirage cannot be updated if updating a participant instance. - Tabular inlines are fixed: - the one which is used for spectacles with listing only propose choices in spectacle with listing - same with the other (spectacles without listing) - Still does too much request (because of tabularinlines) --- bda/admin.py | 86 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 65 insertions(+), 21 deletions(-) diff --git a/bda/admin.py b/bda/admin.py index 8926c554..56c958dc 100644 --- a/bda/admin.py +++ b/bda/admin.py @@ -13,32 +13,85 @@ from bda.models import Spectacle, Salle, Participant, ChoixSpectacle,\ Attribution, Tirage, Quote, CategorieSpectacle, SpectacleRevente +class ReadOnlyMixin(object): + readonly_fields_update = () + + def get_readonly_fields(self, request, obj=None): + readonly_fields = super().get_readonly_fields(request, obj) + if obj is None: + return readonly_fields + else: + return readonly_fields + self.readonly_fields_update + + class ChoixSpectacleInline(admin.TabularInline): model = ChoixSpectacle sortable_field_name = "priority" -class AttributionInline(admin.TabularInline): - model = Attribution - extra = 0 +class AttributionAdminForm(forms.ModelForm): - def get_queryset(self, request): - qs = super(AttributionInline, self).get_queryset(request) - return qs.filter(spectacle__listing=False) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields['spectacle'].queryset = ( + Spectacle.objects + .select_related('location') + ) + + +class AttributionNoListingAdminForm(AttributionAdminForm): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields['spectacle'].queryset = ( + self.fields['spectacle'].queryset + .filter(listing=False) + ) + + +class AttributionListingAdminForm(AttributionAdminForm): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields['spectacle'].queryset = ( + self.fields['spectacle'].queryset + .filter(listing=True) + ) class AttributionInlineListing(admin.TabularInline): model = Attribution - exclude = ('given', ) extra = 0 + form = AttributionListingAdminForm def get_queryset(self, request): - qs = super(AttributionInlineListing, self).get_queryset(request) + qs = super().get_queryset(request) return qs.filter(spectacle__listing=True) -class ParticipantAdmin(admin.ModelAdmin): - inlines = [AttributionInline, AttributionInlineListing] +class AttributionInlineNoListing(admin.TabularInline): + model = Attribution + exclude = ('given', ) + extra = 0 + form = AttributionNoListingAdminForm + + def get_queryset(self, request): + qs = super().get_queryset(request) + return qs.filter(spectacle__listing=False) + + +class ParticipantAdminForm(forms.ModelForm): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields['choicesrevente'].queryset = ( + Spectacle.objects + .select_related('location') + ) + + +class ParticipantAdmin(ReadOnlyMixin, admin.ModelAdmin): + inlines = [AttributionInlineListing, AttributionInlineNoListing] def get_queryset(self, request): return Participant.objects.annotate(nb_places=Count('attributions'), @@ -65,6 +118,8 @@ class ParticipantAdmin(admin.ModelAdmin): actions_on_bottom = True list_per_page = 400 readonly_fields = ("total",) + readonly_fields_update = ('user', 'tirage') + form = ParticipantAdminForm def send_attribs(self, request, queryset): datatuple = [] @@ -120,17 +175,6 @@ class AttributionAdminForm(forms.ModelForm): return cleaned_data -class ReadOnlyMixin(object): - readonly_fields_update = () - - def get_readonly_fields(self, request, obj=None): - readonly_fields = super().get_readonly_fields(request, obj) - if obj is None: - return readonly_fields - else: - return readonly_fields + self.readonly_fields_update - - class AttributionAdmin(ReadOnlyMixin, admin.ModelAdmin): def paid(self, obj): return obj.participant.paid From ce70269e7bd513e12b671952d6e2755ce8a47536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Sat, 8 Apr 2017 17:50:36 +0200 Subject: [PATCH 23/43] Fewer requests on create/update spectaclerevente in bda admin. - O(#participant) -> O(1) requests --- bda/admin.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/bda/admin.py b/bda/admin.py index 56c958dc..02eebad5 100644 --- a/bda/admin.py +++ b/bda/admin.py @@ -230,6 +230,24 @@ class SalleAdmin(admin.ModelAdmin): search_fields = ('name', 'address') +class SpectacleReventeAdminForm(forms.ModelForm): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields['answered_mail'].queryset = ( + Participant.objects + .select_related('user', 'tirage') + ) + self.fields['seller'].queryset = ( + Participant.objects + .select_related('user', 'tirage') + ) + self.fields['soldTo'].queryset = ( + Participant.objects + .select_related('user', 'tirage') + ) + + class SpectacleReventeAdmin(admin.ModelAdmin): """ Administration des reventes de spectacles @@ -252,6 +270,7 @@ class SpectacleReventeAdmin(admin.ModelAdmin): actions = ['transfer', 'reinit'] actions_on_bottom = True + form = SpectacleReventeAdminForm def transfer(self, request, queryset): """ From 8870b5ace2cad03e9d44e8a730ac6b37a32660f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Sun, 9 Apr 2017 17:37:15 +0200 Subject: [PATCH 24/43] Fewer queries on poll view --- gestioncof/views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gestioncof/views.py b/gestioncof/views.py index 944d9dc2..457a99c4 100644 --- a/gestioncof/views.py +++ b/gestioncof/views.py @@ -94,7 +94,10 @@ def logout(request): @login_required def survey(request, survey_id): - survey = get_object_or_404(Survey, id=survey_id) + survey = get_object_or_404( + Survey.objects.prefetch_related('questions', 'questions__answers'), + id=survey_id, + ) if not survey.survey_open or survey.old: raise Http404 success = False From 3dc91e30bd798cdd6c18c928727f5b8da218ea9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Sun, 9 Apr 2017 17:51:40 +0200 Subject: [PATCH 25/43] Fewer requests on petit cours list management. --- gestioncof/petits_cours_views.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/gestioncof/petits_cours_views.py b/gestioncof/petits_cours_views.py index 332e156c..ca0b55af 100644 --- a/gestioncof/petits_cours_views.py +++ b/gestioncof/petits_cours_views.py @@ -24,13 +24,14 @@ from gestioncof.shared import lock_table, unlock_tables class DemandeListView(ListView): - model = PetitCoursDemande + queryset = ( + PetitCoursDemande.objects + .prefetch_related('matieres') + .order_by('traitee', '-id') + ) template_name = "petits_cours_demandes_list.html" paginate_by = 20 - def get_queryset(self): - return PetitCoursDemande.objects.order_by('traitee', '-id').all() - class DemandeDetailView(DetailView): model = PetitCoursDemande From 6ce2f178bf29c4c87a14de858037e3b66b27d4bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Sun, 9 Apr 2017 17:57:11 +0200 Subject: [PATCH 26/43] Fewer requests on petit cours details management. --- gestioncof/petits_cours_views.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gestioncof/petits_cours_views.py b/gestioncof/petits_cours_views.py index ca0b55af..3fa0dc57 100644 --- a/gestioncof/petits_cours_views.py +++ b/gestioncof/petits_cours_views.py @@ -35,6 +35,11 @@ class DemandeListView(ListView): class DemandeDetailView(DetailView): model = PetitCoursDemande + queryset = ( + PetitCoursDemande.objects + .prefetch_related('petitcoursattribution_set', + 'matieres') + ) template_name = "gestioncof/details_demande_petit_cours.html" context_object_name = "demande" From e97e0081d7bfb4e1890ebe0b4bc1131639879214 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Tue, 11 Apr 2017 23:13:54 +0200 Subject: [PATCH 27/43] Fewer queries on stats of an account balance. - Remove labels, should be replaced to an anchor to the relative operation in history. - Add select_related as necessary. --- kfet/views.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/kfet/views.py b/kfet/views.py index 2c4a4f73..cfc58aa0 100644 --- a/kfet/views.py +++ b/kfet/views.py @@ -2219,10 +2219,13 @@ class AccountStatBalance(PkUrlMixin, JSONDetailView): # prepare querysets # TODO: retirer les opgroup dont tous les op sont annulées opegroups = OperationGroup.objects.filter(on_acc=account) - recv_transfers = Transfer.objects.filter(to_acc=account, - canceled_at=None) - sent_transfers = Transfer.objects.filter(from_acc=account, - canceled_at=None) + transfers = ( + Transfer.objects + .filter(canceled_at=None) + .select_related('group') + ) + recv_transfers = transfers.filter(to_acc=account) + sent_transfers = transfers.filter(from_acc=account) # apply filters if begin_date is not None: @@ -2250,13 +2253,11 @@ class AccountStatBalance(PkUrlMixin, JSONDetailView): actions.append({ 'at': (begin_date or account.created_at).isoformat(), 'amount': 0, - 'label': 'début', 'balance': 0, }) actions.append({ 'at': (end_date or timezone.now()).isoformat(), 'amount': 0, - 'label': 'fin', 'balance': 0, }) @@ -2264,21 +2265,18 @@ class AccountStatBalance(PkUrlMixin, JSONDetailView): { 'at': ope_grp.at.isoformat(), 'amount': ope_grp.amount, - 'label': str(ope_grp), 'balance': 0, } for ope_grp in opegroups ] + [ { 'at': tr.group.at.isoformat(), 'amount': tr.amount, - 'label': str(tr), 'balance': 0, } for tr in recv_transfers ] + [ { 'at': tr.group.at.isoformat(), 'amount': -tr.amount, - 'label': str(tr), 'balance': 0, } for tr in sent_transfers ] From 3f4a1adbb9cd083731b3ad340c47dae51c461586 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Wed, 12 Apr 2017 18:03:31 +0200 Subject: [PATCH 28/43] Fewer queries on stats/scales + Fix Scales: - Fix #chunks when used with std_chunk=True (there was one too many at the beginning) - Scale.end gives the end of the last chunk (instead of its start) So scale.begin -> scale.end gives the full range of the scale. `kfet_day` now returns an aware datetime. ScaleMixin: - new method `get_by_chunks` which use only one query and ranks elements according to the scale. Elements are returned by a generator for each scale chunk (and all chunks are returned as a generator too). ArticlesStatSales and AccountStatOperations use this new method to avoid issuing #scale_chunks queries. ArticleStat: - fixed on Chrome --- kfet/static/kfet/js/statistic.js | 4 +- kfet/statistic.py | 103 ++++++++++++++++++++++++-- kfet/templates/kfet/article_read.html | 2 +- kfet/views.py | 63 ++++++++++------ 4 files changed, 140 insertions(+), 32 deletions(-) diff --git a/kfet/static/kfet/js/statistic.js b/kfet/static/kfet/js/statistic.js index f210c11d..db31e0e8 100644 --- a/kfet/static/kfet/js/statistic.js +++ b/kfet/static/kfet/js/statistic.js @@ -61,7 +61,7 @@ var chart = charts[i]; // format the data - var chart_data = is_time_chart ? handleTimeChart(chart.values) : dictToArray(chart.values, 1); + var chart_data = is_time_chart ? handleTimeChart(chart.values) : dictToArray(chart.values, 0); chart_datasets.push( { @@ -132,7 +132,7 @@ type: 'line', options: chart_options, data: { - labels: (data.labels || []).slice(1), + labels: data.labels || [], datasets: chart_datasets, } }; diff --git a/kfet/statistic.py b/kfet/statistic.py index fe948f73..5ff169ff 100644 --- a/kfet/statistic.py +++ b/kfet/statistic.py @@ -4,6 +4,7 @@ from datetime import date, datetime, time, timedelta from dateutil.relativedelta import relativedelta from dateutil.parser import parse as dateutil_parse +import pytz from django.utils import timezone from django.db.models import Sum @@ -13,7 +14,8 @@ KFET_WAKES_UP_AT = time(7, 0) def kfet_day(year, month, day, start_at=KFET_WAKES_UP_AT): """datetime wrapper with time offset.""" - return datetime.combine(date(year, month, day), start_at) + naive = datetime.combine(date(year, month, day), start_at) + return pytz.timezone('Europe/Paris').localize(naive, is_dst=None) def to_kfet_day(dt, start_at=KFET_WAKES_UP_AT): @@ -32,16 +34,21 @@ class Scale(object): self.std_chunk = std_chunk if last: end = timezone.now() + if std_chunk: + if begin is not None: + begin = self.get_chunk_start(begin) + if end is not None: + end = self.do_step(self.get_chunk_start(end)) if begin is not None and n_steps != 0: - self.begin = self.get_from(begin) + self.begin = begin self.end = self.do_step(self.begin, n_steps=n_steps) elif end is not None and n_steps != 0: - self.end = self.get_from(end) + self.end = end self.begin = self.do_step(self.end, n_steps=-n_steps) elif begin is not None and end is not None: - self.begin = self.get_from(begin) - self.end = self.get_from(end) + self.begin = begin + self.end = end else: raise Exception('Two of these args must be specified: ' 'n_steps, begin, end; ' @@ -71,7 +78,7 @@ class Scale(object): def get_datetimes(self): datetimes = [self.begin] tmp = self.begin - while tmp <= self.end: + while tmp < self.end: tmp = self.do_step(tmp) datetimes.append(tmp) return datetimes @@ -232,3 +239,87 @@ class ScaleMixin(object): qs.filter(**{begin_f: begin, end_f: end}) for begin, end in scale ] + + def get_by_chunks(self, qs, scale, field_callback=None, field_db='at'): + """Objects of queryset ranked according to a given scale. + + Returns a generator whose each item, corresponding to a scale chunk, + is a generator of objects from qs for this chunk. + + Args: + qs: Queryset of source objects, must be ordered *first* on the + same field returned by `field_callback`. + scale: Used to rank objects. + field_callback: Callable which gives value from an object used + to compare against limits of the scale chunks. + Default to: lambda obj: getattr(obj, field_db) + field_db: Used to filter against `scale` limits. + Default to 'at'. + + Examples: + If queryset `qs` use `values()`, `field_callback` must be set and + could be: `lambda d: d['at']` + If `field_db` use foreign attributes (eg with `__`), it should be + something like: `lambda obj: obj.group.at`. + + """ + if field_callback is None: + def field_callback(obj): + return getattr(obj, field_db) + + begin_f = '{}__gte'.format(field_db) + end_f = '{}__lte'.format(field_db) + + qs = ( + qs + .filter(**{begin_f: scale.begin, end_f: scale.end}) + ) + + obj_iter = iter(qs) + + last_obj = None + + def _objects_until(obj_iter, field_callback, end): + """Generator of objects until `end`. + + Ends if objects source is empty or when an object not verifying + field_callback(obj) <= end is met. + + If this object exists, it is stored in `last_obj` which is found + from outer scope. + Also, if this same variable is non-empty when the function is + called, it first yields its content. + + Args: + obj_iter: Source used to get objects. + field_callback: Returned value, when it is called on an object + will be used to test ordering against `end`. + end + + """ + nonlocal last_obj + + if last_obj is not None: + yield last_obj + last_obj = None + + for obj in obj_iter: + if field_callback(obj) <= end: + yield obj + else: + last_obj = obj + return + + for begin, end in scale: + # forward last seen object, if it exists, to the right chunk, + # and fill with empty generators for intermediate chunks of scale + if last_obj is not None: + if field_callback(last_obj) > end: + yield iter(()) + continue + + # yields generator for this chunk + # this set last_obj to None if obj_iter reach its end, otherwise + # it's set to the first met object from obj_iter which doesn't + # belong to this chunk + yield _objects_until(obj_iter, field_callback, end) diff --git a/kfet/templates/kfet/article_read.html b/kfet/templates/kfet/article_read.html index 6fe025f6..19a11094 100644 --- a/kfet/templates/kfet/article_read.html +++ b/kfet/templates/kfet/article_read.html @@ -104,7 +104,7 @@ $(document).ready(function() { var stat_last = new StatsGroup( "{% url 'kfet.article.stat.sales.list' article.id %}", - $("#stat_last"), + $("#stat_last") ); }); diff --git a/kfet/views.py b/kfet/views.py index cfc58aa0..1df78d1e 100644 --- a/kfet/views.py +++ b/kfet/views.py @@ -2369,13 +2369,19 @@ class AccountStatOperation(ScaleMixin, PkUrlMixin, JSONDetailView): # à l'article en question et qui ne sont pas annulées # puis on choisi pour chaques intervalle les opérations # effectuées dans ces intervalles de temps - all_operations = (Operation.objects - .filter(group__on_acc=self.object) - .filter(canceled_at=None) - ) + all_operations = ( + Operation.objects + .filter(group__on_acc=self.object, + canceled_at=None) + .values('article_nb', 'group__at') + .order_by('group__at') + ) if types is not None: all_operations = all_operations.filter(type__in=types) - chunks = self.chunkify_qs(all_operations, scale, field='group__at') + chunks = self.get_by_chunks( + all_operations, scale, field_db='group__at', + field_callback=(lambda d: d['group__at']), + ) return chunks def get_context_data(self, *args, **kwargs): @@ -2391,7 +2397,8 @@ class AccountStatOperation(ScaleMixin, PkUrlMixin, JSONDetailView): # On compte les opérations nb_ventes = [] for chunk in operations: - nb_ventes.append(tot_ventes(chunk)) + ventes = sum(ope['article_nb'] for ope in chunk) + nb_ventes.append(ventes) context['charts'] = [{"color": "rgb(255, 99, 132)", "label": "NB items achetés", @@ -2442,29 +2449,39 @@ class ArticleStatSales(ScaleMixin, JSONDetailView): context = {'labels': old_ctx['labels']} scale = self.scale - # On selectionne les opérations qui correspondent - # à l'article en question et qui ne sont pas annulées - # puis on choisi pour chaques intervalle les opérations - # effectuées dans ces intervalles de temps - all_operations = ( + all_purchases = ( Operation.objects - .filter(type=Operation.PURCHASE, - article=self.object, - canceled_at=None, - ) + .filter( + type=Operation.PURCHASE, + article=self.object, + canceled_at=None, + ) + .values('group__at', 'article_nb') + .order_by('group__at') ) - chunks = self.chunkify_qs(all_operations, scale, field='group__at') + liq_only = all_purchases.filter(group__on_acc__trigramme='LIQ') + liq_exclude = all_purchases.exclude(group__on_acc__trigramme='LIQ') + + chunks_liq = self.get_by_chunks( + liq_only, scale, field_db='group__at', + field_callback=lambda d: d['group__at'], + ) + chunks_no_liq = self.get_by_chunks( + liq_exclude, scale, field_db='group__at', + field_callback=lambda d: d['group__at'], + ) + # On compte les opérations nb_ventes = [] nb_accounts = [] nb_liq = [] - for qs in chunks: - nb_ventes.append( - tot_ventes(qs)) - nb_liq.append( - tot_ventes(qs.filter(group__on_acc__trigramme='LIQ'))) - nb_accounts.append( - tot_ventes(qs.exclude(group__on_acc__trigramme='LIQ'))) + for chunk_liq, chunk_no_liq in zip(chunks_liq, chunks_no_liq): + sum_accounts = sum(ope['article_nb'] for ope in chunk_no_liq) + sum_liq = sum(ope['article_nb'] for ope in chunk_liq) + nb_ventes.append(sum_accounts + sum_liq) + nb_accounts.append(sum_accounts) + nb_liq.append(sum_liq) + context['charts'] = [{"color": "rgb(255, 99, 132)", "label": "Toutes consommations", "values": nb_ventes}, From 06572f0bb5b7994910e7bed6be29dcc90aa7d25c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Thu, 13 Apr 2017 14:11:44 +0200 Subject: [PATCH 29/43] Order create use Scale. Order create view use WeekScale. No query improvements, only shorter code. Scale/ScaleMixin: - Two methods directly relative to the Scale class move to... the Scale class. - Fix order create on Chrome. --- kfet/statistic.py | 187 +++++++++++----------- kfet/templates/kfet/inventory_create.html | 2 + kfet/views.py | 113 +++++++------ 3 files changed, 149 insertions(+), 153 deletions(-) diff --git a/kfet/statistic.py b/kfet/statistic.py index 5ff169ff..8ffb7db5 100644 --- a/kfet/statistic.py +++ b/kfet/statistic.py @@ -88,6 +88,99 @@ class Scale(object): label_fmt = self.label_fmt return [begin.strftime(label_fmt) for begin, end in self] + def chunkify_qs(self, qs, field=None): + if field is None: + field = 'at' + begin_f = '{}__gte'.format(field) + end_f = '{}__lte'.format(field) + return [ + qs.filter(**{begin_f: begin, end_f: end}) + for begin, end in self + ] + + def get_by_chunks(self, qs, field_callback=None, field_db='at'): + """Objects of queryset ranked according to the scale. + + Returns a generator whose each item, corresponding to a scale chunk, + is a generator of objects from qs for this chunk. + + Args: + qs: Queryset of source objects, must be ordered *first* on the + same field returned by `field_callback`. + field_callback: Callable which gives value from an object used + to compare against limits of the scale chunks. + Default to: lambda obj: getattr(obj, field_db) + field_db: Used to filter against `scale` limits. + Default to 'at'. + + Examples: + If queryset `qs` use `values()`, `field_callback` must be set and + could be: `lambda d: d['at']` + If `field_db` use foreign attributes (eg with `__`), it should be + something like: `lambda obj: obj.group.at`. + + """ + if field_callback is None: + def field_callback(obj): + return getattr(obj, field_db) + + begin_f = '{}__gte'.format(field_db) + end_f = '{}__lte'.format(field_db) + + qs = ( + qs + .filter(**{begin_f: self.begin, end_f: self.end}) + ) + + obj_iter = iter(qs) + + last_obj = None + + def _objects_until(obj_iter, field_callback, end): + """Generator of objects until `end`. + + Ends if objects source is empty or when an object not verifying + field_callback(obj) <= end is met. + + If this object exists, it is stored in `last_obj` which is found + from outer scope. + Also, if this same variable is non-empty when the function is + called, it first yields its content. + + Args: + obj_iter: Source used to get objects. + field_callback: Returned value, when it is called on an object + will be used to test ordering against `end`. + end + + """ + nonlocal last_obj + + if last_obj is not None: + yield last_obj + last_obj = None + + for obj in obj_iter: + if field_callback(obj) <= end: + yield obj + else: + last_obj = obj + return + + for begin, end in self: + # forward last seen object, if it exists, to the right chunk, + # and fill with empty generators for intermediate chunks of scale + if last_obj is not None: + if field_callback(last_obj) > end: + yield iter(()) + continue + + # yields generator for this chunk + # this set last_obj to None if obj_iter reach its end, otherwise + # it's set to the first met object from obj_iter which doesn't + # belong to this chunk + yield _objects_until(obj_iter, field_callback, end) + class DayScale(Scale): name = 'day' @@ -229,97 +322,3 @@ class ScaleMixin(object): def get_default_scale(self): return DayScale(n_steps=7, last=True) - - def chunkify_qs(self, qs, scale, field=None): - if field is None: - field = 'at' - begin_f = '{}__gte'.format(field) - end_f = '{}__lte'.format(field) - return [ - qs.filter(**{begin_f: begin, end_f: end}) - for begin, end in scale - ] - - def get_by_chunks(self, qs, scale, field_callback=None, field_db='at'): - """Objects of queryset ranked according to a given scale. - - Returns a generator whose each item, corresponding to a scale chunk, - is a generator of objects from qs for this chunk. - - Args: - qs: Queryset of source objects, must be ordered *first* on the - same field returned by `field_callback`. - scale: Used to rank objects. - field_callback: Callable which gives value from an object used - to compare against limits of the scale chunks. - Default to: lambda obj: getattr(obj, field_db) - field_db: Used to filter against `scale` limits. - Default to 'at'. - - Examples: - If queryset `qs` use `values()`, `field_callback` must be set and - could be: `lambda d: d['at']` - If `field_db` use foreign attributes (eg with `__`), it should be - something like: `lambda obj: obj.group.at`. - - """ - if field_callback is None: - def field_callback(obj): - return getattr(obj, field_db) - - begin_f = '{}__gte'.format(field_db) - end_f = '{}__lte'.format(field_db) - - qs = ( - qs - .filter(**{begin_f: scale.begin, end_f: scale.end}) - ) - - obj_iter = iter(qs) - - last_obj = None - - def _objects_until(obj_iter, field_callback, end): - """Generator of objects until `end`. - - Ends if objects source is empty or when an object not verifying - field_callback(obj) <= end is met. - - If this object exists, it is stored in `last_obj` which is found - from outer scope. - Also, if this same variable is non-empty when the function is - called, it first yields its content. - - Args: - obj_iter: Source used to get objects. - field_callback: Returned value, when it is called on an object - will be used to test ordering against `end`. - end - - """ - nonlocal last_obj - - if last_obj is not None: - yield last_obj - last_obj = None - - for obj in obj_iter: - if field_callback(obj) <= end: - yield obj - else: - last_obj = obj - return - - for begin, end in scale: - # forward last seen object, if it exists, to the right chunk, - # and fill with empty generators for intermediate chunks of scale - if last_obj is not None: - if field_callback(last_obj) > end: - yield iter(()) - continue - - # yields generator for this chunk - # this set last_obj to None if obj_iter reach its end, otherwise - # it's set to the first met object from obj_iter which doesn't - # belong to this chunk - yield _objects_until(obj_iter, field_callback, end) diff --git a/kfet/templates/kfet/inventory_create.html b/kfet/templates/kfet/inventory_create.html index d8109f8e..0192d4ad 100644 --- a/kfet/templates/kfet/inventory_create.html +++ b/kfet/templates/kfet/inventory_create.html @@ -161,6 +161,8 @@ $(document).ready(function() { $('input[type="submit"]').on("click", function(e) { e.preventDefault(); + var content; + if (conflicts.size) { content = ''; content += "Conflits possibles :" diff --git a/kfet/views.py b/kfet/views.py index 1df78d1e..45d9d1cb 100644 --- a/kfet/views.py +++ b/kfet/views.py @@ -50,7 +50,7 @@ from decimal import Decimal import django_cas_ng import heapq import statistics -from kfet.statistic import ScaleMixin, last_stats_manifest, tot_ventes +from kfet.statistic import ScaleMixin, last_stats_manifest, tot_ventes, WeekScale class Home(TemplateView): @@ -1798,68 +1798,60 @@ class OrderList(ListView): context['suppliers'] = Supplier.objects.order_by('name') return context + @teamkfet_required def order_create(request, pk): supplier = get_object_or_404(Supplier, pk=pk) - articles = (Article.objects - .filter(suppliers=supplier.pk) - .distinct() - .select_related('category') - .order_by('category__name', 'name')) + articles = ( + Article.objects + .filter(suppliers=supplier.pk) + .distinct() + .select_related('category') + .order_by('category__name', 'name') + ) - initial = [] - today = timezone.now().date() - sales_q = (Operation.objects + # Force hit to cache + articles = list(articles) + + sales_q = ( + Operation.objects .select_related('group') .filter(article__in=articles, canceled_at=None) - .values('article')) - sales_s1 = (sales_q - .filter( - group__at__gte = today-timedelta(weeks=5), - group__at__lt = today-timedelta(weeks=4)) + .values('article') .annotate(nb=Sum('article_nb')) ) - sales_s1 = { d['article']:d['nb'] for d in sales_s1 } - sales_s2 = (sales_q - .filter( - group__at__gte = today-timedelta(weeks=4), - group__at__lt = today-timedelta(weeks=3)) - .annotate(nb=Sum('article_nb')) - ) - sales_s2 = { d['article']:d['nb'] for d in sales_s2 } - sales_s3 = (sales_q - .filter( - group__at__gte = today-timedelta(weeks=3), - group__at__lt = today-timedelta(weeks=2)) - .annotate(nb=Sum('article_nb')) - ) - sales_s3 = { d['article']:d['nb'] for d in sales_s3 } - sales_s4 = (sales_q - .filter( - group__at__gte = today-timedelta(weeks=2), - group__at__lt = today-timedelta(weeks=1)) - .annotate(nb=Sum('article_nb')) - ) - sales_s4 = { d['article']:d['nb'] for d in sales_s4 } - sales_s5 = (sales_q - .filter(group__at__gte = today-timedelta(weeks=1)) - .annotate(nb=Sum('article_nb')) - ) - sales_s5 = { d['article']:d['nb'] for d in sales_s5 } + scale = WeekScale(last=True, n_steps=5, std_chunk=False) + chunks = scale.chunkify_qs(sales_q, field='group__at') + sales = [] + + for chunk in chunks: + sales.append( + {d['article']: d['nb'] for d in chunk} + ) + + initial = [] for article in articles: - v_s1 = sales_s1.get(article.pk, 0) - v_s2 = sales_s2.get(article.pk, 0) - v_s3 = sales_s3.get(article.pk, 0) - v_s4 = sales_s4.get(article.pk, 0) - v_s5 = sales_s5.get(article.pk, 0) + # Get sales for each 5 last weeks + v_s1 = sales[0].get(article.pk, 0) + v_s2 = sales[1].get(article.pk, 0) + v_s3 = sales[2].get(article.pk, 0) + v_s4 = sales[3].get(article.pk, 0) + v_s5 = sales[4].get(article.pk, 0) v_all = [v_s1, v_s2, v_s3, v_s4, v_s5] + # Take the 3 greatest (eg to avoid 2 weeks of vacations) v_3max = heapq.nlargest(3, v_all) + # Get average and standard deviation v_moy = statistics.mean(v_3max) v_et = statistics.pstdev(v_3max, v_moy) + # Expected sales for next week v_prev = v_moy + v_et + # We want to have 1.5 * the expected sales in stock + # (because sometimes some articles are not delivered) c_rec_tot = max(v_prev * 1.5 - article.stock, 0) + # If ordered quantity is close enough to a level which can led to free + # boxes, we increase it to this level. if article.box_capacity: c_rec_temp = c_rec_tot / article.box_capacity if c_rec_temp >= 10: @@ -1889,8 +1881,9 @@ def order_create(request, pk): }) cls_formset = formset_factory( - form = OrderArticleForm, - extra = 0) + form=OrderArticleForm, + extra=0, + ) if request.POST: formset = cls_formset(request.POST, initial=initial) @@ -1907,14 +1900,15 @@ def order_create(request, pk): order.save() saved = True - article = articles.get(pk=form.cleaned_data['article'].pk) + article = form.cleaned_data['article'] q_ordered = form.cleaned_data['quantity_ordered'] if article.box_capacity: q_ordered *= article.box_capacity OrderArticle.objects.create( - order = order, - article = article, - quantity_ordered = q_ordered) + order=order, + article=article, + quantity_ordered=q_ordered, + ) if saved: messages.success(request, 'Commande créée') return redirect('kfet.order.read', order.pk) @@ -1926,9 +1920,10 @@ def order_create(request, pk): return render(request, 'kfet/order_create.html', { 'supplier': supplier, - 'formset' : formset, + 'formset': formset, }) + class OrderRead(DetailView): model = Order template_name = 'kfet/order_read.html' @@ -2378,8 +2373,8 @@ class AccountStatOperation(ScaleMixin, PkUrlMixin, JSONDetailView): ) if types is not None: all_operations = all_operations.filter(type__in=types) - chunks = self.get_by_chunks( - all_operations, scale, field_db='group__at', + chunks = scale.get_by_chunks( + all_operations, field_db='group__at', field_callback=(lambda d: d['group__at']), ) return chunks @@ -2462,12 +2457,12 @@ class ArticleStatSales(ScaleMixin, JSONDetailView): liq_only = all_purchases.filter(group__on_acc__trigramme='LIQ') liq_exclude = all_purchases.exclude(group__on_acc__trigramme='LIQ') - chunks_liq = self.get_by_chunks( - liq_only, scale, field_db='group__at', + chunks_liq = scale.get_by_chunks( + liq_only, field_db='group__at', field_callback=lambda d: d['group__at'], ) - chunks_no_liq = self.get_by_chunks( - liq_exclude, scale, field_db='group__at', + chunks_no_liq = scale.get_by_chunks( + liq_exclude, field_db='group__at', field_callback=lambda d: d['group__at'], ) From 7db497d09593bf3e5990e399bb110fd012421cb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Thu, 13 Apr 2017 16:34:29 +0200 Subject: [PATCH 30/43] Less articles prices history - Prices given with order_to_inventory are saved to db only if they are updated (it doesn't create a new price row each time) Fixes #142. --- kfet/views.py | 94 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 61 insertions(+), 33 deletions(-) diff --git a/kfet/views.py b/kfet/views.py index 330b195a..a33e3fc0 100644 --- a/kfet/views.py +++ b/kfet/views.py @@ -1946,6 +1946,7 @@ class OrderRead(DetailView): context['mail'] = mail return context + @teamkfet_required def order_to_inventory(request, pk): order = get_object_or_404(Order, pk=pk) @@ -1953,28 +1954,36 @@ def order_to_inventory(request, pk): if hasattr(order, 'inventory'): raise Http404 - articles = (Article.objects - .filter(orders=order.pk) - .select_related('category') - .prefetch_related(Prefetch('orderarticle_set', - queryset = OrderArticle.objects.filter(order=order), - to_attr = 'order')) - .prefetch_related(Prefetch('supplierarticle_set', - queryset = (SupplierArticle.objects - .filter(supplier=order.supplier) - .order_by('-at')), - to_attr = 'supplier')) - .order_by('category__name', 'name')) + supplier_prefetch = Prefetch( + 'article__supplierarticle_set', + queryset=( + SupplierArticle.objects + .filter(supplier=order.supplier) + .order_by('-at') + ), + to_attr='supplier', + ) + + order_articles = ( + OrderArticle.objects + .filter(order=order.pk) + .select_related('article', 'article__category') + .prefetch_related( + supplier_prefetch, + ) + .order_by('article__category__name', 'article__name') + ) initial = [] - for article in articles: + for order_article in order_articles: + article = order_article.article initial.append({ 'article': article.pk, 'name': article.name, 'category': article.category_id, 'category__name': article.category.name, - 'quantity_ordered': article.order[0].quantity_ordered, - 'quantity_received': article.order[0].quantity_ordered, + 'quantity_ordered': order_article.quantity_ordered, + 'quantity_received': order_article.quantity_ordered, 'price_HT': article.supplier[0].price_HT, 'TVA': article.supplier[0].TVA, 'rights': article.supplier[0].rights, @@ -1989,31 +1998,50 @@ def order_to_inventory(request, pk): messages.error(request, 'Permission refusée') elif formset.is_valid(): with transaction.atomic(): - inventory = Inventory() - inventory.order = order - inventory.by = request.user.profile.account_kfet - inventory.save() + inventory = Inventory.objects.create( + order=order, by=request.user.profile.account_kfet, + ) + new_supplierarticle = [] + new_inventoryarticle = [] for form in formset: q_received = form.cleaned_data['quantity_received'] article = form.cleaned_data['article'] - SupplierArticle.objects.create( - supplier = order.supplier, - article = article, - price_HT = form.cleaned_data['price_HT'], - TVA = form.cleaned_data['TVA'], - rights = form.cleaned_data['rights']) - (OrderArticle.objects - .filter(order=order, article=article) - .update(quantity_received = q_received)) - InventoryArticle.objects.create( - inventory = inventory, - article = article, - stock_old = article.stock, - stock_new = article.stock + q_received) + + price_HT = form.cleaned_data['price_HT'] + TVA = form.cleaned_data['TVA'] + rights = form.cleaned_data['rights'] + + if any((form.initial['price_HT'] != price_HT, + form.initial['TVA'] != TVA, + form.initial['rights'] != rights)): + new_supplierarticle.append( + SupplierArticle( + supplier=order.supplier, + article=article, + price_HT=price_HT, + TVA=TVA, + rights=rights, + ) + ) + ( + OrderArticle.objects + .filter(order=order, article=article) + .update(quantity_received=q_received) + ) + new_inventoryarticle.append( + InventoryArticle( + inventory=inventory, + article=article, + stock_old=article.stock, + stock_new=article.stock + q_received, + ) + ) article.stock += q_received if q_received > 0: article.is_sold = True article.save() + SupplierArticle.objects.bulk_create(new_supplierarticle) + InventoryArticle.objects.bulk_create(new_inventoryarticle) messages.success(request, "C'est tout bon !") return redirect('kfet.order') else: From 8622002e8df3f7c98a40b7f68fb45c5222c9e43e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Mon, 17 Apr 2017 20:40:54 +0200 Subject: [PATCH 31/43] minor change --- cof/settings_dev.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cof/settings_dev.py b/cof/settings_dev.py index fe4fcdea..c4a46011 100644 --- a/cof/settings_dev.py +++ b/cof/settings_dev.py @@ -192,9 +192,7 @@ def show_toolbar(request): machine physique n'est pas forcément connue, et peut difficilement être mise dans les INTERNAL_IPS. """ - if not DEBUG: - return False - return True + return DEBUG DEBUG_TOOLBAR_CONFIG = { 'SHOW_TOOLBAR_CALLBACK': show_toolbar, From 0d8a613f28223e756f660caa2ba8c000ba3cc596 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Fri, 21 Apr 2017 18:22:53 +0200 Subject: [PATCH 32/43] improve bda inscription form/view code --- bda/forms.py | 36 ++++++++++++++++++++++++++++++++---- bda/views.py | 50 +++++++++----------------------------------------- 2 files changed, 41 insertions(+), 45 deletions(-) diff --git a/bda/forms.py b/bda/forms.py index 3565bedf..c0417d1e 100644 --- a/bda/forms.py +++ b/bda/forms.py @@ -1,14 +1,42 @@ # -*- coding: utf-8 -*- -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - from django import forms +from django.forms.models import BaseInlineFormSet from django.utils import timezone + from bda.models import Attribution, Spectacle +class InscriptionInlineFormSet(BaseInlineFormSet): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # self.instance is a Participant object + tirage = self.instance.tirage + + # set once for all "spectacle" field choices + # - restrict choices to the spectacles of this tirage + # - force_choices avoid many db requests + spectacles = tirage.spectacle_set.select_related('location') + choices = [(sp.pk, str(sp)) for sp in spectacles] + self.force_choices('spectacle', choices) + + def force_choices(self, name, choices): + """Set choices of a field. + + As ModelChoiceIterator (default use to get choices of a + ModelChoiceField), it appends an empty selection if requested. + + """ + for form in self.forms: + field = form.fields[name] + if field.empty_label is not None: + field.choices = [('', field.empty_label)] + choices + else: + field.choices = choices + + class TokenForm(forms.Form): token = forms.CharField(widget=forms.widgets.Textarea()) diff --git a/bda/views.py b/bda/views.py index 6e0e73b6..00a1b300 100644 --- a/bda/views.py +++ b/bda/views.py @@ -32,6 +32,7 @@ from bda.models import ( from bda.algorithm import Algorithm from bda.forms import ( TokenForm, ResellForm, AnnulForm, InscriptionReventeForm, SoldForm, + InscriptionInlineFormSet, ) @@ -157,60 +158,27 @@ def inscription(request, tirage_id): messages.error(request, "Le tirage n'est pas encore ouvert : " "ouverture le {:s}".format(opening)) return render(request, 'bda/resume-inscription-tirage.html', {}) + + participant, _ = ( + Participant.objects.select_related('tirage') + .get_or_create(user=request.user, tirage=tirage) + ) + if timezone.now() > tirage.fermeture: # Le tirage est fermé. - participant, _ = ( - Participant.objects - .get_or_create(user=request.user, tirage=tirage) - ) choices = participant.choixspectacle_set.order_by("priority") messages.error(request, " C'est fini : tirage au sort dans la journée !") return render(request, "bda/resume-inscription-tirage.html", {"choices": choices}) - def force_for(f, to_choices=None, **kwargs): - """Overrides choices for ModelChoiceField. - - Args: - f (models.Field): To render as forms.Field - to_choices (dict): If a key `f.name` exists, f._choices is set to - its value. - - """ - formfield = f.formfield(**kwargs) - if to_choices: - if f.name in to_choices: - choices = [('', '---------')] + to_choices[f.name] - formfield._choices = choices - return formfield - - # Restrict spectacles choices to spectacles for this tirage. - spectacles = ( - tirage.spectacle_set - .select_related('location') - ) - spectacles_field_choices = [(sp.pk, str(sp)) for sp in spectacles] - - # Allow for spectacle choices to be set once for all. - # Form display use 1 request instead of (#forms of formset * #spectacles). - # FIXME: Validation still generates too much requests... - formfield_callback = partial( - force_for, - to_choices={ - 'spectacle': spectacles_field_choices, - }, - ) BdaFormSet = inlineformset_factory( Participant, ChoixSpectacle, fields=("spectacle", "double_choice", "priority"), - formfield_callback=formfield_callback, - ) - participant, _ = ( - Participant.objects - .get_or_create(user=request.user, tirage=tirage) + formset=InscriptionInlineFormSet, ) + success = False stateerror = False if request.method == "POST": From 2aee43e01a67d07f0392f230d83f6e5db10f2de7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20P=C3=A9pin?= Date: Mon, 24 Apr 2017 21:27:01 +0100 Subject: [PATCH 33/43] Add more configuration options for redis - `REDIS_HOST` can be specified in the secrets - Two new secrets: `REDIS_PASSWD` and `REDIS_DB` --- cof/settings/common.py | 8 ++++++-- cof/settings/secret_example.py | 3 +++ provisioning/bootstrap.sh | 10 +++++----- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/cof/settings/common.py b/cof/settings/common.py index 93b11dae..612d52fc 100644 --- a/cof/settings/common.py +++ b/cof/settings/common.py @@ -22,7 +22,8 @@ except KeyError: # Other secrets try: from .secret import ( - SECRET_KEY, RECAPTCHA_PUBLIC_KEY, RECAPTCHA_PRIVATE_KEY, ADMINS + SECRET_KEY, RECAPTCHA_PUBLIC_KEY, RECAPTCHA_PRIVATE_KEY, ADMINS, + REDIS_PASSWD, REDIS_DB, REDIS_HOST ) except ImportError: raise RuntimeError("Secrets missing") @@ -159,7 +160,10 @@ CHANNEL_LAYERS = { "default": { "BACKEND": "asgi_redis.RedisChannelLayer", "CONFIG": { - "hosts": [(os.environ.get("REDIS_HOST", "localhost"), 6379)], + "hosts": [( + "redis://:{}@{}:6379/{}" + .format(REDIS_PASSWD, REDIS_HOST, REDIS_DB) + )], }, "ROUTING": "cof.routing.channel_routing", } diff --git a/cof/settings/secret_example.py b/cof/settings/secret_example.py index 36a8e932..3dc5de4b 100644 --- a/cof/settings/secret_example.py +++ b/cof/settings/secret_example.py @@ -1,4 +1,7 @@ SECRET_KEY = 'q()(zn4m63i%5cp4)f+ww4-28_w+ly3q9=6imw2ciu&_(5_4ah' RECAPTCHA_PUBLIC_KEY = "DUMMY" RECAPTCHA_PRIVATE_KEY = "DUMMY" +REDIS_PASSWD = "dummy" +REDIS_DB = 0 +REDIS_HOST = "127.0.0.1" ADMINS = None diff --git a/provisioning/bootstrap.sh b/provisioning/bootstrap.sh index c8f73ab6..33ae8308 100644 --- a/provisioning/bootstrap.sh +++ b/provisioning/bootstrap.sh @@ -23,6 +23,9 @@ apt-get install -y mysql-server mysql -uroot -p$DBPASSWD -e "CREATE DATABASE $DBNAME; GRANT ALL PRIVILEGES ON $DBNAME.* TO '$DBUSER'@'localhost' IDENTIFIED BY '$DBPASSWD'" mysql -uroot -p$DBPASSWD -e "GRANT ALL PRIVILEGES ON test_$DBNAME.* TO '$DBUSER'@'localhost'" +# Configuration de redis +echo "requirepass dummy" >> /etc/redis/redis.conf + # Installation et configuration d'Apache apt-get install -y apache2 a2enmod proxy proxy_http proxy_wstunnel headers @@ -36,7 +39,7 @@ chown -R ubuntu:www-data /var/www/static # Mise en place du .bash_profile pour tout configurer lors du `vagrant ssh` cat >> ~ubuntu/.bashrc < Date: Mon, 24 Apr 2017 21:52:40 +0100 Subject: [PATCH 34/43] Fix settings in the provisioning script --- provisioning/bootstrap.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/provisioning/bootstrap.sh b/provisioning/bootstrap.sh index 33ae8308..1e576a65 100644 --- a/provisioning/bootstrap.sh +++ b/provisioning/bootstrap.sh @@ -25,6 +25,7 @@ mysql -uroot -p$DBPASSWD -e "GRANT ALL PRIVILEGES ON test_$DBNAME.* TO '$DBUSER' # Configuration de redis echo "requirepass dummy" >> /etc/redis/redis.conf +service redis restart # Installation et configuration d'Apache apt-get install -y apache2 @@ -39,7 +40,7 @@ chown -R ubuntu:www-data /var/www/static # Mise en place du .bash_profile pour tout configurer lors du `vagrant ssh` cat >> ~ubuntu/.bashrc < Date: Tue, 25 Apr 2017 20:23:21 +0100 Subject: [PATCH 35/43] Add REDIS_PORT to the settings and secrets --- cof/settings/common.py | 7 ++++--- cof/settings/secret_example.py | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cof/settings/common.py b/cof/settings/common.py index 612d52fc..261760d6 100644 --- a/cof/settings/common.py +++ b/cof/settings/common.py @@ -23,7 +23,7 @@ except KeyError: try: from .secret import ( SECRET_KEY, RECAPTCHA_PUBLIC_KEY, RECAPTCHA_PRIVATE_KEY, ADMINS, - REDIS_PASSWD, REDIS_DB, REDIS_HOST + REDIS_PASSWD, REDIS_DB, REDIS_HOST, REDIS_PORT ) except ImportError: raise RuntimeError("Secrets missing") @@ -161,8 +161,9 @@ CHANNEL_LAYERS = { "BACKEND": "asgi_redis.RedisChannelLayer", "CONFIG": { "hosts": [( - "redis://:{}@{}:6379/{}" - .format(REDIS_PASSWD, REDIS_HOST, REDIS_DB) + "redis://:{passwd}@{host}:{port}/{db}" + .format(passwd=REDIS_PASSWD, host=REDIS_HOST, + port=REDIS_PORT, db=REDIS_DB) )], }, "ROUTING": "cof.routing.channel_routing", diff --git a/cof/settings/secret_example.py b/cof/settings/secret_example.py index 3dc5de4b..eeb5271c 100644 --- a/cof/settings/secret_example.py +++ b/cof/settings/secret_example.py @@ -2,6 +2,7 @@ SECRET_KEY = 'q()(zn4m63i%5cp4)f+ww4-28_w+ly3q9=6imw2ciu&_(5_4ah' RECAPTCHA_PUBLIC_KEY = "DUMMY" RECAPTCHA_PRIVATE_KEY = "DUMMY" REDIS_PASSWD = "dummy" +REDIS_PORT = 6379 REDIS_DB = 0 REDIS_HOST = "127.0.0.1" ADMINS = None From fb4258f821b20db969935d55dcff55d112b842eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20P=C3=A9pin?= Date: Tue, 25 Apr 2017 20:23:51 +0100 Subject: [PATCH 36/43] Set the redis passwd properly in bootstrap.sh --- provisioning/bootstrap.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/provisioning/bootstrap.sh b/provisioning/bootstrap.sh index 1e576a65..38efdfb5 100644 --- a/provisioning/bootstrap.sh +++ b/provisioning/bootstrap.sh @@ -24,8 +24,9 @@ mysql -uroot -p$DBPASSWD -e "CREATE DATABASE $DBNAME; GRANT ALL PRIVILEGES ON $D mysql -uroot -p$DBPASSWD -e "GRANT ALL PRIVILEGES ON test_$DBNAME.* TO '$DBUSER'@'localhost'" # Configuration de redis -echo "requirepass dummy" >> /etc/redis/redis.conf -service redis restart +REDIS_PASSWD="dummy" +redis-cli CONFIG SET requirepass $REDIS_PASSWD +redis-cli -a $REDIS_PASSWD CONFIG REWRITE # Installation et configuration d'Apache apt-get install -y apache2 From 6cdb79198935191074cd0cfa1c82f2ee7b8c286c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Wed, 10 May 2017 12:39:56 +0200 Subject: [PATCH 37/43] fix class name conflicts --- bda/admin.py | 57 ++++++++++++++++++++++------------------------------ 1 file changed, 24 insertions(+), 33 deletions(-) diff --git a/bda/admin.py b/bda/admin.py index 02eebad5..0cc66d43 100644 --- a/bda/admin.py +++ b/bda/admin.py @@ -29,55 +29,46 @@ class ChoixSpectacleInline(admin.TabularInline): sortable_field_name = "priority" -class AttributionAdminForm(forms.ModelForm): +class AttributionTabularAdminForm(forms.ModelForm): + listing = None def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields['spectacle'].queryset = ( - Spectacle.objects - .select_related('location') - ) + spectacles = Spectacle.objects.select_related('location') + if self.listing is not None: + spectacles = spectacles.filter(listing=self.listing) + self.fields['spectacle'].queryset = spectacles -class AttributionNoListingAdminForm(AttributionAdminForm): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.fields['spectacle'].queryset = ( - self.fields['spectacle'].queryset - .filter(listing=False) - ) +class WithoutListingAttributionTabularAdminForm(AttributionTabularAdminForm): + listing = False -class AttributionListingAdminForm(AttributionAdminForm): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.fields['spectacle'].queryset = ( - self.fields['spectacle'].queryset - .filter(listing=True) - ) +class WithListingAttributionTabularAdminForm(AttributionTabularAdminForm): + listing = True -class AttributionInlineListing(admin.TabularInline): +class AttributionInline(admin.TabularInline): model = Attribution extra = 0 - form = AttributionListingAdminForm + listing = None def get_queryset(self, request): qs = super().get_queryset(request) - return qs.filter(spectacle__listing=True) + if self.listing is not None: + qs.filter(spectacle__listing=self.listing) + return qs -class AttributionInlineNoListing(admin.TabularInline): - model = Attribution +class WithListingAttributionInline(AttributionInline): + form = WithListingAttributionTabularAdminForm + listing = True + + +class WithoutListingAttributionInline(AttributionInline): exclude = ('given', ) - extra = 0 - form = AttributionNoListingAdminForm - - def get_queryset(self, request): - qs = super().get_queryset(request) - return qs.filter(spectacle__listing=False) + form = WithoutListingAttributionTabularAdminForm + listing = False class ParticipantAdminForm(forms.ModelForm): @@ -91,7 +82,7 @@ class ParticipantAdminForm(forms.ModelForm): class ParticipantAdmin(ReadOnlyMixin, admin.ModelAdmin): - inlines = [AttributionInlineListing, AttributionInlineNoListing] + inlines = [WithListingAttributionInline, WithoutListingAttributionInline] def get_queryset(self, request): return Participant.objects.annotate(nb_places=Count('attributions'), From b0e7ebfbc5ba73f0ea233439e9d0038d552d239f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Wed, 10 May 2017 12:49:14 +0200 Subject: [PATCH 38/43] fix typo + pep8 + del future imports --- kfet/backends.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/kfet/backends.py b/kfet/backends.py index c0aec699..fb9538d0 100644 --- a/kfet/backends.py +++ b/kfet/backends.py @@ -1,15 +1,12 @@ # -*- coding: utf-8 -*- -from __future__ import (absolute_import, division, - print_function, unicode_literals) -from builtins import * - import hashlib from django.contrib.auth.models import User, Permission from gestioncof.models import CofProfile from kfet.models import Account, GenericTeamToken + class KFetBackend(object): def authenticate(self, request): password = request.POST.get('KFETPASSWORD', '') @@ -18,13 +15,15 @@ class KFetBackend(object): return None try: - password_sha256 = hashlib.sha256(password.encode('utf-8')).hexdigest() + password_sha256 = ( + hashlib.sha256(password.encode('utf-8')) + .hexdigest() + ) account = Account.objects.get(password=password_sha256) - user = account.cofprofile.user + return account.cofprofile.user except Account.DoesNotExist: return None - return user class GenericTeamBackend(object): def authenticate(self, username=None, token=None): @@ -48,7 +47,7 @@ class GenericTeamBackend(object): try: return ( User.objects - .select_related('profile__acount_kfet') + .select_related('profile__account_kfet') .get(pk=user_id) ) except User.DoesNotExist: From b1e46792c8958c153b81248532d43697bafdace5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Wed, 10 May 2017 13:11:47 +0200 Subject: [PATCH 39/43] (little) cleaning of order_create view --- kfet/views.py | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/kfet/views.py b/kfet/views.py index c522bb85..7f06b80e 100644 --- a/kfet/views.py +++ b/kfet/views.py @@ -1823,23 +1823,17 @@ def order_create(request, pk): ) scale = WeekScale(last=True, n_steps=5, std_chunk=False) chunks = scale.chunkify_qs(sales_q, field='group__at') - sales = [] - for chunk in chunks: - sales.append( - {d['article']: d['nb'] for d in chunk} - ) + sales = [ + {d['article']: d['nb'] for d in chunk} + for chunk in chunks + ] initial = [] for article in articles: # Get sales for each 5 last weeks - v_s1 = sales[0].get(article.pk, 0) - v_s2 = sales[1].get(article.pk, 0) - v_s3 = sales[2].get(article.pk, 0) - v_s4 = sales[3].get(article.pk, 0) - v_s5 = sales[4].get(article.pk, 0) - v_all = [v_s1, v_s2, v_s3, v_s4, v_s5] + v_all = [chunk.get(article.pk, 0) for chunk in sales] # Take the 3 greatest (eg to avoid 2 weeks of vacations) v_3max = heapq.nlargest(3, v_all) # Get average and standard deviation @@ -1869,11 +1863,7 @@ def order_create(request, pk): 'category__name': article.category.name, 'stock': article.stock, 'box_capacity': article.box_capacity, - 'v_s1': v_s1, - 'v_s2': v_s2, - 'v_s3': v_s3, - 'v_s4': v_s4, - 'v_s5': v_s5, + 'v_all': v_all, 'v_moy': round(v_moy), 'v_et': round(v_et), 'v_prev': round(v_prev), From 4ac7b30bdd351ed4e4670e7288746d4e710518f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Fri, 12 May 2017 16:55:18 +0200 Subject: [PATCH 40/43] Fix UserGroupForm + tests for this form. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Non-K-Fêt group membership is no longer erased by the account edit form. - Add some tests to ensure proposed choices in this form corresponds to K-Fêt groups + test case for #161. Fixes #161 --- kfet/forms.py | 13 ++++++----- kfet/tests/test_forms.py | 48 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 6 deletions(-) create mode 100644 kfet/tests/test_forms.py diff --git a/kfet/forms.py b/kfet/forms.py index f89b8f08..826df257 100644 --- a/kfet/forms.py +++ b/kfet/forms.py @@ -2,6 +2,7 @@ from datetime import timedelta from decimal import Decimal +from itertools import chain from django import forms from django.core.exceptions import ValidationError @@ -134,6 +135,7 @@ class UserRestrictTeamForm(UserForm): class Meta(UserForm.Meta): fields = ['first_name', 'last_name', 'email'] + class UserGroupForm(forms.ModelForm): groups = forms.ModelMultipleChoiceField( Group.objects.filter(name__icontains='K-Fêt'), @@ -141,16 +143,15 @@ class UserGroupForm(forms.ModelForm): required=False) def clean_groups(self): - groups = self.cleaned_data.get('groups') - # Si aucun groupe, on le dénomme - if not groups: - groups = self.instance.groups.exclude(name__icontains='K-Fêt') - return groups + kfet_groups = self.cleaned_data.get('groups') + other_groups = self.instance.groups.exclude(name__icontains='K-Fêt') + return chain(kfet_groups, other_groups) class Meta: - model = User + model = User fields = ['groups'] + class GroupForm(forms.ModelForm): permissions = forms.ModelMultipleChoiceField( queryset= Permission.objects.filter(content_type__in= diff --git a/kfet/tests/test_forms.py b/kfet/tests/test_forms.py new file mode 100644 index 00000000..27c7b3d8 --- /dev/null +++ b/kfet/tests/test_forms.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +from django.test import TestCase +from django.contrib.auth.models import User, Group + +from kfet.forms import UserGroupForm + + +class UserGroupFormTests(TestCase): + """Test suite for UserGroupForm.""" + + def setUp(self): + # create user + self.user = User.objects.create(username="foo", password="foo") + + # create some K-Fêt groups + prefix_name = "K-Fêt " + names = ["Group 1", "Group 2", "Group 3"] + self.kfet_groups = [ + Group.objects.create(name=prefix_name+name) + for name in names + ] + + # create a non-K-Fêt group + self.other_group = Group.objects.create(name="Other group") + + def test_choices(self): + """Only K-Fêt groups are selectable.""" + form = UserGroupForm(instance=self.user) + groups_field = form.fields['groups'] + self.assertEqual(len(groups_field.choices), len(self.kfet_groups)) + + def test_keep_others(self): + """User stays in its non-K-Fêt groups.""" + user = self.user + + # add user to a non-K-Fêt group + user.groups.add(self.other_group) + + # add user to some K-Fêt groups through UserGroupForm + data = { + 'groups': [group.pk for group in self.kfet_groups], + } + form = UserGroupForm(data, instance=user) + + form.is_valid() + form.save() + self.assertEqual(len(user.groups.all()), 1+len(self.kfet_groups)) From e0b1db1e1e14341c512b4bddbe29a871004ac977 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delobelle?= Date: Sun, 14 May 2017 22:19:25 +0200 Subject: [PATCH 41/43] more robust tests --- kfet/forms.py | 3 +-- kfet/tests/test_forms.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/kfet/forms.py b/kfet/forms.py index 826df257..96787ddd 100644 --- a/kfet/forms.py +++ b/kfet/forms.py @@ -2,7 +2,6 @@ from datetime import timedelta from decimal import Decimal -from itertools import chain from django import forms from django.core.exceptions import ValidationError @@ -145,7 +144,7 @@ class UserGroupForm(forms.ModelForm): def clean_groups(self): kfet_groups = self.cleaned_data.get('groups') other_groups = self.instance.groups.exclude(name__icontains='K-Fêt') - return chain(kfet_groups, other_groups) + return list(kfet_groups) + list(other_groups) class Meta: model = User diff --git a/kfet/tests/test_forms.py b/kfet/tests/test_forms.py index 27c7b3d8..7f129a3f 100644 --- a/kfet/tests/test_forms.py +++ b/kfet/tests/test_forms.py @@ -28,7 +28,11 @@ class UserGroupFormTests(TestCase): """Only K-Fêt groups are selectable.""" form = UserGroupForm(instance=self.user) groups_field = form.fields['groups'] - self.assertEqual(len(groups_field.choices), len(self.kfet_groups)) + self.assertQuerysetEqual( + groups_field.queryset, + [repr(g) for g in self.kfet_groups], + ordered=False, + ) def test_keep_others(self): """User stays in its non-K-Fêt groups.""" @@ -45,4 +49,8 @@ class UserGroupFormTests(TestCase): form.is_valid() form.save() - self.assertEqual(len(user.groups.all()), 1+len(self.kfet_groups)) + self.assertQuerysetEqual( + user.groups.all(), + [repr(g) for g in [self.other_group] + self.kfet_groups], + ordered=False, + ) From bf0e3453018724f19e6305709b0b0ff997ce04c8 Mon Sep 17 00:00:00 2001 From: Ludovic Stephan Date: Sun, 14 May 2017 23:50:09 -0300 Subject: [PATCH 42/43] Fix asynchronous calls with resetSettings The `resetSettings` function now returns the ajax object, allowing to chain other calls to it, e.g. the `getHistory` function. --- kfet/templates/kfet/kpsul.html | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kfet/templates/kfet/kpsul.html b/kfet/templates/kfet/kpsul.html index 45416d06..7d943977 100644 --- a/kfet/templates/kfet/kpsul.html +++ b/kfet/templates/kfet/kpsul.html @@ -182,7 +182,7 @@ $(document).ready(function() { settings = {} function resetSettings() { - $.ajax({ + return $.ajax({ dataType: "json", url : "{% url 'kfet.kpsul.get_settings' %}", method : "POST", @@ -1431,9 +1431,10 @@ $(document).ready(function() { resetArticles(); resetPreviousOp(); khistory.reset(); - resetSettings(); - getArticles(); - getHistory(); + resetSettings().done(function (){ + getArticles(); + getHistory(); + }); } function resetSelectable() { From 647b32e727fd6930c72ce5c2df608690cbf4fd06 Mon Sep 17 00:00:00 2001 From: Ludovic Stephan Date: Mon, 15 May 2017 17:52:49 -0300 Subject: [PATCH 43/43] Move displayAddcost function --- kfet/templates/kfet/kpsul.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kfet/templates/kfet/kpsul.html b/kfet/templates/kfet/kpsul.html index 7d943977..80a384fe 100644 --- a/kfet/templates/kfet/kpsul.html +++ b/kfet/templates/kfet/kpsul.html @@ -191,7 +191,6 @@ $(document).ready(function() { settings['addcost_for'] = data['addcost_for']; settings['addcost_amount'] = parseFloat(data['addcost_amount']); settings['subvention_cof'] = parseFloat(data['subvention_cof']); - displayAddcost(); }); } @@ -1434,6 +1433,7 @@ $(document).ready(function() { resetSettings().done(function (){ getArticles(); getHistory(); + displayAddcost(); }); }