From 137e105f725d8e2334c0b3cde0243de4b1e1e2f4 Mon Sep 17 00:00:00 2001 From: Tomasz Nowak Date: Sun, 3 Dec 2023 00:47:31 +0100 Subject: [PATCH 1/4] add metrics page: "Product Tag Count" It is fully based on "Product Type Count" metrics page. --- docs/content/en/usage/features.md | 3 + dojo/forms.py | 18 +++- dojo/locale/en/LC_MESSAGES/django.po | 4 + dojo/metrics/urls.py | 2 + dojo/metrics/views.py | 156 ++++++++++++++++++++++++++- dojo/templates/base.html | 5 + dojo/templates/dojo/pt_counts.html | 10 +- dojo/utils.py | 10 +- 8 files changed, 195 insertions(+), 13 deletions(-) diff --git a/docs/content/en/usage/features.md b/docs/content/en/usage/features.md index fdd3e19480d..470c009bf71 100644 --- a/docs/content/en/usage/features.md +++ b/docs/content/en/usage/features.md @@ -557,6 +557,9 @@ Product Type Counts ![Product Type Counts](../../images/met_2.png) +Product Tag Counts +: Same as above, but for a group of products sharing a tag. + Simple Metrics : Provides tabular data for all Product Types. The data displayed in this view is the total number of S0, S1, S2, S3, S4, Opened This diff --git a/dojo/forms.py b/dojo/forms.py index 94c1e6ee9df..1014675bf69 100755 --- a/dojo/forms.py +++ b/dojo/forms.py @@ -2099,21 +2099,35 @@ def get_years(): return [(now.year, now.year), (now.year - 1, now.year - 1), (now.year - 2, now.year - 2)] -class ProductTypeCountsForm(forms.Form): +class ProductCountsFormBase(forms.Form): month = forms.ChoiceField(choices=list(MONTHS.items()), required=True, error_messages={ 'required': '*'}) year = forms.ChoiceField(choices=get_years, required=True, error_messages={ 'required': '*'}) + + +class ProductTypeCountsForm(ProductCountsFormBase): product_type = forms.ModelChoiceField(required=True, queryset=Product_Type.objects.none(), error_messages={ 'required': '*'}) def __init__(self, *args, **kwargs): - super(ProductTypeCountsForm, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.fields['product_type'].queryset = get_authorized_product_types(Permissions.Product_Type_View) +class ProductTagCountsForm(ProductCountsFormBase): + product_tag = forms.ModelChoiceField(required=True, + queryset=Product.tags.tag_model.objects.none().order_by('name'), + error_messages={ + 'required': '*'}) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields['product_tag'].queryset = Product.tags.tag_model.objects.all() + + class APIKeyForm(forms.ModelForm): id = forms.IntegerField(required=True, widget=forms.widgets.HiddenInput()) diff --git a/dojo/locale/en/LC_MESSAGES/django.po b/dojo/locale/en/LC_MESSAGES/django.po index dbb9e756559..ab26c8cbdb4 100644 --- a/dojo/locale/en/LC_MESSAGES/django.po +++ b/dojo/locale/en/LC_MESSAGES/django.po @@ -2692,6 +2692,10 @@ msgstr "" msgid "Product Type Counts" msgstr "" +#: dojo/templates/base.html +msgid "Product Tag Counts" +msgstr "" + #: dojo/templates/base.html msgid "Users" msgstr "" diff --git a/dojo/metrics/urls.py b/dojo/metrics/urls.py index 06b0726a56a..7b2683cf6f7 100644 --- a/dojo/metrics/urls.py +++ b/dojo/metrics/urls.py @@ -18,6 +18,8 @@ views.metrics, name='product_type_metrics'), re_path(r'^metrics/product/type/counts$', views.product_type_counts, name='product_type_counts'), + re_path(r'^metrics/product/tag/counts$', + views.product_tag_counts, name='product_tag_counts'), re_path(r'^metrics/engineer$', views.engineer_metrics, name='engineer_metrics'), re_path(r'^metrics/engineer/(?P\d+)$', views.view_engineer, diff --git a/dojo/metrics/views.py b/dojo/metrics/views.py index 342c7b1229c..07ce44c58ca 100644 --- a/dojo/metrics/views.py +++ b/dojo/metrics/views.py @@ -21,7 +21,7 @@ from django.utils import timezone from dojo.filters import MetricsFindingFilter, UserFilter, MetricsEndpointFilter -from dojo.forms import SimpleMetricsForm, ProductTypeCountsForm +from dojo.forms import SimpleMetricsForm, ProductTypeCountsForm, ProductTagCountsForm from dojo.models import Product_Type, Finding, Product, Engagement, Test, \ Risk_Acceptance, Dojo_User, Endpoint_Status from dojo.utils import get_page_items, add_breadcrumb, findings_this_period, opened_in_period, count_findings, \ @@ -587,13 +587,13 @@ def product_type_counts(request): end_date.month, end_date.day, tzinfo=timezone.get_current_timezone()) - oip = opened_in_period(start_date, end_date, pt) + oip = opened_in_period(start_date, end_date, test__engagement__product__prod_type=pt) # trending data - 12 months for x in range(12, 0, -1): opened_in_period_list.append( opened_in_period(start_date + relativedelta(months=-x), end_of_month + relativedelta(months=-x), - pt)) + test__engagement__product__prod_type=pt)) opened_in_period_list.append(oip) @@ -698,6 +698,156 @@ def product_type_counts(request): ) +def product_tag_counts(request): + form = ProductTagCountsForm() + opened_in_period_list = [] + oip = None + cip = None + aip = None + all_current_in_pt = None + top_ten = None + pt = None + today = timezone.now() + first_of_month = today.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + mid_month = first_of_month.replace(day=15, hour=23, minute=59, second=59, microsecond=999999) + end_of_month = mid_month.replace(day=monthrange(today.year, today.month)[1], hour=23, minute=59, second=59, + microsecond=999999) + start_date = first_of_month + end_date = end_of_month + + if request.method == 'GET' and 'month' in request.GET and 'year' in request.GET and 'product_tag' in request.GET: + form = ProductTagCountsForm(request.GET) + if form.is_valid(): + pt = form.cleaned_data['product_tag'] + user_has_permission_or_403(request.user, pt, Permissions.Product_Type_View) # FIXME + month = int(form.cleaned_data['month']) + year = int(form.cleaned_data['year']) + first_of_month = first_of_month.replace(month=month, year=year) + + month_requested = datetime(year, month, 1) + + end_of_month = month_requested.replace(day=monthrange(month_requested.year, month_requested.month)[1], + hour=23, minute=59, second=59, microsecond=999999) + start_date = first_of_month + start_date = datetime(start_date.year, + start_date.month, start_date.day, + tzinfo=timezone.get_current_timezone()) + end_date = end_of_month + end_date = datetime(end_date.year, + end_date.month, end_date.day, + tzinfo=timezone.get_current_timezone()) + + oip = opened_in_period(start_date, end_date, test__engagement__product__tags__name=pt) + + # trending data - 12 months + for x in range(12, 0, -1): + opened_in_period_list.append( + opened_in_period(start_date + relativedelta(months=-x), end_of_month + relativedelta(months=-x), + test__engagement__product__tags__name=pt)) + + opened_in_period_list.append(oip) + + closed_in_period = Finding.objects.filter(mitigated__date__range=[start_date, end_date], + test__engagement__product__tags__name=pt, + severity__in=('Critical', 'High', 'Medium', 'Low')).values( + 'numerical_severity').annotate(Count('numerical_severity')).order_by('numerical_severity') + + total_closed_in_period = Finding.objects.filter(mitigated__date__range=[start_date, end_date], + test__engagement__product__tags__name=pt, + severity__in=( + 'Critical', 'High', 'Medium', 'Low')).aggregate( + total=Sum( + Case(When(severity__in=('Critical', 'High', 'Medium', 'Low'), + then=Value(1)), + output_field=IntegerField())))['total'] + + overall_in_pt = Finding.objects.filter(date__lt=end_date, + verified=True, + false_p=False, + duplicate=False, + out_of_scope=False, + mitigated__isnull=True, + test__engagement__product__tags__name=pt, + severity__in=('Critical', 'High', 'Medium', 'Low')).values( + 'numerical_severity').annotate(Count('numerical_severity')).order_by('numerical_severity') + + total_overall_in_pt = Finding.objects.filter(date__lte=end_date, + verified=True, + false_p=False, + duplicate=False, + out_of_scope=False, + mitigated__isnull=True, + test__engagement__product__tags__name=pt, + severity__in=('Critical', 'High', 'Medium', 'Low')).aggregate( + total=Sum( + Case(When(severity__in=('Critical', 'High', 'Medium', 'Low'), + then=Value(1)), + output_field=IntegerField())))['total'] + + all_current_in_pt = Finding.objects.filter(date__lte=end_date, + verified=True, + false_p=False, + duplicate=False, + out_of_scope=False, + mitigated__isnull=True, + test__engagement__product__tags__name=pt, + severity__in=( + 'Critical', 'High', 'Medium', 'Low')).prefetch_related( + 'test__engagement__product', + 'test__engagement__product__prod_type', + 'test__engagement__risk_acceptance', + 'reporter').order_by( + 'numerical_severity') + + top_ten = Product.objects.filter(engagement__test__finding__date__lte=end_date, + engagement__test__finding__verified=True, + engagement__test__finding__false_p=False, + engagement__test__finding__duplicate=False, + engagement__test__finding__out_of_scope=False, + engagement__test__finding__mitigated__isnull=True, + engagement__test__finding__severity__in=( + 'Critical', 'High', 'Medium', 'Low'), + tags__name=pt) + top_ten = severity_count(top_ten, 'annotate', 'engagement__test__finding__severity').order_by('-critical', '-high', '-medium', '-low')[:10] + + cip = {'S0': 0, + 'S1': 0, + 'S2': 0, + 'S3': 0, + 'Total': total_closed_in_period} + + aip = {'S0': 0, + 'S1': 0, + 'S2': 0, + 'S3': 0, + 'Total': total_overall_in_pt} + + for o in closed_in_period: + cip[o['numerical_severity']] = o['numerical_severity__count'] + + for o in overall_in_pt: + aip[o['numerical_severity']] = o['numerical_severity__count'] + else: + messages.add_message(request, messages.ERROR, _("Please choose month and year and the Product Type."), + extra_tags='alert-danger') + + add_breadcrumb(title=_("Bi-Weekly Metrics"), top_level=True, request=request) + + return render(request, + 'dojo/pt_counts.html', + {'form': form, + 'start_date': start_date, + 'end_date': end_date, + 'opened_in_period': oip, + 'trending_opened': opened_in_period_list, + 'closed_in_period': cip, + 'overall_in_pt': aip, + 'all_current_in_pt': all_current_in_pt, + 'top_ten': top_ten, + 'pt': pt} + ) + + def engineer_metrics(request): # only superusers can select other users to view if request.user.is_superuser: diff --git a/dojo/templates/base.html b/dojo/templates/base.html index 7157a738964..7a4200ea9cd 100644 --- a/dojo/templates/base.html +++ b/dojo/templates/base.html @@ -413,6 +413,11 @@ {% trans "Product Type Counts" %} +
  • + + {% trans "Product Tag Counts" %} + +
  • {% trans "Simple Metrics" %} diff --git a/dojo/templates/dojo/pt_counts.html b/dojo/templates/dojo/pt_counts.html index 0c8728b42c1..5cfc6a96934 100644 --- a/dojo/templates/dojo/pt_counts.html +++ b/dojo/templates/dojo/pt_counts.html @@ -12,7 +12,7 @@ {% block content %} {{ block.super }} -
    + {{ form.as_p }}
    @@ -20,8 +20,12 @@ {% if pt %}

    {% blocktrans with start_date=start_date.date end_date=end_date.date%}Finding Information For Period of {{ start_date }} - {{ end_date }} {% endblocktrans %}

    -

    {{ pt.name }}

    [ -     {% trans "View Details" %}] +

    {{ pt.name }}

    + {% if pt|class_name == "Product_Type" %} + [{% trans "View Details" %}] + {% elif pt|class_name == "Tagulous_Product_tags" %} + [{% trans "View Details" %}] + {% endif %}

    {% trans "Total Security Bug Count In Period" %}

    diff --git a/dojo/utils.py b/dojo/utils.py index 40cc68f192d..fa17fc3e4c6 100644 --- a/dojo/utils.py +++ b/dojo/utils.py @@ -1085,7 +1085,7 @@ def get_period_counts(findings, } -def opened_in_period(start_date, end_date, pt): +def opened_in_period(start_date, end_date, **kwargs): start_date = datetime( start_date.year, start_date.month, @@ -1098,7 +1098,7 @@ def opened_in_period(start_date, end_date, pt): tzinfo=timezone.get_current_timezone()) opened_in_period = Finding.objects.filter( date__range=[start_date, end_date], - test__engagement__product__prod_type=pt, + **kwargs, verified=True, false_p=False, duplicate=False, @@ -1110,7 +1110,7 @@ def opened_in_period(start_date, end_date, pt): Count('numerical_severity')).order_by('numerical_severity') total_opened_in_period = Finding.objects.filter( date__range=[start_date, end_date], - test__engagement__product__prod_type=pt, + **kwargs, verified=True, false_p=False, duplicate=False, @@ -1142,7 +1142,7 @@ def opened_in_period(start_date, end_date, pt): 'closed': Finding.objects.filter( mitigated__date__range=[start_date, end_date], - test__engagement__product__prod_type=pt, + **kwargs, severity__in=('Critical', 'High', 'Medium', 'Low')).aggregate( total=Sum( Case( @@ -1158,7 +1158,7 @@ def opened_in_period(start_date, end_date, pt): duplicate=False, out_of_scope=False, mitigated__isnull=True, - test__engagement__product__prod_type=pt, + **kwargs, severity__in=('Critical', 'High', 'Medium', 'Low')).count() } From 7b86aa84111d4857cbea9cf8dd9a8b1e9988a368 Mon Sep 17 00:00:00 2001 From: Tomasz Nowak Date: Thu, 4 Jan 2024 20:36:21 +0100 Subject: [PATCH 2/4] fixup! add metrics page: "Product Tag Count" --- dojo/forms.py | 4 +++- dojo/metrics/views.py | 19 ++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/dojo/forms.py b/dojo/forms.py index 1014675bf69..abf491d6b85 100755 --- a/dojo/forms.py +++ b/dojo/forms.py @@ -2125,7 +2125,9 @@ class ProductTagCountsForm(ProductCountsFormBase): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields['product_tag'].queryset = Product.tags.tag_model.objects.all() + prods = get_authorized_products(Permissions.Product_View) + tags_available_to_user = Product.tags.tag_model.objects.filter(product__in=prods) + self.fields['product_tag'].queryset = tags_available_to_user class APIKeyForm(forms.ModelForm): diff --git a/dojo/metrics/views.py b/dojo/metrics/views.py index 07ce44c58ca..e21639b0591 100644 --- a/dojo/metrics/views.py +++ b/dojo/metrics/views.py @@ -718,8 +718,9 @@ def product_tag_counts(request): if request.method == 'GET' and 'month' in request.GET and 'year' in request.GET and 'product_tag' in request.GET: form = ProductTagCountsForm(request.GET) if form.is_valid(): + prods = get_authorized_products(Permissions.Product_View) + pt = form.cleaned_data['product_tag'] - user_has_permission_or_403(request.user, pt, Permissions.Product_Type_View) # FIXME month = int(form.cleaned_data['month']) year = int(form.cleaned_data['year']) first_of_month = first_of_month.replace(month=month, year=year) @@ -737,23 +738,28 @@ def product_tag_counts(request): end_date.month, end_date.day, tzinfo=timezone.get_current_timezone()) - oip = opened_in_period(start_date, end_date, test__engagement__product__tags__name=pt) + oip = opened_in_period(start_date, end_date, + test__engagement__product__tags__name=pt, + test__engagement__product__in=prods, + ) # trending data - 12 months for x in range(12, 0, -1): opened_in_period_list.append( opened_in_period(start_date + relativedelta(months=-x), end_of_month + relativedelta(months=-x), - test__engagement__product__tags__name=pt)) + test__engagement__product__tags__name=pt, test__engagement__product__in=prods)) opened_in_period_list.append(oip) closed_in_period = Finding.objects.filter(mitigated__date__range=[start_date, end_date], test__engagement__product__tags__name=pt, + test__engagement__product__in=prods, severity__in=('Critical', 'High', 'Medium', 'Low')).values( 'numerical_severity').annotate(Count('numerical_severity')).order_by('numerical_severity') total_closed_in_period = Finding.objects.filter(mitigated__date__range=[start_date, end_date], test__engagement__product__tags__name=pt, + test__engagement__product__in=prods, severity__in=( 'Critical', 'High', 'Medium', 'Low')).aggregate( total=Sum( @@ -768,6 +774,7 @@ def product_tag_counts(request): out_of_scope=False, mitigated__isnull=True, test__engagement__product__tags__name=pt, + test__engagement__product__in=prods, severity__in=('Critical', 'High', 'Medium', 'Low')).values( 'numerical_severity').annotate(Count('numerical_severity')).order_by('numerical_severity') @@ -778,6 +785,7 @@ def product_tag_counts(request): out_of_scope=False, mitigated__isnull=True, test__engagement__product__tags__name=pt, + test__engagement__product__in=prods, severity__in=('Critical', 'High', 'Medium', 'Low')).aggregate( total=Sum( Case(When(severity__in=('Critical', 'High', 'Medium', 'Low'), @@ -791,6 +799,7 @@ def product_tag_counts(request): out_of_scope=False, mitigated__isnull=True, test__engagement__product__tags__name=pt, + test__engagement__product__in=prods, severity__in=( 'Critical', 'High', 'Medium', 'Low')).prefetch_related( 'test__engagement__product', @@ -807,7 +816,7 @@ def product_tag_counts(request): engagement__test__finding__mitigated__isnull=True, engagement__test__finding__severity__in=( 'Critical', 'High', 'Medium', 'Low'), - tags__name=pt) + tags__name=pt, engagement__product__in=prods) top_ten = severity_count(top_ten, 'annotate', 'engagement__test__finding__severity').order_by('-critical', '-high', '-medium', '-low')[:10] cip = {'S0': 0, @@ -828,7 +837,7 @@ def product_tag_counts(request): for o in overall_in_pt: aip[o['numerical_severity']] = o['numerical_severity__count'] else: - messages.add_message(request, messages.ERROR, _("Please choose month and year and the Product Type."), + messages.add_message(request, messages.ERROR, _("Please choose month and year and the Product Tag."), extra_tags='alert-danger') add_breadcrumb(title=_("Bi-Weekly Metrics"), top_level=True, request=request) From a7ad33b0080b521678a2f961ee600ae267a168b4 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Wed, 31 Jan 2024 15:13:14 -0600 Subject: [PATCH 3/4] Fix Flake8 --- dojo/metrics/views.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dojo/metrics/views.py b/dojo/metrics/views.py index e21639b0591..37105d4dc34 100644 --- a/dojo/metrics/views.py +++ b/dojo/metrics/views.py @@ -20,7 +20,7 @@ from django.views.decorators.cache import cache_page from django.utils import timezone -from dojo.filters import MetricsFindingFilter, UserFilter, MetricsEndpointFilter +from dojo.filters import MetricsFindingFilter, UserFilter, MetricsEndpointF ilter from dojo.forms import SimpleMetricsForm, ProductTypeCountsForm, ProductTagCountsForm from dojo.models import Product_Type, Finding, Product, Engagement, Test, \ Risk_Acceptance, Dojo_User, Endpoint_Status @@ -740,8 +740,7 @@ def product_tag_counts(request): oip = opened_in_period(start_date, end_date, test__engagement__product__tags__name=pt, - test__engagement__product__in=prods, - ) + test__engagement__product__in=prods) # trending data - 12 months for x in range(12, 0, -1): From 8b08a88b1b5401e457877f7619c7eb35ff0d5471 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Wed, 31 Jan 2024 15:17:23 -0600 Subject: [PATCH 4/4] Update views.py --- dojo/metrics/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dojo/metrics/views.py b/dojo/metrics/views.py index 37105d4dc34..a80467188aa 100644 --- a/dojo/metrics/views.py +++ b/dojo/metrics/views.py @@ -20,7 +20,7 @@ from django.views.decorators.cache import cache_page from django.utils import timezone -from dojo.filters import MetricsFindingFilter, UserFilter, MetricsEndpointF ilter +from dojo.filters import MetricsFindingFilter, UserFilter, MetricsEndpointFilter from dojo.forms import SimpleMetricsForm, ProductTypeCountsForm, ProductTagCountsForm from dojo.models import Product_Type, Finding, Product, Engagement, Test, \ Risk_Acceptance, Dojo_User, Endpoint_Status