Add: http custom error pages, more checks js possible error, check if file delete from storage by cli

Dockerfile

@@ -29,7 +29,8 @@ ARG VERSION=0.1.0 \
     USER_NAME=collector \
     USER_GROUP=collector \
     APP_UID=1000 \
-    APP_GID=1000
+    APP_GID=1000 \
+    HEALTHCHECK_PATH=api/v1/check/health/
 
 # copy app dependences
 COPY --from=base /usr/local/lib/python3.10/site-packages/ /usr/local/lib/python3.10/site-packages/
@@ -61,7 +62,7 @@ LABEL maintainer="s.zhukovskii@ispsystem.com"
 LABEL me.zhukovsky.logs-collector.version=v${VERSION}
 
 # call the health check endpoint of app
-HEALTHCHECK CMD curl --fail http://localhost:${WEB_PORT} || exit 1
+HEALTHCHECK CMD curl --fail http://localhost:${WEB_PORT}/${HEALTHCHECK_PATH} || exit 1
 
 # run app
 ENTRYPOINT [ "sh", "entrypoint.sh" ]

logs_collector/collector/api/permissions.py

@@ -11,3 +11,15 @@ class IsGuestUpload(permissions.BasePermission):
             return True
 
         return request.user.is_authenticated
+
+
+class IsGuestCheckUrls(permissions.BasePermission):
+    """
+    Special permission class for the ability to upload attachments
+    to an unauthorized user using a ticket token
+    """
+    def has_permission(self, request, view):
+        if request.method in ('HEAD', 'OPTIONS', 'GET',):
+            return True
+
+        return request.user.is_authenticated

logs_collector/collector/api/serializers.py

@@ -67,3 +67,25 @@ class StorageInfoSerializer(serializers.Serializer):
     free = serializers.IntegerField(read_only=True)
     used_percent = serializers.IntegerField(read_only=True)
     status = serializers.CharField(read_only=True)
+
+
+class TokenStateRootSerializer(serializers.Serializer):
+    info = serializers.CharField(read_only=True, default="manual message")
+
+
+class TokenStateSerializer(serializers.ModelSerializer):
+    token = serializers.UUIDField(read_only=True)
+    attempts = serializers.IntegerField(read_only=True)
+    resolved = serializers.BooleanField(read_only=True)
+
+    class Meta:
+        model = Ticket
+        fields = [
+            'token',
+            'attempts',
+            'resolved'
+        ]
+
+
+class AppHealthInfoSerializer(serializers.Serializer):
+    status = serializers.CharField(read_only=True, default="ok")

logs_collector/collector/api/urls.py

@@ -15,8 +15,31 @@ router.register(r'archives', views.ArchiveViewSet)
 router.register(r'platforms', views.PlatformViewSet)
 router.register(r'tickets', views.TicketViewSet)
 
+check_urlpatterns = [
+    path(
+        'health/',
+        views.AppHealthInfo.as_view(),
+        name='app-info'
+    ),
+    path(
+        'storage/',
+        views.StorageInfo.as_view(),
+        name='storage-info'
+    ),
+    path(
+        'token/',
+        views.TokenStateRoot.as_view(),
+        name='token-root'
+    ),
+    path(
+        'token/<str:token>',
+        views.TokenStateInfo.as_view(),
+        name='token-info'
+    ),
+]
+
 urlpatterns = [
     # CRUD:
     path('v1/', include(router.urls)),
-    path('v1/storage/', views.StorageInfo.as_view(), name='storage-info'),
+    path('v1/check/', include(check_urlpatterns)),
 ]

logs_collector/collector/api/views.py

@@ -10,9 +10,7 @@ from rest_framework.parsers import (
 )
 from rest_framework.permissions import IsAuthenticated
 from rest_framework.response import Response
-from rest_framework import viewsets
-from rest_framework import views
-from rest_framework import filters
+from rest_framework import filters, generics, views, viewsets
 
 from django_filters.rest_framework import DjangoFilterBackend
 
@@ -23,13 +21,16 @@ from collector.models import Archive, Ticket, Platform
 from collector.utils.helpers import get_mount_fs_info
 
 from .filters import ArchiveFilter, TicketFilter
-from .permissions import IsGuestUpload
+from .permissions import IsGuestUpload, IsGuestCheckUrls
 from .serializers import (
     PublicArchiveUploadSerializer,
     ArchiveSerializer,
     PlatformSerializer,
     TicketSerializer,
     StorageInfoSerializer,
+    TokenStateSerializer,
+    AppHealthInfoSerializer,
+    TokenStateRootSerializer,
 )
 
 
@@ -82,29 +83,35 @@ class ArchiveViewSet(viewsets.ModelViewSet):
                 bound_ticket = Ticket.objects.get(token=upload_token)
                 if bound_ticket.resolved:
                     return Response(
-                        {'error': f'ticket {bound_ticket} already resolved'},
+                        {'detail': f'ticket {bound_ticket} already resolved'},
                         status=status.HTTP_423_LOCKED
                     )
                 if bound_ticket.attempts <= 0:
                     return Response(
-                        {'error': f'token {upload_token} expired'},
+                        {'detail': f'token {upload_token} expired'},
                         status=status.HTTP_423_LOCKED
                     )
                 bound_ticket.attempts -= 1
                 bound_ticket.save()
                 # ? mixin bound ticket number to request.data from user
-                request.data['ticket'] = bound_ticket.number
+                try:
+                    request.data['ticket'] = bound_ticket.number
+                except AttributeError:
+                    return Response(
+                        {'detail': 'Bad Request'},
+                        status=status.HTTP_400_BAD_REQUEST
+                    )
                 # ? change serializer for guest user
                 if not request.user.is_authenticated:
                     self.serializer_class = PublicArchiveUploadSerializer
             except (ValidationError, ObjectDoesNotExist,):
                 return Response(
-                    {'error': f'token {upload_token} is not valid'},
+                    {'detail': f'token {upload_token} is not valid'},
                     status=status.HTTP_403_FORBIDDEN
                 )
         else:
             return Response(
-                {'error': 'Header Upload-Token is required'},
+                {'detail': 'Header Upload-Token is required'},
                 status=status.HTTP_401_UNAUTHORIZED
             )
         # ! default create method:
@@ -172,3 +179,40 @@ class StorageInfo(views.APIView):
     )
     def get(self, request):
         return Response(get_mount_fs_info(settings.DATA_DIR))
+
+
+class TokenStateRoot(views.APIView):
+    """ Show the message of a specific upload token URL"""
+    permission_classes = (IsGuestCheckUrls,)
+
+    @extend_schema(
+        responses=TokenStateRootSerializer,
+        summary='Show info message how get token status'
+    )
+    def get(self, request):
+        message = "to find out the status of the token, place it in the URL"
+        return Response({"detail": message}, status=status.HTTP_303_SEE_OTHER)
+
+
+@extend_schema_view(
+    get=extend_schema(
+        summary='Show the status of a specific upload token'
+    )
+)
+class TokenStateInfo(generics.RetrieveAPIView):
+    """ Show the status of a specific upload token"""
+    queryset = Ticket.objects.order_by('-time_create')
+    lookup_field = 'token'
+    serializer_class = TokenStateSerializer
+    permission_classes = (IsGuestCheckUrls,)
+
+
+class AppHealthInfo(views.APIView):
+    permission_classes = (IsGuestCheckUrls,)
+
+    @extend_schema(
+        responses=AppHealthInfoSerializer,
+        summary='Show app status'
+    )
+    def get(self, request):
+        return Response({'status': 'ok'}, status=status.HTTP_200_OK)

logs_collector/collector/management/commands/cleanup_media.py

@@ -0,0 +1,59 @@
+import os
+from django.core.management.base import BaseCommand
+from django.apps import apps
+from django.db.models import Q
+from django.conf import settings
+from django.db.models import FileField
+
+
+class Command(BaseCommand):
+    # HELP MESSAGE:
+    help_part1 = 'This command deletes all media files from'
+    help_part2 = 'the MEDIA_ROOT directory which are no longer referenced'
+    help_part3 = 'by any of the models from installed_apps'
+    help = f'{help_part1} {help_part2} {help_part3}'
+
+    def handle(self, *args, **options):
+        all_models = apps.get_models()
+        physical_files = set()
+        db_files = set()
+        # Get all files from the database
+        for model in all_models:
+            file_fields = []
+            filters = Q()
+            for f_ in model._meta.fields:
+                if isinstance(f_, FileField):
+                    file_fields.append(f_.name)
+                    is_null = {'{}__isnull'.format(f_.name): True}
+                    is_empty = {'{}__exact'.format(f_.name): ''}
+                    filters &= Q(**is_null) | Q(**is_empty)
+            # only retrieve the models which have non-empty,
+            # non-null file fields
+            if file_fields:
+                files = model.objects.exclude(filters).values_list(
+                    *file_fields,
+                    flat=True
+                ).distinct()
+                db_files.update(files)
+        # Get all files from the MEDIA_ROOT, recursively
+        media_root = getattr(settings, 'MEDIA_ROOT', None)
+        if media_root is not None:
+            for relative_root, dirs, files in os.walk(media_root):
+                for file_ in files:
+                    # Compute the relative file path to the media directory,
+                    # so it can be compared to the values from the db
+                    relative_file = os.path.join(
+                        os.path.relpath(relative_root, media_root), file_
+                    )
+                    physical_files.add(relative_file)
+        # Compute the difference and delete those files
+        deletables = physical_files - db_files
+        if deletables:
+            for file_ in deletables:
+                os.remove(os.path.join(media_root, file_))
+            # Bottom-up - delete all empty folders
+            for relative_root, dirs, files in os.walk(
+                    media_root, topdown=False):
+                for dir_ in dirs:
+                    if not os.listdir(os.path.join(relative_root, dir_)):
+                        os.rmdir(os.path.join(relative_root, dir_))

logs_collector/collector/middleware.py

@@ -0,0 +1,24 @@
+from django.http import HttpResponse
+from django.template import loader
+
+
+class HttpResponseNotAllowedMiddleware:
+    def __init__(self, get_response):
+        self.get_response = get_response
+        # One-time configuration and initialization.
+
+    def __call__(self, request):
+
+        # Code to be executed for each request before
+        # the view (and later middleware) are called.
+
+        response = self.get_response(request)
+
+        # Code to be executed for each request/response after
+        # the view is called.
+        if response.status_code == 405:
+            context = {}
+            template = loader.get_template('405.html')
+            return HttpResponse(template.render(context, request))
+
+        return response

logs_collector/collector/static/collector/js/helpers.js

@@ -66,4 +66,18 @@ const updateStorageInfo = () => {
     });
 };
 
-export {sizify, updateBsTooltip, updateStorageInfo};
+const genAlertMessage = (
+        alertMessage='Success message',
+        alertType='success',
+        extraClass=''
+        ) => {
+    let alertMessageHTML = [
+        `<div class="alert alert-${alertType} alert-dismissible ${extraClass}" role="alert">`,
+        `   <div>${alertMessage}</div>`,
+        '   <button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>',
+        '</div>'
+    ].join('')
+    return alertMessageHTML
+}
+
+export {sizify, updateBsTooltip, updateStorageInfo, genAlertMessage};

logs_collector/collector/static/collector/js/jq.upload.progress.js

@@ -1,88 +1,138 @@
-import {updateStorageInfo} from "./helpers.js";
+import {updateStorageInfo, genAlertMessage} from "./helpers.js";
 
 $(function () {
+    // set global variables:
     const uploadForm = document.getElementById('upload_form');
-    const input_file = document.getElementById('id_file');
-    const progress_bar = document.getElementById('progress');
-    const alert_container = document.getElementById('alert');
-    
+    const inputFile = document.getElementById('id_file');
+    const progressBar = document.getElementById('progress');
+    const alertContainer = document.getElementById('alert');
+    // get upload form:
     $("#upload_form").submit(function(e){
         e.preventDefault();
-        // $form = $(this)
+        // collect request data:
         let formData = new FormData(this);
-        let upload_token = formData.get("token")
-        const media_data = input_file.files[0];
-        if(media_data != null){
-            progress_bar.classList.remove("not-visible");
-        }
-        $.ajax({
-            type: 'POST',
-            url: progress_bar.getAttribute("upload-url"),
-            data: formData,
-            dataType: 'json',
-            xhr:function(){
-                const xhr = new window.XMLHttpRequest();
-                xhr.timeout = 3600000; // increase request timeout to 1 hour
-                xhr.upload.addEventListener('progress', e=>{
-                    if(e.lengthComputable){
-                        const percentProgress = (e.loaded/e.total)*100;
-                        console.log(percentProgress);
-                        progress_bar.innerHTML = `
-                        <div
-                            class="progress-bar progress-bar-striped progress-bar-animated"
-                            style="width: ${percentProgress}%"
-                        >
-                        </div>`
+        let uploadToken = formData.get("token")
+        // generate the URL for token validation:
+        let tokenStatusUrl = [
+            progressBar.getAttribute('token-status-url'),
+            uploadToken
+        ].join('')
+        // init upload file func:
+        const uploadFile = () => {
+            // toggle visible progress bar:
+            const mediaData = inputFile.files[0];
+            if(mediaData != null){
+                progressBar.classList.remove("not-visible");
+            }
+            // upload file (chunk) xrh request:
+            $.ajax({
+                type: 'POST',
+                url: progressBar.getAttribute("upload-url"),
+                data: formData,
+                dataType: 'json',
+                xhr:function(){
+                    const xhr = new window.XMLHttpRequest();
+                    xhr.timeout = 3600000; // increase request timeout to 1 hour
+                    xhr.upload.addEventListener('progress', e=>{
+                        if(e.lengthComputable){
+                            const percentProgress = (e.loaded/e.total)*100;
+                            console.log(percentProgress);
+                            progressBar.innerHTML = `
+                            <div
+                                class="progress-bar progress-bar-striped progress-bar-animated"
+                                style="width: ${percentProgress}%"
+                            >
+                            </div>`
+                        }
+                    });
+                    return xhr
+                },
+                // set auth method:
+                beforeSend: function(xhr) {
+                    if (uploadToken) {
+                        xhr.setRequestHeader("Upload-Token", uploadToken);
                     }
-                });
-                return xhr
-            },
-            beforeSend: function(xhr) {
-                if (upload_token) {
-                    xhr.setRequestHeader("Upload-Token", upload_token);
+                },
+                success: function(data, textStatus, jqXHR){
+                    alertContainer.innerHTML = genAlertMessage(
+                        'The file has been successfully uploaded to the server. Thank you!',
+                        'success',
+                        'col-lg-6'
+                    )
+                    uploadForm.reset()
+                    progressBar.classList.add('not-visible')
+                    try {
+                        updateStorageInfo();
+                    } catch (error) {
+                        console.log(error)
+                    };
+                },
+                error: function(jqXHR, textStatus, errorThrown){
+                    let errorMessage = "Unexpected error. Try again please"
+                    if (jqXHR.status === 423 || jqXHR.status === 403) {
+                        errorMessage = `Error ${jqXHR.status} <br> ${jqXHR.responseJSON.detail}`
+                    }
+                    if (jqXHR.status === 401) {
+                        errorMessage = `Error ${jqXHR.status} <br> The token field cannot be empty`
+                    }
+                    if (jqXHR.status === 400) {
+                        errorMessage = `Error ${jqXHR.status} <br> ${jqXHR.responseJSON.detail}`
+                    }
+                    alertContainer.innerHTML = genAlertMessage(
+                        errorMessage,
+                        'danger',
+                        'col-lg-6'
+                    )
+                    progressBar.classList.add('not-visible')
+                },
+                cache: false,
+                contentType: false,
+                processData: false,
+            });
+        }
+        // check token status and upload file if token valid:
+        $.ajax({
+            type: 'GET',
+            url: tokenStatusUrl,
+            dataType: "json",
+            success: function (data, textStatus, jqXHR) {
+                if (data.attempts === 0) {
+                    alertContainer.innerHTML = genAlertMessage(
+                        `Error 423 <br> Token: ${uploadToken} expired`,
+                        'danger',
+                        'col-lg-6'
+                    );
                 }
-            },
-            success: function(data, textStatus, jqXHR){
-                console.log(jqXHR.status);
-                let type = "success";
-                alert_container.innerHTML = [
-                    `<div class="alert alert-${type} alert-dismissible col-lg-6" role="alert">`,
-                    `   <div>The file has been successfully uploaded to the server. Thank you!</div>`,
-                    '   <button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>',
-                    '</div>'
-                ].join('')
-                uploadForm.reset()
-                progress_bar.classList.add('not-visible')
-                try {
-                    updateStorageInfo();
-                } catch (error) {
-                    console.log(error)
+                else if (data.resolved === true) {
+                    alertContainer.innerHTML = genAlertMessage(
+                        `Error 423 <br> Ticket bound with token: ${uploadToken} <br> already resolved`,
+                        'danger',
+                        'col-lg-6'
+                    );
+                } else {
+                    alertContainer.innerHTML = genAlertMessage(
+                        `Token: ${uploadToken} is valid. <br> Starting to upload...`,
+                        'success',
+                        'col-lg-6'
+                    );
+                    uploadFile();
                 };
             },
-            error: function(jqXHR, textStatus, errorThrown){
-                console.log(jqXHR);
-                let type = "danger";
-                let error_message = "Unexpected error. Try again please"
-                if (jqXHR.status === 423) {
-                    error_message = `Error ${jqXHR.status}: ${jqXHR.responseJSON.error}`
+            error: function(jqXHR){
+                if (jqXHR.responseJSON.detail) {
+                    alertContainer.innerHTML = genAlertMessage(
+                        `Error 403 <br> Token: ${uploadToken} is not valid`,
+                        'danger',
+                        'col-lg-6'
+                    )
+                } else {
+                    alertContainer.innerHTML = genAlertMessage(
+                        `Unexpected error. Try again please`,
+                        'danger',
+                        'col-lg-6'
+                    )
                 }
-                if (jqXHR.status === 403) {
-                    error_message = `Error ${jqXHR.status}: ${jqXHR.responseJSON.error}`
-                }
-                if (jqXHR.status === 401) {
-                    error_message = 'The token field cannot be empty'
-                }
-                alert_container.innerHTML = [
-                    `<div class="alert alert-${type} alert-dismissible col-lg-6" role="alert">`,
-                    `   <div>${error_message}</div>`,
-                    '   <button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>',
-                    '</div>'
-                ].join('')
-                progress_bar.classList.add('not-visible')
             },
-            cache: false,
-            contentType: false,
-            processData: false,
         });
     });
 });

logs_collector/collector/templates/collector/archive_upload.html

@@ -35,7 +35,8 @@
           <div
             id="progress"
             upload-url="{% url 'collector_api:archive-list' %}"
-            class="progress"
+            token-status-url="{% url 'collector_api:token-root' %}"
+            class="progress not-visible"
             role="progressbar"
             aria-label="Example 20px high"
             aria-valuenow="25"

logs_collector/collector/views.py

@@ -1,5 +1,5 @@
 from django.contrib.auth.mixins import LoginRequiredMixin
-from django.http import FileResponse
+from django.http import FileResponse, Http404
 from django.views import generic
 from django.views.generic.detail import SingleObjectMixin
 from django.db.models import Q
@@ -35,6 +35,10 @@ class ArchiveHandlerView(
 
     def get(self, request, path):
         self.object = self.get_object()
+        try:
+            self.object.file.size
+        except FileNotFoundError:
+            raise Http404(f'File: {self.object.file} not found')
         return FileResponse(self.object.file)
 
 

logs_collector/logs_collector/settings.py

@@ -89,6 +89,7 @@ MIDDLEWARE = [
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'whitenoise.middleware.WhiteNoiseMiddleware',
+    'collector.middleware.HttpResponseNotAllowedMiddleware',
 ]
 
 ROOT_URLCONF = 'logs_collector.urls'

logs_collector/templates/403_csrf.html

@@ -0,0 +1,10 @@
+{% extends 'errors.html' %}
+{% load static %}
+
+{% block title %} Logs Collector - CSRF error {% endblock title %}
+
+{% block status_code %}403{% endblock status_code %}
+{% block error_message %}
+<p class="fs-3"> <span class="text-danger">Opps!</span> CSRF verification failed.</p>
+<p class="lead">Request aborted</p>
+{% endblock error_message %}

logs_collector/templates/404.html

@@ -0,0 +1,10 @@
+{% extends 'errors.html' %}
+{% load static %}
+
+{% block title %} Logs Collector - Not Found {% endblock title %}
+
+{% block status_code %}404{% endblock status_code %}
+{% block error_message %}
+<p class="fs-3"> <span class="text-danger">Opps!</span> Page not found.</p>
+<p class="lead">The content you're looking for doesn't exist.</p>
+{% endblock error_message %}

logs_collector/templates/405.html

@@ -0,0 +1,14 @@
+{% extends 'errors.html' %}
+{% load static %}
+
+{% block title %} Logs Collector - Method not allowed {% endblock title %}
+
+{% block status_code %}405{% endblock status_code %}
+{% block error_message %}
+<p class="fs-3"> <span class="text-danger">Opps!</span> Method not allowed</p>
+<p class="lead">
+  Request method: 
+  <span class="text-danger">{{ request.method }}</span> 
+  isn't allowed for this URL
+</p>
+{% endblock error_message %}

logs_collector/templates/500.html

@@ -0,0 +1,10 @@
+{% extends 'errors.html' %}
+{% load static %}
+
+{% block title %} Logs Collector - Server error {% endblock title %}
+
+{% block status_code %}500{% endblock status_code %}
+{% block error_message %}
+<p class="fs-3"> <span class="text-danger">Opps!</span> Server error</p>
+<p class="lead">Unexpected error, please try again or contact system admin </p>
+{% endblock error_message %}

logs_collector/templates/base.html

@@ -34,10 +34,12 @@
     rel="stylesheet"
     href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.10.5/font/bootstrap-icons.css"
     >
+    {% block errors_head %}{% endblock errors_head %}
     {% block collector_head %}{% endblock collector_head %}
     {% block account_head %}{% endblock account_head %}
   </head>
   <body class="d-flex flex-column min-vh-100">
+    {% block http_errors %}{% endblock http_errors %}
     {% block collector_content %}{% endblock collector_content %}
     {% block account_content %}{% endblock account_content %}
     <!-- BS dependences JS-->

logs_collector/templates/errors.html

@@ -0,0 +1,20 @@
+{% extends 'base.html' %}
+{% load static %}
+
+{% block errors_head %}
+  <title>{% block title %}{% endblock title %}</title>
+{% endblock errors_head %}
+
+{% block http_errors %}
+<div class="d-flex align-items-center justify-content-center vh-100" >
+  <div class="text-center">
+      <h1 class="display-1 fw-bold">{% block status_code %}{% endblock status_code %}</h1>
+      {% block error_message %}{% endblock error_message %}
+      <a href="{% url 'collector:index' %}" class="btn btn-secondary">Go Home</a>
+  </div>
+  <!-- Theme switcher -->
+  <div class="dropdown position-fixed bottom-0 end-0 mb-3 me-3 bd-mode-toggle">
+    {% include 'includes/theme_switcher.html' %}
+  </div>
+</div>
+{% endblock http_errors %}

logs_collector/templates/includes/footer.html

@@ -1,21 +1,29 @@
 <nav class="navbar navbar-expand-lg bg-body-tertiary">
-  <div class="container d-flex justify-content-between">
+  <div class="container">
     <ul class="navbar-nav">
       <li class="nav-item">
-        <span class="text-muted">
+        <button class="btn">
           v{{ version }}
           {% if environment != 'Production' %}
             Staging: {{ environment }}
           {% endif %}
-          | © {{ author }} 
-        </span>
+        </button>
+      </li>
+      <!-- Separator -->
+      <li class="nav-item py-2 py-lg-1 col-12 col-lg-auto">
+        <div class="vr d-none d-lg-flex h-100 mx-lg-2 text-white"></div>
+        <hr class="d-lg-none my-2 text-white-50">
+      </li>
+      <li class="nav-item">
+        <button class='btn'> © {{ author }} </button>
       </li>
     </ul>
+    {% if request.user.is_authenticated %}
     <ul class="navbar-nav">
       <li class="nav-item">
         <li>
           <a
-            class="nav-link"
+            class="btn"
             type="button"
             href="{% url 'swagger-ui' %}"
             target="_blank"
@@ -28,5 +36,6 @@
         </a>
       </li>
     </ul>
+    {% endif %}
   </div>
 </nav>