python3 uwsgi plugin

Merge branch 'development'

.gitignore

@@ -1,3 +1,13 @@
+# project specific
+#settings.json
+db.sqlite3
+dummyusers.json
+*.prof
+
+
+# IDE
+.vscode
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

CalibreWebCompanion/CalibreWebCompanion/settings.py

@@ -12,41 +12,36 @@ https://docs.djangoproject.com/en/3.0/ref/settings/
 
 import os
 import json
+import logging
+
+logger = logging.getLogger(__name__)
 
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
-with open( BASE_DIR + "/settings.json", "r") as userfile:
+with open(BASE_DIR + "/settings.json", "r") as userfile:
     usersettings = json.load(userfile)
     CALIBRE_DIR = os.path.abspath(usersettings["CALIBRE_DIR"])
     SECRET_KEY = usersettings["SECRET_KEY"]
     ALLOWED_HOSTS = usersettings["ALLOWED_HOSTS"]
     INTERNAL_IPS = usersettings["INTERNAL_IPS"]
-
-
-# CALIBRE_DIR = os.path.abspath(
-#     "C:\\Users\\MassiveAtoms\\Documents\\Calibre Library")
-# SECRET_KEY = 'u(8^+rb%rz5hsx4v^^y(ul7g(4n7a8!db@s*9(m5cs*2_ppy8+'
-# ALLOWED_HOSTS = ['127.0.0.1', ]
-# INTERNAL_IPS = ['127.0.0.1', ]
-
+    DEBUG = usersettings["DEBUG"]
 
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 
 
-
 EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
 
 # optimisation stuff ###############################################3
 #                                                                    #
-CONN_MAX_AGE = 60 * 5 
+CONN_MAX_AGE = 60 * 5
 
 CACHES = {
     'default': {
         'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
         'LOCATION': 'unique-snowflake',
-        "TIMEOUT" : 60 * 5,
-        
+        "TIMEOUT": 60 * 5,
+
     }
 }
 
@@ -58,17 +53,57 @@ CACHES = {
 # https://docs.djangoproject.com/en/3.0/howto/static-files/
 
 STATICFILES_DIRS = [
-    os.path.abspath(CALIBRE_DIR),
+    # os.path.abspath(CALIBRE_DIR),
     # '/static/',
 ]
 
 STATIC_URL = '/static/'
+
+STATIC_ROOT = BASE_DIR + "/static/"
+##                                                                     ##
+#########################################################################
+# LOGGING
+
+
+
+logfile = usersettings["LOGFOLDER"] + "django.log"
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "root": {"level": "INFO", "handlers": ["file"]},
+    "handlers": {
+        "file": {
+            "level": "INFO",
+            "class": "logging.FileHandler",
+            "filename": logfile,
+            "formatter": "app",
+        },
+    },
+    "loggers": {
+        "django": {
+            "handlers": ["file"],
+            "level": "INFO",
+            "propagate": True
+        },
+    },
+    "formatters": {
+        "app": {
+            "format": (
+                u"%(asctime)s [%(levelname)-8s] "
+                "(%(module)s.%(funcName)s) %(message)s"
+            ),
+            "datefmt": "%Y-%m-%d %H:%M:%S",
+        },
+    },
+}
+
+
 ##                                                                    ##
 ########################################################################
 ##                    DERUG                                           ##
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+
 
 DEBUG_TOOLBAR_PANELS = [
     'debug_toolbar.panels.timer.TimerPanel',
@@ -87,6 +122,12 @@ DEBUG_TOOLBAR_PANELS = [
 ########################################################################
 ##                    DERUG                                           ##
 
+
+# SILKY_PYTHON_PROFILER = True
+# SILKY_PYTHON_PROFILER_BINARY = True
+# SILKY_PYTHON_PROFILER_RESULT_PATH = BASE_DIR + "/profiler"
+# SILKY_META = True
+
 LOGIN_REDIRECT_URL = '/books'
 
 # Application definition
@@ -99,12 +140,14 @@ INSTALLED_APPS = [
     'django.contrib.messages',
     'django.contrib.staticfiles',
     "library",
-    'debug_toolbar', # DEBUG  purposes
+    # "silk", # DEBUG/profilling purposes
+    # 'debug_toolbar', # DEBUG  purposes
 ]
 
 MIDDLEWARE = [
-    'debug_toolbar.middleware.DebugToolbarMiddleware', # DEBUG purposes
-    'django.middleware.cache.UpdateCacheMiddleware', # cache
+    # 'silk.middleware.SilkyMiddleware', # DEBUG/profiling purposes
+    # 'debug_toolbar.middleware.DebugToolbarMiddleware', # DEBUG purposes
+    'django.middleware.cache.UpdateCacheMiddleware',  # cache
     'django.middleware.security.SecurityMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.common.CommonMiddleware',
@@ -112,8 +155,11 @@ MIDDLEWARE = [
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
-    'django.middleware.cache.FetchFromCacheMiddleware', # cache
+    'django.middleware.cache.FetchFromCacheMiddleware',  # cache
 ]
+##                                                                    ##
+########################################################################
+DEFAULT_CHARSET = "utf-8"
 
 ROOT_URLCONF = 'CalibreWebCompanion.urls'
 
@@ -138,14 +184,20 @@ TEMPLATES = [
 
 WSGI_APPLICATION = 'CalibreWebCompanion.wsgi.application'
 
-
-# Database
+##                                                                    ##
+########################################################################
+##                    DATBASE                                         ##
 # https://docs.djangoproject.com/en/3.0/ref/settings/#databases
 
+if usersettings["ISDOCKER"]:
+    defaultdb_path = "calibre"
+else:
+    defaultdb_path = BASE_DIR
+
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+        'NAME': os.path.join(defaultdb_path, 'db.sqlite3'),
     },
     'calibre': {
         'ENGINE': 'django.db.backends.sqlite3',
@@ -154,7 +206,7 @@ DATABASES = {
 }
 
 
-DATABASE_ROUTERS = ["db_routers.DjangoRouter", "db_routers.CalibreRouter"]
+DATABASE_ROUTERS = ["db_routers.CalibreRouter", "db_routers.DjangoRouter"]
 
 # Password validation
 # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators

CalibreWebCompanion/CalibreWebCompanion/urls.py

@@ -22,6 +22,7 @@ from django.conf import settings
 from django.urls import include, path
 
 
+
 urlpatterns = [
     path('admin/', admin.site.urls),
     path('accounts/', include('django.contrib.auth.urls')),
@@ -30,8 +31,9 @@ urlpatterns = [
 ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
 
 
-if settings.DEBUG: # DEBUG purposes
-    import debug_toolbar
-    urlpatterns = [
-        path('__debug__/', include(debug_toolbar.urls)),
-    ] + urlpatterns
+# if settings.DEBUG: # DEBUG purposes
+#     urlpatterns+= [path('silk/', include('silk.urls', namespace='silk'))]
+    # import debug_toolbar
+    # urlpatterns = [
+    #     path('__debug__/', include(debug_toolbar.urls)),
+    # ] + urlpatterns

CalibreWebCompanion/CalibreWebCompanion/uwsgi.ini

@@ -0,0 +1,21 @@
+[uwsgi]
+base = /cwebcomp
+chdir = %(base)
+home = %(base)
+pidfile= %(base)/cwebcomp.pid
+pythonpath= /usr/local
+uid = www-data
+gid = www-data
+module = wsgi:application # path to wsgy.py file
+socket = :8000
+processes = 8
+threads = 4
+master = true
+chmod-socket = 660
+vacuum = true
+die-on-term = true
+harakiri = 20 
+max-requests = 5000
+logs = %(base)/uwsgi_info.logs
+daemonize = %(base)/uwsgi.logs
+plugins = python3

CalibreWebCompanion/db_routers/routers.py

@@ -1,26 +1,51 @@
+import logging
 
+logger = logging.getLogger(__name__)
 
 class DjangoRouter:
     """
     A router to control all database operations on models in the
     auth and contenttypes applications.
     """
-    route_app_labels = {'auth', 'contenttypes', "sessions", "sites", "admin", "flatpages"}
+    def db_for_read(self, model, **hints):
+        """
+        Attempts to read anything else goes to calibre
+        """
+        return 'default'
+
+    def db_for_write(self, model, **hints): 
+        """
+        Attempts to write auth and contenttypes models go to 'calibre'.
+        """
+        return 'default'
+
+    def allow_relation(self, obj1, obj2, **hints):
+        """
+        Allow relations.
+        """
+        return True
+
+    def allow_migrate(self, db, app_label, model_name=None, **hints): 
+        """
+        Yes
+        """
+        return True
+
+
+
+class CalibreRouter:
+    """
+    A router to control all database operations on models in the
+    auth and contenttypes applications.
+    """
+    route_app_labels = {"library"}
 
     def db_for_read(self, model, **hints):
         """
         Attempts to read auth and contenttypes models go to default.
         """
         if model._meta.app_label in self.route_app_labels:
-            return 'default'
-        return None
-
-    def db_for_write(self, model, **hints):
-        """
-        Attempts to write auth and contenttypes models go to django.
-        """
-        if model._meta.app_label in self.route_app_labels:
-            return 'default'
+            return 'calibre'
         return None
 
     def allow_relation(self, obj1, obj2, **hints):
@@ -35,42 +60,19 @@ class DjangoRouter:
            return True
         return None
 
-    def allow_migrate(self, db, app_label, model_name=None, **hints):
-        """
-        Make sure the auth and contenttypes apps only appear in the
-        'django' database.
-        """
-        if app_label in self.route_app_labels:
-            return db == 'default'
-        return None
-
-
-
-class CalibreRouter:
-    """
-    A router to control all database operations on models in the
-    auth and contenttypes applications.
-    """
-    def db_for_read(self, model, **hints):
-        """
-        Attempts to read anything else goes to calibre
-        """
-        return 'calibre'
-
-    # def db_for_write(self, model, **hints): # might be prudent not to allow writes
+    # def allow_migrate(self, db, app_label, model_name=None, **hints):
     #     """
-    #     Attempts to write auth and contenttypes models go to 'calibre'.
+    #     Make sure the auth and contenttypes apps only appear in the
+    #     'django' database.
     #     """
-    #     return 'calibre'
+    #     if app_label in self.route_app_labels:
+    #         return db == 'default'
+    #     return None
 
-    def allow_relation(self, obj1, obj2, **hints):
-        """
-        Allow relations.
-        """
-        return True
-
-    # def allow_migrate(self, db, app_label, model_name=None, **hints): # might be prudent not to allow migrations
+    # def db_for_write(self, model, **hints):
     #     """
-    #     Yes
+    #     Attempts to write auth and contenttypes models go to django.
     #     """
-    #     return True
+    #     if model._meta.app_label in self.route_app_labels:
+    #         return 'default'
+    #     return None

CalibreWebCompanion/gunicorn.conf.py

@@ -0,0 +1,33 @@
+import multiprocessing
+import os
+import json
+
+bind = "127.0.0.1:8000"
+workers = multiprocessing.cpu_count() * 2 + 1
+preload_app = True # By preloading an application you can save some RAM resources as well as speed up server boot times
+keepalive = 5
+# daemon = True # Detaches the server from the controlling terminal and enters the background. disabled for now
+# logging
+
+with open("settings.json", "r") as jfile:
+    settings = json.load(jfile)
+
+errorlog = settings["LOGFOLDER"] + "/gunicorn_error.log"
+loglevel = "warning"
+accesslog = settings["LOGFOLDER"] + "/gunicorn_access.log"
+
+if not os.path.isdir("/usr/src/app/data/logs"):
+    os.mkdir("/usr/src/app/data/logs")
+if not os.path.isfile(errorlog):
+    os.system('touch {}'.format(errorlog))
+if not os.path.isfile(accesslog):
+    os.system('touch {}'.format(accesslog))
+capture_output = True
+
+# debug settings which need to be commented out in prod
+# reload=True
+# reload_engine = "inotify"
+
+
+# I only went till the section https://docs.gunicorn.org/en/latest/settings.html#logging there are more settings
+# some of them might be useful

CalibreWebCompanion/library/context_processors.py

@@ -1,6 +1,8 @@
 from .models import Author, Tag, Publisher, Language, Rating, Series
 from django.db.models import Count
+import logging
 
+logger = logging.getLogger(__name__)
 
 def filters(request):
     # unique_authors = Author.objects.all().order_by('sort')
@@ -13,7 +15,7 @@ def filters(request):
     unique_authors = Author.objects.only('name', "id").annotate(num_books=Count('book')).order_by('name')
     unique_tags = Tag.objects.annotate(num_books=Count('book')).order_by('name')
     unique_publishers = Publisher.objects.annotate(num_books=Count('book')).order_by('name')
-    unique_languages = Language.objects.annotate(num_books=Count('book')).order_by('rating')
+    unique_languages = Language.objects.annotate(num_books=Count('book')).order_by('lang_code')
     unique_ratings = Rating.objects.annotate(num_books=Count('book'))
     unique_series = Series.objects.annotate(num_books=Count('book')).order_by('sort')
     
@@ -25,4 +27,4 @@ def filters(request):
         "unique_languages": unique_languages,
         "unique_ratings": unique_ratings,
         "unique_series": unique_series
-    }
+    }

CalibreWebCompanion/library/forms.py

@@ -1,7 +1,9 @@
 from django import forms
 from django.contrib.auth.forms import UserCreationForm
 from django.contrib.auth.models import User
+import logging
 
+logger = logging.getLogger(__name__)
 
 class SearchForm(forms.Form):
     title = forms.CharField(label="Title", max_length=200)

CalibreWebCompanion/library/models.py

@@ -8,7 +8,9 @@
 from django.db import models
 from django.urls import reverse
 from django.utils.functional import cached_property
+import logging
 
+logger = logging.getLogger(__name__)
 
 class Author(models.Model):
     name = models.TextField()

CalibreWebCompanion/library/templates/base.html

@@ -3,10 +3,8 @@
 
 <head>
   {% block title %}<title>Local Library</title>{% endblock %}
-
   <meta charset="utf-8">
   <meta name="viewport" content="width=device-width, initial-scale=1">
-
   <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
   <link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
   <!-- Compiled and minified CSS -->

CalibreWebCompanion/library/templates/library/book_detail.html

@@ -1,12 +1,12 @@
 {% extends "base.html" %}
-
+{% block title %}<title>{{book.title}}</title>{% endblock %}
 {% block content %}
 {% load static %}
 
 <div class="col s12 m7">
   <div class="card z-depth-0 horizontal">
     <div class="card-image">
-      <a style="padding-top:15%" href=" {% static "" %}{{download}}"><img src=" {% static "" %}{{imgpath}}"
+      <a style="padding-top:15%" href=" /download/{{download}}"><img src=" /download/{{imgpath}}"
           alt="download" srcset=""></a>
     </div>
     <div class="card-stacked">

CalibreWebCompanion/library/tests.py

@@ -1,3 +1,166 @@
-from django.test import TestCase
+from django.test import Client, TestCase
+from pprint import pprint
+from django.test.utils import setup_test_environment
+from .models import Book, Author, Publisher, Series, Rating, Tag, Identifier
+from django.db.models import Count
 
-# Create your tests here.
+
+client = Client()
+client.login(username="testuser", password="dumbeasypassword")
+
+def booklisttest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    res = c.get("/books/")
+    assert res.status_code == 200
+    context = dict(res.context)
+    assert sorted(context["book_list"], key=lambda x: x.id)== sorted(Book.objects.all(), key=lambda x: x.id)
+
+def authorlisttest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    res = c.get("/authors/")
+    assert res.status_code == 200
+    context = dict(res.context)
+    assert sorted(context["author_list"], key=lambda x: x.id)== sorted(Author.objects.all(), key=lambda x: x.id)
+
+
+def publisherlisttest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    res = c.get("/publishers/")
+    assert res.status_code == 200
+    context = dict(res.context)
+    assert sorted(context["publisher_list"], key=lambda x: x.id)== sorted(Publisher.objects.all(), key=lambda x: x.id)
+
+def serieslisttest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    res = c.get("/series/")
+    assert res.status_code == 200
+    context = dict(res.context)
+    assert sorted(context["series_list"], key=lambda x: x.id)== sorted(Series.objects.all(), key=lambda x: x.id)
+
+
+def ratinglisttest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    res = c.get("/ratings/")
+    assert res.status_code == 200
+    context = dict(res.context)
+    assert sorted(context["rating_list"], key=lambda x: x.id)== sorted(Rating.objects.all(), key=lambda x: x.id)
+
+def taglisttest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    res = c.get("/tags/")
+    assert res.status_code == 200
+    context = dict(res.context)
+    assert sorted(context["tag_list"], key=lambda x: x.id)== sorted(Tag.objects.all(), key=lambda x: x.id)
+
+
+def bookdetailtest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    ids = [i.id for i in Book.objects.all()][:10]
+    for i in ids:
+        res = c.get(f"/book/{i}")
+        assert res.status_code == 200
+        context = dict(res.context)
+        assert context["book"] == Book.objects.get(id=i)
+
+
+def authordetailtest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    ids = [i.id for i in Author.objects.all()][:10]
+    for i in ids:
+        res = c.get(f"/author/{i}")
+        assert res.status_code == 200
+        context = dict(res.context)
+        assert context["author"] == Author.objects.get(id=i)
+
+def publisherdetailtest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    ids = [i.id for i in Publisher.objects.all()][:10]
+    for i in ids:
+        res = c.get(f"/publisher/{i}")
+        assert res.status_code == 200
+        context = dict(res.context)
+        assert context["publisher"] == Publisher.objects.get(id=i)
+
+def seriesdetailtest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    ids = [i.id for i in Series.objects.all()][:10]
+    for i in ids:
+        res = c.get(f"/series/{i}")
+        assert res.status_code == 200
+        context = dict(res.context)
+        assert context["series"] == Series.objects.get(id=i)
+
+def ratingdetailtest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    ids = [i.id for i in Rating.objects.all()][:10]
+    for i in ids:
+        res = c.get(f"/rating/{i}")
+        assert res.status_code == 200
+        context = dict(res.context)
+        assert context["rating"] == Rating.objects.get(id=i)        
+
+
+def tagdetailtest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    ids = [i.id for i in Tag.objects.all()][:10]
+    for i in ids:
+        res = c.get(f"/tag/{i}")
+        assert res.status_code == 200
+        context = dict(res.context)
+        assert context["tag"] == Tag.objects.get(id=i) 
+
+
+def search_partial(key, value, book=None):
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    res = c.get("/results/", {key : value})
+    if not book:
+        return dict(res.context)["book_list"]
+    return book in dict(res.context)["book_list"]
+
+
+def searchtest():
+    books = [i for i in Book.objects.all()][:10]
+    for i in books:
+        assert search_partial("title", i.title, i)
+        assert search_partial("generic", i.title, i)
+        assert search_partial("author", i.author_sort, i)
+        author = i.authors.first()
+        if author:
+            assert search_partial("author", author.name, i)
+            assert search_partial("generic", author.name, i)
+
+        assert search_partial("generic", i.author_sort, i)
+        id = Identifier.objects.filter(book=i.id).first()
+        if id:
+            assert search_partial("identifier", id, i)
+            assert search_partial("generic", id, i)
+
+
+    
+
+booklisttest()
+bookdetailtest()
+authorlisttest()
+authordetailtest()
+publisherdetailtest()
+publisherlisttest()
+seriesdetailtest()
+serieslisttest()
+ratingdetailtest()
+ratinglisttest()
+tagdetailtest()
+taglisttest()
+searchtest()

CalibreWebCompanion/library/urls.py

@@ -1,7 +1,9 @@
 from django.urls import path
 from . import views
 from django.views.decorators.cache import cache_page
+import logging
 
+logger = logging.getLogger(__name__)
 
 urlpatterns = [
     path('authors/', views.AuthorListView.as_view(), name='authors'),
@@ -10,19 +12,20 @@ urlpatterns = [
     path('ratings/', views.RatingListView.as_view(), name='ratings'),
     path('tags/', views.TagListView.as_view(), name='tags'),
     path('series/', views.SeriesListView.as_view(), name='series'),
-
-
-    path('author/<int:pk>', views.AuthorDetailView.as_view(), name='author-detail-view'),
+    path('author/<int:pk>', views.AuthorDetailView.as_view(),
+         name='author-detail-view'),
     path('book/<int:pk>', views.BookDetailView.as_view(), name='book-detail-view'),
-    path('publisher/<int:pk>', views.PublisherDetailView.as_view(), name='publisher-detail-view'),
-    path('rating/<int:pk>', views.RatingDetailView.as_view(), name='rating-detail-view'),
-    path('series/<int:pk>', views.SeriesDetailView.as_view(), name='series-detail-view'),
+    path('publisher/<int:pk>', views.PublisherDetailView.as_view(),
+         name='publisher-detail-view'),
+    path('rating/<int:pk>', views.RatingDetailView.as_view(),
+         name='rating-detail-view'),
+    path('series/<int:pk>', views.SeriesDetailView.as_view(),
+         name='series-detail-view'),
     path('tag/<int:pk>', views.TagDetailView.as_view(), name='tag-detail-view'),
 
     path('results/', views.ResultsView.as_view(), name='results'),
     path('search/', views.SearchView.as_view(), name='search'),
-
-    path('accounts/sign_up/',views.sign_up,name="sign-up")
+    path('accounts/sign_up/', views.sign_up, name="sign-up")
 
 
-]
+]

CalibreWebCompanion/library/views.py

@@ -1,3 +1,4 @@
+from django.utils.decorators import method_decorator
 from django.shortcuts import render
 from django.views import generic
 from .models import Author, Book, Comment, Rating, BookAuthorLink, Publisher, Tag, BookTagLink, BookRatingLink, Data, Identifier, Series
@@ -8,7 +9,11 @@ from django.db.models import Q
 from django.contrib.auth.models import User
 from django.contrib.auth import login
 from django.contrib.auth.decorators import login_required
+import logging
 
+logger = logging.getLogger(__name__)
+
+# might be helpful for vary headers later
 
 
 @login_required
@@ -31,6 +36,9 @@ def sign_up(request):
 class SearchView(generic.TemplateView):
     template_name = 'search.html'
 
+    def dispatch(self, *args, **kwargs):
+        return super(SearchView, self).dispatch(*args, **kwargs)
+
 
 class ResultsView(generic.ListView):  # no clue if this is secure.
     # according to this https://stackoverflow.com/questions/13574043/how-do-django-forms-sanitize-text-input-to-prevent-sql-injection-xss-etc
@@ -38,6 +46,9 @@ class ResultsView(generic.ListView):  # no clue if this is secure.
     model = Book
     template_name = 'results.html'
 
+    def dispatch(self, *args, **kwargs):
+        return super(ResultsView, self).dispatch(*args, **kwargs)
+
     def get_queryset(self):  # new
         title = self.request.GET.get('title')
         author = self.request.GET.get('author')
@@ -47,13 +58,29 @@ class ResultsView(generic.ListView):  # no clue if this is secure.
         if title:
             books = books.filter(sort__icontains=title)
         if author:
-            books = books.filter(author_sort__icontains=author)
+            # authors are stored as author_sort and author, needs to be slightly more complex
+            author_obj = Author.objects.filter(name__icontains=author).first()
+            if not author_obj:
+                author_id = -1
+            else:
+                author_id = author_obj.id
+
+            books = books.filter(
+                Q(author_sort__icontains=author) |
+                Q(authors__id=author_id)
+            )
         if identifier:
             books = books.filter(identifier__val=identifier)
         if generic:
+            author_obj = Author.objects.filter(name__icontains=generic).first()
+            if not author_obj:
+                author_id = -1
+            else:
+                author_id = author_obj.id
             books = books.filter(
-                Q(sort__icontains=generic) | 
-                Q(author_sort__icontains=generic) | 
+                Q(sort__icontains=generic) |
+                Q(author_sort__icontains=generic) |
+                Q(authors__id=author_id) |
                 Q(identifier__val=generic)
             )
         return books
@@ -62,10 +89,16 @@ class ResultsView(generic.ListView):  # no clue if this is secure.
 class AuthorListView(generic.ListView):
     model = Author
 
+    def dispatch(self, *args, **kwargs):
+        return super(AuthorListView, self).dispatch(*args, **kwargs)
+
 
 class BookListView(generic.ListView):
     model = Book
 
+    def dispatch(self, *args, **kwargs):
+        return super(BookListView, self).dispatch(*args, **kwargs)
+
     def get_queryset(self):
         # Annotate the books with ratings, tags, etc
         # books = Book.objects.annotate(
@@ -76,21 +109,37 @@ class BookListView(generic.ListView):
 class PublisherListView(generic.ListView):
     model = Publisher
 
+    def dispatch(self, *args, **kwargs):
+        return super(PublisherListView, self).dispatch(*args, **kwargs)
+
 
 class RatingListView(generic.ListView):
     model = Rating
 
-class SeriesListView(generic.ListView): # make url entry and template, sometime
+    def dispatch(self, *args, **kwargs):
+        return super(RatingListView, self).dispatch(*args, **kwargs)
+
+
+class SeriesListView(generic.ListView):  # make url entry and template, sometime
     model = Series
 
+    def dispatch(self, *args, **kwargs):
+        return super(SeriesListView, self).dispatch(*args, **kwargs)
+
 
 class TagListView(generic.ListView):
     model = Tag
 
+    def dispatch(self, *args, **kwargs):
+        return super(TagListView, self).dispatch(*args, **kwargs)
+
 
 class AuthorDetailView(generic.DetailView):
     model = Author
 
+    def dispatch(self, *args, **kwargs):
+        return super(AuthorDetailView, self).dispatch(*args, **kwargs)
+
     def get_context_data(self, **kwargs):
         # Call the base implementation first to get the context
         context = super(AuthorDetailView, self).get_context_data(**kwargs)
@@ -104,6 +153,9 @@ class AuthorDetailView(generic.DetailView):
 class BookDetailView(generic.DetailView):
     model = Book
 
+    def dispatch(self, *args, **kwargs):
+        return super(BookDetailView, self).dispatch(*args, **kwargs)
+
     def get_context_data(self, **kwargs):
         # Call the base implementation first to get the context
         context = super(BookDetailView, self).get_context_data(**kwargs)
@@ -115,13 +167,16 @@ class BookDetailView(generic.DetailView):
             pass
         context["imgpath"] = context["object"].path + "/cover.jpg"
         download = Data.objects.get(book=context["object"].id)
-        context["download"] = f"{context['object'].path}/{download.name}.{download.format}"
+        context["download"] = f"{context['object'].path}/{download.name}.{download.format.lower()}"
         return context
 
 
 class PublisherDetailView(generic.DetailView):
     model = Publisher
 
+    def dispatch(self, *args, **kwargs):
+        return super(PublisherDetailView, self).dispatch(*args, **kwargs)
+
     def get_context_data(self, **kwargs):
         # Call the base implementation first to get the context
         context = super(PublisherDetailView, self).get_context_data(**kwargs)
@@ -135,6 +190,9 @@ class PublisherDetailView(generic.DetailView):
 class RatingDetailView(generic.DetailView):
     model = Rating
 
+    def dispatch(self, *args, **kwargs):
+        return super(RatingDetailView, self).dispatch(*args, **kwargs)
+
     def get_context_data(self, **kwargs):
         # Call the base implementation first to get the context
         context = super(RatingDetailView, self).get_context_data(**kwargs)
@@ -148,6 +206,9 @@ class RatingDetailView(generic.DetailView):
 class TagDetailView(generic.DetailView):
     model = Tag
 
+    def dispatch(self, *args, **kwargs):
+        return super(TagDetailView, self).dispatch(*args, **kwargs)
+
     def get_context_data(self, **kwargs):
         # Call the base implementation first to get the context
         context = super(TagDetailView, self).get_context_data(**kwargs)
@@ -157,9 +218,13 @@ class TagDetailView(generic.DetailView):
         context['books'] = sorted(books,  key=lambda x: x.title)
         return context
 
+
 class SeriesDetailView(generic.DetailView):
     model = Series
 
+    def dispatch(self, *args, **kwargs):
+        return super(SeriesDetailView, self).dispatch(*args, **kwargs)
+
     def get_context_data(self, **kwargs):
         # Call the base implementation first to get the context
         context = super(SeriesDetailView, self).get_context_data(**kwargs)
@@ -167,4 +232,4 @@ class SeriesDetailView(generic.DetailView):
         books = Book.objects.prefetch_related("tags", "ratings")
         books = books.filter(series=context["object"].id)
         context['books'] = sorted(books,  key=lambda x: x.title)
-        return context
+        return context

CalibreWebCompanion/settings.json

@@ -0,0 +1,15 @@
+{
+    "CALIBRE_DIR": "calibre",
+    "SECRET_KEY": "u(8^+rb%rz5hsx4v^^y(ul7g(4n7a8!db@s*9(m5cs*2_ppy8+",
+    "ALLOWED_HOSTS": [
+        "127.0.0.1"
+    ],
+    "INTERNAL_IPS": [
+        "127.0.0.1"
+    ],
+    "DEBUG" : true,
+    "LOGFOLDER" : "/cwebcomp/logs",
+    "ISDOCKER" : true
+
+
+}

Dockerfile

@@ -0,0 +1,35 @@
+FROM python:3.9.1-slim-buster
+
+RUN apt-get clean && \
+    apt-get update && \
+    apt-get install -y nginx smbclient default-libmysqlclient-dev \
+     gcc python3-cffi libcairo2 libpango-1.0-0 libpangocairo-1.0-0 \
+     libgdk-pixbuf2.0-0 libffi-dev shared-mime-info uwsgi-core uwsgi-plugin-python3
+
+# set environment variables
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+RUN mkdir /cwebcomp
+WORKDIR /cwebcomp
+ADD . /cwebcomp/
+# only add this next one if you have static files
+RUN mkdir static
+
+RUN pip install -r requirements.txt
+RUN python CalibreWebCompanion/manage.py collectstatic
+
+# only if you need celery 
+#RUN useradd -ms /bin/bash celery 
+#COPY broker/init.d_celeryd /etc/init.d/celeryd
+#COPY broker/celeryd /etc/default/celeryd
+
+# nginx config and script to be run
+COPY deployment/docker/nginx.conf /etc/nginx/sites-available/default
+COPY deployment/docker/start.sh /usr/local/bin/start.sh
+
+# set proper file permissions
+RUN chmod u+x /usr/local/bin/start.sh
+
+EXPOSE 80
+CMD ["/bin/bash", "-c", "start.sh"]

README.md

@@ -1,22 +1,69 @@
+# What is CalibreWebAlternative? 
+This is a web server to the popular book management application Calibre. We found that the builtin webserver was kinda shit, so we're building our own. (make this friendlier later)
+
+
+
+# Features
+
+-  navbar with tags, series, authors, etc
+-  Search by author, identifier, title
+- authentication 
+
+
+# Some screenshots
+Here's how the various lists look like 
+![booklist](./screenshots/booklist.png)
+Book detail
+![bookdetail](./screenshots/bookdetail.png)
+navbar
+![nav](./screenshots/navbar.png)
+Adanced search
+![booklist](./screenshots/search.png)
 # requirements
-Django 3.0
+Django 3.0  
+Calibre 4.13 (I have not tested it with anything else atm, will be resolved later)
 
 # how to use:
-Edit `./CalibreWebCompanion/CalibreWebCompanion/settings`.  
-Set CALIBRE_DIR to the path of your library  
-`./CalibreWebCompanion`    
-run `./manage.py runserver`  
+1. [Docker setup](./deployment/instructions.md#user-content-docker-detup)
+2. [Non Docker setup](./deployment/instructions.md#user-content-non-docker-detup)
 
-this is in development mode. don't actually use it or release it like this. The debug info it shows is spicy.  
-# Features
+
+# Ignore pretty much everything below if you're not working on the project
+
+# Profiling
+
+To do profiling, you have to create some dummy users  
+Unbakify a file `./loadtesting/dummyusers.json.bak`  and fill in the credentials for the dummy users  
+
+While django is running, open another shell and cd to `./loadtesting` and run `./bench.py`
+To have a more interactive session, 
+comment out 
+```
+run-time = 2m
+headless = true
+``` 
+in `locust.conf`, and then run `./bench.py`
+You can then go to [http://localhost:8089/](http://localhost:8089/) to see live graphs, tweak the number of users and more.
+
+
+# Finished Features
 
 - [x] Books
 - [x] navbar with tags, series, authors, etc
 - [x] Search
 - [x] authentication 
+- [x] Cache 
+- [x] logging
+- [x]  deploy instructions
+
+# TODO ROADMAP
+- [ ] cache with vary headers
+- [ ] localisation
+- [ ] Beautifying template (only works well on 720p, no other viewports)
+- [ ]  Setup email functionality (atm, there's only a dummy one, and you can't reset passwords)
+- [ ]  isolate the styling and templates, so we can swap them out by just swapping directory content
+
 
 
-# TODO
 
-- [ ] fix author_detail_view with annotate instead of current implementation
  

deployment/Dockerfile

@@ -0,0 +1,29 @@
+
+## pull official base image
+FROM python:slim-buster
+EXPOSE 8080
+## set work directory
+WORKDIR /usr/src/app
+## install dependencies
+RUN pip install --upgrade pip
+COPY ./requirements.txt .
+RUN pip install -r requirements.txt
+RUN apk add nginx supervisor
+
+# do nginx stuff
+RUN adduser -D -g 'www' www
+RUN mkdir -p /run/nginx
+COPY ./deployment/nginx.conf /etc/nginx/
+
+## copy project
+COPY ./CalibreWebCompanion ./CalibreWebCompanion
+COPY ./deployment/startupscript.py ./
+## gunicorn borks started with supervisord
+COPY ./deployment/supervisord.conf /etc/
+ENTRYPOINT /usr/bin/supervisord -c /etc/supervisord.conf
+
+
+# docker run --publish 8000:80 \
+# -v '/home/massiveatoms/Desktop/logs:/usr/src/app/data' \
+# -v '/run/media/massiveatoms/1AEEEA6EEEEA421D1/Documents and Settings/MassiveAtoms/Documents/Calibre Library/:/usr/src/app/calibredir' \
+# --name cw calibreweb:1.0.1

deployment/deploy.py

@@ -0,0 +1,4 @@
+from os import environ
+
+
+

deployment/docker/nginx.conf

@@ -0,0 +1,20 @@
+server {
+    listen 80;
+    server_name 127.0.0.1;
+    charset utf-8;
+    client_max_body_size 75M;
+
+
+    location /static/ {
+        alias /cwebcomp/static/;
+    }
+
+    location /media/ {
+        alias /cwebcomp/media/;
+    }
+
+     location / {
+        include     uwsgi_params;
+        uwsgi_pass  127.0.0.1:8000;
+    }
+}

deployment/docker/start.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+uwsgi --ini CalibreWebCompanion/CalibreWebCompanion/uwsgi.ini
+nginx -g 'daemon off;'

deployment/instructions.md

@@ -0,0 +1,45 @@
+
+# Docker setup (no provided docker image atm)
+1. clone the repo
+2. rename ./calireWebCompanion/settings.json.bak to settings.json
+3. change the secret key
+4. run `build --tag calibreweb:1.0  . -f ./deployment/Dockerfile` to build the image
+5. run your container with your bind/mount your volumes/paths/things 
+Here's an example of step 5
+```
+docker run --publish 80:80\ 
+-v '/home/massiveatoms/Desktop/logs:/usr/src/app/data' \
+-v '/run/media/massiveatoms/1AEEEA6EEEEA421D/Documents and Settings/MassiveAtoms/Documents/Calibre Library/:/usr/src/app/calibredir' \
+--name cw calibreweb:1.0.1
+```
+
+your Calibre path/volume/whatever needs to be mounted at `/usr/src/app/calibredir`, and you need to mount a volume for the db and logs at `/usr/src/app/data`
+
+Issues with it at the moment:
+1. we still need to do something to create a random secret key. Atm, this would still 
+
+# Docker (provided image)
+not done yet
+
+# non docker setup
+this might need to be modified, since some things have changed to adapt it for docker setup
+1. clone repo
+2. pip install -r requirements.txt
+3. rename the settings.json.bak to settings.json, change logging folder, change secret key, set isdocker to false
+4. install gunicorn and nginx
+5. move this nginx.conf to /etc/nginx
+6. create a user and group `www`
+7. make whatever user nginx runs as (for now, www) the owner of calibredir
+8. give execute permissions to parent of calibredir
+9. cd to repo, run `gunicorn CalibreWebCompanion.wsgi`
+10. start nginx `sudo systemctl restart nginx`
+11. make steps 9 and 10 happen on startup?
+
+Slight issues with this atm:
+1. where to do ssl?
+
+
+Suggestions: 
+1. We might want to use sockets instead of ip/port?
+2. autostart gunicorn/nginx
+3. some extra instrumentation for gunicorn https://docs.gunicorn.org/en/latest/deploy.html

deployment/nginx.conf

@@ -0,0 +1,81 @@
+worker_processes 1;
+
+# user nobody nogroup;
+user www www;  # TEMP disabled
+# user nobody nobody; # for systems with 'nobody' as a group instead
+error_log  /usr/src/app/data/logs/nginx.log warn;
+# pid /var/run/nginx.pid;
+
+events {
+  worker_connections 1024; # increase if you have lots of clients
+  accept_mutex off; # set to 'on' if nginx worker_processes > 1
+  use epoll; # to enable for Linux 2.6+ MASSIVEATOMS 
+  # 'use kqueue;' to enable for FreeBSD, OSX
+}
+
+http {
+  include mime.types;
+  # fallback in case we can't determine a type
+  default_type application/octet-stream;
+  access_log /var/log/nginx/access.log combined;
+  sendfile on;
+
+  upstream app_server {
+    # fail_timeout=0 means we always retry an upstream even if it failed
+    # to return a good HTTP response
+
+    # for UNIX domain socket setups
+    # server unix:/tmp/gunicorn.sock fail_timeout=0;
+
+    # for a TCP configuration
+    server 127.0.0.1:8000 fail_timeout=0;
+  }
+
+  server {
+    # if no Host match, close the connection to prevent host spoofing
+    listen 80 default_server;
+    return 444;
+  }
+
+  server {
+    listen 80 deferred;  # for Linux massiveatoms
+    # use 'listen 80 accept_filter=httpready;' for FreeBSD
+    # listen 80;
+    client_max_body_size 4G;
+
+    # set the correct host(s) for your site
+    server_name localhost  0.0.0.0; # set this to the server url? or ip? we'll see MASSIVEATOMS
+
+    keepalive_timeout 5;
+
+    # # MASSIVEATOMS
+      location /download/ {
+        alias "/usr/src/app/calibredir/";
+        # Never forget the fact that this little statement being root instead of alias caused us to lose more than a day troubleshooting
+    }
+
+          location /static/ {
+        alias "/usr/src/app/CalibreWebCompanion/static/";
+        # Never forget the fact that this little statement being root instead of alias caused us to lose more than a day troubleshooting
+    }
+
+    location / {
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      # proxy_set_header Host $http_host;
+      # we don't want nginx trying to do something clever with
+      # redirects, we set the Host: header above already.
+      proxy_redirect off;
+      proxy_pass http://127.0.0.1:8000;
+    }
+
+
+
+
+
+    error_page 500 502 503 504 /500.html;
+    location = /500.html {
+      root /path/to/app/current/public;
+    }
+  }
+}

deployment/startupscript.py

@@ -0,0 +1,11 @@
+from os import system, chdir
+
+
+# system("chown -R www:www /usr/src/app/calibredir")
+# print("ownership of calibredir changed")
+chdir("/usr/src/app/CalibreWebCompanion")
+system("python ./manage.py makemigrations")
+print("ran makemigrations")
+system("python ./manage.py migrate")
+print("migrate")
+

deployment/supervisord.conf

@@ -0,0 +1,36 @@
+[supervisord]
+nodaemon=true
+logfile=/tmp/supervisord.log
+childlogdir=/tmp
+pidfile = /tmp/supervisord.pid
+
+[program:gunicorn]
+directory=/usr/src/app/CalibreWebCompanion
+command=gunicorn CalibreWebCompanion.wsgi
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autorestart=false
+startretries=0
+startsecs = 0
+
+[program:nginx]
+# user=www
+command=nginx
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autorestart=false
+startretries=0
+
+[program:startupscript]
+directory=/usr/src/app
+command=python ./startupscript.py
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autorestart=false
+startretries=0

loadtesting/bench.py

@@ -0,0 +1,34 @@
+import csv
+from rich.console import Console
+from rich.table import Column, Table
+import subprocess
+subprocess.run(["locust"])
+
+def floatify(mystring):  # floatify probable floats
+    return f"{float(mystring):3.3f}"
+
+
+results = dict()
+with open("calibre_stats.csv", "r") as cfile:
+    reader = csv.reader(cfile, delimiter=",")
+    for row in reader:
+        if not len(row) or row[0] == "Type":
+            continue
+        results[row[0] + " " + row[1]] = {
+            "median": floatify(row[4]),
+            "avg": floatify(row[5]),
+            "min": floatify(row[6]),
+            "max": floatify(row[7]),
+        }
+
+console = Console()
+table = Table(show_header=True, header_style="bold green")
+table.add_column("Action/url")
+table.add_column("min")
+table.add_column("avg")
+table.add_column("median")  # destination
+table.add_column("max")  # source
+
+for k, v in results.items():
+    table.add_row(k, v["min"], v["avg"], v["median"], v["max"])
+console.print(table)

loadtesting/calibre_failures.csv

@@ -0,0 +1,2 @@
+Method,Name,Error,Occurrences
+GET,/book/<id>,500 Server Error: Internal Server Error for url: /book/<id>,3

loadtesting/calibre_stats.csv

@@ -0,0 +1,20 @@
+Type,Name,Request Count,Failure Count,Median Response Time,Average Response Time,Min Response Time,Max Response Time,Average Content Size,Requests/s,Failures/s,50%,66%,75%,80%,90%,95%,98%,99%,99.9%,99.99%,99.999%,100%
+GET,/accounts/login/,20,0,15,14.807796478271484,8.188486099243164,21.71945571899414,2024.0,0.16739911642581679,0.0,16,17,18,18,20,22,22,22,22,22,22,22
+POST,/accounts/login/,20,0,320.0,335.94770431518555,302.3109436035156,504.45032119750977,76378.0,0.16739911642581679,0.0,330,340,340,350,370,500,500,500,500,500,500,500
+GET,/author/<id>,40,0,27,27.45213508605957,20.0350284576416,67.98744201660156,30708.725,0.33479823285163357,0.0,27,28,29,29,32,34,68,68,68,68,68,68
+GET,/authors/,53,0,33,32.4410987350176,3.3960342407226562,151.21984481811523,39339.0,0.4436076585284145,0.0,33,33,35,36,43,47,73,150,150,150,150,150
+GET,/book/<id>,63,3,31,30.65679186866397,20.604372024536133,50.59003829956055,30451.74603174603,0.5273072167413229,0.025109867463872518,31,32,33,33,35,37,37,51,51,51,51,51
+GET,/books/,42,0,130.0,110.17770994277228,3.3218860626220703,274.7330665588379,76378.0,0.3515381444942153,0.0,130,130,130,140,160,190,270,270,270,270,270,270
+GET,/publisher/<id>,48,0,27,26.85883641242981,17.744064331054688,69.56601142883301,30890.270833333332,0.4017578794219603,0.0,27,27,29,29,31,34,70,70,70,70,70,70
+GET,/publishers/,40,0,24,22.84235954284668,4.957437515258789,34.004926681518555,33158.0,0.33479823285163357,0.0,24,26,27,27,28,31,34,34,34,34,34,34
+GET,/rating/<id>,53,0,30,36.589348091269436,9.972572326660156,93.70565414428711,35127.301886792455,0.4436076585284145,0.0,30,40,46,51,57,68,85,94,94,94,94,94
+GET,/ratings/,42,0,23,21.207468850272043,6.12950325012207,37.114858627319336,30028.0,0.3515381444942153,0.0,23,23,24,24,26,28,37,37,37,37,37,37
+GET,/search/,54,0,17,15.807677198339391,3.0889511108398438,24.329662322998047,30164.0,0.45197761434970535,0.0,18,19,20,21,22,23,23,24,24,24,24,24
+GET,/series/,44,0,20,19.72411437468095,5.149126052856445,28.244972229003906,29798.0,0.3682780561367969,0.0,21,23,23,24,25,26,28,28,28,28,28,28
+GET,/tag/<id>,45,0,26,26.674440171983505,16.248226165771484,37.3835563659668,31041.577777777777,0.3766480119580878,0.0,26,27,28,29,30,35,37,37,37,37,37,37
+GET,/tags/,45,0,32,29.666270150078667,3.5097599029541016,77.78573036193848,38073.0,0.3766480119580878,0.0,32,32,33,34,36,39,78,78,78,78,78,78
+GET,search_by_author,55,0,22,22.336192564530805,12.028932571411133,51.23710632324219,30187.963636363635,0.4603475701709962,0.0,22,24,24,25,28,30,30,51,51,51,51,51
+GET,search_by_identifier,45,0,26,24.89140298631456,17.200946807861328,34.82961654663086,30556.533333333333,0.3766480119580878,0.0,26,27,27,28,30,31,35,35,35,35,35,35
+GET,search_by_title,43,0,26,24.867362754289495,12.270927429199219,32.17816352844238,30419.883720930233,0.35990810031550613,0.0,26,27,27,28,29,30,32,32,32,32,32,32
+GET,search_generic,62,0,26,26.363442021031535,16.860246658325195,69.95797157287598,30512.548387096773,0.518937260920032,0.0,26,27,28,28,32,34,38,70,70,70,70,70
+,Aggregated,814,3,26,37.75617238637563,3.0889511108398438,504.45032119750977,34674.45577395577,6.813144038530743,0.025109867463872518,26,28,31,32,43,130,310,330,500,500,500,500

loadtesting/calibre_stats_history.csv

@@ -0,0 +1,62 @@
+"Timestamp","User Count","Type","Name","Requests/s","Failures/s","50%","66%","75%","80%","90%","95%","98%","99%","99.9%","99.99%","99.999%","100%","Total Request Count","Total Failure Count","Total Median Response Time","Total Average Response Time","Total Min Response Time","Total Max Response Time","Total Average Content Size"
+"1596336131","1","","Aggregated",0.00,0.00,"N/A","N/A","N/A","N/A","N/A","N/A","N/A","N/A","N/A","N/A","N/A","N/A",0,0,0,0,0,0,0
+"1596336133","4","","Aggregated",0.00,0.00,18,19,300,300,330,330,330,330,330,330,330,330,12,0,18,115,8,333,37040
+"1596336135","8","","Aggregated",6.00,0.00,19,22,300,320,330,330,340,340,340,340,340,340,26,0,19,110,8,344,36889
+"1596336137","12","","Aggregated",6.00,0.00,22,28,300,320,330,350,370,370,370,370,370,370,45,0,22,104,3,373,37421
+"1596336139","16","","Aggregated",7.33,0.00,23,28,190,310,330,350,370,370,370,370,370,370,63,0,23,99,3,373,37468
+"1596336141","20","","Aggregated",7.14,0.00,23,27,51,310,330,340,370,370,370,370,370,370,80,0,23,94,3,373,36325
+"1596336143","20","","Aggregated",8.00,0.00,26,30,51,270,330,350,370,500,500,500,500,500,96,0,24,91,3,504,36582
+"1596336145","20","","Aggregated",8.30,0.00,26,30,32,40,310,340,370,500,500,500,500,500,111,0,26,82,3,504,35951
+"1596336147","20","","Aggregated",8.60,0.00,27,30,32,35,270,340,370,500,500,500,500,500,127,0,25,76,3,504,35750
+"1596336149","20","","Aggregated",7.90,0.00,27,31,33,36,130,270,340,500,500,500,500,500,140,0,26,73,3,504,35941
+"1596336151","20","","Aggregated",7.90,0.00,27,31,34,35,44,130,150,240,240,240,240,240,153,0,26,72,3,504,35966
+"1596336153","20","","Aggregated",7.10,0.00,28,33,35,36,78,130,150,240,240,240,240,240,168,0,26,69,3,504,36026
+"1596336155","20","","Aggregated",7.30,0.00,28,33,36,36,78,130,150,240,240,240,240,240,177,0,26,67,3,504,35843
+"1596336157","20","","Aggregated",7.30,0.00,28,34,36,36,78,130,150,240,240,240,240,240,188,1,26,64,3,504,35397
+"1596336159","20","","Aggregated",6.50,0.10,26,32,35,36,78,140,150,240,240,240,240,240,205,1,26,62,3,504,35527
+"1596336161","20","","Aggregated",6.50,0.10,26,29,33,35,47,73,130,140,140,140,140,140,218,1,26,60,3,504,35650
+"1596336163","20","","Aggregated",6.60,0.10,25,27,29,32,35,68,140,160,160,160,160,160,235,1,26,58,3,504,35495
+"1596336165","20","","Aggregated",6.60,0.10,26,28,32,33,42,130,140,160,160,160,160,160,244,1,26,57,3,504,35601
+"1596336167","20","","Aggregated",6.70,0.10,26,29,32,33,55,130,130,160,160,160,160,160,255,2,26,56,3,504,35369
+"1596336169","20","","Aggregated",6.60,0.10,26,29,32,32,43,58,130,160,160,160,160,160,269,2,26,54,3,504,35229
+"1596336171","20","","Aggregated",6.40,0.10,26,28,31,32,39,55,58,130,130,130,130,130,284,2,26,53,3,504,35026
+"1596336173","20","","Aggregated",6.70,0.10,26,29,31,32,39,58,130,130,130,130,130,130,296,2,26,52,3,504,35030
+"1596336175","20","","Aggregated",6.10,0.10,26,29,31,32,35,55,130,130,130,130,130,130,310,2,26,51,3,504,34991
+"1596336177","20","","Aggregated",6.40,0.10,25,29,30,31,35,51,130,130,130,130,130,130,320,2,26,50,3,504,34890
+"1596336179","20","","Aggregated",6.50,0.00,25,28,29,30,33,120,130,130,130,130,130,130,336,2,26,49,3,504,34830
+"1596336181","20","","Aggregated",6.70,0.00,26,27,29,30,33,44,130,140,140,140,140,140,351,2,26,48,3,504,34829
+"1596336183","20","","Aggregated",6.10,0.00,25,27,29,30,35,94,130,140,140,140,140,140,363,2,26,48,3,504,34787
+"1596336185","20","","Aggregated",6.70,0.00,26,28,29,30,37,120,150,190,190,190,190,190,378,2,26,48,3,504,34880
+"1596336187","20","","Aggregated",6.40,0.00,26,28,30,33,85,130,150,190,190,190,190,190,391,2,26,47,3,504,34874
+"1596336189","20","","Aggregated",6.80,0.00,27,29,33,34,94,140,150,190,190,190,190,190,400,2,26,47,3,504,34951
+"1596336191","20","","Aggregated",6.60,0.00,27,28,32,33,46,130,150,190,190,190,190,190,417,2,26,46,3,504,34925
+"1596336193","20","","Aggregated",6.90,0.00,26,28,29,31,37,85,130,130,130,130,130,130,430,2,26,46,3,504,34937
+"1596336195","20","","Aggregated",6.60,0.00,26,29,30,32,46,120,130,130,130,130,130,130,443,2,26,45,3,504,34959
+"1596336197","20","","Aggregated",6.60,0.00,26,28,29,30,35,46,120,130,130,130,130,130,454,2,26,45,3,504,34988
+"1596336199","20","","Aggregated",6.50,0.00,26,28,29,30,32,37,120,130,130,130,130,130,473,2,26,44,3,504,34866
+"1596336201","20","","Aggregated",7.10,0.00,26,28,29,30,32,35,120,130,130,130,130,130,485,2,26,44,3,504,34795
+"1596336203","20","","Aggregated",6.80,0.00,26,29,30,30,32,33,34,38,38,38,38,38,501,2,26,43,3,504,34747
+"1596336205","20","","Aggregated",7.20,0.00,27,29,30,31,32,33,34,43,43,43,43,43,512,2,26,43,3,504,34754
+"1596336207","20","","Aggregated",7.10,0.00,27,28,30,32,33,40,56,130,130,130,130,130,529,2,26,42,3,504,34777
+"1596336209","20","","Aggregated",7.20,0.00,27,29,32,33,40,57,130,130,130,130,130,130,541,2,26,42,3,504,34936
+"1596336211","20","","Aggregated",6.80,0.00,27,29,32,33,43,130,130,130,130,130,130,130,554,2,26,42,3,504,34959
+"1596336213","20","","Aggregated",7.10,0.00,27,29,32,33,57,130,130,130,130,130,130,130,572,2,26,42,3,504,35016
+"1596336215","20","","Aggregated",6.90,0.00,27,28,31,33,57,130,130,130,130,130,130,130,579,2,26,42,3,504,34976
+"1596336217","20","","Aggregated",6.80,0.00,27,28,31,32,50,130,130,130,130,130,130,130,594,2,26,41,3,504,34913
+"1596336219","20","","Aggregated",6.70,0.00,27,27,29,31,45,120,130,140,140,140,140,140,613,2,26,41,3,504,34870
+"1596336221","20","","Aggregated",6.70,0.00,26,27,28,29,32,46,130,140,140,140,140,140,625,2,26,41,3,504,34803
+"1596336223","20","","Aggregated",6.80,0.00,25,27,28,30,32,33,50,140,140,140,140,140,638,2,26,40,3,504,34731
+"1596336225","20","","Aggregated",6.80,0.00,25,27,30,31,32,33,50,140,140,140,140,140,648,2,26,40,3,504,34695
+"1596336227","20","","Aggregated",6.80,0.00,26,27,31,31,33,34,36,140,140,140,140,140,663,2,26,40,3,504,34632
+"1596336229","20","","Aggregated",6.70,0.00,26,28,31,32,34,36,36,44,44,44,44,44,673,2,26,40,3,504,34673
+"1596336231","20","","Aggregated",6.60,0.00,26,29,31,33,35,36,44,120,120,120,120,120,687,2,26,39,3,504,34659
+"1596336233","20","","Aggregated",6.40,0.00,27,29,31,33,35,44,68,120,120,120,120,120,698,2,26,39,3,504,34624
+"1596336235","20","","Aggregated",5.90,0.00,26,28,30,32,35,56,120,120,120,120,120,120,714,2,26,39,3,504,34690
+"1596336237","20","","Aggregated",6.30,0.00,26,28,30,32,44,120,120,130,130,130,130,130,724,2,26,39,3,504,34719
+"1596336239","20","","Aggregated",6.00,0.00,24,27,28,30,34,56,120,130,130,130,130,130,737,2,26,39,3,504,34667
+"1596336241","20","","Aggregated",6.20,0.00,24,27,28,29,34,68,120,130,130,130,130,130,750,2,26,38,3,504,34630
+"1596336243","20","","Aggregated",6.20,0.00,24,26,27,28,31,36,58,130,130,130,130,130,765,2,26,38,3,504,34688
+"1596336245","20","","Aggregated",6.90,0.00,24,26,28,28,32,36,58,130,130,130,130,130,778,2,26,38,3,504,34700
+"1596336247","20","","Aggregated",6.50,0.00,25,27,28,30,34,37,58,130,130,130,130,130,790,3,26,38,3,504,34677
+"1596336249","20","","Aggregated",6.70,0.10,25,27,30,31,34,37,58,130,130,130,130,130,802,3,26,38,3,504,34713
+"1596336250","0","","Aggregated",6.10,0.10,24,27,29,30,33,37,58,130,130,130,130,130,814,3,26,37,3,504,34674

loadtesting/dummyusers.json.bak

@@ -0,0 +1,22 @@
+[
+    {
+        "pw": "insertpasswordhere",
+        "user": "insertuserhere1"
+    },
+    {
+        "pw": "insertpasswordhere",
+        "user": "insertuserhere2"
+    },
+    {
+        "pw": "insertpasswordhere",
+        "user": "insertuserhere3"
+    },
+    {
+        "pw": "insertpasswordhere",
+        "user": "insertuserhere4"
+    },
+    {
+        "pw": "insertpasswordhere",
+        "user": "insertuserhere5"
+    }
+]

loadtesting/locust.conf

@@ -0,0 +1,9 @@
+locustfile = locustfile.py
+expect-workers = 200
+host = http://localhost
+users = 20
+hatch-rate = 2
+run-time = 2m
+headless = true
+csv=calibre
+only-summary=true

loadtesting/locustfile.py

@@ -0,0 +1,187 @@
+from locust import HttpUser, task, between
+import random
+from sqlalchemy import create_engine
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+import json
+
+# -------------------------------- fetching data to test with
+
+with open("./../CalibreWebCompanion/settings.json", "r") as jfile:
+    calpath = json.load(jfile)["CALIBRE_DIR"] + "/metadata.db"
+
+with open("dummyusers.json", "r") as jfile:
+    users = json.load(jfile)
+
+engine = create_engine(f'sqlite:///{calpath}')
+Base = declarative_base(engine)
+
+
+class Author(Base):  # needed
+    """"""
+    __tablename__ = 'authors'
+    __table_args__ = {'autoload': True}
+    # has int id, text name, sort
+
+
+class Identifier(Base):  # needed
+    """"""
+    __tablename__ = 'identifiers'
+    __table_args__ = {'autoload': True}
+    # has int id, int book, text value
+
+
+class Publisher(Base):  # needed
+    """"""
+    __tablename__ = 'publishers'
+    __table_args__ = {'autoload': True}
+    # has int id, text name
+
+
+class Rating(Base):  # needed
+    """"""
+    __tablename__ = 'ratings'
+    __table_args__ = {'autoload': True}
+    # has int id,  int rating
+
+
+class Series(Base):  # needed
+    """"""
+    __tablename__ = 'series'
+    __table_args__ = {'autoload': True}
+    # has int id,  text name
+
+
+class Tag(Base):  # needed
+    """"""
+    __tablename__ = 'tags'
+    __table_args__ = {'autoload': True}
+    # has int id,  text name
+
+
+class Book(Base):  # needed
+    """"""
+    __tablename__ = 'books'
+    __table_args__ = {'autoload': True}
+    # has int id,  text title, text sort, time timestamp, time pubdate,
+    # float series_index, text path
+
+
+def loadSession():
+    """"""
+    metadata = Base.metadata
+    Session = sessionmaker(bind=engine)
+    session = Session()
+    return session
+
+
+session = loadSession()
+
+titles = [i.title for i in session.query(Book).all()]
+authors = [i.name for i in session.query(Author).all()]
+identifiers = [i.val for i in session.query(Identifier).all()]
+book_ids = [i.id for i in session.query(Book).all()]
+author_ids = [i.id for i in session.query(Author).all()]
+publisher_ids = [i.id for i in session.query(Publisher).all()]
+rating_ids = [i.id for i in session.query(Rating).all()]
+series_ids = [i.id for i in session.query(Series).all()]
+tag_ids = [i.id for i in session.query(Tag).all()]
+
+
+def randlist(mylist):
+    return mylist[random.randint(0, len(mylist) - 1)]
+
+
+class UserBehavior(HttpUser):
+    wait_time = between(1, 5)
+
+    def on_start(self):
+        """ on_start is called when a Locust start before any task is scheduled """
+        r = self.client.get('/accounts/login/')
+        self.client.headers['Referer'] = self.client.base_url
+        user = randlist(users)
+        self.client.post('/accounts/login/',
+                         {
+                             "username": user["user"],
+                             "password": user["pw"],
+                             'csrfmiddlewaretoken': r.cookies["csrftoken"]
+                         })
+
+    @task(1)
+    def search_by_title(self):
+        title = randlist(titles)
+        self.client.get(f"/results/?title={title}", name="search_by_title")
+
+    @task(1)
+    def booklist(self):
+        self.client.get("/books/")
+
+    @task(1)
+    def bookdetail(self):
+        pk = randlist(book_ids)
+        self.client.get(f"/book/{pk}", name="/book/<id>")
+
+    @task(1)
+    def search_by_author(self):
+        author = randlist(authors)
+        self.client.get(f"/results/?author={author}", name="search_by_author")
+
+    @task(1)
+    def authorlist(self):
+        self.client.get("/authors/")
+
+    @task(1)
+    def authordetail(self):
+        pk = randlist(author_ids)
+        self.client.get(f"/author/{pk}", name="/author/<id>")
+
+    @task(1)
+    def search_by_id(self):
+        id_ = randlist(identifiers)
+        self.client.get(f"/results/?identifier={id_}", name="search_by_identifier")
+
+    @task(1)
+    def search_generic(self):
+        t = random.randint(0, 3)
+        if not t:
+            term = randlist(titles)
+        elif t == 1:
+            term = randlist(authors)
+        else:
+            term = randlist(identifiers)
+        self.client.get(f"/results/?generic={term}", name="search_generic")
+
+    @task(1)
+    def searchbad(self):
+        self.client.get("/search/")
+
+    @task(1)
+    def ratingslist(self):
+        self.client.get("/ratings/")
+
+    @task(1)
+    def ratingdetail(self):
+        pk = randlist(rating_ids)
+        self.client.get(f"/rating/{pk}", name="/rating/<id>")
+
+    @task(1)
+    def taglist(self):
+        self.client.get("/tags/")
+
+    @task(1)
+    def tagdetail(self):
+        pk = randlist(tag_ids)
+        self.client.get(f"/tag/{pk}", name="/tag/<id>")
+
+    @task(1)
+    def serieslist(self):
+        self.client.get("/series/")
+
+    @task(1)
+    def publisherlist(self):
+        self.client.get("/publishers/")
+
+    @task(1)
+    def publisherdetail(self):
+        pk = randlist(publisher_ids)
+        self.client.get(f"/publisher/{pk}", name="/publisher/<id>")

requirements.txt

@@ -1 +1,9 @@
-django==3.0.8
+django>=3.0.8
+inotify>=0.2.10
+gunicorn>=20.0 
+# development
+# django-debug-toolbar>=2.2
+# django-silk>=4.0
+# locust>=1.1
+# sqlalchemy>=1.3.15
+# rich>=3.0