fixed nginx, bug in context processors

I removed some hardcoded paths for logging and where the default db should be located.
I also organized deployment stuff a bit

CalibreWebCompanion/CalibreWebCompanion/settings.py

@@ -66,7 +66,7 @@ STATIC_ROOT = BASE_DIR + "/static/"
 
 
 
-logfile = usersettings["LOGFILE"]
+logfile = usersettings["LOGFOLDER"] + "django.log"
 LOGGING = {
     "version": 1,
     "disable_existing_loggers": False,
@@ -75,7 +75,7 @@ LOGGING = {
         "file": {
             "level": "INFO",
             "class": "logging.FileHandler",
-            "filename": usersettings["LOGFILE"],
+            "filename": logfile,
             "formatter": "app",
         },
     },
@@ -97,6 +97,7 @@ LOGGING = {
     },
 }
 
+
 ##                                                                    ##
 ########################################################################
 ##                    DERUG                                           ##
@@ -188,10 +189,15 @@ WSGI_APPLICATION = 'CalibreWebCompanion.wsgi.application'
 ##                    DATBASE                                         ##
 # https://docs.djangoproject.com/en/3.0/ref/settings/#databases
 
+if usersettings["ISDOCKER"]:
+    defaultdb_path = "/usr/src/app/data/"
+else:
+    defaultdb_path = BASE_DIR
+
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+        'NAME': os.path.join(defaultdb_path, 'db.sqlite3'),
     },
     'calibre': {
         'ENGINE': 'django.db.backends.sqlite3',

CalibreWebCompanion/gunicorn.conf.py

@@ -1,4 +1,6 @@
 import multiprocessing
+import os
+import json
 
 bind = "127.0.0.1:8000"
 workers = multiprocessing.cpu_count() * 2 + 1
@@ -7,10 +9,20 @@ keepalive = 5
 # daemon = True # Detaches the server from the controlling terminal and enters the background. disabled for now
 # logging
 
-errorlog = "/home/massiveatoms/Desktop/logs/gunicorn_error.log"
+with open("settings.json", "r") as jfile:
+    settings = json.load(jfile)
+
+errorlog = settings["LOGFOLDER"] + "/gunicorn_error.log"
 loglevel = "warning"
-accesslog = "/home/massiveatoms/Desktop/logs/gunicorn_access.log"
+accesslog = settings["LOGFOLDER"] + "/gunicorn_access.log"
-# capture_output = True
+
+if not os.path.isdir("/usr/src/app/data/logs"):
+    os.mkdir("/usr/src/app/data/logs")
+if not os.path.isfile(errorlog):
+    os.system('touch {}'.format(errorlog))
+if not os.path.isfile(accesslog):
+    os.system('touch {}'.format(accesslog))
+capture_output = True
 
 # debug settings which need to be commented out in prod
 # reload=True

CalibreWebCompanion/library/context_processors.py

@@ -15,7 +15,7 @@ def filters(request):
     unique_authors = Author.objects.only('name', "id").annotate(num_books=Count('book')).order_by('name')
     unique_tags = Tag.objects.annotate(num_books=Count('book')).order_by('name')
     unique_publishers = Publisher.objects.annotate(num_books=Count('book')).order_by('name')
-    unique_languages = Language.objects.annotate(num_books=Count('book')).order_by('rating')
+    unique_languages = Language.objects.annotate(num_books=Count('book')).order_by('lang_code')
     unique_ratings = Rating.objects.annotate(num_books=Count('book'))
     unique_series = Series.objects.annotate(num_books=Count('book')).order_by('sort')
     
@@ -27,4 +27,4 @@ def filters(request):
         "unique_languages": unique_languages,
         "unique_ratings": unique_ratings,
         "unique_series": unique_series
-    }
+    }

CalibreWebCompanion/library/tests.py

@@ -1,3 +1,166 @@
-from django.test import TestCase
+from django.test import Client, TestCase
+from pprint import pprint
+from django.test.utils import setup_test_environment
+from .models import Book, Author, Publisher, Series, Rating, Tag, Identifier
+from django.db.models import Count
 
-# Create your tests here.
+
+client = Client()
+client.login(username="testuser", password="dumbeasypassword")
+
+def booklisttest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    res = c.get("/books/")
+    assert res.status_code == 200
+    context = dict(res.context)
+    assert sorted(context["book_list"], key=lambda x: x.id)== sorted(Book.objects.all(), key=lambda x: x.id)
+
+def authorlisttest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    res = c.get("/authors/")
+    assert res.status_code == 200
+    context = dict(res.context)
+    assert sorted(context["author_list"], key=lambda x: x.id)== sorted(Author.objects.all(), key=lambda x: x.id)
+
+
+def publisherlisttest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    res = c.get("/publishers/")
+    assert res.status_code == 200
+    context = dict(res.context)
+    assert sorted(context["publisher_list"], key=lambda x: x.id)== sorted(Publisher.objects.all(), key=lambda x: x.id)
+
+def serieslisttest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    res = c.get("/series/")
+    assert res.status_code == 200
+    context = dict(res.context)
+    assert sorted(context["series_list"], key=lambda x: x.id)== sorted(Series.objects.all(), key=lambda x: x.id)
+
+
+def ratinglisttest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    res = c.get("/ratings/")
+    assert res.status_code == 200
+    context = dict(res.context)
+    assert sorted(context["rating_list"], key=lambda x: x.id)== sorted(Rating.objects.all(), key=lambda x: x.id)
+
+def taglisttest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    res = c.get("/tags/")
+    assert res.status_code == 200
+    context = dict(res.context)
+    assert sorted(context["tag_list"], key=lambda x: x.id)== sorted(Tag.objects.all(), key=lambda x: x.id)
+
+
+def bookdetailtest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    ids = [i.id for i in Book.objects.all()][:10]
+    for i in ids:
+        res = c.get(f"/book/{i}")
+        assert res.status_code == 200
+        context = dict(res.context)
+        assert context["book"] == Book.objects.get(id=i)
+
+
+def authordetailtest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    ids = [i.id for i in Author.objects.all()][:10]
+    for i in ids:
+        res = c.get(f"/author/{i}")
+        assert res.status_code == 200
+        context = dict(res.context)
+        assert context["author"] == Author.objects.get(id=i)
+
+def publisherdetailtest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    ids = [i.id for i in Publisher.objects.all()][:10]
+    for i in ids:
+        res = c.get(f"/publisher/{i}")
+        assert res.status_code == 200
+        context = dict(res.context)
+        assert context["publisher"] == Publisher.objects.get(id=i)
+
+def seriesdetailtest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    ids = [i.id for i in Series.objects.all()][:10]
+    for i in ids:
+        res = c.get(f"/series/{i}")
+        assert res.status_code == 200
+        context = dict(res.context)
+        assert context["series"] == Series.objects.get(id=i)
+
+def ratingdetailtest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    ids = [i.id for i in Rating.objects.all()][:10]
+    for i in ids:
+        res = c.get(f"/rating/{i}")
+        assert res.status_code == 200
+        context = dict(res.context)
+        assert context["rating"] == Rating.objects.get(id=i)        
+
+
+def tagdetailtest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    ids = [i.id for i in Tag.objects.all()][:10]
+    for i in ids:
+        res = c.get(f"/tag/{i}")
+        assert res.status_code == 200
+        context = dict(res.context)
+        assert context["tag"] == Tag.objects.get(id=i) 
+
+
+def search_partial(key, value, book=None):
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    res = c.get("/results/", {key : value})
+    if not book:
+        return dict(res.context)["book_list"]
+    return book in dict(res.context)["book_list"]
+
+
+def searchtest():
+    books = [i for i in Book.objects.all()][:10]
+    for i in books:
+        assert search_partial("title", i.title, i)
+        assert search_partial("generic", i.title, i)
+        assert search_partial("author", i.author_sort, i)
+        author = i.authors.first()
+        if author:
+            assert search_partial("author", author.name, i)
+            assert search_partial("generic", author.name, i)
+
+        assert search_partial("generic", i.author_sort, i)
+        id = Identifier.objects.filter(book=i.id).first()
+        if id:
+            assert search_partial("identifier", id, i)
+            assert search_partial("generic", id, i)
+
+
+    
+
+booklisttest()
+bookdetailtest()
+authorlisttest()
+authordetailtest()
+publisherdetailtest()
+publisherlisttest()
+seriesdetailtest()
+serieslisttest()
+ratingdetailtest()
+ratinglisttest()
+tagdetailtest()
+taglisttest()
+searchtest()

CalibreWebCompanion/library/views.py

@@ -1,3 +1,4 @@
+from django.utils.decorators import method_decorator
 from django.shortcuts import render
 from django.views import generic
 from .models import Author, Book, Comment, Rating, BookAuthorLink, Publisher, Tag, BookTagLink, BookRatingLink, Data, Identifier, Series
@@ -13,7 +14,6 @@ import logging
 logger = logging.getLogger(__name__)
 
 # might be helpful for vary headers later
-from django.utils.decorators import method_decorator
 
 
 @login_required
@@ -58,13 +58,29 @@ class ResultsView(generic.ListView):  # no clue if this is secure.
         if title:
             books = books.filter(sort__icontains=title)
         if author:
-            books = books.filter(author_sort__icontains=author)
+            # authors are stored as author_sort and author, needs to be slightly more complex
+            author_obj = Author.objects.filter(name__icontains=author).first()
+            if not author_obj:
+                author_id = -1
+            else:
+                author_id = author_obj.id
+
+            books = books.filter(
+                Q(author_sort__icontains=author) |
+                Q(authors__id=author_id)
+            )
         if identifier:
             books = books.filter(identifier__val=identifier)
         if generic:
+            author_obj = Author.objects.filter(name__icontains=generic).first()
+            if not author_obj:
+                author_id = -1
+            else:
+                author_id = author_obj.id
             books = books.filter(
                 Q(sort__icontains=generic) |
                 Q(author_sort__icontains=generic) |
+                Q(authors__id=author_id) |
                 Q(identifier__val=generic)
             )
         return books

deployment/Dockerfile

@@ -0,0 +1,29 @@
+
+## pull official base image
+FROM python:3.8.3-alpine
+EXPOSE 8080
+## set work directory
+WORKDIR /usr/src/app
+## install dependencies
+RUN pip install --upgrade pip
+COPY ./requirements.txt .
+RUN pip install -r requirements.txt
+RUN apk add nginx supervisor
+
+# do nginx stuff
+RUN adduser -D -g 'www' www
+RUN mkdir -p /run/nginx
+COPY ./deployment/nginx.conf /etc/nginx/
+
+## copy project
+COPY ./CalibreWebCompanion ./CalibreWebCompanion
+copy ./deployment/startupscript.py ./
+## gunicorn borks started with supervisord
+COPY ./deployment/supervisord.conf /etc/
+ENTRYPOINT /usr/bin/supervisord -c /etc/supervisord.conf
+
+
+# docker run --publish 8000:80 \
+# -v '/home/massiveatoms/Desktop/logs:/usr/src/app/data' \
+# -v '/run/media/massiveatoms/1AEEEA6EEEEA421D1/Documents and Settings/MassiveAtoms/Documents/Calibre Library/:/usr/src/app/calibredir' \
+# --name cw calibreweb:1.0.1

deployment/deploy.py

@@ -0,0 +1,4 @@
+from os import environ
+
+
+

deployment/instructions.md

@@ -1,4 +1,4 @@
-
+# non docker stuff
 1. clone repo
 2. pip install -r requirements.txt
 3. install gunicorn and nginx
@@ -18,4 +18,19 @@ Suggestions:
 1. We might want to use sockets instead of ip/port?
 2. logging
 3. autostart gunicorn/nginx
-4. some extra instrumentation for gunicorn https://docs.gunicorn.org/en/latest/deploy.html
+4. some extra instrumentation for gunicorn https://docs.gunicorn.org/en/latest/deploy.html
+
+# docker stuff
+here are the commands i use to build and run:
+1. build --tag calibreweb:1.0.1  . -f ./deployment/Dockerfile
+2. 
+```
+docker run --publish 8000:80\ 
+-v '/home/massiveatoms/Desktop/logs:/usr/src/app/data' \
+-v '/run/media/massiveatoms/1AEEEA6EEEEA421D/Documents and Settings/MassiveAtoms/Documents/Calibre Library/:/usr/src/app/calibredir' \
+--name cw calibreweb:1.0.1
+```
+
+Docker stuff to fix:
+1. permission/ownership issue with volumes and nginx
+2. we need a setup script that changes secretkey, and generates the default db

deployment/nginx.conf

@@ -1,9 +1,9 @@
 worker_processes 1;
 
 # user nobody nogroup;
-user massiveatoms; 
+user www www;  # TEMP disabled
 # user nobody nobody; # for systems with 'nobody' as a group instead
-error_log  /home/massiveatoms/Desktop/logs/nginx.log warn;
+error_log  /usr/src/app/data/logs/nginx.log warn;
 # pid /var/run/nginx.pid;
 
 events {
@@ -44,18 +44,18 @@ http {
     client_max_body_size 4G;
 
     # set the correct host(s) for your site
-    server_name localhost  192.168.1.4; # set this to the server url? or ip? we'll see MASSIVEATOMS
+    server_name localhost  0.0.0.0; # set this to the server url? or ip? we'll see MASSIVEATOMS
 
     keepalive_timeout 5;
 
-    # MASSIVEATOMS
+    # # MASSIVEATOMS
       location /download/ {
-        alias "/run/media/massiveatoms/1AEEEA6EEEEA421D/Documents and Settings/MassiveAtoms/Documents/Calibre Library/";
+        alias "/usr/src/app/calibredir/";
         # Never forget the fact that this little statement being root instead of alias caused us to lose more than a day troubleshooting
     }
 
           location /static/ {
-        alias "/home/massiveatoms/Desktop/calibre-web-companion/CalibreWebCompanion/static/";
+        alias "/usr/src/app/CalibreWebCompanion/static/";
         # Never forget the fact that this little statement being root instead of alias caused us to lose more than a day troubleshooting
     }
 

deployment/startupscript.py

@@ -0,0 +1,11 @@
+from os import system, chdir
+
+
+# system("chown -R www:www /usr/src/app/calibredir")
+# print("ownership of calibredir changed")
+chdir("/usr/src/app/CalibreWebCompanion")
+system("python ./manage.py makemigrations")
+print("ran makemigrations")
+system("python ./manage.py migrate")
+print("migrate")
+

deployment/supervisord.conf

@@ -0,0 +1,36 @@
+[supervisord]
+nodaemon=true
+logfile=/tmp/supervisord.log
+childlogdir=/tmp
+pidfile = /tmp/supervisord.pid
+
+[program:gunicorn]
+directory=/usr/src/app/CalibreWebCompanion
+command=gunicorn CalibreWebCompanion.wsgi
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autorestart=false
+startretries=0
+startsecs = 0
+
+[program:nginx]
+# user=www
+command=nginx
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autorestart=false
+startretries=0
+
+[program:startupscript]
+directory=/usr/src/app
+command=python ./startupscript.py
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autorestart=false
+startretries=0

requirements.txt

@@ -1,8 +1,9 @@
 django>=3.0.8
 inotify>=0.2.10
+gunicorn>=20.0 
 # development
-django-debug-toolbar>=2.2
+# django-debug-toolbar>=2.2
-django-silk>=4.0
+# django-silk>=4.0
-locust>=1.1
+# locust>=1.1
-sqlalchemy>=1.3.15
+# sqlalchemy>=1.3.15
-rich>=3.0
+# rich>=3.0