changed instructions

Merge branch 'development'

CalibreWebCompanion/CalibreWebCompanion/settings.py

@@ -66,7 +66,7 @@ STATIC_ROOT = BASE_DIR + "/static/"
 
 
 
-logfile = usersettings["LOGFILE"]
+logfile = usersettings["LOGFOLDER"] + "django.log"
 LOGGING = {
     "version": 1,
     "disable_existing_loggers": False,
@@ -75,7 +75,7 @@ LOGGING = {
         "file": {
             "level": "INFO",
             "class": "logging.FileHandler",
-            "filename": usersettings["LOGFILE"],
+            "filename": logfile,
             "formatter": "app",
         },
     },
@@ -97,6 +97,7 @@ LOGGING = {
     },
 }
 
+
 ##                                                                    ##
 ########################################################################
 ##                    DERUG                                           ##
@@ -158,7 +159,7 @@ MIDDLEWARE = [
 ]
 ##                                                                    ##
 ########################################################################
-
+DEFAULT_CHARSET = "utf-8"
 
 ROOT_URLCONF = 'CalibreWebCompanion.urls'
 
@@ -188,10 +189,15 @@ WSGI_APPLICATION = 'CalibreWebCompanion.wsgi.application'
 ##                    DATBASE                                         ##
 # https://docs.djangoproject.com/en/3.0/ref/settings/#databases
 
+if usersettings["ISDOCKER"]:
+    defaultdb_path = "/usr/src/app/data/"
+else:
+    defaultdb_path = BASE_DIR
+
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+        'NAME': os.path.join(defaultdb_path, 'db.sqlite3'),
     },
     'calibre': {
         'ENGINE': 'django.db.backends.sqlite3',

CalibreWebCompanion/gunicorn.conf.py

@@ -1,4 +1,6 @@
 import multiprocessing
+import os
+import json
 
 bind = "127.0.0.1:8000"
 workers = multiprocessing.cpu_count() * 2 + 1
@@ -7,10 +9,20 @@ keepalive = 5
 # daemon = True # Detaches the server from the controlling terminal and enters the background. disabled for now
 # logging
 
-errorlog = "/home/massiveatoms/Desktop/logs/gunicorn_error.log"
+with open("settings.json", "r") as jfile:
+    settings = json.load(jfile)
+
+errorlog = settings["LOGFOLDER"] + "/gunicorn_error.log"
 loglevel = "warning"
-accesslog = "/home/massiveatoms/Desktop/logs/gunicorn_access.log"
+accesslog = settings["LOGFOLDER"] + "/gunicorn_access.log"
-# capture_output = True
+
+if not os.path.isdir("/usr/src/app/data/logs"):
+    os.mkdir("/usr/src/app/data/logs")
+if not os.path.isfile(errorlog):
+    os.system('touch {}'.format(errorlog))
+if not os.path.isfile(accesslog):
+    os.system('touch {}'.format(accesslog))
+capture_output = True
 
 # debug settings which need to be commented out in prod
 # reload=True

CalibreWebCompanion/library/context_processors.py

@@ -15,7 +15,7 @@ def filters(request):
     unique_authors = Author.objects.only('name', "id").annotate(num_books=Count('book')).order_by('name')
     unique_tags = Tag.objects.annotate(num_books=Count('book')).order_by('name')
     unique_publishers = Publisher.objects.annotate(num_books=Count('book')).order_by('name')
-    unique_languages = Language.objects.annotate(num_books=Count('book')).order_by('rating')
+    unique_languages = Language.objects.annotate(num_books=Count('book')).order_by('lang_code')
     unique_ratings = Rating.objects.annotate(num_books=Count('book'))
     unique_series = Series.objects.annotate(num_books=Count('book')).order_by('sort')
     

CalibreWebCompanion/library/tests.py

@@ -1,3 +1,166 @@
-from django.test import TestCase
+from django.test import Client, TestCase
+from pprint import pprint
+from django.test.utils import setup_test_environment
+from .models import Book, Author, Publisher, Series, Rating, Tag, Identifier
+from django.db.models import Count
 
-# Create your tests here.
+
+client = Client()
+client.login(username="testuser", password="dumbeasypassword")
+
+def booklisttest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    res = c.get("/books/")
+    assert res.status_code == 200
+    context = dict(res.context)
+    assert sorted(context["book_list"], key=lambda x: x.id)== sorted(Book.objects.all(), key=lambda x: x.id)
+
+def authorlisttest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    res = c.get("/authors/")
+    assert res.status_code == 200
+    context = dict(res.context)
+    assert sorted(context["author_list"], key=lambda x: x.id)== sorted(Author.objects.all(), key=lambda x: x.id)
+
+
+def publisherlisttest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    res = c.get("/publishers/")
+    assert res.status_code == 200
+    context = dict(res.context)
+    assert sorted(context["publisher_list"], key=lambda x: x.id)== sorted(Publisher.objects.all(), key=lambda x: x.id)
+
+def serieslisttest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    res = c.get("/series/")
+    assert res.status_code == 200
+    context = dict(res.context)
+    assert sorted(context["series_list"], key=lambda x: x.id)== sorted(Series.objects.all(), key=lambda x: x.id)
+
+
+def ratinglisttest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    res = c.get("/ratings/")
+    assert res.status_code == 200
+    context = dict(res.context)
+    assert sorted(context["rating_list"], key=lambda x: x.id)== sorted(Rating.objects.all(), key=lambda x: x.id)
+
+def taglisttest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    res = c.get("/tags/")
+    assert res.status_code == 200
+    context = dict(res.context)
+    assert sorted(context["tag_list"], key=lambda x: x.id)== sorted(Tag.objects.all(), key=lambda x: x.id)
+
+
+def bookdetailtest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    ids = [i.id for i in Book.objects.all()][:10]
+    for i in ids:
+        res = c.get(f"/book/{i}")
+        assert res.status_code == 200
+        context = dict(res.context)
+        assert context["book"] == Book.objects.get(id=i)
+
+
+def authordetailtest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    ids = [i.id for i in Author.objects.all()][:10]
+    for i in ids:
+        res = c.get(f"/author/{i}")
+        assert res.status_code == 200
+        context = dict(res.context)
+        assert context["author"] == Author.objects.get(id=i)
+
+def publisherdetailtest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    ids = [i.id for i in Publisher.objects.all()][:10]
+    for i in ids:
+        res = c.get(f"/publisher/{i}")
+        assert res.status_code == 200
+        context = dict(res.context)
+        assert context["publisher"] == Publisher.objects.get(id=i)
+
+def seriesdetailtest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    ids = [i.id for i in Series.objects.all()][:10]
+    for i in ids:
+        res = c.get(f"/series/{i}")
+        assert res.status_code == 200
+        context = dict(res.context)
+        assert context["series"] == Series.objects.get(id=i)
+
+def ratingdetailtest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    ids = [i.id for i in Rating.objects.all()][:10]
+    for i in ids:
+        res = c.get(f"/rating/{i}")
+        assert res.status_code == 200
+        context = dict(res.context)
+        assert context["rating"] == Rating.objects.get(id=i)        
+
+
+def tagdetailtest():
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    ids = [i.id for i in Tag.objects.all()][:10]
+    for i in ids:
+        res = c.get(f"/tag/{i}")
+        assert res.status_code == 200
+        context = dict(res.context)
+        assert context["tag"] == Tag.objects.get(id=i) 
+
+
+def search_partial(key, value, book=None):
+    c = Client()
+    c.login(username="testuser", password="dumbeasypassword")
+    res = c.get("/results/", {key : value})
+    if not book:
+        return dict(res.context)["book_list"]
+    return book in dict(res.context)["book_list"]
+
+
+def searchtest():
+    books = [i for i in Book.objects.all()][:10]
+    for i in books:
+        assert search_partial("title", i.title, i)
+        assert search_partial("generic", i.title, i)
+        assert search_partial("author", i.author_sort, i)
+        author = i.authors.first()
+        if author:
+            assert search_partial("author", author.name, i)
+            assert search_partial("generic", author.name, i)
+
+        assert search_partial("generic", i.author_sort, i)
+        id = Identifier.objects.filter(book=i.id).first()
+        if id:
+            assert search_partial("identifier", id, i)
+            assert search_partial("generic", id, i)
+
+
+    
+
+booklisttest()
+bookdetailtest()
+authorlisttest()
+authordetailtest()
+publisherdetailtest()
+publisherlisttest()
+seriesdetailtest()
+serieslisttest()
+ratingdetailtest()
+ratinglisttest()
+tagdetailtest()
+taglisttest()
+searchtest()

CalibreWebCompanion/library/views.py

@@ -1,3 +1,4 @@
+from django.utils.decorators import method_decorator
 from django.shortcuts import render
 from django.views import generic
 from .models import Author, Book, Comment, Rating, BookAuthorLink, Publisher, Tag, BookTagLink, BookRatingLink, Data, Identifier, Series
@@ -13,7 +14,6 @@ import logging
 logger = logging.getLogger(__name__)
 
 # might be helpful for vary headers later
-from django.utils.decorators import method_decorator
 
 
 @login_required
@@ -58,13 +58,29 @@ class ResultsView(generic.ListView):  # no clue if this is secure.
         if title:
             books = books.filter(sort__icontains=title)
         if author:
-            books = books.filter(author_sort__icontains=author)
+            # authors are stored as author_sort and author, needs to be slightly more complex
+            author_obj = Author.objects.filter(name__icontains=author).first()
+            if not author_obj:
+                author_id = -1
+            else:
+                author_id = author_obj.id
+
+            books = books.filter(
+                Q(author_sort__icontains=author) |
+                Q(authors__id=author_id)
+            )
         if identifier:
             books = books.filter(identifier__val=identifier)
         if generic:
+            author_obj = Author.objects.filter(name__icontains=generic).first()
+            if not author_obj:
+                author_id = -1
+            else:
+                author_id = author_obj.id
             books = books.filter(
                 Q(sort__icontains=generic) |
                 Q(author_sort__icontains=generic) |
+                Q(authors__id=author_id) |
                 Q(identifier__val=generic)
             )
         return books

CalibreWebCompanion/settings.json.bak

@@ -1,12 +1,15 @@
 {
-    "CALIBRE_DIR": "PATH\\TO\\your\\calibre\library",
+    "CALIBRE_DIR": "/usr/src/app/calibredir",
     "SECRET_KEY": "u(8^+rb%rz5hsx4v^^y(ul7g(4n7a8!db@s*9(m5cs*2_ppy8+",
-   "ALLOWED_HOSTS": [
+    "ALLOWED_HOSTS": [
         "127.0.0.1"
     ],
     "INTERNAL_IPS": [
         "127.0.0.1"
     ],
-    "DEBUG" : false,
+    "DEBUG" : true,
-    "LOGFILE" : "/home/massiveatoms/Desktop/logs/django.log"
+    "LOGFOLDER" : "/usr/src/app/data/logs/",
+    "ISDOCKER" : true
+
+
 }

README.md

@@ -24,24 +24,10 @@ Django 3.0
 Calibre 4.13 (I have not tested it with anything else atm, will be resolved later)
 
 # how to use:
-1. clone repo
+1. [Docker setup](./deployment/instructions.md#user-content-docker-detup)
-2. Remove the `.bak` from `./CalibreWebCompanion/settings.json.bak`  and `db.sqlite3.bak`
+2. [Non Docker setup](./deployment/instructions.md#user-content-non-docker-detup)
-3. Edit `./CalibreWebCompanion/settings.json`. Definitely change the secret key
-4. Not sure if the db needs to be regenerated, but we'll see later __!!!!!!!!!!__
-5. pip install -r requirements.txt
-6. install gunicorn and nginx
-7. move this nginx.conf to /etc/nginx
-8. make whatever user nginx runs as (in this case, massiveatoms) the owner of calibredir
-9. give execute permissions to parent of calibredir
-10. cd to repo, run `gunicorn CalibreWebCompanion.wsgi`
-11. start nginx `sudo systemctl restart nginx`
 
 
- 
- 
-`./CalibreWebCompanion`     
-run `./manage.py runserver`    
-
 # Ignore pretty much everything below if you're not working on the project
 
 # Profiling
@@ -74,7 +60,8 @@ You can then go to [http://localhost:8089/](http://localhost:8089/) to see live
 - [ ] cache with vary headers
 - [ ] localisation
 - [ ] Beautifying template (only works well on 720p, no other viewports)
-- [ ]  Setup email functionality
+- [ ]  Setup email functionality (atm, there's only a dummy one, and you can't reset passwords)
+- [ ]  isolate the styling and templates, so we can swap them out by just swapping directory content
 
 
 

deployment/Dockerfile

@@ -0,0 +1,29 @@
+
+## pull official base image
+FROM python:3.8.3-alpine
+EXPOSE 8080
+## set work directory
+WORKDIR /usr/src/app
+## install dependencies
+RUN pip install --upgrade pip
+COPY ./requirements.txt .
+RUN pip install -r requirements.txt
+RUN apk add nginx supervisor
+
+# do nginx stuff
+RUN adduser -D -g 'www' www
+RUN mkdir -p /run/nginx
+COPY ./deployment/nginx.conf /etc/nginx/
+
+## copy project
+COPY ./CalibreWebCompanion ./CalibreWebCompanion
+copy ./deployment/startupscript.py ./
+## gunicorn borks started with supervisord
+COPY ./deployment/supervisord.conf /etc/
+ENTRYPOINT /usr/bin/supervisord -c /etc/supervisord.conf
+
+
+# docker run --publish 8000:80 \
+# -v '/home/massiveatoms/Desktop/logs:/usr/src/app/data' \
+# -v '/run/media/massiveatoms/1AEEEA6EEEEA421D1/Documents and Settings/MassiveAtoms/Documents/Calibre Library/:/usr/src/app/calibredir' \
+# --name cw calibreweb:1.0.1

deployment/deploy.py

@@ -0,0 +1,4 @@
+from os import environ
+
+
+

deployment/instructions.md

@@ -1,21 +1,45 @@
 
+# Docker setup (no provided docker image atm)
+1. clone the repo
+2. rename ./calireWebCompanion/settings.json.bak to settings.json
+3. change the secret key
+4. run `build --tag calibreweb:1.0  . -f ./deployment/Dockerfile` to build the image
+5. run your container with your bind/mount your volumes/paths/things 
+Here's an example of step 5
+```
+docker run --publish 80:80\ 
+-v '/home/massiveatoms/Desktop/logs:/usr/src/app/data' \
+-v '/run/media/massiveatoms/1AEEEA6EEEEA421D/Documents and Settings/MassiveAtoms/Documents/Calibre Library/:/usr/src/app/calibredir' \
+--name cw calibreweb:1.0.1
+```
+
+your Calibre path/volume/whatever needs to be mounted at `/usr/src/app/calibredir`, and you need to mount a volume for the db and logs at `/usr/src/app/data`
+
+Issues with it at the moment:
+1. we still need to do something to create a random secret key. Atm, this would still 
+
+# Docker (provided image)
+not done yet
+
+# non docker setup
+this might need to be modified, since some things have changed to adapt it for docker setup
 1. clone repo
 2. pip install -r requirements.txt
-3. install gunicorn and nginx
+3. rename the settings.json.bak to settings.json, change logging folder, change secret key, set isdocker to false
-4. move this nginx.conf to /etc/nginx
+4. install gunicorn and nginx
-5. make whatever user nginx runs as (in this case, massiveatoms) the owner of calibredir
+5. move this nginx.conf to /etc/nginx
-6. give execute permissions to parent of calibredir
+6. create a user and group `www`
-7. cd to repo, run `gunicorn CalibreWebCompanion.wsgi`
+7. make whatever user nginx runs as (for now, www) the owner of calibredir
-8. start nginx `sudo systemctl restart nginx`
+8. give execute permissions to parent of calibredir
+9. cd to repo, run `gunicorn CalibreWebCompanion.wsgi`
+10. start nginx `sudo systemctl restart nginx`
+11. make steps 9 and 10 happen on startup?
 
 Slight issues with this atm:
-1. server_name in nginx.conf needs to be changed
+1. where to do ssl?
-2. User needs to be edited in nginx.conf, now it's just my user acc. This affectd step 4-6
-3. where to do ssl?
 
 
 Suggestions: 
 1. We might want to use sockets instead of ip/port?
-2. logging
+2. autostart gunicorn/nginx
-3. autostart gunicorn/nginx
+3. some extra instrumentation for gunicorn https://docs.gunicorn.org/en/latest/deploy.html
-4. some extra instrumentation for gunicorn https://docs.gunicorn.org/en/latest/deploy.html

deployment/nginx.conf

@@ -1,9 +1,9 @@
 worker_processes 1;
 
 # user nobody nogroup;
-user massiveatoms; 
+user www www;  # TEMP disabled
 # user nobody nobody; # for systems with 'nobody' as a group instead
-error_log  /home/massiveatoms/Desktop/logs/nginx.log warn;
+error_log  /usr/src/app/data/logs/nginx.log warn;
 # pid /var/run/nginx.pid;
 
 events {
@@ -44,18 +44,18 @@ http {
     client_max_body_size 4G;
 
     # set the correct host(s) for your site
-    server_name localhost  192.168.1.4; # set this to the server url? or ip? we'll see MASSIVEATOMS
+    server_name localhost  0.0.0.0; # set this to the server url? or ip? we'll see MASSIVEATOMS
 
     keepalive_timeout 5;
 
-    # MASSIVEATOMS
+    # # MASSIVEATOMS
       location /download/ {
-        alias "/run/media/massiveatoms/1AEEEA6EEEEA421D/Documents and Settings/MassiveAtoms/Documents/Calibre Library/";
+        alias "/usr/src/app/calibredir/";
         # Never forget the fact that this little statement being root instead of alias caused us to lose more than a day troubleshooting
     }
 
           location /static/ {
-        alias "/home/massiveatoms/Desktop/calibre-web-companion/CalibreWebCompanion/static/";
+        alias "/usr/src/app/CalibreWebCompanion/static/";
         # Never forget the fact that this little statement being root instead of alias caused us to lose more than a day troubleshooting
     }
 

deployment/startupscript.py

@@ -0,0 +1,11 @@
+from os import system, chdir
+
+
+# system("chown -R www:www /usr/src/app/calibredir")
+# print("ownership of calibredir changed")
+chdir("/usr/src/app/CalibreWebCompanion")
+system("python ./manage.py makemigrations")
+print("ran makemigrations")
+system("python ./manage.py migrate")
+print("migrate")
+

deployment/supervisord.conf

@@ -0,0 +1,36 @@
+[supervisord]
+nodaemon=true
+logfile=/tmp/supervisord.log
+childlogdir=/tmp
+pidfile = /tmp/supervisord.pid
+
+[program:gunicorn]
+directory=/usr/src/app/CalibreWebCompanion
+command=gunicorn CalibreWebCompanion.wsgi
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autorestart=false
+startretries=0
+startsecs = 0
+
+[program:nginx]
+# user=www
+command=nginx
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autorestart=false
+startretries=0
+
+[program:startupscript]
+directory=/usr/src/app
+command=python ./startupscript.py
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autorestart=false
+startretries=0

requirements.txt

@@ -1,8 +1,9 @@
 django>=3.0.8
 inotify>=0.2.10
+gunicorn>=20.0 
 # development
-django-debug-toolbar>=2.2
+# django-debug-toolbar>=2.2
-django-silk>=4.0
+# django-silk>=4.0
-locust>=1.1
+# locust>=1.1
-sqlalchemy>=1.3.15
+# sqlalchemy>=1.3.15
-rich>=3.0
+# rich>=3.0