Compare commits

..

4 Commits

Author SHA1 Message Date
TinyAtoms
49ca4bccdc fixed nginx, bug in context processors 2020-08-16 01:45:28 -03:00
TinyAtoms
9843299ef6 fixed some db stuff 2020-08-15 11:00:25 -03:00
TinyAtoms
75099ca05e Removed hardcoded paths, make workflow changes
I removed some hardcoded paths for logging and where the default db should be located.
I also organized deployment stuff a bit
2020-08-15 02:40:03 -03:00
TinyAtoms
1b8d81bd3c pre fixing problems 2020-08-15 01:56:27 -03:00
14 changed files with 318 additions and 25 deletions

View File

@ -66,7 +66,7 @@ STATIC_ROOT = BASE_DIR + "/static/"
logfile = usersettings["LOGFILE"]
logfile = usersettings["LOGFOLDER"] + "django.log"
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
@ -75,7 +75,7 @@ LOGGING = {
"file": {
"level": "INFO",
"class": "logging.FileHandler",
"filename": usersettings["LOGFILE"],
"filename": logfile,
"formatter": "app",
},
},
@ -97,6 +97,7 @@ LOGGING = {
},
}
## ##
########################################################################
## DERUG ##
@ -188,10 +189,15 @@ WSGI_APPLICATION = 'CalibreWebCompanion.wsgi.application'
## DATBASE ##
# https://docs.djangoproject.com/en/3.0/ref/settings/#databases
if usersettings["ISDOCKER"]:
defaultdb_path = "/usr/src/app/data/"
else:
defaultdb_path = BASE_DIR
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
'NAME': os.path.join(defaultdb_path, 'db.sqlite3'),
},
'calibre': {
'ENGINE': 'django.db.backends.sqlite3',

View File

@ -1,4 +1,6 @@
import multiprocessing
import os
import json
bind = "127.0.0.1:8000"
workers = multiprocessing.cpu_count() * 2 + 1
@ -7,10 +9,20 @@ keepalive = 5
# daemon = True # Detaches the server from the controlling terminal and enters the background. disabled for now
# logging
errorlog = "/home/massiveatoms/Desktop/logs/gunicorn_error.log"
with open("settings.json", "r") as jfile:
settings = json.load(jfile)
errorlog = settings["LOGFOLDER"] + "/gunicorn_error.log"
loglevel = "warning"
accesslog = "/home/massiveatoms/Desktop/logs/gunicorn_access.log"
# capture_output = True
accesslog = settings["LOGFOLDER"] + "/gunicorn_access.log"
if not os.path.isdir("/usr/src/app/data/logs"):
os.mkdir("/usr/src/app/data/logs")
if not os.path.isfile(errorlog):
os.system('touch {}'.format(errorlog))
if not os.path.isfile(accesslog):
os.system('touch {}'.format(accesslog))
capture_output = True
# debug settings which need to be commented out in prod
# reload=True

View File

@ -15,7 +15,7 @@ def filters(request):
unique_authors = Author.objects.only('name', "id").annotate(num_books=Count('book')).order_by('name')
unique_tags = Tag.objects.annotate(num_books=Count('book')).order_by('name')
unique_publishers = Publisher.objects.annotate(num_books=Count('book')).order_by('name')
unique_languages = Language.objects.annotate(num_books=Count('book')).order_by('rating')
unique_languages = Language.objects.annotate(num_books=Count('book')).order_by('lang_code')
unique_ratings = Rating.objects.annotate(num_books=Count('book'))
unique_series = Series.objects.annotate(num_books=Count('book')).order_by('sort')
@ -27,4 +27,4 @@ def filters(request):
"unique_languages": unique_languages,
"unique_ratings": unique_ratings,
"unique_series": unique_series
}
}

View File

@ -1,3 +1,166 @@
from django.test import TestCase
from django.test import Client, TestCase
from pprint import pprint
from django.test.utils import setup_test_environment
from .models import Book, Author, Publisher, Series, Rating, Tag, Identifier
from django.db.models import Count
# Create your tests here.
client = Client()
client.login(username="testuser", password="dumbeasypassword")
def booklisttest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
res = c.get("/books/")
assert res.status_code == 200
context = dict(res.context)
assert sorted(context["book_list"], key=lambda x: x.id)== sorted(Book.objects.all(), key=lambda x: x.id)
def authorlisttest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
res = c.get("/authors/")
assert res.status_code == 200
context = dict(res.context)
assert sorted(context["author_list"], key=lambda x: x.id)== sorted(Author.objects.all(), key=lambda x: x.id)
def publisherlisttest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
res = c.get("/publishers/")
assert res.status_code == 200
context = dict(res.context)
assert sorted(context["publisher_list"], key=lambda x: x.id)== sorted(Publisher.objects.all(), key=lambda x: x.id)
def serieslisttest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
res = c.get("/series/")
assert res.status_code == 200
context = dict(res.context)
assert sorted(context["series_list"], key=lambda x: x.id)== sorted(Series.objects.all(), key=lambda x: x.id)
def ratinglisttest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
res = c.get("/ratings/")
assert res.status_code == 200
context = dict(res.context)
assert sorted(context["rating_list"], key=lambda x: x.id)== sorted(Rating.objects.all(), key=lambda x: x.id)
def taglisttest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
res = c.get("/tags/")
assert res.status_code == 200
context = dict(res.context)
assert sorted(context["tag_list"], key=lambda x: x.id)== sorted(Tag.objects.all(), key=lambda x: x.id)
def bookdetailtest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
ids = [i.id for i in Book.objects.all()][:10]
for i in ids:
res = c.get(f"/book/{i}")
assert res.status_code == 200
context = dict(res.context)
assert context["book"] == Book.objects.get(id=i)
def authordetailtest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
ids = [i.id for i in Author.objects.all()][:10]
for i in ids:
res = c.get(f"/author/{i}")
assert res.status_code == 200
context = dict(res.context)
assert context["author"] == Author.objects.get(id=i)
def publisherdetailtest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
ids = [i.id for i in Publisher.objects.all()][:10]
for i in ids:
res = c.get(f"/publisher/{i}")
assert res.status_code == 200
context = dict(res.context)
assert context["publisher"] == Publisher.objects.get(id=i)
def seriesdetailtest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
ids = [i.id for i in Series.objects.all()][:10]
for i in ids:
res = c.get(f"/series/{i}")
assert res.status_code == 200
context = dict(res.context)
assert context["series"] == Series.objects.get(id=i)
def ratingdetailtest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
ids = [i.id for i in Rating.objects.all()][:10]
for i in ids:
res = c.get(f"/rating/{i}")
assert res.status_code == 200
context = dict(res.context)
assert context["rating"] == Rating.objects.get(id=i)
def tagdetailtest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
ids = [i.id for i in Tag.objects.all()][:10]
for i in ids:
res = c.get(f"/tag/{i}")
assert res.status_code == 200
context = dict(res.context)
assert context["tag"] == Tag.objects.get(id=i)
def search_partial(key, value, book=None):
c = Client()
c.login(username="testuser", password="dumbeasypassword")
res = c.get("/results/", {key : value})
if not book:
return dict(res.context)["book_list"]
return book in dict(res.context)["book_list"]
def searchtest():
books = [i for i in Book.objects.all()][:10]
for i in books:
assert search_partial("title", i.title, i)
assert search_partial("generic", i.title, i)
assert search_partial("author", i.author_sort, i)
author = i.authors.first()
if author:
assert search_partial("author", author.name, i)
assert search_partial("generic", author.name, i)
assert search_partial("generic", i.author_sort, i)
id = Identifier.objects.filter(book=i.id).first()
if id:
assert search_partial("identifier", id, i)
assert search_partial("generic", id, i)
booklisttest()
bookdetailtest()
authorlisttest()
authordetailtest()
publisherdetailtest()
publisherlisttest()
seriesdetailtest()
serieslisttest()
ratingdetailtest()
ratinglisttest()
tagdetailtest()
taglisttest()
searchtest()

View File

@ -1,3 +1,4 @@
from django.utils.decorators import method_decorator
from django.shortcuts import render
from django.views import generic
from .models import Author, Book, Comment, Rating, BookAuthorLink, Publisher, Tag, BookTagLink, BookRatingLink, Data, Identifier, Series
@ -13,7 +14,6 @@ import logging
logger = logging.getLogger(__name__)
# might be helpful for vary headers later
from django.utils.decorators import method_decorator
@login_required
@ -58,13 +58,29 @@ class ResultsView(generic.ListView): # no clue if this is secure.
if title:
books = books.filter(sort__icontains=title)
if author:
books = books.filter(author_sort__icontains=author)
# authors are stored as author_sort and author, needs to be slightly more complex
author_obj = Author.objects.filter(name__icontains=author).first()
if not author_obj:
author_id = -1
else:
author_id = author_obj.id
books = books.filter(
Q(author_sort__icontains=author) |
Q(authors__id=author_id)
)
if identifier:
books = books.filter(identifier__val=identifier)
if generic:
author_obj = Author.objects.filter(name__icontains=generic).first()
if not author_obj:
author_id = -1
else:
author_id = author_obj.id
books = books.filter(
Q(sort__icontains=generic) |
Q(author_sort__icontains=generic) |
Q(authors__id=author_id) |
Q(identifier__val=generic)
)
return books

0
CalibreWebCompanion/manage.py Normal file → Executable file
View File

0
Running Normal file
View File

29
deployment/Dockerfile Normal file
View File

@ -0,0 +1,29 @@
## pull official base image
FROM python:3.8.3-alpine
EXPOSE 8080
## set work directory
WORKDIR /usr/src/app
## install dependencies
RUN pip install --upgrade pip
COPY ./requirements.txt .
RUN pip install -r requirements.txt
RUN apk add nginx supervisor
# do nginx stuff
RUN adduser -D -g 'www' www
RUN mkdir -p /run/nginx
COPY ./deployment/nginx.conf /etc/nginx/
## copy project
COPY ./CalibreWebCompanion ./CalibreWebCompanion
copy ./deployment/startupscript.py ./
## gunicorn borks started with supervisord
COPY ./deployment/supervisord.conf /etc/
ENTRYPOINT /usr/bin/supervisord -c /etc/supervisord.conf
# docker run --publish 8000:80 \
# -v '/home/massiveatoms/Desktop/logs:/usr/src/app/data' \
# -v '/run/media/massiveatoms/1AEEEA6EEEEA421D1/Documents and Settings/MassiveAtoms/Documents/Calibre Library/:/usr/src/app/calibredir' \
# --name cw calibreweb:1.0.1

4
deployment/deploy.py Normal file
View File

@ -0,0 +1,4 @@
from os import environ

View File

@ -1,4 +1,4 @@
# non docker stuff
1. clone repo
2. pip install -r requirements.txt
3. install gunicorn and nginx
@ -18,4 +18,19 @@ Suggestions:
1. We might want to use sockets instead of ip/port?
2. logging
3. autostart gunicorn/nginx
4. some extra instrumentation for gunicorn https://docs.gunicorn.org/en/latest/deploy.html
4. some extra instrumentation for gunicorn https://docs.gunicorn.org/en/latest/deploy.html
# docker stuff
here are the commands i use to build and run:
1. build --tag calibreweb:1.0.1 . -f ./deployment/Dockerfile
2.
```
docker run --publish 8000:80\
-v '/home/massiveatoms/Desktop/logs:/usr/src/app/data' \
-v '/run/media/massiveatoms/1AEEEA6EEEEA421D/Documents and Settings/MassiveAtoms/Documents/Calibre Library/:/usr/src/app/calibredir' \
--name cw calibreweb:1.0.1
```
Docker stuff to fix:
1. permission/ownership issue with volumes and nginx
2. we need a setup script that changes secretkey, and generates the default db

View File

@ -1,9 +1,9 @@
worker_processes 1;
# user nobody nogroup;
user massiveatoms;
user www www; # TEMP disabled
# user nobody nobody; # for systems with 'nobody' as a group instead
error_log /home/massiveatoms/Desktop/logs/nginx.log warn;
error_log /usr/src/app/data/logs/nginx.log warn;
# pid /var/run/nginx.pid;
events {
@ -44,18 +44,18 @@ http {
client_max_body_size 4G;
# set the correct host(s) for your site
server_name localhost 192.168.1.4; # set this to the server url? or ip? we'll see MASSIVEATOMS
server_name localhost 0.0.0.0; # set this to the server url? or ip? we'll see MASSIVEATOMS
keepalive_timeout 5;
# MASSIVEATOMS
# # MASSIVEATOMS
location /download/ {
alias "/run/media/massiveatoms/1AEEEA6EEEEA421D/Documents and Settings/MassiveAtoms/Documents/Calibre Library/";
alias "/usr/src/app/calibredir/";
# Never forget the fact that this little statement being root instead of alias caused us to lose more than a day troubleshooting
}
location /static/ {
alias "/home/massiveatoms/Desktop/calibre-web-companion/CalibreWebCompanion/static/";
alias "/usr/src/app/CalibreWebCompanion/static/";
# Never forget the fact that this little statement being root instead of alias caused us to lose more than a day troubleshooting
}

View File

@ -0,0 +1,11 @@
from os import system, chdir
# system("chown -R www:www /usr/src/app/calibredir")
# print("ownership of calibredir changed")
chdir("/usr/src/app/CalibreWebCompanion")
system("python ./manage.py makemigrations")
print("ran makemigrations")
system("python ./manage.py migrate")
print("migrate")

View File

@ -0,0 +1,36 @@
[supervisord]
nodaemon=true
logfile=/tmp/supervisord.log
childlogdir=/tmp
pidfile = /tmp/supervisord.pid
[program:gunicorn]
directory=/usr/src/app/CalibreWebCompanion
command=gunicorn CalibreWebCompanion.wsgi
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
autorestart=false
startretries=0
startsecs = 0
[program:nginx]
# user=www
command=nginx
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
autorestart=false
startretries=0
[program:startupscript]
directory=/usr/src/app
command=python ./startupscript.py
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
autorestart=false
startretries=0

View File

@ -1,8 +1,9 @@
django>=3.0.8
inotify>=0.2.10
gunicorn>=20.0
# development
django-debug-toolbar>=2.2
django-silk>=4.0
locust>=1.1
sqlalchemy>=1.3.15
rich>=3.0
# django-debug-toolbar>=2.2
# django-silk>=4.0
# locust>=1.1
# sqlalchemy>=1.3.15
# rich>=3.0