13 Commits

Author SHA1 Message Date
MassiveAtoms
6e99404134 mend
rm a file
2021-03-05 10:33:29 -03:00
MassiveAtoms
7b033454b3 nginx unit works 2021-03-05 10:32:42 -03:00
MassiveAtoms
f34200901e i give up 2021-03-02 00:31:38 -03:00
MassiveAtoms
330a36556c static file issue, otherwise working 2021-03-02 00:09:15 -03:00
TinyAtoms
d7a385fd45 changed instructions 2020-08-16 02:11:33 -03:00
TinyAtoms
edc9366a5b changed instructions 2020-08-16 02:06:22 -03:00
TinyAtoms
88fcb17dc5 dev fixes
Merge branch 'development'
2020-08-16 01:47:38 -03:00
TinyAtoms
49ca4bccdc fixed nginx, bug in context processors 2020-08-16 01:45:28 -03:00
TinyAtoms
3985a5635d Merge branch 'development' of https://git.tau.aperturect.com/MassiveAtoms/calibre-web-companion into development 2020-08-15 11:04:49 -03:00
TinyAtoms
9843299ef6 fixed some db stuff 2020-08-15 11:00:25 -03:00
TinyAtoms
75099ca05e Removed hardcoded paths, make workflow changes
I removed some hardcoded paths for logging and where the default db should be located.
I also organized deployment stuff a bit
2020-08-15 02:40:03 -03:00
TinyAtoms
1b8d81bd3c pre fixing problems 2020-08-15 01:56:27 -03:00
MassiveAtoms
75e78d606f added tests, fixed search bug, and 'global context variable' bug 2020-08-04 12:21:17 -03:00
18 changed files with 408 additions and 185 deletions

3
.gitignore vendored
View File

@@ -3,7 +3,8 @@ settings.json
db.sqlite3
dummyusers.json
*.prof
statictest.txt
./test.sh
# IDE
.vscode

View File

@@ -23,9 +23,10 @@ with open(BASE_DIR + "/settings.json", "r") as userfile:
CALIBRE_DIR = os.path.abspath(usersettings["CALIBRE_DIR"])
SECRET_KEY = usersettings["SECRET_KEY"]
ALLOWED_HOSTS = usersettings["ALLOWED_HOSTS"]
INTERNAL_IPS = usersettings["INTERNAL_IPS"]
DEBUG = usersettings["DEBUG"]
DOCKER = os.environ.get('AM_DOCKER_INSTANCE', False)
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -64,39 +65,43 @@ STATIC_ROOT = BASE_DIR + "/static/"
#########################################################################
# LOGGING
# TODO: think about the issue for a bit. no write access to file on first run
# but startscript doesn't run to give access if it can't start django
# if not DOCKER:
# logfile = "/app//data//django.log"
# if not os.path.isfile(logfile):
# os.mknod(logfile)
# LOGGING = {
# "version": 1,
# "disable_existing_loggers": False,
# "root": {"level": "INFO", "handlers": ["file"]},
# "handlers": {
# "file": {
# "level": "INFO",
# "class": "logging.FileHandler",
# "filename": logfile,
# "formatter": "app",
# },
# },
# "loggers": {
# "django": {
# "handlers": ["file"],
# "level": "INFO",
# "propagate": True
# },
# },
# "formatters": {
# "app": {
# "format": (
# u"%(asctime)s [%(levelname)-8s] "
# "(%(module)s.%(funcName)s) %(message)s"
# ),
# "datefmt": "%Y-%m-%d %H:%M:%S",
# },
# },
# }
logfile = usersettings["LOGFILE"]
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"root": {"level": "INFO", "handlers": ["file"]},
"handlers": {
"file": {
"level": "INFO",
"class": "logging.FileHandler",
"filename": usersettings["LOGFILE"],
"formatter": "app",
},
},
"loggers": {
"django": {
"handlers": ["file"],
"level": "INFO",
"propagate": True
},
},
"formatters": {
"app": {
"format": (
u"%(asctime)s [%(levelname)-8s] "
"(%(module)s.%(funcName)s) %(message)s"
),
"datefmt": "%Y-%m-%d %H:%M:%S",
},
},
}
## ##
########################################################################
## DERUG ##
@@ -139,6 +144,7 @@ INSTALLED_APPS = [
'django.contrib.messages',
'django.contrib.staticfiles',
"library",
"django_extensions"
# "silk", # DEBUG/profilling purposes
# 'debug_toolbar', # DEBUG purposes
]
@@ -158,7 +164,7 @@ MIDDLEWARE = [
]
## ##
########################################################################
DEFAULT_CHARSET = "utf-8"
ROOT_URLCONF = 'CalibreWebCompanion.urls'
@@ -188,14 +194,23 @@ WSGI_APPLICATION = 'CalibreWebCompanion.wsgi.application'
## DATBASE ##
# https://docs.djangoproject.com/en/3.0/ref/settings/#databases
if DOCKER:
djangodb_path = "/app/CalibreWebCompanion"
calibredb_path = "/app/content/"
else:
djangodb_path = BASE_DIR
calibredb_path = CALIBRE_DIR
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
'NAME': os.path.join(djangodb_path, 'db.sqlite3'),
},
'calibre': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(CALIBRE_DIR, 'metadata.db'),
'NAME': os.path.join(calibredb_path, 'metadata.db'),
}
}

View File

@@ -1,21 +0,0 @@
import multiprocessing
bind = "127.0.0.1:8000"
workers = multiprocessing.cpu_count() * 2 + 1
preload_app = True # By preloading an application you can save some RAM resources as well as speed up server boot times
keepalive = 5
# daemon = True # Detaches the server from the controlling terminal and enters the background. disabled for now
# logging
errorlog = "/home/massiveatoms/Desktop/logs/gunicorn_error.log"
loglevel = "warning"
accesslog = "/home/massiveatoms/Desktop/logs/gunicorn_access.log"
# capture_output = True
# debug settings which need to be commented out in prod
# reload=True
# reload_engine = "inotify"
# I only went till the section https://docs.gunicorn.org/en/latest/settings.html#logging there are more settings
# some of them might be useful

View File

@@ -15,7 +15,7 @@ def filters(request):
unique_authors = Author.objects.only('name', "id").annotate(num_books=Count('book')).order_by('name')
unique_tags = Tag.objects.annotate(num_books=Count('book')).order_by('name')
unique_publishers = Publisher.objects.annotate(num_books=Count('book')).order_by('name')
unique_languages = Language.objects.annotate(num_books=Count('book')).order_by('rating')
unique_languages = Language.objects.annotate(num_books=Count('book')).order_by('lang_code')
unique_ratings = Rating.objects.annotate(num_books=Count('book'))
unique_series = Series.objects.annotate(num_books=Count('book')).order_by('sort')

View File

@@ -6,7 +6,7 @@
<div class="col s12 m7">
<div class="card z-depth-0 horizontal">
<div class="card-image">
<a style="padding-top:15%" href=" /download/{{download}}"><img src=" /download/{{imgpath}}"
<a style="padding-top:15%" href=" /content/{{download}}"><img src=" /content/{{imgpath}}"
alt="download" srcset=""></a>
</div>
<div class="card-stacked">

View File

@@ -1,3 +1,166 @@
from django.test import TestCase
from django.test import Client, TestCase
from pprint import pprint
from django.test.utils import setup_test_environment
from .models import Book, Author, Publisher, Series, Rating, Tag, Identifier
from django.db.models import Count
# Create your tests here.
client = Client()
client.login(username="testuser", password="dumbeasypassword")
def booklisttest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
res = c.get("/books/")
assert res.status_code == 200
context = dict(res.context)
assert sorted(context["book_list"], key=lambda x: x.id)== sorted(Book.objects.all(), key=lambda x: x.id)
def authorlisttest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
res = c.get("/authors/")
assert res.status_code == 200
context = dict(res.context)
assert sorted(context["author_list"], key=lambda x: x.id)== sorted(Author.objects.all(), key=lambda x: x.id)
def publisherlisttest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
res = c.get("/publishers/")
assert res.status_code == 200
context = dict(res.context)
assert sorted(context["publisher_list"], key=lambda x: x.id)== sorted(Publisher.objects.all(), key=lambda x: x.id)
def serieslisttest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
res = c.get("/series/")
assert res.status_code == 200
context = dict(res.context)
assert sorted(context["series_list"], key=lambda x: x.id)== sorted(Series.objects.all(), key=lambda x: x.id)
def ratinglisttest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
res = c.get("/ratings/")
assert res.status_code == 200
context = dict(res.context)
assert sorted(context["rating_list"], key=lambda x: x.id)== sorted(Rating.objects.all(), key=lambda x: x.id)
def taglisttest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
res = c.get("/tags/")
assert res.status_code == 200
context = dict(res.context)
assert sorted(context["tag_list"], key=lambda x: x.id)== sorted(Tag.objects.all(), key=lambda x: x.id)
def bookdetailtest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
ids = [i.id for i in Book.objects.all()][:10]
for i in ids:
res = c.get(f"/book/{i}")
assert res.status_code == 200
context = dict(res.context)
assert context["book"] == Book.objects.get(id=i)
def authordetailtest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
ids = [i.id for i in Author.objects.all()][:10]
for i in ids:
res = c.get(f"/author/{i}")
assert res.status_code == 200
context = dict(res.context)
assert context["author"] == Author.objects.get(id=i)
def publisherdetailtest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
ids = [i.id for i in Publisher.objects.all()][:10]
for i in ids:
res = c.get(f"/publisher/{i}")
assert res.status_code == 200
context = dict(res.context)
assert context["publisher"] == Publisher.objects.get(id=i)
def seriesdetailtest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
ids = [i.id for i in Series.objects.all()][:10]
for i in ids:
res = c.get(f"/series/{i}")
assert res.status_code == 200
context = dict(res.context)
assert context["series"] == Series.objects.get(id=i)
def ratingdetailtest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
ids = [i.id for i in Rating.objects.all()][:10]
for i in ids:
res = c.get(f"/rating/{i}")
assert res.status_code == 200
context = dict(res.context)
assert context["rating"] == Rating.objects.get(id=i)
def tagdetailtest():
c = Client()
c.login(username="testuser", password="dumbeasypassword")
ids = [i.id for i in Tag.objects.all()][:10]
for i in ids:
res = c.get(f"/tag/{i}")
assert res.status_code == 200
context = dict(res.context)
assert context["tag"] == Tag.objects.get(id=i)
def search_partial(key, value, book=None):
c = Client()
c.login(username="testuser", password="dumbeasypassword")
res = c.get("/results/", {key : value})
if not book:
return dict(res.context)["book_list"]
return book in dict(res.context)["book_list"]
def searchtest():
books = [i for i in Book.objects.all()][:10]
for i in books:
assert search_partial("title", i.title, i)
assert search_partial("generic", i.title, i)
assert search_partial("author", i.author_sort, i)
author = i.authors.first()
if author:
assert search_partial("author", author.name, i)
assert search_partial("generic", author.name, i)
assert search_partial("generic", i.author_sort, i)
id = Identifier.objects.filter(book=i.id).first()
if id:
assert search_partial("identifier", id, i)
assert search_partial("generic", id, i)
booklisttest()
bookdetailtest()
authorlisttest()
authordetailtest()
publisherdetailtest()
publisherlisttest()
seriesdetailtest()
serieslisttest()
ratingdetailtest()
ratinglisttest()
tagdetailtest()
taglisttest()
searchtest()

View File

@@ -1,3 +1,4 @@
from django.utils.decorators import method_decorator
from django.shortcuts import render
from django.views import generic
from .models import Author, Book, Comment, Rating, BookAuthorLink, Publisher, Tag, BookTagLink, BookRatingLink, Data, Identifier, Series
@@ -13,7 +14,6 @@ import logging
logger = logging.getLogger(__name__)
# might be helpful for vary headers later
from django.utils.decorators import method_decorator
@login_required
@@ -58,13 +58,29 @@ class ResultsView(generic.ListView): # no clue if this is secure.
if title:
books = books.filter(sort__icontains=title)
if author:
books = books.filter(author_sort__icontains=author)
# authors are stored as author_sort and author, needs to be slightly more complex
author_obj = Author.objects.filter(name__icontains=author).first()
if not author_obj:
author_id = -1
else:
author_id = author_obj.id
books = books.filter(
Q(author_sort__icontains=author) |
Q(authors__id=author_id)
)
if identifier:
books = books.filter(identifier__val=identifier)
if generic:
author_obj = Author.objects.filter(name__icontains=generic).first()
if not author_obj:
author_id = -1
else:
author_id = author_obj.id
books = books.filter(
Q(sort__icontains=generic) |
Q(author_sort__icontains=generic) |
Q(authors__id=author_id) |
Q(identifier__val=generic)
)
return books

0
CalibreWebCompanion/manage.py Normal file → Executable file
View File

View File

@@ -1,12 +1,13 @@
{
"CALIBRE_DIR": "PATH\\TO\\your\\calibre\library",
"CALIBRE_DIR": "/app/content",
"LOGFOLDER" : "/usr/src/app/data/logs/",
"SECRET_KEY": "u(8^+rb%rz5hsx4v^^y(ul7g(4n7a8!db@s*9(m5cs*2_ppy8+",
"ALLOWED_HOSTS": [
"127.0.0.1"
"ALLOWED_HOSTS": [
"*"
],
"INTERNAL_IPS": [
"127.0.0.1"
"127.0.0.1", "localhost"
],
"DEBUG" : false,
"LOGFILE" : "/home/massiveatoms/Desktop/logs/django.log"
"DEBUG" : false
}

View File

@@ -24,24 +24,10 @@ Django 3.0
Calibre 4.13 (I have not tested it with anything else atm, will be resolved later)
# how to use:
1. clone repo
2. Remove the `.bak` from `./CalibreWebCompanion/settings.json.bak` and `db.sqlite3.bak`
3. Edit `./CalibreWebCompanion/settings.json`. Definitely change the secret key
4. Not sure if the db needs to be regenerated, but we'll see later __!!!!!!!!!!__
5. pip install -r requirements.txt
6. install gunicorn and nginx
7. move this nginx.conf to /etc/nginx
8. make whatever user nginx runs as (in this case, massiveatoms) the owner of calibredir
9. give execute permissions to parent of calibredir
10. cd to repo, run `gunicorn CalibreWebCompanion.wsgi`
11. start nginx `sudo systemctl restart nginx`
1. [Docker setup](./deployment/instructions.md#user-content-docker-detup)
2. [Non Docker setup](./deployment/instructions.md#user-content-non-docker-detup)
`./CalibreWebCompanion`
run `./manage.py runserver`
# Ignore pretty much everything below if you're not working on the project
# Profiling
@@ -74,7 +60,8 @@ You can then go to [http://localhost:8089/](http://localhost:8089/) to see live
- [ ] cache with vary headers
- [ ] localisation
- [ ] Beautifying template (only works well on 720p, no other viewports)
- [ ] Setup email functionality
- [ ] Setup email functionality (atm, there's only a dummy one, and you can't reset passwords)
- [ ] isolate the styling and templates, so we can swap them out by just swapping directory content

0
Running Normal file
View File

23
deployment/Dockerfile Normal file
View File

@@ -0,0 +1,23 @@
## pull official base image
FROM nginx/unit:1.22.0-python3.9
EXPOSE 80
ENV AM_DOCKER_INSTANCE Yes
## set work directory
WORKDIR /app
## install dependencies
COPY ./requirements.txt .
RUN pip install -r requirements.txt
## copy project
COPY ./CalibreWebCompanion ./CalibreWebCompanion
# perms
RUN chown -R unit:unit /app
COPY ./deployment/entrypoints/* /docker-entrypoint.d/
RUN chmod +x /docker-entrypoint.d/start.sh
# docker run --publish 80:80 \
# -v '/home/MassiveAtoms/Desktop/logs:/app/data' \
# -v '/home/MassiveAtoms/windows/Users/MassiveAtoms/Documents/Calibre Library/:/app/calibredir' \
# --name cw calibreweb:1.0

View File

@@ -0,0 +1,54 @@
{
"listeners": {
"*:80": {
"pass": "routes"
}
},
"routes": [
{
"match": {
"uri": [
"/content/*",
"!~\\.db"
]
},
"action": {
"share": "/app/"
}
},
{
"match": {
"uri": "/static/*"
},
"action": {
"share": "/app/CalibreWebCompanion/"
}
},
{
"match": {
"uri": [
"!~\\.db", "*"
]
},
"action": {
"pass": "applications/django"
}
}
],
"applications": {
"django": {
"type": "python 3",
"path": "/app/CalibreWebCompanion/",
"module": "CalibreWebCompanion.wsgi",
"environment": {
"DJANGO_SETTINGS_MODULE": "CalibreWebCompanion.settings",
"DB_ENGINE": "django.db.backends.sqlite3"
},
"processes" : {
"max": 25,
"spare": 1,
"idle_timeout": 20
}
}
}
}

View File

@@ -0,0 +1,22 @@
CAL_DIR="/app/content/"
DATA_DIR="/app/data/"
CWC_PATH="/app/CalibreWebCompanion"
if [ ! -d "$CAL_DIR" ]; then
echo "Calibre Library not mounted at the correct location."
echo "Mount it at /app/content/"
echo "Exiting..."
exit 1
fi
if [ ! -d "$DATA_DIR" ]; then
echo "A data directory not mounted at the correct location, exiting"
echo "This is used to store the database and logs"
echo "mount something at /app/data/"
echo "exiting"
exit 1
fi
cp -R -u -p "/app/CalibreWebCompanion/db.sqlite3" "/app/data/"
ls -l /app
python "${CWC_PATH}/manage.py" makemigrations
python "${CWC_PATH}/manage.py" migrate

View File

@@ -1,21 +1,45 @@
# Docker setup (no provided docker image atm)
1. clone the repo
2. rename ./calireWebCompanion/settings.json.bak to settings.json
3. change the secret key
4. run `build --tag calibreweb:1.0 . -f ./deployment/Dockerfile` to build the image
5. run your container with your bind/mount your volumes/paths/things
Here's an example of step 5
```
docker run --publish 80:80 \
-v '/home/MassiveAtoms/Desktop/logs:/app/data' \
-v '/home/MassiveAtoms/windows/Users/MassiveAtoms/Documents/Calibre\ Library/:/app/calibredir' \
--name cw calibreweb:1.0
```
your Calibre path/volume/whatever needs to be mounted at `/app/calibredir`, and you need to mount a volume for the db and logs at `/app/data`
Issues with it at the moment:
1. we still need to do something to create a random secret key. Atm, this would still
# Docker (provided image)
not done yet
# non docker setup
this might need to be modified, since some things have changed to adapt it for docker setup
1. clone repo
2. pip install -r requirements.txt
3. install gunicorn and nginx
4. move this nginx.conf to /etc/nginx
5. make whatever user nginx runs as (in this case, massiveatoms) the owner of calibredir
6. give execute permissions to parent of calibredir
7. cd to repo, run `gunicorn CalibreWebCompanion.wsgi`
8. start nginx `sudo systemctl restart nginx`
3. rename the settings.json.bak to settings.json, change logging folder, change secret key, set isdocker to false
4. install gunicorn and nginx
5. move this nginx.conf to /etc/nginx
6. create a user and group `www`
7. make whatever user nginx runs as (for now, www) the owner of calibredir
8. give execute permissions to parent of calibredir
9. cd to repo, run `gunicorn CalibreWebCompanion.wsgi`
10. start nginx `sudo systemctl restart nginx`
11. make steps 9 and 10 happen on startup?
Slight issues with this atm:
1. server_name in nginx.conf needs to be changed
2. User needs to be edited in nginx.conf, now it's just my user acc. This affectd step 4-6
3. where to do ssl?
1. where to do ssl?
Suggestions:
1. We might want to use sockets instead of ip/port?
2. logging
3. autostart gunicorn/nginx
4. some extra instrumentation for gunicorn https://docs.gunicorn.org/en/latest/deploy.html
2. autostart gunicorn/nginx
3. some extra instrumentation for gunicorn https://docs.gunicorn.org/en/latest/deploy.html

View File

@@ -1,81 +0,0 @@
worker_processes 1;
# user nobody nogroup;
user massiveatoms;
# user nobody nobody; # for systems with 'nobody' as a group instead
error_log /home/massiveatoms/Desktop/logs/nginx.log warn;
# pid /var/run/nginx.pid;
events {
worker_connections 1024; # increase if you have lots of clients
accept_mutex off; # set to 'on' if nginx worker_processes > 1
use epoll; # to enable for Linux 2.6+ MASSIVEATOMS
# 'use kqueue;' to enable for FreeBSD, OSX
}
http {
include mime.types;
# fallback in case we can't determine a type
default_type application/octet-stream;
access_log /var/log/nginx/access.log combined;
sendfile on;
upstream app_server {
# fail_timeout=0 means we always retry an upstream even if it failed
# to return a good HTTP response
# for UNIX domain socket setups
# server unix:/tmp/gunicorn.sock fail_timeout=0;
# for a TCP configuration
server 127.0.0.1:8000 fail_timeout=0;
}
server {
# if no Host match, close the connection to prevent host spoofing
listen 80 default_server;
return 444;
}
server {
listen 80 deferred; # for Linux massiveatoms
# use 'listen 80 accept_filter=httpready;' for FreeBSD
# listen 80;
client_max_body_size 4G;
# set the correct host(s) for your site
server_name localhost 192.168.1.4; # set this to the server url? or ip? we'll see MASSIVEATOMS
keepalive_timeout 5;
# MASSIVEATOMS
location /download/ {
alias "/run/media/massiveatoms/1AEEEA6EEEEA421D/Documents and Settings/MassiveAtoms/Documents/Calibre Library/";
# Never forget the fact that this little statement being root instead of alias caused us to lose more than a day troubleshooting
}
location /static/ {
alias "/home/massiveatoms/Desktop/calibre-web-companion/CalibreWebCompanion/static/";
# Never forget the fact that this little statement being root instead of alias caused us to lose more than a day troubleshooting
}
location / {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# proxy_set_header Host $http_host;
# we don't want nginx trying to do something clever with
# redirects, we set the Host: header above already.
proxy_redirect off;
proxy_pass http://127.0.0.1:8000;
}
error_page 500 502 503 504 /500.html;
location = /500.html {
root /path/to/app/current/public;
}
}
}

View File

@@ -1,8 +1,10 @@
django>=3.0.8
inotify>=0.2.10
gunicorn>=20.0
django-extensions
# development
django-debug-toolbar>=2.2
django-silk>=4.0
locust>=1.1
sqlalchemy>=1.3.15
rich>=3.0
# django-debug-toolbar>=2.2
# django-silk>=4.0
# locust>=1.1
# sqlalchemy>=1.3.15
# rich>=3.0

17
test.sh Executable file
View File

@@ -0,0 +1,17 @@
docker build --tag calibreweb:1.0 . -f ./deployment/Dockerfile
docker stop cw
docker rm cw
docker run --publish 80:80 \
-v '/home/MassiveAtoms/Desktop/logs:/app/data' \
-v '/home/MassiveAtoms/windows/Users/MassiveAtoms/Documents/Calibre Library/:/app/content' \
--name cw calibreweb:1.0
sleep 8
echo "download/test"
curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1/download/statictest.txt
echo " "
echo "/test"
curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1/statictest.txt
echo " "
echo "/static/test"
curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1/static/statictest.txt
echo " "