mend

rm a file

.gitignore

@@ -3,7 +3,8 @@ settings.json
 db.sqlite3
 dummyusers.json
 *.prof
-
+statictest.txt
+./test.sh
 
 # IDE
 .vscode

CalibreWebCompanion/CalibreWebCompanion/settings.py

@@ -23,9 +23,10 @@ with open(BASE_DIR + "/settings.json", "r") as userfile:
     CALIBRE_DIR = os.path.abspath(usersettings["CALIBRE_DIR"])
     SECRET_KEY = usersettings["SECRET_KEY"]
     ALLOWED_HOSTS = usersettings["ALLOWED_HOSTS"]
-    INTERNAL_IPS = usersettings["INTERNAL_IPS"]
     DEBUG = usersettings["DEBUG"]
 
+DOCKER = os.environ.get('AM_DOCKER_INSTANCE', False)
+
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 
@@ -64,38 +65,41 @@ STATIC_ROOT = BASE_DIR + "/static/"
 #########################################################################
 # LOGGING
 
-
+# TODO: think about the issue for a bit. no write access to file on first run
-
+# but startscript doesn't run to give access if it can't start django
-logfile = usersettings["LOGFOLDER"] + "django.log"
+# if not DOCKER: 
-LOGGING = {
+#     logfile = "/app//data//django.log"
-    "version": 1,
+#     if not os.path.isfile(logfile):
-    "disable_existing_loggers": False,
+#         os.mknod(logfile)
-    "root": {"level": "INFO", "handlers": ["file"]},
+#     LOGGING = {
-    "handlers": {
+#         "version": 1,
-        "file": {
+#         "disable_existing_loggers": False,
-            "level": "INFO",
+#         "root": {"level": "INFO", "handlers": ["file"]},
-            "class": "logging.FileHandler",
+#         "handlers": {
-            "filename": logfile,
+#             "file": {
-            "formatter": "app",
+#                 "level": "INFO",
-        },
+#                 "class": "logging.FileHandler",
-    },
+#                 "filename": logfile,
-    "loggers": {
+#                 "formatter": "app",
-        "django": {
+#             },
-            "handlers": ["file"],
+#         },
-            "level": "INFO",
+#         "loggers": {
-            "propagate": True
+#             "django": {
-        },
+#                 "handlers": ["file"],
-    },
+#                 "level": "INFO",
-    "formatters": {
+#                 "propagate": True
-        "app": {
+#             },
-            "format": (
+#         },
-                u"%(asctime)s [%(levelname)-8s] "
+#         "formatters": {
-                "(%(module)s.%(funcName)s) %(message)s"
+#             "app": {
-            ),
+#                 "format": (
-            "datefmt": "%Y-%m-%d %H:%M:%S",
+#                     u"%(asctime)s [%(levelname)-8s] "
-        },
+#                     "(%(module)s.%(funcName)s) %(message)s"
-    },
+#                 ),
-}
+#                 "datefmt": "%Y-%m-%d %H:%M:%S",
+#             },
+#         },
+#     }
 
 
 ##                                                                    ##
@@ -140,6 +144,7 @@ INSTALLED_APPS = [
     'django.contrib.messages',
     'django.contrib.staticfiles',
     "library",
+    "django_extensions"
     # "silk", # DEBUG/profilling purposes
     # 'debug_toolbar', # DEBUG  purposes
 ]
@@ -189,19 +194,23 @@ WSGI_APPLICATION = 'CalibreWebCompanion.wsgi.application'
 ##                    DATBASE                                         ##
 # https://docs.djangoproject.com/en/3.0/ref/settings/#databases
 
-if usersettings["ISDOCKER"]:
+
-    defaultdb_path = "/usr/src/app/data/"
+if DOCKER:
+    djangodb_path = "/app/CalibreWebCompanion"
+    calibredb_path = "/app/content/"
+    
 else:
-    defaultdb_path = BASE_DIR
+    djangodb_path = BASE_DIR
+    calibredb_path = CALIBRE_DIR
 
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(defaultdb_path, 'db.sqlite3'),
+        'NAME': os.path.join(djangodb_path, 'db.sqlite3'),
     },
     'calibre': {
         'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(CALIBRE_DIR, 'metadata.db'),
+        'NAME': os.path.join(calibredb_path, 'metadata.db'),
     }
 }
 

CalibreWebCompanion/gunicorn.conf.py

@@ -1,33 +0,0 @@
-import multiprocessing
-import os
-import json
-
-bind = "127.0.0.1:8000"
-workers = multiprocessing.cpu_count() * 2 + 1
-preload_app = True # By preloading an application you can save some RAM resources as well as speed up server boot times
-keepalive = 5
-# daemon = True # Detaches the server from the controlling terminal and enters the background. disabled for now
-# logging
-
-with open("settings.json", "r") as jfile:
-    settings = json.load(jfile)
-
-errorlog = settings["LOGFOLDER"] + "/gunicorn_error.log"
-loglevel = "warning"
-accesslog = settings["LOGFOLDER"] + "/gunicorn_access.log"
-
-if not os.path.isdir("/usr/src/app/data/logs"):
-    os.mkdir("/usr/src/app/data/logs")
-if not os.path.isfile(errorlog):
-    os.system('touch {}'.format(errorlog))
-if not os.path.isfile(accesslog):
-    os.system('touch {}'.format(accesslog))
-capture_output = True
-
-# debug settings which need to be commented out in prod
-# reload=True
-# reload_engine = "inotify"
-
-
-# I only went till the section https://docs.gunicorn.org/en/latest/settings.html#logging there are more settings
-# some of them might be useful

CalibreWebCompanion/library/templates/library/book_detail.html

@@ -6,7 +6,7 @@
 <div class="col s12 m7">
   <div class="card z-depth-0 horizontal">
     <div class="card-image">
-      <a style="padding-top:15%" href=" /download/{{download}}"><img src=" /download/{{imgpath}}"
+      <a style="padding-top:15%" href=" /content/{{download}}"><img src=" /content/{{imgpath}}"
           alt="download" srcset=""></a>
     </div>
     <div class="card-stacked">

CalibreWebCompanion/settings.json.bak

@@ -1,12 +1,13 @@
 {
-    "CALIBRE_DIR": "PATH\\TO\\your\\calibre\library",
+    "CALIBRE_DIR": "/app/content",
+    "LOGFOLDER" : "/usr/src/app/data/logs/",
     "SECRET_KEY": "u(8^+rb%rz5hsx4v^^y(ul7g(4n7a8!db@s*9(m5cs*2_ppy8+",
-   "ALLOWED_HOSTS": [
+    "ALLOWED_HOSTS": [
-        "127.0.0.1"
+        "*"
     ],
     "INTERNAL_IPS": [
-        "127.0.0.1"
+        "127.0.0.1", "localhost"
     ],
-    "DEBUG" : false,
+    "DEBUG" : false
-    "LOGFILE" : "/home/massiveatoms/Desktop/logs/django.log"
+
 }

README.md

@@ -24,24 +24,10 @@ Django 3.0
 Calibre 4.13 (I have not tested it with anything else atm, will be resolved later)
 
 # how to use:
-1. clone repo
+1. [Docker setup](./deployment/instructions.md#user-content-docker-detup)
-2. Remove the `.bak` from `./CalibreWebCompanion/settings.json.bak`  and `db.sqlite3.bak`
+2. [Non Docker setup](./deployment/instructions.md#user-content-non-docker-detup)
-3. Edit `./CalibreWebCompanion/settings.json`. Definitely change the secret key
-4. Not sure if the db needs to be regenerated, but we'll see later __!!!!!!!!!!__
-5. pip install -r requirements.txt
-6. install gunicorn and nginx
-7. move this nginx.conf to /etc/nginx
-8. make whatever user nginx runs as (in this case, massiveatoms) the owner of calibredir
-9. give execute permissions to parent of calibredir
-10. cd to repo, run `gunicorn CalibreWebCompanion.wsgi`
-11. start nginx `sudo systemctl restart nginx`
 
 
- 
- 
-`./CalibreWebCompanion`     
-run `./manage.py runserver`    
-
 # Ignore pretty much everything below if you're not working on the project
 
 # Profiling
@@ -74,7 +60,8 @@ You can then go to [http://localhost:8089/](http://localhost:8089/) to see live
 - [ ] cache with vary headers
 - [ ] localisation
 - [ ] Beautifying template (only works well on 720p, no other viewports)
-- [ ]  Setup email functionality
+- [ ]  Setup email functionality (atm, there's only a dummy one, and you can't reset passwords)
+- [ ]  isolate the styling and templates, so we can swap them out by just swapping directory content
 
 
 

deployment/Dockerfile

@@ -1,29 +1,23 @@
 
 ## pull official base image
-FROM python:3.8.3-alpine
+FROM nginx/unit:1.22.0-python3.9
-EXPOSE 8080
+EXPOSE 80
+ENV AM_DOCKER_INSTANCE Yes
 ## set work directory
-WORKDIR /usr/src/app
+WORKDIR /app
+
 ## install dependencies
-RUN pip install --upgrade pip
 COPY ./requirements.txt .
 RUN pip install -r requirements.txt
-RUN apk add nginx supervisor
-
-# do nginx stuff
-RUN adduser -D -g 'www' www
-RUN mkdir -p /run/nginx
-COPY ./deployment/nginx.conf /etc/nginx/
-
 ## copy project
 COPY ./CalibreWebCompanion ./CalibreWebCompanion
-copy ./deployment/startupscript.py ./
-## gunicorn borks started with supervisord
-COPY ./deployment/supervisord.conf /etc/
-ENTRYPOINT /usr/bin/supervisord -c /etc/supervisord.conf
 
+# perms
+RUN chown -R unit:unit /app 
+COPY ./deployment/entrypoints/* /docker-entrypoint.d/
+RUN chmod +x /docker-entrypoint.d/start.sh
 
-# docker run --publish 8000:80 \
+# docker run --publish 80:80 \
-# -v '/home/massiveatoms/Desktop/logs:/usr/src/app/data' \
+# -v '/home/MassiveAtoms/Desktop/logs:/app/data' \
-# -v '/run/media/massiveatoms/1AEEEA6EEEEA421D1/Documents and Settings/MassiveAtoms/Documents/Calibre Library/:/usr/src/app/calibredir' \
+# -v '/home/MassiveAtoms/windows/Users/MassiveAtoms/Documents/Calibre Library/:/app/calibredir' \
-# --name cw calibreweb:1.0.1
+# --name cw calibreweb:1.0

deployment/deploy.py

@@ -1,4 +0,0 @@
-from os import environ
-
-
-

deployment/entrypoints/config.json

@@ -0,0 +1,54 @@
+{
+    "listeners": {
+        "*:80": {
+            "pass": "routes"
+        }
+    },
+    "routes": [
+        {
+            "match": {
+                "uri": [
+                    "/content/*",
+                    "!~\\.db"
+                ]
+            },
+            "action": {
+                "share": "/app/"
+            }
+        },
+        {
+            "match": {
+                "uri": "/static/*"
+            },
+            "action": {
+                "share": "/app/CalibreWebCompanion/"
+            }
+        },
+        {
+            "match": {
+                "uri": [
+                    "!~\\.db", "*"
+                ]
+            },
+            "action": {
+                "pass": "applications/django"
+            }
+        }
+    ],
+    "applications": {
+        "django": {
+            "type": "python 3",
+            "path": "/app/CalibreWebCompanion/",
+            "module": "CalibreWebCompanion.wsgi",
+            "environment": {
+                "DJANGO_SETTINGS_MODULE": "CalibreWebCompanion.settings",
+                "DB_ENGINE": "django.db.backends.sqlite3"
+            },
+            "processes" : {
+                "max": 25,
+                "spare": 1,
+                "idle_timeout": 20
+            }
+        }
+    }
+}

deployment/entrypoints/start.sh

@@ -0,0 +1,22 @@
+CAL_DIR="/app/content/"
+DATA_DIR="/app/data/"
+CWC_PATH="/app/CalibreWebCompanion"
+if [ ! -d "$CAL_DIR" ]; then
+  echo "Calibre Library not mounted at the correct location."
+  echo "Mount it at /app/content/"
+  echo "Exiting..."
+  exit 1
+fi
+
+if [ ! -d "$DATA_DIR" ]; then
+  echo "A data directory not mounted at the correct location, exiting"
+  echo "This is used to store the database and logs"
+  echo "mount something at /app/data/"
+  echo "exiting"
+  exit 1
+fi
+
+cp -R -u -p "/app/CalibreWebCompanion/db.sqlite3" "/app/data/"
+ls -l /app
+python "${CWC_PATH}/manage.py" makemigrations
+python "${CWC_PATH}/manage.py" migrate

deployment/instructions.md

@@ -1,36 +1,45 @@
-# non docker stuff
+
+# Docker setup (no provided docker image atm)
+1. clone the repo
+2. rename ./calireWebCompanion/settings.json.bak to settings.json
+3. change the secret key
+4. run `build --tag calibreweb:1.0  . -f ./deployment/Dockerfile` to build the image
+5. run your container with your bind/mount your volumes/paths/things 
+Here's an example of step 5
+```
+docker run --publish 80:80 \
+-v '/home/MassiveAtoms/Desktop/logs:/app/data' \
+-v '/home/MassiveAtoms/windows/Users/MassiveAtoms/Documents/Calibre\ Library/:/app/calibredir' \
+--name cw calibreweb:1.0
+```
+
+your Calibre path/volume/whatever needs to be mounted at `/app/calibredir`, and you need to mount a volume for the db and logs at `/app/data`
+
+Issues with it at the moment:
+1. we still need to do something to create a random secret key. Atm, this would still 
+
+# Docker (provided image)
+not done yet
+
+# non docker setup
+this might need to be modified, since some things have changed to adapt it for docker setup
 1. clone repo
 2. pip install -r requirements.txt
-3. install gunicorn and nginx
+3. rename the settings.json.bak to settings.json, change logging folder, change secret key, set isdocker to false
-4. move this nginx.conf to /etc/nginx
+4. install gunicorn and nginx
-5. make whatever user nginx runs as (in this case, massiveatoms) the owner of calibredir
+5. move this nginx.conf to /etc/nginx
-6. give execute permissions to parent of calibredir
+6. create a user and group `www`
-7. cd to repo, run `gunicorn CalibreWebCompanion.wsgi`
+7. make whatever user nginx runs as (for now, www) the owner of calibredir
-8. start nginx `sudo systemctl restart nginx`
+8. give execute permissions to parent of calibredir
+9. cd to repo, run `gunicorn CalibreWebCompanion.wsgi`
+10. start nginx `sudo systemctl restart nginx`
+11. make steps 9 and 10 happen on startup?
 
 Slight issues with this atm:
-1. server_name in nginx.conf needs to be changed
+1. where to do ssl?
-2. User needs to be edited in nginx.conf, now it's just my user acc. This affectd step 4-6
-3. where to do ssl?
 
 
 Suggestions: 
 1. We might want to use sockets instead of ip/port?
-2. logging
+2. autostart gunicorn/nginx
-3. autostart gunicorn/nginx
+3. some extra instrumentation for gunicorn https://docs.gunicorn.org/en/latest/deploy.html
-4. some extra instrumentation for gunicorn https://docs.gunicorn.org/en/latest/deploy.html
-
-# docker stuff
-here are the commands i use to build and run:
-1. build --tag calibreweb:1.0.1  . -f ./deployment/Dockerfile
-2. 
-```
-docker run --publish 8000:80\ 
--v '/home/massiveatoms/Desktop/logs:/usr/src/app/data' \
--v '/run/media/massiveatoms/1AEEEA6EEEEA421D/Documents and Settings/MassiveAtoms/Documents/Calibre Library/:/usr/src/app/calibredir' \
---name cw calibreweb:1.0.1
-```
-
-Docker stuff to fix:
-1. permission/ownership issue with volumes and nginx
-2. we need a setup script that changes secretkey, and generates the default db

deployment/nginx.conf

@@ -1,81 +0,0 @@
-worker_processes 1;
-
-# user nobody nogroup;
-user www www;  # TEMP disabled
-# user nobody nobody; # for systems with 'nobody' as a group instead
-error_log  /usr/src/app/data/logs/nginx.log warn;
-# pid /var/run/nginx.pid;
-
-events {
-  worker_connections 1024; # increase if you have lots of clients
-  accept_mutex off; # set to 'on' if nginx worker_processes > 1
-  use epoll; # to enable for Linux 2.6+ MASSIVEATOMS 
-  # 'use kqueue;' to enable for FreeBSD, OSX
-}
-
-http {
-  include mime.types;
-  # fallback in case we can't determine a type
-  default_type application/octet-stream;
-  access_log /var/log/nginx/access.log combined;
-  sendfile on;
-
-  upstream app_server {
-    # fail_timeout=0 means we always retry an upstream even if it failed
-    # to return a good HTTP response
-
-    # for UNIX domain socket setups
-    # server unix:/tmp/gunicorn.sock fail_timeout=0;
-
-    # for a TCP configuration
-    server 127.0.0.1:8000 fail_timeout=0;
-  }
-
-  server {
-    # if no Host match, close the connection to prevent host spoofing
-    listen 80 default_server;
-    return 444;
-  }
-
-  server {
-    listen 80 deferred;  # for Linux massiveatoms
-    # use 'listen 80 accept_filter=httpready;' for FreeBSD
-    # listen 80;
-    client_max_body_size 4G;
-
-    # set the correct host(s) for your site
-    server_name localhost  0.0.0.0; # set this to the server url? or ip? we'll see MASSIVEATOMS
-
-    keepalive_timeout 5;
-
-    # # MASSIVEATOMS
-      location /download/ {
-        alias "/usr/src/app/calibredir/";
-        # Never forget the fact that this little statement being root instead of alias caused us to lose more than a day troubleshooting
-    }
-
-          location /static/ {
-        alias "/usr/src/app/CalibreWebCompanion/static/";
-        # Never forget the fact that this little statement being root instead of alias caused us to lose more than a day troubleshooting
-    }
-
-    location / {
-      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      proxy_set_header X-Forwarded-Proto $scheme;
-      # proxy_set_header Host $http_host;
-      # we don't want nginx trying to do something clever with
-      # redirects, we set the Host: header above already.
-      proxy_redirect off;
-      proxy_pass http://127.0.0.1:8000;
-    }
-
-
-
-
-
-    error_page 500 502 503 504 /500.html;
-    location = /500.html {
-      root /path/to/app/current/public;
-    }
-  }
-}

deployment/startupscript.py

@@ -1,11 +0,0 @@
-from os import system, chdir
-
-
-# system("chown -R www:www /usr/src/app/calibredir")
-# print("ownership of calibredir changed")
-chdir("/usr/src/app/CalibreWebCompanion")
-system("python ./manage.py makemigrations")
-print("ran makemigrations")
-system("python ./manage.py migrate")
-print("migrate")
-

deployment/supervisord.conf

@@ -1,36 +0,0 @@
-[supervisord]
-nodaemon=true
-logfile=/tmp/supervisord.log
-childlogdir=/tmp
-pidfile = /tmp/supervisord.pid
-
-[program:gunicorn]
-directory=/usr/src/app/CalibreWebCompanion
-command=gunicorn CalibreWebCompanion.wsgi
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-autorestart=false
-startretries=0
-startsecs = 0
-
-[program:nginx]
-# user=www
-command=nginx
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-autorestart=false
-startretries=0
-
-[program:startupscript]
-directory=/usr/src/app
-command=python ./startupscript.py
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-autorestart=false
-startretries=0

requirements.txt

@@ -1,6 +1,7 @@
 django>=3.0.8
 inotify>=0.2.10
 gunicorn>=20.0 
+django-extensions
 # development
 # django-debug-toolbar>=2.2
 # django-silk>=4.0

test.sh

@@ -0,0 +1,17 @@
+docker build --tag calibreweb:1.0  . -f ./deployment/Dockerfile
+docker stop cw
+docker rm cw
+docker run --publish 80:80 \
+-v '/home/MassiveAtoms/Desktop/logs:/app/data' \
+-v '/home/MassiveAtoms/windows/Users/MassiveAtoms/Documents/Calibre Library/:/app/content' \
+--name cw calibreweb:1.0
+sleep 8
+echo "download/test"
+curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1/download/statictest.txt
+echo " "
+echo "/test"
+curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1/statictest.txt
+echo " "
+echo "/static/test"
+curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1/static/statictest.txt
+echo " "