From 69212c44b603d2c3952527bb41d04db88421a62b Mon Sep 17 00:00:00 2001 From: Martin Winkler Date: Thu, 15 Dec 2022 22:02:48 +0100 Subject: [PATCH] paperless - new prefered seq for paperless using a venv --- seqs/paperless-venv.cfg.example | 8 + seqs/paperless-venv.sh | 341 ++++++++++++++++++++++++++++++++ seqs/paperless.sh | 6 +- 3 files changed, 351 insertions(+), 4 deletions(-) create mode 100644 seqs/paperless-venv.cfg.example create mode 100755 seqs/paperless-venv.sh diff --git a/seqs/paperless-venv.cfg.example b/seqs/paperless-venv.cfg.example new file mode 100644 index 0000000..e66d2d9 --- /dev/null +++ b/seqs/paperless-venv.cfg.example @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +sc_paperlessHome="/opt/paperless" +sc_paperlessVenv="${sc_paperlessHome}/venv" +sc_paperlessDir="${sc_paperlessHome}/paperless-ngx" +sc_paperlessBackupDir="${HOME}/backup" +sc_paperlessPython="python3" +sc_paperlessUser="paperless" diff --git a/seqs/paperless-venv.sh b/seqs/paperless-venv.sh new file mode 100755 index 0000000..0deb66d --- /dev/null +++ b/seqs/paperless-venv.sh @@ -0,0 +1,341 @@ +#!/bin/bash + +readonly toolName="paperless-ngx" +readonly toolDeps="python3 python3-pip python3-dev default-libmysqlclient-dev imagemagick fonts-liberation gnupg libpq-dev libmagic-dev mime-support libzbar0 poppler-utils redis-server" +readonly toolDepsOcr="unpaper ghostscript icc-profiles-free qpdf liblept5 libxml2 pngquant zlib1g tesseract-ocr" +readonly versionUrl="https://api.github.com/repos/paperless-ngx/paperless-ngx/releases/latest" + +versionNew= +versionNow= +downUrl= + +sq_paperlessDownLoc="/tmp/paperless_latest.tar.xz" +sq_aptOpt= + +seq_config() { + ## or to use sequencer api with global config file: + if initSeqConfig "${seq_configName:?}" "${seq_configTemplate:?}" ; then + sq_config=1 + else + # End if no configuration file exists + dry || return 1 + fi + + ## Apt cmdline option to suppress user interaction + interactive || sq_aptOpt="-y" + + ## Disable error checks if external scripts are used + ## e.g. error on unbound variables + #disableErrorCheck + + ## Return of non zero value will abort the sequence + return 0 +} + +getVersions() { + versionNew="${versionNew:-$(curl --silent "$versionUrl" | grep -Po '"tag_name": "v\K.*?(?=")')}" + versionNow="${versionNow:-$(grep -Po '"Paperless-ngx",version:"\K.*?(?=")' 2>/dev/null < "${sc_paperlessDir}/static/frontend/de-DE/main.js")}" + downUrl="${downUrl:-"https://github.com/paperless-ngx/paperless-ngx/releases/download/v${versionNew:?}/paperless-ngx-v${versionNew:?}.tar.xz"}" +} + +step_1_info() { echo "Status of ${toolName}"; } +step_1_alias() { echo "status"; } +step_1() { + getVersions + info -n "${toolName} " + if [[ -n "${versionNow}" ]] ; then + color green + info -d "version ${versionNow} installed" + if [[ ! ${versionNow:-} == ${versionNew} ]] ; then + info -a "Update available to version ${versionNew} from:" + info -a "${downUrl}" + fi + color none + step service + else + color yellow + info -d "not installed" + info -a "Version ${versionNew} available for installation from:" + info -a "${downUrl}" + fi +} + +step_3_info() { echo "Manage ${toolName} services"; } +step_3_options() { echo "[status|start|stop|restart]"; } +step_3_alias() { echo "service"; } +step_3() { + shift + local serviceCommand="is-active" + case "${1:-"status"}" in + start) + serviceCommand="${1}" ;; + stop) + serviceCommand="${1}" ;; + restart) + serviceCommand="${1}" ;; + status) + serviceCommand="is-active" + info -n "paperless-webserver: " + exe systemctl "${serviceCommand:?}" paperless-webserver + info -na "paperless-scheduler: " + exe systemctl "${serviceCommand:?}" paperless-scheduler + info -na "paperless-consumer : " + exe systemctl "${serviceCommand:?}" paperless-consumer + info -na "paperless-task-queue: " + exe systemctl "${serviceCommand:?}" paperless-task-queue.service + return 0 ;; + "") + ;; + *) + error "Unknown command ${1:-"-"}" + return 1 ;; + esac + exe systemctl "${serviceCommand:?}" paperless-webserver + exe systemctl "${serviceCommand:?}" paperless-scheduler + exe systemctl "${serviceCommand:?}" paperless-consumer + exe systemctl "${serviceCommand:?}" paperless-task-queue +} + +step_10_info() { echo "Install python3"; } +step_10_alias() { echo "install"; } +step_10() { + exe apt update + exe apt install ${toolDeps} ${aptOpt:-} + exe apt install ${toolDepsOcr} ${aptOpt:-} +} + +step_11_info() { echo "Add system user"; } +step_11() { + if id "${sc_paperlessUser}" >/dev/null 2>&1 ; then + endReturn -o 1 "User ${sc_paperlessUser} already exists" + fi + exe adduser --disabled-password --disabled-login --gecos "" --home "${sc_paperlessHome}" "${sc_paperlessUser:?}" +} + +step_12_info() { echo "Install/upgrade ${toolName}"; } +step_12_alias() { echo "upgrade"; } +step_12() { + local toolUpgrade=0 + getVersions + if [[ ${versionNow:-} == ${versionNew} ]] ; then + endReturn -o 1 "Latest version ${versionNow} already installed" + else + if [ -n "${versionNow:-}" ] ; then + info "Upgrading $toolName from ${versionNow} to ${versionNew}" + else + info "Installing $toolName version ${versionNew}" + fi + fi + exe wget ${downUrl} -q -O "${sq_paperlessDownLoc}" + endReturn -o $? "Download failed" + + if [ -e "${sc_paperlessDir}" ] ; then + if step backup --nostart; then + info "Backup successful" + else + endReturn -o 1 "Backup failed." + fi + toolUpgrade=1 + exe mv "${sc_paperlessDir}" "${sc_paperlessDir}_bu" + fi + + exe mkdir -p "${sc_paperlessHome}" + exe tar -xf "${sq_paperlessDownLoc}" -C "${sc_paperlessHome}" + exe chown -R "${sc_paperlessUser}": "${sc_paperlessHome}" + + if ((toolUpgrade)) ; then + info "Moving over ${toolName} and gunicorn configuration" + addConf -c -f "${sc_paperlessDir}_bu/paperless.conf" "${sc_paperlessDir}/paperless.conf" + saveReturn $? + addConf -c -f "${sc_paperlessDir}_bu/gunicorn.conf.py" "${sc_paperlessDir}/gunicorn.conf.py" + saveReturn $? + endReturn "${toolName} configuration could not be transferred. ${sc_paperlessDir}_bu not deleted." + exe rm -rf "${sc_paperlessDir}_bu" + info "Starting post upgrade procedure" + step postupgrade + step service start + die "Upgrade finished" + fi +} + +step_13_info() { echo "Install venv"; } +step_13_alias() { echo "venv"; } +step_13() { + exe "${sc_paperlessPython}" -m venv "${sc_paperlessVenv}" +} + +step_14_info() { echo "Install other requirements"; } +step_14_alias() { echo "requirements"; } +step_14() { + exe cd "${sc_paperlessHome}" + exe "${sc_paperlessVenv}/bin/pip3" install --upgrade pip + exe "${sc_paperlessVenv}/bin/pip3" install --requirement "${sc_paperlessDir}/requirements.txt" +} + +step_15_info() { echo "Initialization"; } +step_15() { + exe install --owner="${sc_paperlessUser}" --group="${sc_paperlessUser}" -d "${sc_paperlessHome}/"{consume,data,media} + exe cd "${sc_paperlessDir}/src" + exe "${sc_paperlessVenv}/bin/python3" manage.py migrate + exe "${sc_paperlessVenv}/bin/python3" manage.py createsuperuser + exe chown -R "${sc_paperlessUser}": "${sc_paperless}"{consume,data,media} +} + +step_17_info() { echo "Post upgrade steps"; } +step_17_alias() { echo "postupgrade"; } +step_17() { + step requirements + exe cd "${sc_paperlessDir}/src" + exe "${sc_paperlessVenv}/bin/python3" manage.py migrate +} + +step_30_info() { + echo "Retag existing documents" + echoinfo "OPTIONS" + echoinfo " -c, --correspondent" + echoinfo " -T, --tags" + echoinfo " -t, --document_type" + echoinfo " -i, --inbox-only" + echoinfo " --use-first" + echoinfo " -f, --overwrite" +} +step_30_options() { echo "[OPTIONS]"; } +step_30_alias() { echo "retag"; } +step_30() { + shift + exe cd "${sc_paperlessDir}/src" + exe sudo -u "${sc_paperlessUser}" "${sc_paperlessVenv}/bin/python3" manage.py document_retagger "$@" +} + +step_32_info() { + echo "Manage document search index" + echoinfo " reindex - (default) create index from scratch" + echoinfo " optimize - updates index to increase search speed and" + echoinfo " ensures autocompletion" +} +step_32_options() { echo "[reindex|optimize]"; } +step_32_alias() { echo "reindex"; } +step_32() { + shift + local command="${1:-"reindex"}" + case "${1:-}" in + reindex) ;; + optimize) ;; + "") ;; + *) fatal "Unkown argument ${1}" ;; + esac + exe cd "${sc_paperlessDir}/src" + exe sudo -u "${sc_paperlessUser}" "${sc_paperlessVenv}/bin/python3" manage.py document_index "${command}" +} + +step_34_info() { + echo "Rename (apply storage paths) to existing documents" + echoinfo "Be aware that this command moves files on the file system." + echoinfo "Consider creating a backup before execution." +} +step_34_alias() { echo "rename"; } +step_34() { + exe cd "${sc_paperlessDir}/src" + exe sudo -u "${sc_paperlessUser}" "${sc_paperlessVenv}/bin/python3" manage.py document_renamer +} + +step_40_info() { echo "Backup ${toolName}"; } +step_40_options() { echo "[--nostart]"; } +step_40_alias() { echo "backup"; } +step_40() { + shift + info "Doing backup..." + step service stop + exe cd "$(dirname -- "${sc_paperlessHome}")" + exe tar czf "${sc_paperlessBackupDir:-"${HOME}/backup"}/paperless_backup_$(date +%Y%m%d-%H%M%S).tar.gz" \ + "$(basename -- "${sc_paperlessHome}")" + #--exclude="$(basename -- "${sc_paperlessHome}")/.local" \ + #--exclude="$(basename -- "${sc_paperlessHome}")/.cache" \ + if [[ ${1:-} == "--nostart" ]] ; then + info "Not starting ${toolName} services after backup" + else + step service start + fi +} + +step_100_info() { echo "Notes"; } +step_100_alias() { echo "notes"; } +step_100() { + color green + cat </dev/null && for f in *.service; do echo " $f"; done) + * Modifications: + * WorkingDirectory=/opt/paperless/paperless-ngx/src + * path to celery: ${sc_paperlessHome}/venv/bin/celery +* service files webserver + * Environment="PAPERLESS_BIND_ADDR=0.0.0.0" + Environment="PAPERLESS_PORT=8084" + To chose a custom listen address and port for gunicorn + (default address: [..], default port: 8000) +* /etc/ImageMagick-x/policy.xml + * enable access to pdfs + + +# Nginx proxy + +--- +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name mydomain.com; + + access_log off; #/var/log/nginx/paperless.log; + error_log /var/log/nginx/paperless_error.log; + + include /etc/nginx/ssl.conf; + error_page 497 https://\$host:\$server_port\$request_uri; + + fastcgi_read_timeout 300s; + proxy_read_timeout 65s; + client_max_body_size 20M; + + location / { + proxy_pass http://10.0.0.10/; + + # These configuration options are required for WebSockets to work. + proxy_http_version 1.1; + proxy_set_header Upgrade \$http_upgrade; + proxy_set_header Connection "upgrade"; + + proxy_redirect off; + proxy_set_header Host \$http_host; + proxy_set_header X-Real-IP \$remote_addr; + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto \$scheme; + } +} +--- +EOF_NOTES + color none +} + +# shellcheck disable=SC2034 # Appears unused +readonly sqr_minVersion=16 +# shellcheck disable=SC1091 # Don't follow this source +. /usr/local/bin/sequencer.sh diff --git a/seqs/paperless.sh b/seqs/paperless.sh index 1e7a7fa..0276606 100755 --- a/seqs/paperless.sh +++ b/seqs/paperless.sh @@ -161,9 +161,6 @@ step_13_alias() { echo "requirements"; } step_13() { exe cd "${sc_paperlessHome}" exe sudo -Hu paperless pip3 install --upgrade pip - info "Remove '-e' before the line for django-q" - ask "Press Enter to continue?" - editor "${sc_paperlessDir}/requirements.txt" exe sudo -Hu paperless pip3 install --requirement "${sc_paperlessDir}/requirements.txt" } @@ -184,7 +181,7 @@ step_16() { } step_18_info() { - echoinfoArgs "[OPTIONS]"; echo "Retag existing documents" + echo "Retag existing documents" echoinfo "OPTIONS" echoinfo " -c, --correspondent" echoinfo " -T, --tags" @@ -193,6 +190,7 @@ step_18_info() { echoinfo " --use-first" echoinfo " -f, --overwrite" } +step_18_options() { echo "[OPTIONS]"; } step_18_alias() { echo "retag"; } step_18() { shift