From a665178ee7b565d74219ab5fff4ef94899e672b8 Mon Sep 17 00:00:00 2001 From: Jelte Fennema-Nio Date: Tue, 20 Aug 2024 09:21:49 +0200 Subject: [PATCH v4 2/2] Change to make local development easier, including README This makes a bunch of changes, that are useful to get a local development environment working. The primary change being a README explaining how to do so. --- .gitignore | 1 + README.md | 99 +++++++++++++++++++++++ django/archives/example_settings_local.py | 23 ++++++ django/run_dev.py | 22 +++++ django/uwsgi_dev.ini | 10 +++ loader/archives.ini.sample | 16 ++-- loader/sql/dev_data.sql | 3 + loader/sql/schema.sql | 3 +- pyproject.toml | 21 +++++ 9 files changed, 189 insertions(+), 9 deletions(-) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 django/archives/example_settings_local.py create mode 100755 django/run_dev.py create mode 100644 django/uwsgi_dev.ini create mode 100644 loader/sql/dev_data.sql create mode 100644 pyproject.toml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..11041c7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.egg-info diff --git a/README.md b/README.md new file mode 100644 index 0000000..2d6cc6d --- /dev/null +++ b/README.md @@ -0,0 +1,99 @@ +# PG archives + +This application manages PostgreSQL mailing list archives. + +## The Application + +This is a Django 4.2 application backed by PostgreSQL and running on Python 3.x. + +## Getting Started + +### Ubuntu instructions + +First, prepare your development environment by installing python3, postgresql-server-dev-X.Y, formail and libtidy (use `--no-install-recommends` to avoid installing postfix): + +```bash +sudo apt install python3 postgresql-server-dev-14 procmail libtidy5deb1 --no-install-recommends +``` + +Next, configure your local environment with virtualenv and install local dependencies. + +```bash +python3 -m venv env +source env/bin/activate +pip install -e '.[dev]' +``` + +Install the `pg_tsparser` extension: +```bash +git clone https://github.com/postgrespro/pg_tsparser +cd pg_tsparser +make USE_PGXS=1 +make USE_PGXS=1 install # might need sudo depending on your Postgres installation +cd .. +rm -rf pg_tsparser/ +``` + +Install the `en_us` dictionary for the `pg_tsparser` extension: + +```bash +git clone https://github.com/postgrespro/hunspell_dicts +cd hunspell_dicts/hunspell_en_us +make USE_PGXS=1 +make USE_PGXS=1 install # might need sudo depending on your Postgres installation +cd ../.. +rm -rf hunspell_dicts/ +``` + +Create dummy synonym and stop files: +```bash +touch "$(pg_config --sharedir)/tsearch_data/pg_dict.syn" +touch "$(pg_config --sharedir)/tsearch_data/pg_dict.stop" +``` + +Create a database for the application: + +```bash +createdb archives +psql -d archives -c 'CREATE EXTENSION pg_tsparser' \ + -c 'CREATE EXTENSION hunspell_en_us' \ + -c '\i loader/sql/schema.sql' -c 'COMMIT' \ + -c '\i loader/sql/dev_data.sql' +``` + +Create config for the loader scripts: + +```bash +cp loader/archives.ini.sample loader/archives.ini +``` + +Load some emails from the actual PostgreSQL archives by downloading an mbox +file from (e.g. the one from +the current month) and running the following command. + +NOTE: it's totally fine if some of the emails will fail to load due to a missing Message-ID field. + +```bash +loader/load_message.py --list pgsql-hackers --mbox /path/to/downloaded/mbox/file +``` + +Then go to the `django` directory, that's where the actual web application is. + +```bash +cd django +``` + +Create a local settings file (feel free to edit it): + +```bash +cp archives/example_settings_local.py archives/settings_local.py +``` + +Finally, you're ready to start the web application: + +```bash +./run_dev.py +``` + +Then open to view your local mailing +list archives. diff --git a/django/archives/example_settings_local.py b/django/archives/example_settings_local.py new file mode 100644 index 0000000..230c56b --- /dev/null +++ b/django/archives/example_settings_local.py @@ -0,0 +1,23 @@ +# Enable more debugging information +DEBUG = True +# Prevent logging to try to send emails to postgresql.org admins. +# Use the default Django logging settings instead. +LOGGING = None + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql_psycopg2", + "NAME": "archives", + "USER": "postgres", + "PASSWORD": "postgres", + "HOST": "0.0.0.0", + } +} + +# We don't use HTTPS during development +SESSION_COOKIE_SECURE = False +CSRF_COOKIE_SECURE = False + +# Allow access to all clients +PUBLIC_ARCHIVES = True +ALLOWED_HOSTS = ["*"] diff --git a/django/run_dev.py b/django/run_dev.py new file mode 100755 index 0000000..268cd83 --- /dev/null +++ b/django/run_dev.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +from importlib.machinery import PathFinder +import subprocess +import sys + +django_path = PathFinder().find_spec("django").submodule_search_locations[0] + +django_admin_path = django_path + "/contrib/admin/static/admin" + +if len(sys.argv) > 1: + ini_file = sys.argv[1] +else: + ini_file = "uwsgi_dev.ini" + +subprocess.run( + [ + "uwsgi", + "--static-map", + f"/static/admin={django_path}/contrib/admin/static/admin", + ini_file, + ] +) diff --git a/django/uwsgi_dev.ini b/django/uwsgi_dev.ini new file mode 100644 index 0000000..9ab26b2 --- /dev/null +++ b/django/uwsgi_dev.ini @@ -0,0 +1,10 @@ +[uwsgi] +threads=1 +env=DJANGO_SETTINGS_MODULE=archives.settings +module=archives.wsgi:application +py-autoreload=1 +touch-reload = archives/settings.py +touch-reload = archives/settings_local.py +touch-reload = uwsgi_dev.ini +http=127.0.0.1:8001 +static-map=/media-archives=media diff --git a/loader/archives.ini.sample b/loader/archives.ini.sample index a146f69..087cc43 100644 --- a/loader/archives.ini.sample +++ b/loader/archives.ini.sample @@ -2,16 +2,16 @@ connstr=dbname=archives [varnish] -purgeurl=https://wrigleys.postgresql.org/api/varnish/purge/ +# purgeurl=https://wrigleys.postgresql.org/api/varnish/purge/ [smtp] -server=localhost:9911 -heloname=localhost -resender=noreply@example.com +# server=localhost:9911 +# heloname=localhost +# resender=noreply@example.com [pglister] # synchronize subscribers between pgarchives and pglister -subscribers=0 -root=/path/to/pglister -myname=pgarchives -apikey=CHANGEME +# subscribers=0 +# root=/path/to/pglister +# myname=pgarchives +# apikey=CHANGEME diff --git a/loader/sql/dev_data.sql b/loader/sql/dev_data.sql new file mode 100644 index 0000000..c937f7d --- /dev/null +++ b/loader/sql/dev_data.sql @@ -0,0 +1,3 @@ +INSERT INTO listgroups (groupid, groupname, sortkey) VALUES (1, 'Developer lists', 1); +INSERT INTO lists (listid, listname, shortdesc, description, active, groupid, subscriber_access) VALUES (1, 'pgsql-hackers', 'pgsql-hackers', 'The PostgreSQL developers team lives here. Discussion of current development issues, problems and bugs, and proposed new features. If your question cannot be answered by people in the other lists, and it is likely that only a developer will know the answer, you may re-post your question in this list. You must try elsewhere first!', True, 1, True); + diff --git a/loader/sql/schema.sql b/loader/sql/schema.sql index 4e9508e..7180d34 100644 --- a/loader/sql/schema.sql +++ b/loader/sql/schema.sql @@ -91,6 +91,7 @@ CREATE TABLE loaderrors( err text NOT NULL ); + /* textsearch configs */ CREATE TEXT SEARCH CONFIGURATION pg (PARSER=tsparser); @@ -113,7 +114,7 @@ ALTER TEXT SEARCH CONFIGURATION pg word, hword, hword_part WITH pg_stop, pg_dict, english_ispell, english_stem; ALTER TEXT SEARCH CONFIGURATION pg - DROP MAPPING FOR email, url, url_path, sfloat, float; + DROP MAPPING IF EXISTS FOR email, url, url_path, sfloat, float; CREATE FUNCTION messages_fti_trigger_func() RETURNS trigger AS $$ BEGIN diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..dc1712f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,21 @@ +[project] +name = "pgarchives" +description = "Email archives webapp for the PostgreSQL community" +dynamic = ["version"] +readme = "README.md" +license = "PostgreSQL" +dependencies = [ + "django>=4.2,<5.0", + "psycopg2", + "simplejson", + "requests", + "pycryptodomex", + "python-dateutil", + "pytidylib", +] + +[project.optional-dependencies] +dev = ["uwsgi"] + +[tool.setuptools.packages.find] +include = ["django/archives*"] -- 2.43.0