public inbox for [email protected]help / color / mirror / Atom feed
[PATCH] Add APT repository webpages at /repos/apt/ 2+ messages / 1 participants [nested] [flat]
* [PATCH] Add APT repository webpages at /repos/apt/ @ 2013-03-26 08:33 Adrian Vondendriesch <[email protected]> 0 siblings, 0 replies; 2+ messages in thread From: Adrian Vondendriesch @ 2013-03-26 08:33 UTC (permalink / raw) The repository contents are imported into a (mostly) normalized database schema (apt_* tables), from where the web pages are served. Dependencies: python-apt, postgresql-debversion, pg_trgm Authors: Adrian Vondendriesch and Christoph Berg --- pgweb/apt/__init__.py | 0 pgweb/apt/models.py | 120 ++++++++ pgweb/apt/sql/architecture.sql | 1 + pgweb/apt/sql/package.sql | 2 + pgweb/apt/sql/packagecontents.sql | 2 + pgweb/apt/sql/pkg.sql | 12 + pgweb/apt/sql/release.sql | 2 + pgweb/apt/sql/source.sql | 2 + pgweb/apt/sql/srcsuite.sql | 2 + pgweb/apt/sql/suite.sql | 2 + pgweb/apt/sql/vendor.sql | 1 + pgweb/apt/utils.py | 29 ++ pgweb/apt/views.py | 373 ++++++++++++++++++++++++ pgweb/settings.py | 3 + pgweb/urls.py | 32 +++ pgweb/util/contexts.py | 9 + templates/apt/apt.html | 80 ++++++ templates/apt/binaries.html | 46 +++ templates/apt/binary.html | 167 +++++++++++ templates/apt/binary_contents.html | 14 + templates/apt/dependency.html | 12 + templates/apt/distributions.html | 20 ++ templates/apt/madison.html | 50 ++++ templates/apt/qa.html | 33 +++ templates/apt/search.html | 87 ++++++ templates/apt/source.html | 136 +++++++++ templates/apt/source_file.html | 66 +++++ templates/apt/sources.html | 18 ++ tools/apt/import-packagelist.py | 440 +++++++++++++++++++++++++++++ 29 files changed, 1761 insertions(+) create mode 100644 pgweb/apt/__init__.py create mode 100644 pgweb/apt/models.py create mode 100644 pgweb/apt/sql/architecture.sql create mode 100644 pgweb/apt/sql/package.sql create mode 100644 pgweb/apt/sql/packagecontents.sql create mode 100644 pgweb/apt/sql/pkg.sql create mode 100644 pgweb/apt/sql/release.sql create mode 100644 pgweb/apt/sql/source.sql create mode 100644 pgweb/apt/sql/srcsuite.sql create mode 100644 pgweb/apt/sql/suite.sql create mode 100644 pgweb/apt/sql/vendor.sql create mode 100644 pgweb/apt/utils.py create mode 100644 pgweb/apt/views.py create mode 100644 templates/apt/apt.html create mode 100644 templates/apt/binaries.html create mode 100644 templates/apt/binary.html create mode 100644 templates/apt/binary_contents.html create mode 100644 templates/apt/dependency.html create mode 100644 templates/apt/distributions.html create mode 100644 templates/apt/madison.html create mode 100644 templates/apt/qa.html create mode 100644 templates/apt/search.html create mode 100644 templates/apt/source.html create mode 100644 templates/apt/source_file.html create mode 100644 templates/apt/sources.html create mode 100755 tools/apt/import-packagelist.py diff --git a/pgweb/apt/__init__.py b/pgweb/apt/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pgweb/apt/models.py b/pgweb/apt/models.py new file mode 100644 index 0000000..fbc5fa9 --- /dev/null +++ b/pgweb/apt/models.py @@ -0,0 +1,120 @@ +# APT repository interface for the PostgreSQL website +# Authors: Adrian Vondendriesch <[email protected]> +# Christoph Berg <[email protected]> + +from django.db import models +from datetime import datetime + +class Vendor(models.Model): + vendor = models.TextField(null=False, primary_key=True) + + def __unicode(self): + return vendor; + +class Release(models.Model): + vendor = models.ForeignKey(Vendor) + release = models.TextField(null=False, primary_key=True) + relversion = models.TextField() + active = models.BooleanField(null=False, default=True) + + def __unicode(self): + return release; + +class Architecture(models.Model): + architecture = models.TextField(null=False, primary_key=True) + + def __unicode(self): + return architecture; + +class Component(models.Model): + component = models.TextField(null=False, primary_key=True) + + def __unicode__(self): + return self.component + +class Suite(models.Model): + release = models.ForeignKey(Release) + component = models.ForeignKey(Component) + architecture = models.ForeignKey(Architecture) + last_update = models.DateTimeField(null=True) + active = models.BooleanField(null=False, default=True) + + class Meta: + unique_together = (("release", "component", "architecture"),) + +class Srcsuite(models.Model): + release = models.ForeignKey(Release) + component = models.ForeignKey(Component) + last_update = models.DateTimeField(null=True) + active = models.BooleanField(null=False, default=True) + + class Meta: + unique_together = (("release", "component"),) + +class Package(models.Model): + package = models.TextField(null=False) + version = models.TextField(null=False) + arch = models.ForeignKey(Architecture) + source = models.TextField(null=False) + srcversion = models.TextField(null=False) + maintainer = models.TextField(null=False) + short_description = models.TextField(null=False) + description = models.TextField(null=True) + homepage = models.TextField(null=True) + installed_size = models.IntegerField(null=False) + filename = models.TextField(null=False) + depends = models.TextField(null=True) + recommends = models.TextField(null=True) + suggests = models.TextField(null=True) + size = models.IntegerField(null=False) + + class Meta: + unique_together = (("package", "version", "arch")) + + def __unicode__(self): + return self.package + +class PackageList(models.Model): + suite = models.ForeignKey(Suite) + package = models.ForeignKey(Package) + +class PackageContents(models.Model): + package = models.ForeignKey(Package) + filename = models.TextField(null=False) + +class Source(models.Model): + source = models.TextField(null=False) + srcversion = models.TextField(null=False) + binary = models.TextField(null=False) + architecture = models.TextField(null=False) + maintainer = models.TextField(null=False) + homepage = models.TextField(null=True) + short_description = models.TextField(null=True) + description = models.TextField(null=True) + vcs_browser = models.TextField(null=True) + vcs_repo = models.TextField(null=True) + build_depends = models.TextField(null=True) + directory = models.TextField(null=True) + copyright = models.TextField(null=True) + debchangelog = models.TextField(null=True) + changelog = models.TextField(null=True) + + class Meta: + unique_together = (("source", "srcversion")) + + def __unicode__(self): + return self.source + +class SourceList(models.Model): + suite = models.ForeignKey(Srcsuite) + source = models.ForeignKey(Source) + +# because file is a common identifier I use files as classname +class Files(models.Model): + files = models.TextField(null=False) + md5_hash = models.TextField(null=False) + size = models.IntegerField(null=False) + source = models.ForeignKey(Source) + + unitque_together = (("files", "md5_hash", "size"),) + diff --git a/pgweb/apt/sql/architecture.sql b/pgweb/apt/sql/architecture.sql new file mode 100644 index 0000000..397ba73 --- /dev/null +++ b/pgweb/apt/sql/architecture.sql @@ -0,0 +1 @@ +INSERT INTO apt_architecture VALUES ('all'); diff --git a/pgweb/apt/sql/package.sql b/pgweb/apt/sql/package.sql new file mode 100644 index 0000000..af9f8e4 --- /dev/null +++ b/pgweb/apt/sql/package.sql @@ -0,0 +1,2 @@ +CREATE EXTENSION IF NOT EXISTS debversion; +ALTER TABLE apt_package ALTER COLUMN version type debversion; diff --git a/pgweb/apt/sql/packagecontents.sql b/pgweb/apt/sql/packagecontents.sql new file mode 100644 index 0000000..fbd9fd3 --- /dev/null +++ b/pgweb/apt/sql/packagecontents.sql @@ -0,0 +1,2 @@ +CREATE EXTENSION IF NOT EXISTS pg_trgm; +CREATE INDEX apt_packagecontents_filename_gin ON apt_packagecontents USING gin (filename gin_trgm_ops); diff --git a/pgweb/apt/sql/pkg.sql b/pgweb/apt/sql/pkg.sql new file mode 100644 index 0000000..e7d20aa --- /dev/null +++ b/pgweb/apt/sql/pkg.sql @@ -0,0 +1,12 @@ +CREATE VIEW apt_pkg AS + SELECT s.release_id, + s.component_id, + s.architecture_id, + p.package, + p.version, + p.arch_id, + p.source, + p.srcversion + FROM apt_suite s + JOIN apt_packagelist pl ON s.id = pl.suite_id + JOIN apt_package p ON pl.package_id = p.id; diff --git a/pgweb/apt/sql/release.sql b/pgweb/apt/sql/release.sql new file mode 100644 index 0000000..6d49134 --- /dev/null +++ b/pgweb/apt/sql/release.sql @@ -0,0 +1,2 @@ +ALTER TABLE apt_release + ALTER COLUMN active SET DEFAULT true; diff --git a/pgweb/apt/sql/source.sql b/pgweb/apt/sql/source.sql new file mode 100644 index 0000000..f6ab987 --- /dev/null +++ b/pgweb/apt/sql/source.sql @@ -0,0 +1,2 @@ +CREATE EXTENSION IF NOT EXISTS debversion; +ALTER TABLE apt_source ALTER COLUMN srcversion TYPE debversion; diff --git a/pgweb/apt/sql/srcsuite.sql b/pgweb/apt/sql/srcsuite.sql new file mode 100644 index 0000000..f7a4627 --- /dev/null +++ b/pgweb/apt/sql/srcsuite.sql @@ -0,0 +1,2 @@ +ALTER TABLE apt_srcsuite + ALTER COLUMN active SET DEFAULT true; diff --git a/pgweb/apt/sql/suite.sql b/pgweb/apt/sql/suite.sql new file mode 100644 index 0000000..bfcd3f6 --- /dev/null +++ b/pgweb/apt/sql/suite.sql @@ -0,0 +1,2 @@ +ALTER TABLE apt_suite + ALTER COLUMN active SET DEFAULT true; diff --git a/pgweb/apt/sql/vendor.sql b/pgweb/apt/sql/vendor.sql new file mode 100644 index 0000000..9e59bd9 --- /dev/null +++ b/pgweb/apt/sql/vendor.sql @@ -0,0 +1 @@ +INSERT INTO apt_vendor VALUES ('Debian'), ('Ubuntu'); diff --git a/pgweb/apt/utils.py b/pgweb/apt/utils.py new file mode 100644 index 0000000..5484ec4 --- /dev/null +++ b/pgweb/apt/utils.py @@ -0,0 +1,29 @@ +# APT repository interface for the PostgreSQL website +# Authors: Adrian Vondendriesch <[email protected]> +# Christoph Berg <[email protected]> + +from models import Package, PackageList + +def build_related_list(rel_list, release): + result = [] + if rel_list: + for entry in rel_list.split(","): + entry = entry.strip() + l = [] + for subentry in entry.split("|"): + l.append(build_list_element(subentry, release)) + result.append(l) + return result + +def build_list_element(list_entry, release): + """Returns tuple (full string, package name, dependency package available in our archive)""" + list_entry = list_entry.strip() + s = list_entry.split(" ", 1) + pkg = s[0] + dep = '' + if len(s) > 1: + dep = s[1] + pkg_available = len(PackageList.objects.filter(\ + package__package=pkg, \ + suite__release=release)) + return (list_entry, pkg, dep, pkg_available) diff --git a/pgweb/apt/views.py b/pgweb/apt/views.py new file mode 100644 index 0000000..07f6fd0 --- /dev/null +++ b/pgweb/apt/views.py @@ -0,0 +1,373 @@ +# APT repository interface for the PostgreSQL website +# Authors: Adrian Vondendriesch <[email protected]> +# Christoph Berg <[email protected]> + +from pgweb.util.contexts import render_pgweb +from django.http import HttpResponse, Http404, HttpResponseRedirect +from django.db import connection +from models import * +from utils import * + +def apt(request): + return render_pgweb(request, 'apt', 'apt/apt.html', {}) + +def sources(request, release): + source_list_entries = SourceList.objects.filter(suite__release_id=release).order_by('source__source') + if not len(source_list_entries): + raise Http404 + + sources = [entry.source for entry in source_list_entries] + + return render_pgweb(request, 'apt', 'apt/sources.html', { + 'release': release, + 'sources': sources, + }) + +def source(request, package_name, release='sid-pgdg', component=None, filename=None): + + source_list_entries = SourceList.objects.filter(source__source__exact=package_name) + suites = [e.suite for e in source_list_entries] + + if release: + source_list_entries = source_list_entries.filter(suite__release_id__exact=release) + if component: + source_list_entries = source_list_entries.filter(suite__component_id__exact=component) + + sources = [s.source for s in source_list_entries] + + # check if at least one source package was found + if not len(sources): + raise Http404 + + source = sources[0] + current_component = source_list_entries[0].suite.component_id + current_release = source_list_entries[0].suite.release_id + + binaries = [binary.strip() for binary in source.binary.split(',')] + + if filename: + if filename == 'copyright': + title = 'Copyright file' + content = source.copyright + elif filename == 'debchangelog': + title = 'Debian changelog' + content = source.debchangelog + else: + title = 'Upstream changelog' + content = source.changelog + return render_pgweb(request, 'apt', 'apt/source_file.html', { + 'package': source, + 'filename': filename, + 'title': title, + 'content': content, + 'current_release': current_release, + 'current_component': current_component, + 'binaries': binaries, + }) + + source_list_entries = source_list_entries.filter(suite__release_id__exact=current_release) + components = [e.suite.component_id for e in source_list_entries] + + # create dependency list + build_dependencies = build_related_list(source.build_depends, current_release) + + return render_pgweb(request, 'apt', 'apt/source.html', { + 'package': source, + 'build_dependencies': build_dependencies, + 'distributions': suites, + 'current_release': current_release, + 'components': components, + 'current_component': current_component, + 'files': source.files_set.all(), + 'binaries': binaries + }) + +def binary_contents(request, package_name, release, component, arch): + + # fix contents for all packages + if arch == 'all': + arch = 'amd64' + + binary_list_entries = PackageList.objects.filter(package__package__exact=package_name, suite__release_id__exact=release, suite__component_id__exact=component, suite__architecture_id__exact=arch) + + if not len(binary_list_entries): + raise Http404 + + binary_list = binary_list_entries[0] + binary = binary_list.package + contents = binary.packagecontents_set.all().order_by("filename") + + return render_pgweb(request, 'apt', 'apt/binary_contents.html', { + 'package': binary, + 'current_release': release, + 'current_component': component, + 'contents': contents, + }) + +def binary(request, package_name, release='sid-pgdg', component='main'): + + binary_list_entries = PackageList.objects.filter(package__package__exact=package_name) + suites = [e.suite for e in binary_list_entries.distinct('suite__release')] + + if release: + binary_list_entries = binary_list_entries.filter(suite__release__exact=release) + components = [c.suite.component for c in binary_list_entries.distinct('suite__component')] + if component: + binary_list_entries = binary_list_entries.filter(suite__component_id__exact=component) + + # check if at least on source package was found + if not len(binary_list_entries): + raise Http404 + + binary = binary_list_entries[0].package + current_component = binary_list_entries[0].suite.component_id + current_release = binary_list_entries[0].suite.release_id + + dependencies = build_related_list(binary.depends, current_release) + recommendations = build_related_list(binary.recommends, current_release) + suggestions = build_related_list(binary.suggests, current_release) + + return render_pgweb(request, 'apt', 'apt/binary.html', { + 'package': binary, + 'dependencies': dependencies, + 'recommendations': recommendations, + 'distributions': suites, + 'current_release': current_release, + 'suggestions': suggestions, + 'components': components, + 'current_component': current_component, + 'files': None, + 'downloads': Package.objects.filter(package=package_name, version=binary.version) + }) + +def distributions(request): + releases = Release.objects.order_by("vendor", "relversion", "release") + + return render_pgweb(request, 'apt', 'apt/distributions.html', { + 'releases': releases, + }) + +def binaries(request, release, component_name=None, architecture_name=None): + + if not architecture_name: + architecture_name = 'amd64' + + suites = Suite.objects.filter(release=release, architecture=architecture_name) + components = [d.component for d in suites.distinct('component')] + + if not component_name: + component_name = 'main' + + suites = suites.filter(component_id=component_name) + + if not len(suites): + raise Http404 + + suite = suites[0] + + binary_lists = suite.packagelist_set.all().order_by('package__package') + binaries = [binary_list.package for binary_list in binary_lists] + + return render_pgweb(request, 'apt', 'apt/binaries.html', { + 'suite': suite, + 'components': components, + 'binaries': binaries + }) + +def search(request): + if 'package' in request.GET and request.GET['package']: + package = request.GET['package'] + + result_binaries = Package.objects.distinct('package').filter(package__icontains=package) + result_sources = Source.objects.distinct('source').filter(source__icontains=package) + + return render_pgweb(request, 'apt', 'apt/search.html', { + 'result': True, + 'package': package, + 'result_binaries': result_binaries, + 'result_sources': result_sources, + }) + + elif 'file' in request.GET and request.GET['file']: + filename = request.GET['file'] + + maxresults = 100 + result_filenames = PackageContents.objects.distinct('filename', 'package__package').filter(filename__contains=filename).order_by("filename")[:maxresults] + + return render_pgweb(request, 'apt', 'apt/search.html', { + 'filename': filename, + 'maxresults': maxresults, + 'result_filenames': result_filenames, + }) + + else: + return render_pgweb(request, 'apt', 'apt/search.html', { + }) + +def madison(request): + """Query interface compatible with rmadison(1)""" + + if 'package' in request.GET and request.GET['package']: + package = request.GET['package'] + + sql1 = """SELECT package, version, release_id, component_id, array_agg(architecture_id) AS architecture FROM ( + SELECT package, version, release_id, component_id, architecture_id + FROM apt_packagelist pl + JOIN apt_suite s ON (suite_id = s.id) + JOIN apt_package p ON (package_id = p.id) + UNION ALL SELECT source, srcversion, release_id, component_id, 'source' + FROM apt_sourcelist sl + JOIN apt_srcsuite ss ON (suite_id = ss.id) + JOIN apt_source s ON (source_id = s.id) + ORDER BY package, version, release_id, component_id, architecture_id + ) AS r WHERE """ + sql2 = "GROUP BY package, version, release_id, component_id" + + qual = [] + args = [package] + configdisplay = 'none' + + r = '' + if 'r' in request.GET: + r = 'checked' + qual.append('package ~ %s') + configdisplay = 'block' + else: + qual.append('package = %s') + + a = '' + if 'a' in request.GET and request.GET['a']: + a = request.GET['a'] + qual.append('architecture_id = %s') + args.append(a) + configdisplay = 'block' + + c = '' + if 'c' in request.GET and request.GET['c']: + c = request.GET['c'] + qual.append('component_id = %s') + args.append(c) + configdisplay = 'block' + + s = '' # distributions are called suites in dak + if 's' in request.GET and request.GET['s']: + s = request.GET['s'] + qual.append('release_id = %s') + args.append(s) + configdisplay = 'block' + + cursor = connection.cursor() + cursor.execute(sql1 + " AND ".join(qual) + sql2, args) + return_set = cursor.fetchall() + + plen = 1 + vlen = 1 + rlen = 1 + for (pkg, version, release, component, architecture) in return_set: + if component != 'main': + release += '/' + component + if len(pkg) > plen: + plen = len(pkg) + if len(version) > vlen: + vlen = len(version) + if len(release) > rlen: + rlen = len(release) + fstr = " %%-%ds | %%-%ds | %%-%ds | %%s\n" % (plen, vlen, rlen) + + content = '' + for (pkg, version, release, component, architecture) in return_set: + if component != 'main': + release += '/' + component + content += fstr % (pkg, version, release, ", ".join(architecture)) + + if 'text' in request.GET: + return HttpResponse(content, content_type='text/plain') + + return render_pgweb(request, 'apt', 'apt/madison.html', { + 'result': True, + 'package': package, + 'r': r, + # no need to pass checkbox 'text' value here + 'a': a, + 'c': c, + 's': s, + 'configdisplay': configdisplay, + 'content': content, + }) + + else: + return render_pgweb(request, 'apt', 'apt/madison.html', { + 'configdisplay': 'none', + }) + +def qa(request, query=None): + result = None + + if True or query == None: # TODO: fix the SQL queries + description = "APT Repository Quality Assurance" + sql = None + + elif query == "binary_missing_on_architecture": + description = "Binary package versions that exist only on one architecture" + sql = """select * from apt_pkg a where not exists + (select * from apt_pkg b where a.package = b.package and a.version = b.version and a.architecture_id <> b.architecture_id) + order by release_id, package, version""" + + elif query == "wrong_tag_in_version": + description = "Package versions with wrong pgdg tag" + sql = """select p.* from apt_pkg p join apt_release r on (p.release_id = r.release) + where version !~ case + when release = 'sid-pgdg' then 'pgdg' + when release = 'lenny-pgdg' then 'pg(dg|apt)50' + when release = 'etch-pgdg' then 'pg(dg|apt)40' + when vendor_id = 'Debian' then 'pgdg'||relversion||0 + when release = 'precise-pgdg' then 'pgdg12.4' + when release = 'lucid-pgdg' then 'pgdg10.4' + when vendor_id = 'Ubuntu' then 'pgdg'||relversion + end + and r.active + order by release_id, package, version, architecture_id""" + + elif query == "binary_without_source": + description = "Binary packages without source" + sql = """select * from apt_pkg p where not exists + (select * from apt_source s where (p.source, p.srcversion) = (s.source, s.srcversion)) + order by release_id, package, version, architecture_id""" + + elif query == "binary_in_sid_only": + description = "Binary packages in sid-pgdg missing in other suites" + sql = """SELECT DISTINCT sidpkg.package, sidpkg.version, sidpkg.source, sidpkg.srcversion, + rel.release AS missing_release FROM apt_pkg sidpkg + JOIN apt_release rel ON (true) + LEFT JOIN apt_pkg p ON (rel.release = p.release_id and sidpkg.package = p.package) + WHERE sidpkg.release_id = 'sid-pgdg' AND rel.release <> 'sid-pgdg' AND rel.active + AND p.package IS NULL + ORDER BY sidpkg.package, missing_release""" + + elif query == "binary_with_old_version": + description = "Binary packages with versions different from sid-pgdg" + sql = """SELECT sidpkg.release_id, sidpkg.component_id, sidpkg.package, sidpkg.version, + sidpkg.architecture_id, p.release_id, p.version FROM apt_pkg sidpkg + JOIN apt_pkg p ON (sidpkg.package = p.package AND sidpkg.component_id = p.component_id AND sidpkg.architecture_id = p.architecture_id) + WHERE sidpkg.release_id = 'sid-pgdg' + AND regexp_replace(sidpkg.version, '.pgdg.*', '') <> regexp_replace(p.version, '.pgdg.*', '') + ORDER BY sidpkg.component_id, sidpkg.package, sidpkg.architecture_id, p.release_id""" + + elif query == "source_without_binary": + description = "Source packages without binaries" + sql = """SELECT source, srcversion FROM apt_source s WHERE NOT EXISTS + (SELECT * FROM apt_pkg p WHERE (s.source, s.srcversion) = (p.source, p.srcversion))""" + + else: + raise Http404 + + if sql: + cursor = connection.cursor() + cursor.execute(sql) + result = cursor.fetchall() + + return render_pgweb(request, 'apt', 'apt/qa.html', { + 'query': query, + 'description': description, + 'result': result, + }) diff --git a/pgweb/settings.py b/pgweb/settings.py index 1c72af6..7761f2d 100644 --- a/pgweb/settings.py +++ b/pgweb/settings.py @@ -115,6 +115,7 @@ INSTALLED_APPS = [ 'pgweb.featurematrix', 'pgweb.search', 'pgweb.pugs', + 'pgweb.apt', ] # Default format for date/time (as it changes between machines) @@ -161,5 +162,7 @@ FRONTEND_SMTP_RELAY = "magus.postgresql.org" # Where to relay use OAUTH = {} # OAuth providers and keys PGDG_ORG_ID = -1 # id of the PGDG organisation entry +APT_DIR = "/srv/apt/pub/repos/apt" # Directory containing the dists/ dir from apt.postgresql.org + # Load local settings overrides from settings_local import * diff --git a/pgweb/urls.py b/pgweb/urls.py index d40673f..55ddcbe 100644 --- a/pgweb/urls.py +++ b/pgweb/urls.py @@ -3,6 +3,7 @@ from django.views.generic import RedirectView from pgweb.util.signals import register_basic_signal_handlers +import pgweb.apt.views import pgweb.contributors.views import pgweb.core.views import pgweb.docs.views @@ -110,6 +111,37 @@ urlpatterns = [ url(r'^sitemap.xml', pgweb.core.views.sitemap), url(r'^sitemap_internal.xml', pgweb.core.views.sitemap_internal), + ### + # Packages + ### + url(r'^repos/apt/$', pgweb.apt.views.apt), + url(r'^repos/apt/dists/$', pgweb.apt.views.distributions), + + url(r'^repos/apt/sources/$', RedirectView.as_view(url='/repos/apt/dists/', permanent=True)), + url(r'^repos/apt/sources/(?P<release>[\w.+-]+)/$', pgweb.apt.views.sources), + + url(r'^repos/apt/source/$', RedirectView.as_view(url='/repos/apt/sources', permanent=True)), + url(r'^repos/apt/source/(?P<package_name>[\w.+-]+)/$', pgweb.apt.views.source), + url(r'^repos/apt/source/(?P<package_name>[\w.+-]+)/(?P<release>[\w.+-]+)/$', pgweb.apt.views.source), + url(r'^repos/apt/source/(?P<package_name>[\w.+-]+)/(?P<release>[\w.+-]+)/(?P<filename>copyright|debchangelog|changelog)/$', pgweb.apt.views.source), + url(r'^repos/apt/source/(?P<package_name>[\w.+-]+)/(?P<release>[\w.+-]+)/(?P<component>[\w.+-]+)/$', pgweb.apt.views.source), + url(r'^repos/apt/source/(?P<package_name>[\w.+-]+)/(?P<release>[\w.+-]+)/(?P<component>[\w.+-]+)/(?P<filename>copyright|debchangelog|changelog)/$', pgweb.apt.views.source), + + url(r'^repos/apt/binaries/$', RedirectView.as_view(url='/repos/apt/dists/', permanent=True)), + url(r'^repos/apt/binaries/(?P<release>[\w.+-]+)/$', pgweb.apt.views.binaries), + url(r'^repos/apt/binaries/(?P<release>[\w.+-]+)/(?P<component_name>[\w\-\.]+)/$', pgweb.apt.views.binaries), + + url(r'^repos/apt/binary/$', RedirectView.as_view(url='/repos/apt/dists/', permanent=True)), + url(r'^repos/apt/binary/(?P<package_name>[\w.+-]+)/$', pgweb.apt.views.binary), + url(r'^repos/apt/binary/(?P<package_name>[\w.+-]+)/(?P<release>[\w.+-]+)/$', pgweb.apt.views.binary), + url(r'^repos/apt/binary/(?P<package_name>[\w.+-]+)/(?P<release>[\w.+-]+)/(?P<component>[\w.+-]+)/$', pgweb.apt.views.binary), + url(r'^repos/apt/binary/(?P<package_name>[\w.+-]+)/(?P<release>[\w.+-]+)/(?P<component>[\w.+-]+)/(?P<arch>[\w.+-]+)/contents/$', pgweb.apt.views.binary_contents), + + url(r'^repos/apt/search/$', pgweb.apt.views.search), + url(r'^repos/apt/madison/$', pgweb.apt.views.madison), # machine-readable search interface + url(r'^repos/apt/qa/$', pgweb.apt.views.qa), + url(r'^repos/apt/qa/(?P<query>[\w.+-]+)/$', pgweb.apt.views.qa), + ### # Workaround for broken links pushed in press release ### diff --git a/pgweb/util/contexts.py b/pgweb/util/contexts.py index 43f6c3e..b93cacd 100644 --- a/pgweb/util/contexts.py +++ b/pgweb/util/contexts.py @@ -83,6 +83,15 @@ sitenav = { {'title': 'Change password', 'link': '/account/changepwd/'}, {'title': 'Logout', 'link': '/account/logout'}, ], + 'apt': [ + {'title': 'APT Repository', 'link': '/repos/apt/'}, + {'title': 'Distributions', 'link': '/repos/apt/dists/'}, + {'title': 'Search', 'link': '/repos/apt/search/', 'submenu': [ + {'title': 'Query API', 'link': '/repos/apt/madison/'}, + {'title': 'Repository QA', 'link': '/repos/apt/qa/'}, + ]}, + {'title': 'Wiki', 'link': 'http://wiki.postgresql.org/wiki/Apt';}, + ], } diff --git a/templates/apt/apt.html b/templates/apt/apt.html new file mode 100644 index 0000000..7cfc55a --- /dev/null +++ b/templates/apt/apt.html @@ -0,0 +1,80 @@ +{%extends "base/page.html"%} +{%block title%}Apt Repository{%endblock%} +{%block contents%} +<h1>Apt Repository with PostgreSQL Packages for Debian and Ubuntu</h1> + +<p> +The PostgreSQL Global Development Group (PGDG) maintains an APT repository of +PostgreSQL packages for Debian and Ubuntu located at +http://apt.postgresql.org/pub/repos/apt/. We aim at building PostgreSQL server +packages as well as extensions and modules packages on several Debian/Ubuntu +releases for all PostgreSQL versions supported. +</p> + +<p> +Currently, we support + +<ul> + <li>Debian 6.0 (squeeze), 7.0 (wheezy), and unstable (sid) 64/32 bit (amd64/i386)</li> + <li>Ubuntu 10.04 (lucid) and 12.04 (precise) 64/32 bit (amd64/i386)</li> + <li>PostgreSQL 8.3, 8.4, 9.0, 9.1, 9.2</li> + <li>Several server extensions such as Slony-I, various PL languages, and datatypes</li> + <li>Applications like pgadmin3, pgbouncer, and pgpool-II</li> +</ul> +</p> + +<p> +Packages for older PostgreSQL versions and older Debian distributions will +continue to stay in the repository; updates for those will be provided on an +ad-hoc basis. +</p> + +<script type="text/javascript"> + <!-- + function updateSeries(select) { + var deb = document.getElementById('series-deb'); + deb.innerHTML = select.value; + } + --> +</script> + +<h2>Using the Apt Repository</h2> +<p> +If the version included in your version of Debian/Ubuntu is not the one you want, +you can use the PostgreSQL Apt Repository. This repository will integrate +with your normal systems and patch management, and provide automatic +updates for all supported versions of PostgreSQL throughout the support +<a href="/support/versioning/">lifetime</a> of PostgreSQL. +</p> +<p> +To use the apt repository, follow these steps: +</p> +<ul> + <li> + <select id="field.series" name="field.series" onchange='updateSeries(this);'> + <option selected="selected" value="OS_VERSION">Choose your Debian/Ubuntu version</option> + <option value="wheezy">Debian Wheezy (7.0)</option> + <option value="squeeze">Debian Squeeze (6.0)</option> + <option value="precise">Ubuntu Precise (12.04)</option> + <option value="lucid">Ubuntu Lucid (10.04)</option> + </select> + </li> + <li>Create the file <i>/etc/apt/sources.list.d/pgdg.list</i>, and add a line + for the repository + <code>deb http://apt.postgresql.org/pub/repos/apt/ <span id="series-deb">OS_VERSION</span>-pgdg main</code> + </li> + <li>Import the repository signing key, and update the package lists + <code> +wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | \<br/> + sudo apt-key add -<br/> +sudo apt-get update + </code> + </li> +</ul> +<p> +For more information about the apt repository, including answers to frequent +questions, please see the apt page on +<a href="http://wiki.postgresql.org/wiki/Apt">the wiki</a>. +</p> + +{%endblock%} diff --git a/templates/apt/binaries.html b/templates/apt/binaries.html new file mode 100644 index 0000000..839ae39 --- /dev/null +++ b/templates/apt/binaries.html @@ -0,0 +1,46 @@ +{%extends "base/page.html"%} +{%block title%}Binary Packages in {{suite.release_id}}/{{suite.component_id}}{%endblock%} +{%block contents%} +<div id="pgCommunityWrap"> +<div id="pgCommunity"> +<h1>Binary Packages in {{suite.release_id}}/{{suite.component_id}} ({{binaries|length}})</h1> + +<p> +<ul> +{% for binary in binaries %} + <li> + <b><a href="/repos/apt/binary/{{binary}}/{{suite.release_id}}/{{suite.component_id}}/" + title="click for details to package {{binary}}">{{binary}}</a></b> + {{binary.version}} <br /> + {{binary.short_description}} + </li> +{% endfor %} +</ul> +</p> +</div> +</div> + +{% if components|length > 1 %} +<div id=pgAptWrap> +<div id=pgApt> +<dl> + <dt>Other Components</dt> + <dd> + <ul> + {% for component in components %} + <li> + {% if component.component != suite.component_id %} + <a href="/repos/apt/binaries/{{suite.release_id}}/{{component}}/">{{component}}</a> + {% else %} + <b>{{component}}</b> + {% endif %} + </li> + {% endfor %} + </ul> + </dd> +</dl> +</div> +</div> +{% endif %} + +{%endblock%} diff --git a/templates/apt/binary.html b/templates/apt/binary.html new file mode 100644 index 0000000..6416e1d --- /dev/null +++ b/templates/apt/binary.html @@ -0,0 +1,167 @@ +{%extends "base/page.html"%} +{%block title%}Package {{package}} ({{package.version}}) in {{current_release}}{% if current_component != "main" %}/{{current_component}}{% endif %}{%endblock%} +{%block contents%} +<div id="pgCommunityWrap"> +<div id="pgCommunity"> +<h1>{{package}} ({{package.version}}) in {{current_release}}{% if current_component != "main" %}/{{current_component}}{% endif %}</h1> + +<p> +<b>{{package.short_description}}</b> +</p> + +<p> +{% autoescape off %} +{{package.description}} +{% endautoescape %} +</p> + +{% if package.homepage %} +<p><b>Homepage:</b> + <a href="{{package.homepage}}" + target="_blank" + title="Go to homepage of {{package}}">{{package.homepage}}</a> +{% endif %} + +<p> +<b>Maintainer:</b> {{ package.maintainer }} +</p> + +{% if package.vcs_browser or package.vcs_repo%} +<h2>VCS</h2> + + {% if package.vcs_browser %} +<h3>Browser</h3> +<p> + <a href="{{package.vcs_browser}}" + target="_blank" + title="Go to VCS-Browser of {{package}}">{{package.vcs_browser}}</a> +<p> + {% endif %} + + {% if package.vcs_repo %} +<h3>Repository</h3> +<p> + <a href="{{package.vcs_repo}}" + target="_blank" + title="Get the current version of {{package}}">{{package.vcs_repo}}</a> +<p> + {% endif %} +{% endif %} + +{% if dependencies or recommendations or suggestions %} +<h2>Related Packages</h2> +{% if dependencies %} +<h3>Dependencies {% if package.arch_id != "all" %} (on {{package.arch_id}}) {% endif %}</h3> +<ul> + {% for dep in dependencies %} + {% include "apt/dependency.html" %} + {% endfor %} +</ul> +{% endif %} + +{% if recommendations %} +<h3>Recommendations</h3> +<ul> + {% for dep in recommendations %} + {% include "apt/dependency.html" %} + {% endfor %} +</ul> +{% endif %} + +{% if suggestions %} +<h3>Suggestions</h3> +<ul> + {% for dep in suggestions %} + {% include "apt/dependency.html" %} + {% endfor %} +</ul> +{% endif %} +{% endif %} + +<h2>Download {{package}} for {{current_release}}</h2> +<div class="tblBasic"> +<table> + <tr> + <th>Architecture</th> + <th>Package Size</th> + <th>Installed Size</th> + <th>File List</th> + </tr> + {% for download in downloads %} + <tr> + <td> + <a href="http://apt.postgresql.org/pub/repos/apt/{{download.filename}}" + title="Download {{download}} for {{download.arch_id}}">{{download.arch_id}}</a> + </td> + <td>{{download.size|filesizeformat}}</td> + <td>{{download.installed_size|filesizeformat}}</td> + <td><a href="/repos/apt/binary/{{package}}/{{current_release}}/{{current_component}}/{{download.arch_id}}/contents/">File List</a></td> + </tr> + {% endfor %} +</table> +</div> +</div> +</div> + +<div id=pgAptWrap> +<div id=pgApt> +<dl> + <dt>Source</dt> + <dd> + <ul> + <li><a href="/repos/apt/source/{{package.source}}/{{current_release}}/">{{package.source}}</a></li> + <li><a href="/repos/apt/source/{{package.source}}/{{current_release}}/copyright/">Copyright file</a></li> + <li><a href="/repos/apt/source/{{package.source}}/{{current_release}}/debchangelog/">Package changelog</a></li> + <li><a href="/repos/apt/source/{{package.source}}/{{current_release}}/changelog/">Upstream changelog</a></li> + </ul> + </dd> +</dl> +</div> +</div> + + +{% if components|length > 1 %} +<div id=pgAptWrap> +<div id=pgApt> +<dl> + <dt>This Package in other Components</dt> + <dd> + <ul> +{% for comp in components %} + {% if comp.component == current_component %} + <li><b>{{comp}}</b></li> + {% else %} + <li> + <a href="/repos/apt/binary/{{package}}/{{current_release}}/{{comp}}/">{{comp}}</a> + </li> + {% endif %} +{% endfor %} + </ul> + </dd> +</dl> +</div> +</div> +{% endif %} + +<div id=pgAptWrap> +<div id=pgApt> +<dl> + <dt>Distributions</dt> + <dd> + <ul> + {% for dist in distributions %} + <li> + {% if current_release and current_release == dist.release_id %} + <b>{{dist.release_id}}</b> + {% else %} + <a href="/repos/apt/binary/{{package}}/{{dist.release_id}}/">{{dist.release_id}}</a> + {% endif %} + </li> + {% endfor %} + </ul> + </dd> +</dl> +</div> +</div> + +{%endblock%} diff --git a/templates/apt/binary_contents.html b/templates/apt/binary_contents.html new file mode 100644 index 0000000..385b8a0 --- /dev/null +++ b/templates/apt/binary_contents.html @@ -0,0 +1,14 @@ +{%extends "base/page.html"%} +{%block title%}File List of Package {{package}} ({{package.version}} {{package.arch.architecture}}){%endblock%} +{%block contents%} +<div id="pgCommunityWrap"> +<div id="pgCommunity"> +<h1>{{package}} ({{package.version}} {{package.arch.architecture}}) in {{current_release}} File List</h1> +<p> +<nobr> +{% for file in contents %}{{file.filename}}<br /> +{% endfor %} +</nobr> +</p> + +{%endblock%} diff --git a/templates/apt/dependency.html b/templates/apt/dependency.html new file mode 100644 index 0000000..c7f0709 --- /dev/null +++ b/templates/apt/dependency.html @@ -0,0 +1,12 @@ +<li> +{% for alternative in dep %} + {% if alternative.3 %} + <a href="/repos/apt/binary/{{alternative.1}}/{{current_release}}/" + title="View details for Package {{alternative.1}}">{{alternative.1}}</a> + {{alternative.2}} + {% else %} + {{alternative.0}} + {% endif %} + {% if alternative != dep|last %} | {% endif %} +{% endfor %} +</li> diff --git a/templates/apt/distributions.html b/templates/apt/distributions.html new file mode 100644 index 0000000..82ff46f --- /dev/null +++ b/templates/apt/distributions.html @@ -0,0 +1,20 @@ +{%extends "base/page.html"%} +{%block title%}Distributions on apt.postgresql.org{%endblock%} +{%block contents%} +<h1>Distributions</h1> +<p> +<div class="tblBasic"> + <table> + {% for release in releases %} + <tr> + <th>{{release.vendor_id}} {{release.relversion}}</th> + <th>{{release.release}}</th> + <td><a href="/repos/apt/binaries/{{release.release}}/" title="Binary packages in {{release.release}}">Binaries</a></td> + <td><a href="/repos/apt/sources/{{release.release}}/" title="Source packages in {{release.release}}">Sources</a></td> + </tr> + {% endfor %} + </table> +</div> +</p> + +{%endblock%} diff --git a/templates/apt/madison.html b/templates/apt/madison.html new file mode 100644 index 0000000..3709a14 --- /dev/null +++ b/templates/apt/madison.html @@ -0,0 +1,50 @@ +{%extends "base/page.html"%} +{%block title%}Query apt.postgresql.org{%endblock%} +{%block contents%} + +<script> +function toggle(x) { + var oContent=document.getElementById(x) ; + if (oContent.style.display=="block") { + oContent.style.display="none"; + } else { + oContent.style.display="block"; + } +} +</script> + +<h1>Query apt.postgresql.org</h1> + +<form action="/repos/apt/madison/" method="get"> + <input type="text" name="package" value="{{package}}"> + <input type="submit" value="Search"> <br /> + Enter a package name + <span onclick='javascript:toggle("config")'>(Click for more options)</span><br /> + <div id="config" style="display:{{configdisplay}};"> + <table border="0"> + <tr><td>Regexp search</td><td><input type="checkbox" name="r" {{r}}></td></tr> + <tr><td>Text output</td><td><input type="checkbox" name="text"></td></tr> + <tr><td>Architecture</td><td><input type="text" name="a" value="{{a}}"></td></tr> + <tr><td>Component</td><td><input type="text" name="c" value="{{c}}"></td></tr> + <tr><td>Distribution</td><td><input type="text" name="s" value="{{s}}"></td></tr> + </table> + </div> +</form> + + +{% if package %} +<h2>Results for <span class="pgAptPattern">"{{package}}"</span></h2> + +<pre>{{content}}</pre> + +{% else %} + +<p> +A command line client for this page is +<a href="https://manpages.debian.org/devscripts/rmadison.1.html">rmadison(1)</a; +contained in the +<a href="https://packages.debian.org/devscripts">devscripts</a; package. +</p> +{% endif %} + +{%endblock%} diff --git a/templates/apt/qa.html b/templates/apt/qa.html new file mode 100644 index 0000000..fcf94b2 --- /dev/null +++ b/templates/apt/qa.html @@ -0,0 +1,33 @@ +{%extends "base/page.html"%} +{%block title%}{{description}}{%endblock%} +{%block contents%} +<div id="pgCommunityWrap"> +<div id="pgCommunity"> +<h1>{{description}}</h1> + +{% if query %} + +<table border="0"> +{% for row in result %} +<tr> + {% for column in row %} + <td>{{column}}</td> + {% endfor %} +</tr> +{% endfor %} +</table> + +{% else %} + +<ul> + <li><a href="binary_missing_on_architecture/">Binary missing on architecture</a></li> + <li><a href="wrong_tag_in_version/">Wrong tag in version</a></li> + <li><a href="binary_without_source/">Binary without source</a></li> + <li><a href="binary_in_sid_only/">Binary in sid-pgdg only</a></li> + <li><a href="binary_with_old_version/">Binary with old version</a></li> + <li><a href="source_without_binary/">Source without binaries</a></li> +</ul> + +{% endif %} + +{%endblock%} diff --git a/templates/apt/search.html b/templates/apt/search.html new file mode 100644 index 0000000..4e86963 --- /dev/null +++ b/templates/apt/search.html @@ -0,0 +1,87 @@ +{%extends "base/page.html"%} +{%block title%}Search apt.postgresql.org{%endblock%} +{%block contents%} +<h1>Search on apt.postgresql.org</h1> + +<h2>Search for Packages</h2> + +<form action="/repos/apt/search/" method="get"> + <input type="text" name="package" value="{{package}}"> + <input type="submit" value="Search"> +</form> + +Enter a package name or substring + +{% if package %} +<h2>Results for <span class="pgAptPattern">"{{package}}"</span></h2> + + <div class="pgAptList"> + <h3>Binary Packages: {{result_binaries|length}}</h3> + {% if result_binaries %} + <dl> + + {% for binary in result_binaries %} + <dt><a href="/repos/apt/binary/{{binary}}/">{{binary}}</a></dt> + <dd>{{binary.short_description}}</dd> + {% endfor %} + + </dl> + {% else %} + <p> + No packages found. + </p> + <div> + {% endif %} + + <div class="pgAptList"> + <h3>Source Packages: {{result_sources|length}}</h3> + {% if result_sources %} + <dl> + + {% for source in result_sources %} + <dt><a href="/repos/apt/source/{{source}}/">{{source}}</a></dt> + <dd>{{source.short_description}}</dd> + {% endfor %} + + </dl> + {% else %} + <p> + No packages found. + </p> + <div> + {% endif %} +{% endif %} + +<h2>Search for Files in Packages</h2> + +<form action="/repos/apt/search/" method="get"> + <input type="text" name="file" value="{{filename}}"> + <input type="submit" value="Search"> +</form> + +Enter a filename or substring + +{% if filename %} +<h2>Results for <span class="pgAptPattern">"{{filename}}"</span></h2> + + <div class="pgAptList"> + {% if result_filenames %} + {% if result_filenames|length >= maxresults %} + <h3>Files: Too many results, only {{maxresults}} will be shown</h3> + {% else %} + <h3>Files: {{result_filenames|length}}</h3> + {% endif %} + <table border="0"> + {% for file in result_filenames %} + <tr><td>{{file.filename}}</td><td><a href="/repos/apt/binary/{{file.package}}/">{{file.package}}</a></td></tr> + {% endfor %} + </table> + {% else %} + <p> + No packages found. + </p> + {% endif %} + </div> +{% endif %} + +{%endblock%} diff --git a/templates/apt/source.html b/templates/apt/source.html new file mode 100644 index 0000000..db4a2ac --- /dev/null +++ b/templates/apt/source.html @@ -0,0 +1,136 @@ +{%extends "base/page.html"%} +{%block title%}Source Package {{package}} ({{package.srcversion}}) in {{current_release}}{%endblock%} +{%block contents%} +<div id="pgCommunityWrap"> +<div id="pgCommunity"> +<h1>{{package}} ({{package.srcversion}}) in {{current_release}}</h1> + +<p> +<b>{{package.short_description}}</b> +</p> + +<p> +{% autoescape off %} +{{package.description}} +{% endautoescape %} +</p> + +<p> +<b>Maintainer:</b> {{ package.maintainer }} +</p> + +{% if package.homepage %} +<p><b>Homepage:</b> + <a href="{{package.homepage}}" + target="_blank" + title="Go to homepage of {{package}}">{{package.homepage}}</a> +{% endif %} + +<h2>Files</h2> +<ul> + {% for f in files %} + <li> + <b><a href="http://apt.postgresql.org/pub/repos/apt/{{package.directory}}/{{f.files}}" + title="Download {{f.files}}">{{f.files}}</a></b><br /> + {{f.size|filesizeformat}}, {{f.md5_hash}} + </li> + {% endfor %} +</ul> + +{% if build_dependencies %} +<h2>Build Dependencies</h2> +<ul> + {% for dep in build_dependencies %} + {% include "apt/dependency.html" %} + {% endfor %} +</ul> +{% endif %} + +</div> +</div> + +<div id=pgAptWrap> +<div id=pgApt> +<dl> + <dt>Binaries</dt> + <dd> + <ul> + {% for binary in binaries %} + <li> + <a href="/repos/apt/binary/{{binary}}/{{current_release}}/">{{binary}}</a></li> + {% endfor %} + </ul> + </dd> +</dl> +</div> +</div> + +<div id=pgAptWrap> +<div id=pgApt> +<dl> + <dt>Resources</dt> + <dd> + <ul> + <li><a href="/repos/apt/source/{{package}}/{{current_release}}/copyright/">Copyright file</a></li> + <li><a href="/repos/apt/source/{{package}}/{{current_release}}/debchangelog/">Package changelog</a></li> + <li><a href="/repos/apt/source/{{package}}/{{current_release}}/changelog/">Upstream changelog</a></li> + <li><a href="http://packages.debian.org/{{package}}">{{package}} in Debian</a> + (<a href="http://packages.qa.debian.org/{{package}}">PTS</a>)</li> + <li><a href="https://launchpad.net/ubuntu/+source/{{package}}">{{package}} in Ubuntu</a></li> + {% if package.vcs_browser %} + <li><a href="{{package.vcs_browser}}">Packaging browser</a></li> + {% endif %} + {% if package.vcs_repo %} + <li><a href="{{package.vcs_repo}}">Packaging repository</a></li> + {% endif %} + </ul> + </dd> +</dl> +</div> +</div> + +<div id=pgAptWrap> +<div id=pgApt> +<dl> + <dt>Distributions</dt> + <dd> + <ul> + {% for dist in distributions %} + <li> + {% if current_release and current_release == dist.release_id %} + <b>{{dist.release_id}}</b> + {% else %} + <a href="/repos/apt/source/{{package}}/{{dist.release_id}}/">{{dist.release_id}}</a> + {% endif %} + </li> + {% endfor %} + </ul> + </dd> +</dl> +</div> +</div> + +{% if components|length > 1 %} +<div id=pgAptWrap> +<div id=pgApt> +<dl> + <dt>Components</dt> + <dd> + <ul> +{% for comp in components %} + {% if comp == current_component %} + <li>{{comp}}</li> + {% else %} + <li> + <a href="/repos/apt/source/{{package}}/{{current_release}}/{{comp}}">{{comp}}</a> + </li> + {% endif %} +{% endfor %} + </ul> + </dd> +</dl> +</div> +</div> +{% endif %} + +{%endblock%} diff --git a/templates/apt/source_file.html b/templates/apt/source_file.html new file mode 100644 index 0000000..7051685 --- /dev/null +++ b/templates/apt/source_file.html @@ -0,0 +1,66 @@ +{%extends "base/page.html"%} +{%block title%}{{title}} for {{package}} ({{package.srcversion}}) in {{current_release}}{%endblock%} +{%block contents%} +<div id="pgCommunityWrap"> +<div id="pgCommunity"> +<h1>{{title}} for {{package}} ({{package.srcversion}}) in {{current_release}}</h1> + +<p> +<b>{{package.short_description}}</b> +</p> + +<pre> +{{content}} +</pre> + +</div> +</div> + +<div id=pgAptWrap> +<div id=pgApt> +<dl> + <dt>Source</dt> + <dd> + <ul> + <li><a href="/repos/apt/source/{{package.source}}/{{current_release}}/">{{package.source}}</a></li> + + {% if filename == "copyright" %} + <li><b>Copyright file</b></li> + {% else %} + <li><a href="/repos/apt/source/{{package.source}}/{{current_release}}/copyright/">Copyright file</a></li> + {% endif %} + + {% if filename == "debchangelog" %} + <li><b>Package changelog</b></li> + {% else %} + <li><a href="/repos/apt/source/{{package.source}}/{{current_release}}/debchangelog/">Package changelog</a></li> + {% endif %} + + {% if filename == "changelog" %} + <li><b>Upstream changelog</b></li> + {% else %} + <li><a href="/repos/apt/source/{{package.source}}/{{current_release}}/changelog/">Upstream changelog</a></li> + {% endif %} + </ul> + </dd> +</dl> +</div> +</div> + +<div id=pgAptWrap> +<div id=pgApt> +<dl> + <dt>Binaries</dt> + <dd> + <ul> + {% for binary in binaries %} + <li> + <a href="/repos/apt/binary/{{binary}}/{{current_release}}/">{{binary}}</a></li> + {% endfor %} + </ul> + </dd> +</dl> +</div> +</div> + +{%endblock%} diff --git a/templates/apt/sources.html b/templates/apt/sources.html new file mode 100644 index 0000000..b8f9555 --- /dev/null +++ b/templates/apt/sources.html @@ -0,0 +1,18 @@ +{%extends "base/page.html"%} +{%block title%}Source Packages in {{release}}{%endblock%} +{%block contents%} +<h1>Source Packages in {{release}} ({{sources|length}})</h1> + +<p> +<ul> + {% for source in sources %} + <li> + <b><a href="/repos/apt/source/{{source}}/{{release}}/">{{source}}</a></b> + {{source.srcversion}} <br /> + {{source.short_description}} + </li> + {% endfor %} +</ul> +</p> + +{%endblock%} diff --git a/tools/apt/import-packagelist.py b/tools/apt/import-packagelist.py new file mode 100755 index 0000000..4b1973e --- /dev/null +++ b/tools/apt/import-packagelist.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python + +# debmirror -v -h apt.postgresql.org -r pub/repos/apt --method=http -d sid-pgdg,buster-pgdg --omit-suite-symlinks -s main,9.3,9.4,9.5,9.6,10,11,12 -a source,amd64,i386,ppc64el --exclude='\.deb$' --getcontents --no-check-gpg /srv/repo + +import apt_pkg, markdown, os, re, sys, time, gzip, subprocess + +# Set up for accessing django +import django +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings_local") +import sys +sys.path.append('../../pgweb') +from django.conf import settings +#from apt import models as c + +#from core.models import ImportedRSSFeed, ImportedRSSItem +from django.db import transaction, connection + +#transaction.enter_transaction_management() +#transaction.managed() + +config = { + 'FILTER': [ + 'testing', + 'deprecated' + ] + } + +running_config = { + 'VERBOSE': 0, + 'FORCE': False, + 'DISTONLY': False, + } + +usage = """ +%s [options] + -h show this help + -v verbose + -f force (deletes every package before insert) + -d ignore errors from dscextract (useful for running on dists/ only) +""" % sys.argv[0] + +def dir_filter(string): + for f in config['FILTER']: + if f in string: + return 0 + return 1 + +def getArchitectureNames(directories): + binary_pattern = 'binary-' + binaries = [] + for directory in directories: + if re.match(binary_pattern, directory): + binaries.append(directory.split('-')[1]) + + return binaries + +def buildFileDict(flist): + fdict = {} + for line in flist: + columns = line.split('\t') + if len(columns) > 1 and columns[0] != 'FILE': + filename = '/' + columns[0] + packagename = columns[1].split('/')[-1].strip() + try: + fdict[packagename] + except: + fdict[packagename] = [] + fdict[packagename].append(filename) + #print "P-Name:", packagename + return fdict + +def main(*args, **kwargs): + running_config['VERBOSE'] = 0 + + # args + if len(args) > 0: + for arg in args[0]: + if arg == '-v': + running_config['VERBOSE'] += 1 + if arg == '-f': + running_config['FORCE'] = True + if arg == '-d': + running_config['DISTONLY'] = True + if arg == '-h': + print usage + exit(0) + + cursor = connection.cursor() + + # releases + base_dir = getattr(settings, 'APT_DIR') + "/dists/" + releases = [directory for directory in os.listdir(base_dir) \ + if os.path.isdir(os.path.join(base_dir, directory))] + releases = filter(dir_filter, releases) + if running_config['VERBOSE'] > 0: + print "Releases to scan (%s): %s" % \ + (len(releases), ', '.join(releases)) + + for release in releases: + print "%s:" % (release) + cur_dir = base_dir + release + components = [component for component in os.listdir(cur_dir) \ + if os.path.isdir(os.path.join(cur_dir, component))] + components = filter(dir_filter, components) + if running_config['VERBOSE'] > 0: + print "\tComponents in %s to scan (%s): %s" % \ + (release, len(components), ', '.join(components)) + + for component in components: + if running_config['VERBOSE'] > 0: + print "\t%s:" % (component) + + # insert components, that aren't not yet registered + sql = """ + INSERT INTO apt_component + SELECT * FROM (VALUES (%s)) comp(component) + WHERE NOT EXISTS ( + SELECT * FROM apt_component + WHERE component = comp.component + ) + """ + cursor.execute(sql, (component,)) + + srcsuite_id, last_update = insertSuite(release, component) + parseFile(last_update, srcsuite_id, release, component) + + # read binary architectures + architectures = getArchitectureNames(os.listdir(os.path.join(cur_dir, component))) + + for architecture in architectures: + # insert architectures, that aren't not yet registered + sql = """ + INSERT INTO apt_architecture + SELECT * FROM (VALUES (%s)) arch(architecture) + WHERE NOT EXISTS ( + SELECT * FROM apt_architecture + WHERE architecture = arch.architecture + ) + """ + cursor.execute(sql, (architecture,)) + + suite_id, last_update = insertSuite(release, component, architecture) + parseFile(last_update, suite_id, release, component, architecture) + + # update source package descriptions + sql = """UPDATE apt_source s SET + short_description = (SELECT short_description FROM apt_package p WHERE (p.source, p.srcversion) = (s.source, s.srcversion) ORDER BY source = p.package DESC, p.package LIMIT 1), + description = (SELECT description FROM apt_package p WHERE (p.source, p.srcversion) = (s.source, s.srcversion) ORDER BY source = p.package DESC, p.package LIMIT 1) + WHERE s.short_description IS NULL""" + cursor.execute(sql) + transaction.commit() + + connection.close() + return + + +def insertSuite(release, component, architecture=None): + cursor = connection.cursor() + + sql = "SELECT release FROM apt_release WHERE release = %s" + cursor.execute(sql, (release,)) + if not cursor.fetchone(): + sql = "INSERT INTO apt_release (vendor_id, release, relversion, active) VALUES ('Debian', %s, '', true)" + cursor.execute(sql, (release,)) + + if architecture is None: + tbl = "srcsuite" + src = True + else: + tbl = "suite" + src = False + + # check if srcsuite is already in db + sql = "SELECT id, extract('epoch' from last_update) FROM apt_%s WHERE " % (tbl) + if src: + sql = sql + "(release_id, component_id) = (%s, %s) " + args_tpl = (release, component) + else: + sql = sql + "(release_id, component_id, architecture_id) = (%s, %s, %s)" + args_tpl = (release, component, architecture) + cursor.execute(sql, args_tpl) + + return_set = cursor.fetchone() + + if return_set: + suite_id = return_set[0] + last_update = return_set[1] + + else: + last_update = None + sql = "INSERT INTO apt_%s " % (tbl) + if src: + sql = sql + "(release_id, component_id) " + sql = sql + "VALUES (%s, %s) " + args_tpl = (release, component) + else: + sql = sql + "(release_id, component_id, architecture_id) " + sql = sql + "VALUES (%s, %s, %s) " + args_tpl = (release, component, architecture) + sql = sql + "RETURNING id" + cursor.execute(sql, args_tpl) + + suite_id = cursor.fetchone() + + if running_config['VERBOSE'] and not last_update: + if src: + print "\tinserted new source suite (%s, %s)" % (release, component) + else: + print "\tinserted new suite (%s, %s, %s)" % (release, component, architecture) + + return suite_id, last_update; + +def parseFile(last_update, suite_id, release, component, architecture=None): + verbose = running_config['VERBOSE'] + cursor = connection.cursor() + packagesfile = "" + source_re = re.compile('(.*) \((.*)\)') + #binnmu_re = re.compile('\+b\d+$') + dscname_re = re.compile(' ([^ ]*\.dsc)') + + if architecture is None: + src = True + tbl = 'source' + + packagesfile = settings.APT_DIR + '/dists/%s/%s/source/Sources.gz' \ + % (release, component) + else: + src = False + tbl = 'package' + + packagesfile = settings.APT_DIR + '/dists/%s/%s/binary-%s/Packages.gz' \ + % (release, component, architecture) + + if not os.path.isfile(packagesfile): + raise Exception("%s not found" % (packagesfile,)) + + #check if an update is necessary + mtime = os.path.getmtime(packagesfile) + if last_update and mtime <= last_update + 1 and not running_config['FORCE']: # allow 1s offset for microsecond timestamps + #if verbose: + # print "\t\t%s is uptodate, skipping" % (packagesfile,) + return + + if verbose: + print "\t\tRead file %s" % (packagesfile,) + + if not src: + contents_plain = gzip.open(settings.APT_DIR + '/dists/%s/%s/Contents-%s.gz' \ + % (release, component, architecture)) + contents = buildFileDict(contents_plain) + + sql = "DELETE FROM apt_%slist " % (tbl) + sql = sql + "WHERE suite_id = %s" + cursor.execute(sql, (suite_id,)) + + content = os.popen("zcat '%s'" % (packagesfile)) + package = apt_pkg.TagFile(content) + + # for each package + while package.step(): + insert = True + package_content = {} + package_content['Package'] = package.section.get('Package') + package_content['Version'] = package.section.get('Version') + package_content['Binary'] = package.section.get('Binary') + package_content['Maintainer'] = package.section.get('Maintainer') + if package.section.get('Uploaders'): + package_content['Maintainer'] += ', ' + package.section.get('Uploaders') + package_content['Architecture'] = package.section.get('Architecture') + if package.section.get('Description'): + package_content['Short-Description'] = package.section.get('Description').split('\n')[0] + description = '' + list_needs_newline = False + list_started = False + for line in package.section.get('Description').split('\n ')[1:]: + if line == '.': + line = '' + list_needs_newline = False + elif line[0:3] == ' * ' or line[0:3] == ' - ': + # on seeing a new bullet list, make sure there is a newline before it + if list_needs_newline: + description += '\n' + line = '*' + line[2:] + list_needs_newline = False + list_started = True + elif not list_started: + list_needs_newline = True + description += line + '\n' + package_content['Description'] = markdown.markdown(description.decode("utf-8"), tab_length=1) + package_content['Homepage'] = package.section.get('Homepage') + package_content['VCS-Browser'] = package.section.get('VCS-Browser') + package_content['VCS-Repo'] = package.section.get('VCS-Svn') + if not package_content['VCS-Repo']: + package_content['VCS-Repo'] = package.section.get('VCS-Git') + if not package_content['VCS-Repo']: + package_content['VCS-Repo'] = package.section.get('VCS-Bzr') + package_content['Build-Depends'] = package.section.get('Build-Depends') + package_content['Build-Depends-Indep'] = package.section.get('Build-Depends-Indep') + if package_content['Build-Depends-Indep']: + package_content['Build-Depends'] += ", %s" % package_content['Build-Depends-Indep'] + package_content['Depends'] = package.section.get('Depends') + package_content['Recommends'] = package.section.get('Recommends') + package_content['Suggests'] = package.section.get('Suggests') + package_content['Directory'] = package.section.get('Directory') + package_content['Filename'] = package.section.get('Filename') + package_content['Size'] = package.section.get('Size') + if package.section.has_key('Installed-Size'): + package_content['Installed-Size'] = int(package.section.get('Installed-Size')) * 1024 + package_content['Files'] = package.section.get('Files') + + sourcefield = package.section.get('Source') + if sourcefield: + match = source_re.match(sourcefield) + if match: + package_content['Source'], package_content['Source-Version'] = \ + match.group(1), match.group(2) + else: + package_content['Source'], package_content['Source-Version'] = \ + sourcefield, package_content['Version'] + else: + package_content['Source'], package_content['Source-Version'] = \ + package_content['Package'], package_content['Version'] + + if src: + match = dscname_re.search(package_content['Files']) + if not match: + raise Exception('Files section without .dsc file') + dscfile = settings.APT_DIR + '/' + package_content['Directory'] + '/' + match.group(1) + + package_content['Copyright'] = None + try: + package_content['Copyright'] = subprocess.check_output( + ['dscextract', '-f', dscfile, 'debian/copyright']) + except subprocess.CalledProcessError as e: + if e.returncode != 1 and not running_config['DISTONLY']: + raise + + package_content['Debchangelog'] = None + try: + package_content['Debchangelog'] = subprocess.check_output( + ['dscextract', '-f', dscfile, 'debian/changelog']) + except subprocess.CalledProcessError as e: + if e.returncode != 1 and not running_config['DISTONLY']: + raise + + package_content['Changelog'] = None + try: + package_content['Changelog'] = None # TODO + except subprocess.CalledProcessError as e: + if e.returncode != 1 and not running_config['DISTONLY']: + raise + + # check if the package already exists + sql = "SELECT id FROM apt_%s WHERE " % (tbl) + if src: + sql = sql + "(source, srcversion) = (%(Package)s, %(Version)s)" + else: + sql = sql + "(package, version, arch_id) = (%(Package)s, %(Version)s, %(Architecture)s)" + cursor.execute(sql, package_content) + found = cursor.fetchone() + + if found: + package_id = found[0] + if running_config['FORCE']: + if verbose: + print "\t\tPackage %(Package)s already exists, force delete" % package_content + + # delete list entries + if src: + sql = "DELETE FROM apt_sourcelist WHERE source_id = %s" + else: + sql = "DELETE FROM apt_packagelist WHERE package_id = %s" + cursor.execute(sql, (package_id,)) + + # delete files entries + if src: + sql = "DELETE FROM apt_files WHERE source_id = %s" + cursor.execute(sql, (package_id,)) + + sql = "DELETE FROM apt_%s " % (tbl) + "WHERE id = %s" + cursor.execute(sql, (package_id,)) + insert = True + else: + #if verbose: + # print "Package %(Package)s already exists and up to date, continue" % package_content + insert = False + + if insert: + if verbose: + print "\t\tFound new Package %(Package)s (%(Version)s %(Architecture)s), inserting" % package_content + + # isert new package + sql = "INSERT INTO apt_%s " % (tbl) + if src: + sql = sql + "(source, srcversion, \"binary\", architecture, maintainer, homepage, vcs_browser, vcs_repo, build_depends, directory, copyright, debchangelog, changelog) " + sql = sql + "VALUES (%(Package)s, %(Version)s, %(Binary)s, %(Architecture)s, %(Maintainer)s, %(Homepage)s, %(VCS-Browser)s, %(VCS-Repo)s, %(Build-Depends)s, %(Directory)s, %(Copyright)s, %(Debchangelog)s, %(Changelog)s) " + else: + sql = sql + "(package, version, arch_id, source, srcversion, maintainer, short_description, description, homepage, installed_size, filename, depends, recommends, suggests, size) " + sql = sql + "VALUES (%(Package)s, %(Version)s, %(Architecture)s, %(Source)s, %(Source-Version)s, %(Maintainer)s, %(Short-Description)s, %(Description)s, %(Homepage)s, %(Installed-Size)s, %(Filename)s, %(Depends)s, %(Recommends)s, %(Suggests)s, %(Size)s) " + sql = sql + "RETURNING id" + + cursor.execute(sql, package_content) + package_id = cursor.fetchone()[0] + + if src: + # insert files (dsc, orig.tar.*, ...) + sql = "INSERT INTO apt_files (files, md5_hash, size, source_id) VALUES (%s, %s, %s, %s)" + files_lines = package_content['Files'].split('\n') + for f in files_lines: + f = f.strip().split(" ") + file_md5_hash = f[0] + file_size = f[1] + file_name = f[2] + cursor.execute(sql, (file_name, file_md5_hash, file_size, package_id)) + else: + # insert contents + sql = "INSERT INTO apt_packagecontents (package_id, filename) VALUES (%s, %s)" + for filename in contents[package_content['Package']]: + cursor.execute(sql, (package_id, filename)) + + sql = "INSERT INTO apt_%slist " % (tbl) + if src: + sql = sql + "(suite_id, source_id) " + else: + sql = sql + "(suite_id, package_id) " + sql = sql + "VALUES (%s, %s)" + + cursor.execute(sql, (suite_id, package_id)) + + if src: + sql = "UPDATE apt_srcsuite SET last_update = to_timestamp(%s::int) WHERE (release_id, component_id) = (%s, %s)" + args_tpl = (mtime, release, component) + else: + sql = "UPDATE apt_suite SET last_update = to_timestamp(%s::int) WHERE (release_id, component_id, architecture_id) = (%s, %s, %s)" + args_tpl = (mtime, release, component, architecture) + cursor.execute(sql, args_tpl) + transaction.commit() + +if __name__ == '__main__': + main(sys.argv) -- 2.20.1 --xHFwDpU9dbj6ez1V-- ^ permalink raw reply [nested|flat] 2+ messages in thread
* [PATCH] Add APT repository webpages at /repos/apt/ @ 2013-03-26 08:33 Adrian Vondendriesch <[email protected]> 0 siblings, 0 replies; 2+ messages in thread From: Adrian Vondendriesch @ 2013-03-26 08:33 UTC (permalink / raw) The repository contents are imported into a (mostly) normalized database schema (apt_* tables), from where the web pages are served. Dependencies: python-apt, postgresql-debversion, pg_trgm Authors: Adrian Vondendriesch and Christoph Berg --- pgweb/apt/__init__.py | 0 pgweb/apt/models.py | 120 ++++++++ pgweb/apt/sql/architecture.sql | 1 + pgweb/apt/sql/package.sql | 2 + pgweb/apt/sql/packagecontents.sql | 2 + pgweb/apt/sql/pkg.sql | 12 + pgweb/apt/sql/release.sql | 2 + pgweb/apt/sql/source.sql | 2 + pgweb/apt/sql/srcsuite.sql | 2 + pgweb/apt/sql/suite.sql | 2 + pgweb/apt/sql/vendor.sql | 1 + pgweb/apt/utils.py | 29 ++ pgweb/apt/views.py | 373 ++++++++++++++++++++++++ pgweb/settings.py | 3 + pgweb/urls.py | 32 +++ pgweb/util/contexts.py | 9 + templates/apt/apt.html | 80 ++++++ templates/apt/binaries.html | 46 +++ templates/apt/binary.html | 167 +++++++++++ templates/apt/binary_contents.html | 14 + templates/apt/dependency.html | 12 + templates/apt/distributions.html | 20 ++ templates/apt/madison.html | 50 ++++ templates/apt/qa.html | 33 +++ templates/apt/search.html | 87 ++++++ templates/apt/source.html | 136 +++++++++ templates/apt/source_file.html | 66 +++++ templates/apt/sources.html | 18 ++ tools/apt/import-packagelist.py | 440 +++++++++++++++++++++++++++++ 29 files changed, 1761 insertions(+) create mode 100644 pgweb/apt/__init__.py create mode 100644 pgweb/apt/models.py create mode 100644 pgweb/apt/sql/architecture.sql create mode 100644 pgweb/apt/sql/package.sql create mode 100644 pgweb/apt/sql/packagecontents.sql create mode 100644 pgweb/apt/sql/pkg.sql create mode 100644 pgweb/apt/sql/release.sql create mode 100644 pgweb/apt/sql/source.sql create mode 100644 pgweb/apt/sql/srcsuite.sql create mode 100644 pgweb/apt/sql/suite.sql create mode 100644 pgweb/apt/sql/vendor.sql create mode 100644 pgweb/apt/utils.py create mode 100644 pgweb/apt/views.py create mode 100644 templates/apt/apt.html create mode 100644 templates/apt/binaries.html create mode 100644 templates/apt/binary.html create mode 100644 templates/apt/binary_contents.html create mode 100644 templates/apt/dependency.html create mode 100644 templates/apt/distributions.html create mode 100644 templates/apt/madison.html create mode 100644 templates/apt/qa.html create mode 100644 templates/apt/search.html create mode 100644 templates/apt/source.html create mode 100644 templates/apt/source_file.html create mode 100644 templates/apt/sources.html create mode 100755 tools/apt/import-packagelist.py diff --git a/pgweb/apt/__init__.py b/pgweb/apt/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pgweb/apt/models.py b/pgweb/apt/models.py new file mode 100644 index 0000000..fbc5fa9 --- /dev/null +++ b/pgweb/apt/models.py @@ -0,0 +1,120 @@ +# APT repository interface for the PostgreSQL website +# Authors: Adrian Vondendriesch <[email protected]> +# Christoph Berg <[email protected]> + +from django.db import models +from datetime import datetime + +class Vendor(models.Model): + vendor = models.TextField(null=False, primary_key=True) + + def __unicode(self): + return vendor; + +class Release(models.Model): + vendor = models.ForeignKey(Vendor) + release = models.TextField(null=False, primary_key=True) + relversion = models.TextField() + active = models.BooleanField(null=False, default=True) + + def __unicode(self): + return release; + +class Architecture(models.Model): + architecture = models.TextField(null=False, primary_key=True) + + def __unicode(self): + return architecture; + +class Component(models.Model): + component = models.TextField(null=False, primary_key=True) + + def __unicode__(self): + return self.component + +class Suite(models.Model): + release = models.ForeignKey(Release) + component = models.ForeignKey(Component) + architecture = models.ForeignKey(Architecture) + last_update = models.DateTimeField(null=True) + active = models.BooleanField(null=False, default=True) + + class Meta: + unique_together = (("release", "component", "architecture"),) + +class Srcsuite(models.Model): + release = models.ForeignKey(Release) + component = models.ForeignKey(Component) + last_update = models.DateTimeField(null=True) + active = models.BooleanField(null=False, default=True) + + class Meta: + unique_together = (("release", "component"),) + +class Package(models.Model): + package = models.TextField(null=False) + version = models.TextField(null=False) + arch = models.ForeignKey(Architecture) + source = models.TextField(null=False) + srcversion = models.TextField(null=False) + maintainer = models.TextField(null=False) + short_description = models.TextField(null=False) + description = models.TextField(null=True) + homepage = models.TextField(null=True) + installed_size = models.IntegerField(null=False) + filename = models.TextField(null=False) + depends = models.TextField(null=True) + recommends = models.TextField(null=True) + suggests = models.TextField(null=True) + size = models.IntegerField(null=False) + + class Meta: + unique_together = (("package", "version", "arch")) + + def __unicode__(self): + return self.package + +class PackageList(models.Model): + suite = models.ForeignKey(Suite) + package = models.ForeignKey(Package) + +class PackageContents(models.Model): + package = models.ForeignKey(Package) + filename = models.TextField(null=False) + +class Source(models.Model): + source = models.TextField(null=False) + srcversion = models.TextField(null=False) + binary = models.TextField(null=False) + architecture = models.TextField(null=False) + maintainer = models.TextField(null=False) + homepage = models.TextField(null=True) + short_description = models.TextField(null=True) + description = models.TextField(null=True) + vcs_browser = models.TextField(null=True) + vcs_repo = models.TextField(null=True) + build_depends = models.TextField(null=True) + directory = models.TextField(null=True) + copyright = models.TextField(null=True) + debchangelog = models.TextField(null=True) + changelog = models.TextField(null=True) + + class Meta: + unique_together = (("source", "srcversion")) + + def __unicode__(self): + return self.source + +class SourceList(models.Model): + suite = models.ForeignKey(Srcsuite) + source = models.ForeignKey(Source) + +# because file is a common identifier I use files as classname +class Files(models.Model): + files = models.TextField(null=False) + md5_hash = models.TextField(null=False) + size = models.IntegerField(null=False) + source = models.ForeignKey(Source) + + unitque_together = (("files", "md5_hash", "size"),) + diff --git a/pgweb/apt/sql/architecture.sql b/pgweb/apt/sql/architecture.sql new file mode 100644 index 0000000..397ba73 --- /dev/null +++ b/pgweb/apt/sql/architecture.sql @@ -0,0 +1 @@ +INSERT INTO apt_architecture VALUES ('all'); diff --git a/pgweb/apt/sql/package.sql b/pgweb/apt/sql/package.sql new file mode 100644 index 0000000..af9f8e4 --- /dev/null +++ b/pgweb/apt/sql/package.sql @@ -0,0 +1,2 @@ +CREATE EXTENSION IF NOT EXISTS debversion; +ALTER TABLE apt_package ALTER COLUMN version type debversion; diff --git a/pgweb/apt/sql/packagecontents.sql b/pgweb/apt/sql/packagecontents.sql new file mode 100644 index 0000000..fbd9fd3 --- /dev/null +++ b/pgweb/apt/sql/packagecontents.sql @@ -0,0 +1,2 @@ +CREATE EXTENSION IF NOT EXISTS pg_trgm; +CREATE INDEX apt_packagecontents_filename_gin ON apt_packagecontents USING gin (filename gin_trgm_ops); diff --git a/pgweb/apt/sql/pkg.sql b/pgweb/apt/sql/pkg.sql new file mode 100644 index 0000000..e7d20aa --- /dev/null +++ b/pgweb/apt/sql/pkg.sql @@ -0,0 +1,12 @@ +CREATE VIEW apt_pkg AS + SELECT s.release_id, + s.component_id, + s.architecture_id, + p.package, + p.version, + p.arch_id, + p.source, + p.srcversion + FROM apt_suite s + JOIN apt_packagelist pl ON s.id = pl.suite_id + JOIN apt_package p ON pl.package_id = p.id; diff --git a/pgweb/apt/sql/release.sql b/pgweb/apt/sql/release.sql new file mode 100644 index 0000000..6d49134 --- /dev/null +++ b/pgweb/apt/sql/release.sql @@ -0,0 +1,2 @@ +ALTER TABLE apt_release + ALTER COLUMN active SET DEFAULT true; diff --git a/pgweb/apt/sql/source.sql b/pgweb/apt/sql/source.sql new file mode 100644 index 0000000..f6ab987 --- /dev/null +++ b/pgweb/apt/sql/source.sql @@ -0,0 +1,2 @@ +CREATE EXTENSION IF NOT EXISTS debversion; +ALTER TABLE apt_source ALTER COLUMN srcversion TYPE debversion; diff --git a/pgweb/apt/sql/srcsuite.sql b/pgweb/apt/sql/srcsuite.sql new file mode 100644 index 0000000..f7a4627 --- /dev/null +++ b/pgweb/apt/sql/srcsuite.sql @@ -0,0 +1,2 @@ +ALTER TABLE apt_srcsuite + ALTER COLUMN active SET DEFAULT true; diff --git a/pgweb/apt/sql/suite.sql b/pgweb/apt/sql/suite.sql new file mode 100644 index 0000000..bfcd3f6 --- /dev/null +++ b/pgweb/apt/sql/suite.sql @@ -0,0 +1,2 @@ +ALTER TABLE apt_suite + ALTER COLUMN active SET DEFAULT true; diff --git a/pgweb/apt/sql/vendor.sql b/pgweb/apt/sql/vendor.sql new file mode 100644 index 0000000..9e59bd9 --- /dev/null +++ b/pgweb/apt/sql/vendor.sql @@ -0,0 +1 @@ +INSERT INTO apt_vendor VALUES ('Debian'), ('Ubuntu'); diff --git a/pgweb/apt/utils.py b/pgweb/apt/utils.py new file mode 100644 index 0000000..73b17a9 --- /dev/null +++ b/pgweb/apt/utils.py @@ -0,0 +1,29 @@ +# APT repository interface for the PostgreSQL website +# Authors: Adrian Vondendriesch <[email protected]> +# Christoph Berg <[email protected]> + +from .models import Package, PackageList + +def build_related_list(rel_list, release): + result = [] + if rel_list: + for entry in rel_list.split(","): + entry = entry.strip() + l = [] + for subentry in entry.split("|"): + l.append(build_list_element(subentry, release)) + result.append(l) + return result + +def build_list_element(list_entry, release): + """Returns tuple (full string, package name, dependency package available in our archive)""" + list_entry = list_entry.strip() + s = list_entry.split(" ", 1) + pkg = s[0] + dep = '' + if len(s) > 1: + dep = s[1] + pkg_available = len(PackageList.objects.filter(\ + package__package=pkg, \ + suite__release=release)) + return (list_entry, pkg, dep, pkg_available) diff --git a/pgweb/apt/views.py b/pgweb/apt/views.py new file mode 100644 index 0000000..31cb7e5 --- /dev/null +++ b/pgweb/apt/views.py @@ -0,0 +1,373 @@ +# APT repository interface for the PostgreSQL website +# Authors: Adrian Vondendriesch <[email protected]> +# Christoph Berg <[email protected]> + +from pgweb.util.contexts import render_pgweb +from django.http import HttpResponse, Http404, HttpResponseRedirect +from django.db import connection +from .models import * +from .utils import * + +def apt(request): + return render_pgweb(request, 'apt', 'apt/apt.html', {}) + +def sources(request, release): + source_list_entries = SourceList.objects.filter(suite__release_id=release).order_by('source__source') + if not len(source_list_entries): + raise Http404 + + sources = [entry.source for entry in source_list_entries] + + return render_pgweb(request, 'apt', 'apt/sources.html', { + 'release': release, + 'sources': sources, + }) + +def source(request, package_name, release='sid-pgdg', component=None, filename=None): + + source_list_entries = SourceList.objects.filter(source__source__exact=package_name) + suites = [e.suite for e in source_list_entries] + + if release: + source_list_entries = source_list_entries.filter(suite__release_id__exact=release) + if component: + source_list_entries = source_list_entries.filter(suite__component_id__exact=component) + + sources = [s.source for s in source_list_entries] + + # check if at least one source package was found + if not len(sources): + raise Http404 + + source = sources[0] + current_component = source_list_entries[0].suite.component_id + current_release = source_list_entries[0].suite.release_id + + binaries = [binary.strip() for binary in source.binary.split(',')] + + if filename: + if filename == 'copyright': + title = 'Copyright file' + content = source.copyright + elif filename == 'debchangelog': + title = 'Debian changelog' + content = source.debchangelog + else: + title = 'Upstream changelog' + content = source.changelog + return render_pgweb(request, 'apt', 'apt/source_file.html', { + 'package': source, + 'filename': filename, + 'title': title, + 'content': content, + 'current_release': current_release, + 'current_component': current_component, + 'binaries': binaries, + }) + + source_list_entries = source_list_entries.filter(suite__release_id__exact=current_release) + components = [e.suite.component_id for e in source_list_entries] + + # create dependency list + build_dependencies = build_related_list(source.build_depends, current_release) + + return render_pgweb(request, 'apt', 'apt/source.html', { + 'package': source, + 'build_dependencies': build_dependencies, + 'distributions': suites, + 'current_release': current_release, + 'components': components, + 'current_component': current_component, + 'files': source.files_set.all(), + 'binaries': binaries + }) + +def binary_contents(request, package_name, release, component, arch): + + # fix contents for all packages + if arch == 'all': + arch = 'amd64' + + binary_list_entries = PackageList.objects.filter(package__package__exact=package_name, suite__release_id__exact=release, suite__component_id__exact=component, suite__architecture_id__exact=arch) + + if not len(binary_list_entries): + raise Http404 + + binary_list = binary_list_entries[0] + binary = binary_list.package + contents = binary.packagecontents_set.all().order_by("filename") + + return render_pgweb(request, 'apt', 'apt/binary_contents.html', { + 'package': binary, + 'current_release': release, + 'current_component': component, + 'contents': contents, + }) + +def binary(request, package_name, release='sid-pgdg', component='main'): + + binary_list_entries = PackageList.objects.filter(package__package__exact=package_name) + suites = [e.suite for e in binary_list_entries.distinct('suite__release')] + + if release: + binary_list_entries = binary_list_entries.filter(suite__release__exact=release) + components = [c.suite.component for c in binary_list_entries.distinct('suite__component')] + if component: + binary_list_entries = binary_list_entries.filter(suite__component_id__exact=component) + + # check if at least on source package was found + if not len(binary_list_entries): + raise Http404 + + binary = binary_list_entries[0].package + current_component = binary_list_entries[0].suite.component_id + current_release = binary_list_entries[0].suite.release_id + + dependencies = build_related_list(binary.depends, current_release) + recommendations = build_related_list(binary.recommends, current_release) + suggestions = build_related_list(binary.suggests, current_release) + + return render_pgweb(request, 'apt', 'apt/binary.html', { + 'package': binary, + 'dependencies': dependencies, + 'recommendations': recommendations, + 'distributions': suites, + 'current_release': current_release, + 'suggestions': suggestions, + 'components': components, + 'current_component': current_component, + 'files': None, + 'downloads': Package.objects.filter(package=package_name, version=binary.version) + }) + +def distributions(request): + releases = Release.objects.order_by("vendor", "relversion", "release") + + return render_pgweb(request, 'apt', 'apt/distributions.html', { + 'releases': releases, + }) + +def binaries(request, release, component_name=None, architecture_name=None): + + if not architecture_name: + architecture_name = 'amd64' + + suites = Suite.objects.filter(release=release, architecture=architecture_name) + components = [d.component for d in suites.distinct('component')] + + if not component_name: + component_name = 'main' + + suites = suites.filter(component_id=component_name) + + if not len(suites): + raise Http404 + + suite = suites[0] + + binary_lists = suite.packagelist_set.all().order_by('package__package') + binaries = [binary_list.package for binary_list in binary_lists] + + return render_pgweb(request, 'apt', 'apt/binaries.html', { + 'suite': suite, + 'components': components, + 'binaries': binaries + }) + +def search(request): + if 'package' in request.GET and request.GET['package']: + package = request.GET['package'] + + result_binaries = Package.objects.distinct('package').filter(package__icontains=package) + result_sources = Source.objects.distinct('source').filter(source__icontains=package) + + return render_pgweb(request, 'apt', 'apt/search.html', { + 'result': True, + 'package': package, + 'result_binaries': result_binaries, + 'result_sources': result_sources, + }) + + elif 'file' in request.GET and request.GET['file']: + filename = request.GET['file'] + + maxresults = 100 + result_filenames = PackageContents.objects.distinct('filename', 'package__package').filter(filename__contains=filename).order_by("filename")[:maxresults] + + return render_pgweb(request, 'apt', 'apt/search.html', { + 'filename': filename, + 'maxresults': maxresults, + 'result_filenames': result_filenames, + }) + + else: + return render_pgweb(request, 'apt', 'apt/search.html', { + }) + +def madison(request): + """Query interface compatible with rmadison(1)""" + + if 'package' in request.GET and request.GET['package']: + package = request.GET['package'] + + sql1 = """SELECT package, version, release_id, component_id, array_agg(architecture_id) AS architecture FROM ( + SELECT package, version, release_id, component_id, architecture_id + FROM apt_packagelist pl + JOIN apt_suite s ON (suite_id = s.id) + JOIN apt_package p ON (package_id = p.id) + UNION ALL SELECT source, srcversion, release_id, component_id, 'source' + FROM apt_sourcelist sl + JOIN apt_srcsuite ss ON (suite_id = ss.id) + JOIN apt_source s ON (source_id = s.id) + ORDER BY package, version, release_id, component_id, architecture_id + ) AS r WHERE """ + sql2 = "GROUP BY package, version, release_id, component_id" + + qual = [] + args = [package] + configdisplay = 'none' + + r = '' + if 'r' in request.GET: + r = 'checked' + qual.append('package ~ %s') + configdisplay = 'block' + else: + qual.append('package = %s') + + a = '' + if 'a' in request.GET and request.GET['a']: + a = request.GET['a'] + qual.append('architecture_id = %s') + args.append(a) + configdisplay = 'block' + + c = '' + if 'c' in request.GET and request.GET['c']: + c = request.GET['c'] + qual.append('component_id = %s') + args.append(c) + configdisplay = 'block' + + s = '' # distributions are called suites in dak + if 's' in request.GET and request.GET['s']: + s = request.GET['s'] + qual.append('release_id = %s') + args.append(s) + configdisplay = 'block' + + cursor = connection.cursor() + cursor.execute(sql1 + " AND ".join(qual) + sql2, args) + return_set = cursor.fetchall() + + plen = 1 + vlen = 1 + rlen = 1 + for (pkg, version, release, component, architecture) in return_set: + if component != 'main': + release += '/' + component + if len(pkg) > plen: + plen = len(pkg) + if len(version) > vlen: + vlen = len(version) + if len(release) > rlen: + rlen = len(release) + fstr = " %%-%ds | %%-%ds | %%-%ds | %%s\n" % (plen, vlen, rlen) + + content = '' + for (pkg, version, release, component, architecture) in return_set: + if component != 'main': + release += '/' + component + content += fstr % (pkg, version, release, ", ".join(architecture)) + + if 'text' in request.GET: + return HttpResponse(content, content_type='text/plain') + + return render_pgweb(request, 'apt', 'apt/madison.html', { + 'result': True, + 'package': package, + 'r': r, + # no need to pass checkbox 'text' value here + 'a': a, + 'c': c, + 's': s, + 'configdisplay': configdisplay, + 'content': content, + }) + + else: + return render_pgweb(request, 'apt', 'apt/madison.html', { + 'configdisplay': 'none', + }) + +def qa(request, query=None): + result = None + + if True or query == None: # TODO: fix the SQL queries + description = "APT Repository Quality Assurance" + sql = None + + elif query == "binary_missing_on_architecture": + description = "Binary package versions that exist only on one architecture" + sql = """select * from apt_pkg a where not exists + (select * from apt_pkg b where a.package = b.package and a.version = b.version and a.architecture_id <> b.architecture_id) + order by release_id, package, version""" + + elif query == "wrong_tag_in_version": + description = "Package versions with wrong pgdg tag" + sql = """select p.* from apt_pkg p join apt_release r on (p.release_id = r.release) + where version !~ case + when release = 'sid-pgdg' then 'pgdg' + when release = 'lenny-pgdg' then 'pg(dg|apt)50' + when release = 'etch-pgdg' then 'pg(dg|apt)40' + when vendor_id = 'Debian' then 'pgdg'||relversion||0 + when release = 'precise-pgdg' then 'pgdg12.4' + when release = 'lucid-pgdg' then 'pgdg10.4' + when vendor_id = 'Ubuntu' then 'pgdg'||relversion + end + and r.active + order by release_id, package, version, architecture_id""" + + elif query == "binary_without_source": + description = "Binary packages without source" + sql = """select * from apt_pkg p where not exists + (select * from apt_source s where (p.source, p.srcversion) = (s.source, s.srcversion)) + order by release_id, package, version, architecture_id""" + + elif query == "binary_in_sid_only": + description = "Binary packages in sid-pgdg missing in other suites" + sql = """SELECT DISTINCT sidpkg.package, sidpkg.version, sidpkg.source, sidpkg.srcversion, + rel.release AS missing_release FROM apt_pkg sidpkg + JOIN apt_release rel ON (true) + LEFT JOIN apt_pkg p ON (rel.release = p.release_id and sidpkg.package = p.package) + WHERE sidpkg.release_id = 'sid-pgdg' AND rel.release <> 'sid-pgdg' AND rel.active + AND p.package IS NULL + ORDER BY sidpkg.package, missing_release""" + + elif query == "binary_with_old_version": + description = "Binary packages with versions different from sid-pgdg" + sql = """SELECT sidpkg.release_id, sidpkg.component_id, sidpkg.package, sidpkg.version, + sidpkg.architecture_id, p.release_id, p.version FROM apt_pkg sidpkg + JOIN apt_pkg p ON (sidpkg.package = p.package AND sidpkg.component_id = p.component_id AND sidpkg.architecture_id = p.architecture_id) + WHERE sidpkg.release_id = 'sid-pgdg' + AND regexp_replace(sidpkg.version, '.pgdg.*', '') <> regexp_replace(p.version, '.pgdg.*', '') + ORDER BY sidpkg.component_id, sidpkg.package, sidpkg.architecture_id, p.release_id""" + + elif query == "source_without_binary": + description = "Source packages without binaries" + sql = """SELECT source, srcversion FROM apt_source s WHERE NOT EXISTS + (SELECT * FROM apt_pkg p WHERE (s.source, s.srcversion) = (p.source, p.srcversion))""" + + else: + raise Http404 + + if sql: + cursor = connection.cursor() + cursor.execute(sql) + result = cursor.fetchall() + + return render_pgweb(request, 'apt', 'apt/qa.html', { + 'query': query, + 'description': description, + 'result': result, + }) diff --git a/pgweb/settings.py b/pgweb/settings.py index 540a099..15e72d9 100644 --- a/pgweb/settings.py +++ b/pgweb/settings.py @@ -115,6 +115,7 @@ INSTALLED_APPS = [ 'pgweb.featurematrix', 'pgweb.search', 'pgweb.pugs', + 'pgweb.apt', ] # Default format for date/time (as it changes between machines) @@ -161,5 +162,7 @@ FRONTEND_SMTP_RELAY = "magus.postgresql.org" # Where to relay use OAUTH = {} # OAuth providers and keys PGDG_ORG_ID = -1 # id of the PGDG organisation entry +APT_DIR = "/srv/apt/pub/repos/apt" # Directory containing the dists/ dir from apt.postgresql.org + # Load local settings overrides from .settings_local import * diff --git a/pgweb/urls.py b/pgweb/urls.py index d40673f..55ddcbe 100644 --- a/pgweb/urls.py +++ b/pgweb/urls.py @@ -3,6 +3,7 @@ from django.views.generic import RedirectView from pgweb.util.signals import register_basic_signal_handlers +import pgweb.apt.views import pgweb.contributors.views import pgweb.core.views import pgweb.docs.views @@ -110,6 +111,37 @@ urlpatterns = [ url(r'^sitemap.xml', pgweb.core.views.sitemap), url(r'^sitemap_internal.xml', pgweb.core.views.sitemap_internal), + ### + # Packages + ### + url(r'^repos/apt/$', pgweb.apt.views.apt), + url(r'^repos/apt/dists/$', pgweb.apt.views.distributions), + + url(r'^repos/apt/sources/$', RedirectView.as_view(url='/repos/apt/dists/', permanent=True)), + url(r'^repos/apt/sources/(?P<release>[\w.+-]+)/$', pgweb.apt.views.sources), + + url(r'^repos/apt/source/$', RedirectView.as_view(url='/repos/apt/sources', permanent=True)), + url(r'^repos/apt/source/(?P<package_name>[\w.+-]+)/$', pgweb.apt.views.source), + url(r'^repos/apt/source/(?P<package_name>[\w.+-]+)/(?P<release>[\w.+-]+)/$', pgweb.apt.views.source), + url(r'^repos/apt/source/(?P<package_name>[\w.+-]+)/(?P<release>[\w.+-]+)/(?P<filename>copyright|debchangelog|changelog)/$', pgweb.apt.views.source), + url(r'^repos/apt/source/(?P<package_name>[\w.+-]+)/(?P<release>[\w.+-]+)/(?P<component>[\w.+-]+)/$', pgweb.apt.views.source), + url(r'^repos/apt/source/(?P<package_name>[\w.+-]+)/(?P<release>[\w.+-]+)/(?P<component>[\w.+-]+)/(?P<filename>copyright|debchangelog|changelog)/$', pgweb.apt.views.source), + + url(r'^repos/apt/binaries/$', RedirectView.as_view(url='/repos/apt/dists/', permanent=True)), + url(r'^repos/apt/binaries/(?P<release>[\w.+-]+)/$', pgweb.apt.views.binaries), + url(r'^repos/apt/binaries/(?P<release>[\w.+-]+)/(?P<component_name>[\w\-\.]+)/$', pgweb.apt.views.binaries), + + url(r'^repos/apt/binary/$', RedirectView.as_view(url='/repos/apt/dists/', permanent=True)), + url(r'^repos/apt/binary/(?P<package_name>[\w.+-]+)/$', pgweb.apt.views.binary), + url(r'^repos/apt/binary/(?P<package_name>[\w.+-]+)/(?P<release>[\w.+-]+)/$', pgweb.apt.views.binary), + url(r'^repos/apt/binary/(?P<package_name>[\w.+-]+)/(?P<release>[\w.+-]+)/(?P<component>[\w.+-]+)/$', pgweb.apt.views.binary), + url(r'^repos/apt/binary/(?P<package_name>[\w.+-]+)/(?P<release>[\w.+-]+)/(?P<component>[\w.+-]+)/(?P<arch>[\w.+-]+)/contents/$', pgweb.apt.views.binary_contents), + + url(r'^repos/apt/search/$', pgweb.apt.views.search), + url(r'^repos/apt/madison/$', pgweb.apt.views.madison), # machine-readable search interface + url(r'^repos/apt/qa/$', pgweb.apt.views.qa), + url(r'^repos/apt/qa/(?P<query>[\w.+-]+)/$', pgweb.apt.views.qa), + ### # Workaround for broken links pushed in press release ### diff --git a/pgweb/util/contexts.py b/pgweb/util/contexts.py index 43f6c3e..b93cacd 100644 --- a/pgweb/util/contexts.py +++ b/pgweb/util/contexts.py @@ -83,6 +83,15 @@ sitenav = { {'title': 'Change password', 'link': '/account/changepwd/'}, {'title': 'Logout', 'link': '/account/logout'}, ], + 'apt': [ + {'title': 'APT Repository', 'link': '/repos/apt/'}, + {'title': 'Distributions', 'link': '/repos/apt/dists/'}, + {'title': 'Search', 'link': '/repos/apt/search/', 'submenu': [ + {'title': 'Query API', 'link': '/repos/apt/madison/'}, + {'title': 'Repository QA', 'link': '/repos/apt/qa/'}, + ]}, + {'title': 'Wiki', 'link': 'http://wiki.postgresql.org/wiki/Apt';}, + ], } diff --git a/templates/apt/apt.html b/templates/apt/apt.html new file mode 100644 index 0000000..7cfc55a --- /dev/null +++ b/templates/apt/apt.html @@ -0,0 +1,80 @@ +{%extends "base/page.html"%} +{%block title%}Apt Repository{%endblock%} +{%block contents%} +<h1>Apt Repository with PostgreSQL Packages for Debian and Ubuntu</h1> + +<p> +The PostgreSQL Global Development Group (PGDG) maintains an APT repository of +PostgreSQL packages for Debian and Ubuntu located at +http://apt.postgresql.org/pub/repos/apt/. We aim at building PostgreSQL server +packages as well as extensions and modules packages on several Debian/Ubuntu +releases for all PostgreSQL versions supported. +</p> + +<p> +Currently, we support + +<ul> + <li>Debian 6.0 (squeeze), 7.0 (wheezy), and unstable (sid) 64/32 bit (amd64/i386)</li> + <li>Ubuntu 10.04 (lucid) and 12.04 (precise) 64/32 bit (amd64/i386)</li> + <li>PostgreSQL 8.3, 8.4, 9.0, 9.1, 9.2</li> + <li>Several server extensions such as Slony-I, various PL languages, and datatypes</li> + <li>Applications like pgadmin3, pgbouncer, and pgpool-II</li> +</ul> +</p> + +<p> +Packages for older PostgreSQL versions and older Debian distributions will +continue to stay in the repository; updates for those will be provided on an +ad-hoc basis. +</p> + +<script type="text/javascript"> + <!-- + function updateSeries(select) { + var deb = document.getElementById('series-deb'); + deb.innerHTML = select.value; + } + --> +</script> + +<h2>Using the Apt Repository</h2> +<p> +If the version included in your version of Debian/Ubuntu is not the one you want, +you can use the PostgreSQL Apt Repository. This repository will integrate +with your normal systems and patch management, and provide automatic +updates for all supported versions of PostgreSQL throughout the support +<a href="/support/versioning/">lifetime</a> of PostgreSQL. +</p> +<p> +To use the apt repository, follow these steps: +</p> +<ul> + <li> + <select id="field.series" name="field.series" onchange='updateSeries(this);'> + <option selected="selected" value="OS_VERSION">Choose your Debian/Ubuntu version</option> + <option value="wheezy">Debian Wheezy (7.0)</option> + <option value="squeeze">Debian Squeeze (6.0)</option> + <option value="precise">Ubuntu Precise (12.04)</option> + <option value="lucid">Ubuntu Lucid (10.04)</option> + </select> + </li> + <li>Create the file <i>/etc/apt/sources.list.d/pgdg.list</i>, and add a line + for the repository + <code>deb http://apt.postgresql.org/pub/repos/apt/ <span id="series-deb">OS_VERSION</span>-pgdg main</code> + </li> + <li>Import the repository signing key, and update the package lists + <code> +wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | \<br/> + sudo apt-key add -<br/> +sudo apt-get update + </code> + </li> +</ul> +<p> +For more information about the apt repository, including answers to frequent +questions, please see the apt page on +<a href="http://wiki.postgresql.org/wiki/Apt">the wiki</a>. +</p> + +{%endblock%} diff --git a/templates/apt/binaries.html b/templates/apt/binaries.html new file mode 100644 index 0000000..e6536c2 --- /dev/null +++ b/templates/apt/binaries.html @@ -0,0 +1,46 @@ +{%extends "base/page.html"%} +{%block title%}Binary Packages in {{suite.release_id}}/{{suite.component_id}}{%endblock%} +{%block contents%} +<div id="pgCommunityWrap"> +<div id="pgCommunity"> +<h1>Binary Packages in {{suite.release_id}}/{{suite.component_id}} ({{binaries|length}})</h1> + +<p> +<ul> +{% for binary in binaries %} + <li> + <b><a href="/repos/apt/binary/{{binary.package}}/{{suite.release_id}}/{{suite.component_id}}/" + title="click for details to package {{binary.package}}">{{binary.package}}</a></b> + {{binary.version}} <br /> + {{binary.short_description}} + </li> +{% endfor %} +</ul> +</p> +</div> +</div> + +{% if components|length > 1 %} +<div id=pgAptWrap> +<div id=pgApt> +<dl> + <dt>Other Components</dt> + <dd> + <ul> + {% for component in components %} + <li> + {% if component.component != suite.component_id %} + <a href="/repos/apt/binaries/{{suite.release_id}}/{{component}}/">{{component}}</a> + {% else %} + <b>{{component}}</b> + {% endif %} + </li> + {% endfor %} + </ul> + </dd> +</dl> +</div> +</div> +{% endif %} + +{%endblock%} diff --git a/templates/apt/binary.html b/templates/apt/binary.html new file mode 100644 index 0000000..32f0b2e --- /dev/null +++ b/templates/apt/binary.html @@ -0,0 +1,167 @@ +{%extends "base/page.html"%} +{%block title%}Package {{package.package}} ({{package.version}}) in {{current_release}}{% if current_component != "main" %}/{{current_component}}{% endif %}{%endblock%} +{%block contents%} +<div id="pgCommunityWrap"> +<div id="pgCommunity"> +<h1>{{package.package}} ({{package.version}}) in {{current_release}}{% if current_component != "main" %}/{{current_component}}{% endif %}</h1> + +<p> +<b>{{package.short_description}}</b> +</p> + +<p> +{% autoescape off %} +{{package.description}} +{% endautoescape %} +</p> + +{% if package.homepage %} +<p><b>Homepage:</b> + <a href="{{package.homepage}}" + target="_blank" + title="Go to homepage of {{package.package}}">{{package.homepage}}</a> +{% endif %} + +<p> +<b>Maintainer:</b> {{ package.maintainer }} +</p> + +{% if package.vcs_browser or package.vcs_repo%} +<h2>VCS</h2> + + {% if package.vcs_browser %} +<h3>Browser</h3> +<p> + <a href="{{package.vcs_browser}}" + target="_blank" + title="Go to VCS-Browser of {{package.package}}">{{package.vcs_browser}}</a> +<p> + {% endif %} + + {% if package.vcs_repo %} +<h3>Repository</h3> +<p> + <a href="{{package.vcs_repo}}" + target="_blank" + title="Get the current version of {{package.package}}">{{package.vcs_repo}}</a> +<p> + {% endif %} +{% endif %} + +{% if dependencies or recommendations or suggestions %} +<h2>Related Packages</h2> +{% if dependencies %} +<h3>Dependencies {% if package.arch_id != "all" %} (on {{package.arch_id}}) {% endif %}</h3> +<ul> + {% for dep in dependencies %} + {% include "apt/dependency.html" %} + {% endfor %} +</ul> +{% endif %} + +{% if recommendations %} +<h3>Recommendations</h3> +<ul> + {% for dep in recommendations %} + {% include "apt/dependency.html" %} + {% endfor %} +</ul> +{% endif %} + +{% if suggestions %} +<h3>Suggestions</h3> +<ul> + {% for dep in suggestions %} + {% include "apt/dependency.html" %} + {% endfor %} +</ul> +{% endif %} +{% endif %} + +<h2>Download {{package.package}} for {{current_release}}</h2> +<div class="tblBasic"> +<table> + <tr> + <th>Architecture</th> + <th>Package Size</th> + <th>Installed Size</th> + <th>File List</th> + </tr> + {% for download in downloads %} + <tr> + <td> + <a href="http://apt.postgresql.org/pub/repos/apt/{{download.filename}}" + title="Download {{download}} for {{download.arch_id}}">{{download.arch_id}}</a> + </td> + <td>{{download.size|filesizeformat}}</td> + <td>{{download.installed_size|filesizeformat}}</td> + <td><a href="/repos/apt/binary/{{package.package}}/{{current_release}}/{{current_component}}/{{download.arch_id}}/contents/">File List</a></td> + </tr> + {% endfor %} +</table> +</div> +</div> +</div> + +<div id=pgAptWrap> +<div id=pgApt> +<dl> + <dt>Source</dt> + <dd> + <ul> + <li><a href="/repos/apt/source/{{package.source}}/{{current_release}}/">{{package.source}}</a></li> + <li><a href="/repos/apt/source/{{package.source}}/{{current_release}}/copyright/">Copyright file</a></li> + <li><a href="/repos/apt/source/{{package.source}}/{{current_release}}/debchangelog/">Package changelog</a></li> + <li><a href="/repos/apt/source/{{package.source}}/{{current_release}}/changelog/">Upstream changelog</a></li> + </ul> + </dd> +</dl> +</div> +</div> + + +{% if components|length > 1 %} +<div id=pgAptWrap> +<div id=pgApt> +<dl> + <dt>This Package in other Components</dt> + <dd> + <ul> +{% for comp in components %} + {% if comp.component == current_component %} + <li><b>{{comp}}</b></li> + {% else %} + <li> + <a href="/repos/apt/binary/{{package.package}}/{{current_release}}/{{comp}}/">{{comp}}</a> + </li> + {% endif %} +{% endfor %} + </ul> + </dd> +</dl> +</div> +</div> +{% endif %} + +<div id=pgAptWrap> +<div id=pgApt> +<dl> + <dt>Distributions</dt> + <dd> + <ul> + {% for dist in distributions %} + <li> + {% if current_release and current_release == dist.release_id %} + <b>{{dist.release_id}}</b> + {% else %} + <a href="/repos/apt/binary/{{package.package}}/{{dist.release_id}}/">{{dist.release_id}}</a> + {% endif %} + </li> + {% endfor %} + </ul> + </dd> +</dl> +</div> +</div> + +{%endblock%} diff --git a/templates/apt/binary_contents.html b/templates/apt/binary_contents.html new file mode 100644 index 0000000..c42923b --- /dev/null +++ b/templates/apt/binary_contents.html @@ -0,0 +1,14 @@ +{%extends "base/page.html"%} +{%block title%}File List of Package {{package.package}} ({{package.version}} {{package.arch.architecture}}){%endblock%} +{%block contents%} +<div id="pgCommunityWrap"> +<div id="pgCommunity"> +<h1>{{package.package}} ({{package.version}} {{package.arch.architecture}}) in {{current_release}} File List</h1> +<p> +<nobr> +{% for file in contents %}{{file.filename}}<br /> +{% endfor %} +</nobr> +</p> + +{%endblock%} diff --git a/templates/apt/dependency.html b/templates/apt/dependency.html new file mode 100644 index 0000000..c7f0709 --- /dev/null +++ b/templates/apt/dependency.html @@ -0,0 +1,12 @@ +<li> +{% for alternative in dep %} + {% if alternative.3 %} + <a href="/repos/apt/binary/{{alternative.1}}/{{current_release}}/" + title="View details for Package {{alternative.1}}">{{alternative.1}}</a> + {{alternative.2}} + {% else %} + {{alternative.0}} + {% endif %} + {% if alternative != dep|last %} | {% endif %} +{% endfor %} +</li> diff --git a/templates/apt/distributions.html b/templates/apt/distributions.html new file mode 100644 index 0000000..82ff46f --- /dev/null +++ b/templates/apt/distributions.html @@ -0,0 +1,20 @@ +{%extends "base/page.html"%} +{%block title%}Distributions on apt.postgresql.org{%endblock%} +{%block contents%} +<h1>Distributions</h1> +<p> +<div class="tblBasic"> + <table> + {% for release in releases %} + <tr> + <th>{{release.vendor_id}} {{release.relversion}}</th> + <th>{{release.release}}</th> + <td><a href="/repos/apt/binaries/{{release.release}}/" title="Binary packages in {{release.release}}">Binaries</a></td> + <td><a href="/repos/apt/sources/{{release.release}}/" title="Source packages in {{release.release}}">Sources</a></td> + </tr> + {% endfor %} + </table> +</div> +</p> + +{%endblock%} diff --git a/templates/apt/madison.html b/templates/apt/madison.html new file mode 100644 index 0000000..3709a14 --- /dev/null +++ b/templates/apt/madison.html @@ -0,0 +1,50 @@ +{%extends "base/page.html"%} +{%block title%}Query apt.postgresql.org{%endblock%} +{%block contents%} + +<script> +function toggle(x) { + var oContent=document.getElementById(x) ; + if (oContent.style.display=="block") { + oContent.style.display="none"; + } else { + oContent.style.display="block"; + } +} +</script> + +<h1>Query apt.postgresql.org</h1> + +<form action="/repos/apt/madison/" method="get"> + <input type="text" name="package" value="{{package}}"> + <input type="submit" value="Search"> <br /> + Enter a package name + <span onclick='javascript:toggle("config")'>(Click for more options)</span><br /> + <div id="config" style="display:{{configdisplay}};"> + <table border="0"> + <tr><td>Regexp search</td><td><input type="checkbox" name="r" {{r}}></td></tr> + <tr><td>Text output</td><td><input type="checkbox" name="text"></td></tr> + <tr><td>Architecture</td><td><input type="text" name="a" value="{{a}}"></td></tr> + <tr><td>Component</td><td><input type="text" name="c" value="{{c}}"></td></tr> + <tr><td>Distribution</td><td><input type="text" name="s" value="{{s}}"></td></tr> + </table> + </div> +</form> + + +{% if package %} +<h2>Results for <span class="pgAptPattern">"{{package}}"</span></h2> + +<pre>{{content}}</pre> + +{% else %} + +<p> +A command line client for this page is +<a href="https://manpages.debian.org/devscripts/rmadison.1.html">rmadison(1)</a; +contained in the +<a href="https://packages.debian.org/devscripts">devscripts</a; package. +</p> +{% endif %} + +{%endblock%} diff --git a/templates/apt/qa.html b/templates/apt/qa.html new file mode 100644 index 0000000..fcf94b2 --- /dev/null +++ b/templates/apt/qa.html @@ -0,0 +1,33 @@ +{%extends "base/page.html"%} +{%block title%}{{description}}{%endblock%} +{%block contents%} +<div id="pgCommunityWrap"> +<div id="pgCommunity"> +<h1>{{description}}</h1> + +{% if query %} + +<table border="0"> +{% for row in result %} +<tr> + {% for column in row %} + <td>{{column}}</td> + {% endfor %} +</tr> +{% endfor %} +</table> + +{% else %} + +<ul> + <li><a href="binary_missing_on_architecture/">Binary missing on architecture</a></li> + <li><a href="wrong_tag_in_version/">Wrong tag in version</a></li> + <li><a href="binary_without_source/">Binary without source</a></li> + <li><a href="binary_in_sid_only/">Binary in sid-pgdg only</a></li> + <li><a href="binary_with_old_version/">Binary with old version</a></li> + <li><a href="source_without_binary/">Source without binaries</a></li> +</ul> + +{% endif %} + +{%endblock%} diff --git a/templates/apt/search.html b/templates/apt/search.html new file mode 100644 index 0000000..fbeb67b --- /dev/null +++ b/templates/apt/search.html @@ -0,0 +1,87 @@ +{%extends "base/page.html"%} +{%block title%}Search apt.postgresql.org{%endblock%} +{%block contents%} +<h1>Search on apt.postgresql.org</h1> + +<h2>Search for Packages</h2> + +<form action="/repos/apt/search/" method="get"> + <input type="text" name="package" value="{{package}}"> + <input type="submit" value="Search"> +</form> + +Enter a package name or substring + +{% if package %} +<h2>Results for <span class="pgAptPattern">"{{package}}"</span></h2> + + <div class="pgAptList"> + <h3>Binary Packages: {{result_binaries|length}}</h3> + {% if result_binaries %} + <dl> + + {% for binary in result_binaries %} + <dt><a href="/repos/apt/binary/{{binary.package}}/">{{binary.package}}</a></dt> + <dd>{{binary.short_description}}</dd> + {% endfor %} + + </dl> + {% else %} + <p> + No packages found. + </p> + <div> + {% endif %} + + <div class="pgAptList"> + <h3>Source Packages: {{result_sources|length}}</h3> + {% if result_sources %} + <dl> + + {% for source in result_sources %} + <dt><a href="/repos/apt/source/{{source.source}}/">{{source.source}}</a></dt> + <dd>{{source.short_description}}</dd> + {% endfor %} + + </dl> + {% else %} + <p> + No packages found. + </p> + <div> + {% endif %} +{% endif %} + +<h2>Search for Files in Packages</h2> + +<form action="/repos/apt/search/" method="get"> + <input type="text" name="file" value="{{filename}}"> + <input type="submit" value="Search"> +</form> + +Enter a filename or substring + +{% if filename %} +<h2>Results for <span class="pgAptPattern">"{{filename}}"</span></h2> + + <div class="pgAptList"> + {% if result_filenames %} + {% if result_filenames|length >= maxresults %} + <h3>Files: Too many results, only {{maxresults}} will be shown</h3> + {% else %} + <h3>Files: {{result_filenames|length}}</h3> + {% endif %} + <table border="0"> + {% for file in result_filenames %} + <tr><td>{{file.filename}}</td><td><a href="/repos/apt/binary/{{file.package}}/">{{file.package}}</a></td></tr> + {% endfor %} + </table> + {% else %} + <p> + No packages found. + </p> + {% endif %} + </div> +{% endif %} + +{%endblock%} diff --git a/templates/apt/source.html b/templates/apt/source.html new file mode 100644 index 0000000..cf9768f --- /dev/null +++ b/templates/apt/source.html @@ -0,0 +1,136 @@ +{%extends "base/page.html"%} +{%block title%}Source Package {{package.source}} ({{package.srcversion}}) in {{current_release}}{%endblock%} +{%block contents%} +<div id="pgCommunityWrap"> +<div id="pgCommunity"> +<h1>{{package.source}} ({{package.srcversion}}) in {{current_release}}</h1> + +<p> +<b>{{package.short_description}}</b> +</p> + +<p> +{% autoescape off %} +{{package.description}} +{% endautoescape %} +</p> + +<p> +<b>Maintainer:</b> {{ package.maintainer }} +</p> + +{% if package.homepage %} +<p><b>Homepage:</b> + <a href="{{package.homepage}}" + target="_blank" + title="Go to homepage of {{package.source}}">{{package.homepage}}</a> +{% endif %} + +<h2>Files</h2> +<ul> + {% for f in files %} + <li> + <b><a href="http://apt.postgresql.org/pub/repos/apt/{{package.directory}}/{{f.files}}" + title="Download {{f.files}}">{{f.files}}</a></b><br /> + {{f.size|filesizeformat}}, {{f.md5_hash}} + </li> + {% endfor %} +</ul> + +{% if build_dependencies %} +<h2>Build Dependencies</h2> +<ul> + {% for dep in build_dependencies %} + {% include "apt/dependency.html" %} + {% endfor %} +</ul> +{% endif %} + +</div> +</div> + +<div id=pgAptWrap> +<div id=pgApt> +<dl> + <dt>Binaries</dt> + <dd> + <ul> + {% for binary in binaries %} + <li> + <a href="/repos/apt/binary/{{binary}}/{{current_release}}/">{{binary}}</a></li> + {% endfor %} + </ul> + </dd> +</dl> +</div> +</div> + +<div id=pgAptWrap> +<div id=pgApt> +<dl> + <dt>Resources</dt> + <dd> + <ul> + <li><a href="/repos/apt/source/{{package.source}}/{{current_release}}/copyright/">Copyright file</a></li> + <li><a href="/repos/apt/source/{{package.source}}/{{current_release}}/debchangelog/">Package changelog</a></li> + <li><a href="/repos/apt/source/{{package.source}}/{{current_release}}/changelog/">Upstream changelog</a></li> + <li><a href="https://packages.debian.org/{{package.source}}">{{package.source}} in Debian</a> + (<a href="https://tracker.debian.org/pkg/{{package.source}}">Tracker</a>)</li> + <li><a href="https://launchpad.net/ubuntu/+source/{{package.source}}">{{package.source}} in Ubuntu</a></li> + {% if package.vcs_browser %} + <li><a href="{{package.vcs_browser}}">Packaging browser</a></li> + {% endif %} + {% if package.vcs_repo %} + <li><a href="{{package.vcs_repo}}">Packaging repository</a></li> + {% endif %} + </ul> + </dd> +</dl> +</div> +</div> + +<div id=pgAptWrap> +<div id=pgApt> +<dl> + <dt>Distributions</dt> + <dd> + <ul> + {% for dist in distributions %} + <li> + {% if current_release and current_release == dist.release_id %} + <b>{{dist.release_id}}</b> + {% else %} + <a href="/repos/apt/source/{{package.source}}/{{dist.release_id}}/">{{dist.release_id}}</a> + {% endif %} + </li> + {% endfor %} + </ul> + </dd> +</dl> +</div> +</div> + +{% if components|length > 1 %} +<div id=pgAptWrap> +<div id=pgApt> +<dl> + <dt>Components</dt> + <dd> + <ul> +{% for comp in components %} + {% if comp == current_component %} + <li>{{comp}}</li> + {% else %} + <li> + <a href="/repos/apt/source/{{package.source}}/{{current_release}}/{{comp}}">{{comp}}</a> + </li> + {% endif %} +{% endfor %} + </ul> + </dd> +</dl> +</div> +</div> +{% endif %} + +{%endblock%} diff --git a/templates/apt/source_file.html b/templates/apt/source_file.html new file mode 100644 index 0000000..617b03c --- /dev/null +++ b/templates/apt/source_file.html @@ -0,0 +1,66 @@ +{%extends "base/page.html"%} +{%block title%}{{title}} for {{package.source}} ({{package.srcversion}}) in {{current_release}}{%endblock%} +{%block contents%} +<div id="pgCommunityWrap"> +<div id="pgCommunity"> +<h1>{{title}} for {{package.source}} ({{package.srcversion}}) in {{current_release}}</h1> + +<p> +<b>{{package.short_description}}</b> +</p> + +<pre> +{{content}} +</pre> + +</div> +</div> + +<div id=pgAptWrap> +<div id=pgApt> +<dl> + <dt>Source</dt> + <dd> + <ul> + <li><a href="/repos/apt/source/{{package.source}}/{{current_release}}/">{{package.source}}</a></li> + + {% if filename == "copyright" %} + <li><b>Copyright file</b></li> + {% else %} + <li><a href="/repos/apt/source/{{package.source}}/{{current_release}}/copyright/">Copyright file</a></li> + {% endif %} + + {% if filename == "debchangelog" %} + <li><b>Package changelog</b></li> + {% else %} + <li><a href="/repos/apt/source/{{package.source}}/{{current_release}}/debchangelog/">Package changelog</a></li> + {% endif %} + + {% if filename == "changelog" %} + <li><b>Upstream changelog</b></li> + {% else %} + <li><a href="/repos/apt/source/{{package.source}}/{{current_release}}/changelog/">Upstream changelog</a></li> + {% endif %} + </ul> + </dd> +</dl> +</div> +</div> + +<div id=pgAptWrap> +<div id=pgApt> +<dl> + <dt>Binaries</dt> + <dd> + <ul> + {% for binary in binaries %} + <li> + <a href="/repos/apt/binary/{{binary}}/{{current_release}}/">{{binary}}</a></li> + {% endfor %} + </ul> + </dd> +</dl> +</div> +</div> + +{%endblock%} diff --git a/templates/apt/sources.html b/templates/apt/sources.html new file mode 100644 index 0000000..11c83f3 --- /dev/null +++ b/templates/apt/sources.html @@ -0,0 +1,18 @@ +{%extends "base/page.html"%} +{%block title%}Source Packages in {{release}}{%endblock%} +{%block contents%} +<h1>Source Packages in {{release}} ({{sources|length}})</h1> + +<p> +<ul> + {% for source in sources %} + <li> + <b><a href="/repos/apt/source/{{source.source}}/{{release}}/">{{source.source}}</a></b> + {{source.srcversion}} <br /> + {{source.short_description}} + </li> + {% endfor %} +</ul> +</p> + +{%endblock%} diff --git a/tools/apt/import-packagelist.py b/tools/apt/import-packagelist.py new file mode 100755 index 0000000..4b1973e --- /dev/null +++ b/tools/apt/import-packagelist.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python + +# debmirror -v -h apt.postgresql.org -r pub/repos/apt --method=http -d sid-pgdg,buster-pgdg --omit-suite-symlinks -s main,9.3,9.4,9.5,9.6,10,11,12 -a source,amd64,i386,ppc64el --exclude='\.deb$' --getcontents --no-check-gpg /srv/repo + +import apt_pkg, markdown, os, re, sys, time, gzip, subprocess + +# Set up for accessing django +import django +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings_local") +import sys +sys.path.append('../../pgweb') +from django.conf import settings +#from apt import models as c + +#from core.models import ImportedRSSFeed, ImportedRSSItem +from django.db import transaction, connection + +#transaction.enter_transaction_management() +#transaction.managed() + +config = { + 'FILTER': [ + 'testing', + 'deprecated' + ] + } + +running_config = { + 'VERBOSE': 0, + 'FORCE': False, + 'DISTONLY': False, + } + +usage = """ +%s [options] + -h show this help + -v verbose + -f force (deletes every package before insert) + -d ignore errors from dscextract (useful for running on dists/ only) +""" % sys.argv[0] + +def dir_filter(string): + for f in config['FILTER']: + if f in string: + return 0 + return 1 + +def getArchitectureNames(directories): + binary_pattern = 'binary-' + binaries = [] + for directory in directories: + if re.match(binary_pattern, directory): + binaries.append(directory.split('-')[1]) + + return binaries + +def buildFileDict(flist): + fdict = {} + for line in flist: + columns = line.split('\t') + if len(columns) > 1 and columns[0] != 'FILE': + filename = '/' + columns[0] + packagename = columns[1].split('/')[-1].strip() + try: + fdict[packagename] + except: + fdict[packagename] = [] + fdict[packagename].append(filename) + #print "P-Name:", packagename + return fdict + +def main(*args, **kwargs): + running_config['VERBOSE'] = 0 + + # args + if len(args) > 0: + for arg in args[0]: + if arg == '-v': + running_config['VERBOSE'] += 1 + if arg == '-f': + running_config['FORCE'] = True + if arg == '-d': + running_config['DISTONLY'] = True + if arg == '-h': + print usage + exit(0) + + cursor = connection.cursor() + + # releases + base_dir = getattr(settings, 'APT_DIR') + "/dists/" + releases = [directory for directory in os.listdir(base_dir) \ + if os.path.isdir(os.path.join(base_dir, directory))] + releases = filter(dir_filter, releases) + if running_config['VERBOSE'] > 0: + print "Releases to scan (%s): %s" % \ + (len(releases), ', '.join(releases)) + + for release in releases: + print "%s:" % (release) + cur_dir = base_dir + release + components = [component for component in os.listdir(cur_dir) \ + if os.path.isdir(os.path.join(cur_dir, component))] + components = filter(dir_filter, components) + if running_config['VERBOSE'] > 0: + print "\tComponents in %s to scan (%s): %s" % \ + (release, len(components), ', '.join(components)) + + for component in components: + if running_config['VERBOSE'] > 0: + print "\t%s:" % (component) + + # insert components, that aren't not yet registered + sql = """ + INSERT INTO apt_component + SELECT * FROM (VALUES (%s)) comp(component) + WHERE NOT EXISTS ( + SELECT * FROM apt_component + WHERE component = comp.component + ) + """ + cursor.execute(sql, (component,)) + + srcsuite_id, last_update = insertSuite(release, component) + parseFile(last_update, srcsuite_id, release, component) + + # read binary architectures + architectures = getArchitectureNames(os.listdir(os.path.join(cur_dir, component))) + + for architecture in architectures: + # insert architectures, that aren't not yet registered + sql = """ + INSERT INTO apt_architecture + SELECT * FROM (VALUES (%s)) arch(architecture) + WHERE NOT EXISTS ( + SELECT * FROM apt_architecture + WHERE architecture = arch.architecture + ) + """ + cursor.execute(sql, (architecture,)) + + suite_id, last_update = insertSuite(release, component, architecture) + parseFile(last_update, suite_id, release, component, architecture) + + # update source package descriptions + sql = """UPDATE apt_source s SET + short_description = (SELECT short_description FROM apt_package p WHERE (p.source, p.srcversion) = (s.source, s.srcversion) ORDER BY source = p.package DESC, p.package LIMIT 1), + description = (SELECT description FROM apt_package p WHERE (p.source, p.srcversion) = (s.source, s.srcversion) ORDER BY source = p.package DESC, p.package LIMIT 1) + WHERE s.short_description IS NULL""" + cursor.execute(sql) + transaction.commit() + + connection.close() + return + + +def insertSuite(release, component, architecture=None): + cursor = connection.cursor() + + sql = "SELECT release FROM apt_release WHERE release = %s" + cursor.execute(sql, (release,)) + if not cursor.fetchone(): + sql = "INSERT INTO apt_release (vendor_id, release, relversion, active) VALUES ('Debian', %s, '', true)" + cursor.execute(sql, (release,)) + + if architecture is None: + tbl = "srcsuite" + src = True + else: + tbl = "suite" + src = False + + # check if srcsuite is already in db + sql = "SELECT id, extract('epoch' from last_update) FROM apt_%s WHERE " % (tbl) + if src: + sql = sql + "(release_id, component_id) = (%s, %s) " + args_tpl = (release, component) + else: + sql = sql + "(release_id, component_id, architecture_id) = (%s, %s, %s)" + args_tpl = (release, component, architecture) + cursor.execute(sql, args_tpl) + + return_set = cursor.fetchone() + + if return_set: + suite_id = return_set[0] + last_update = return_set[1] + + else: + last_update = None + sql = "INSERT INTO apt_%s " % (tbl) + if src: + sql = sql + "(release_id, component_id) " + sql = sql + "VALUES (%s, %s) " + args_tpl = (release, component) + else: + sql = sql + "(release_id, component_id, architecture_id) " + sql = sql + "VALUES (%s, %s, %s) " + args_tpl = (release, component, architecture) + sql = sql + "RETURNING id" + cursor.execute(sql, args_tpl) + + suite_id = cursor.fetchone() + + if running_config['VERBOSE'] and not last_update: + if src: + print "\tinserted new source suite (%s, %s)" % (release, component) + else: + print "\tinserted new suite (%s, %s, %s)" % (release, component, architecture) + + return suite_id, last_update; + +def parseFile(last_update, suite_id, release, component, architecture=None): + verbose = running_config['VERBOSE'] + cursor = connection.cursor() + packagesfile = "" + source_re = re.compile('(.*) \((.*)\)') + #binnmu_re = re.compile('\+b\d+$') + dscname_re = re.compile(' ([^ ]*\.dsc)') + + if architecture is None: + src = True + tbl = 'source' + + packagesfile = settings.APT_DIR + '/dists/%s/%s/source/Sources.gz' \ + % (release, component) + else: + src = False + tbl = 'package' + + packagesfile = settings.APT_DIR + '/dists/%s/%s/binary-%s/Packages.gz' \ + % (release, component, architecture) + + if not os.path.isfile(packagesfile): + raise Exception("%s not found" % (packagesfile,)) + + #check if an update is necessary + mtime = os.path.getmtime(packagesfile) + if last_update and mtime <= last_update + 1 and not running_config['FORCE']: # allow 1s offset for microsecond timestamps + #if verbose: + # print "\t\t%s is uptodate, skipping" % (packagesfile,) + return + + if verbose: + print "\t\tRead file %s" % (packagesfile,) + + if not src: + contents_plain = gzip.open(settings.APT_DIR + '/dists/%s/%s/Contents-%s.gz' \ + % (release, component, architecture)) + contents = buildFileDict(contents_plain) + + sql = "DELETE FROM apt_%slist " % (tbl) + sql = sql + "WHERE suite_id = %s" + cursor.execute(sql, (suite_id,)) + + content = os.popen("zcat '%s'" % (packagesfile)) + package = apt_pkg.TagFile(content) + + # for each package + while package.step(): + insert = True + package_content = {} + package_content['Package'] = package.section.get('Package') + package_content['Version'] = package.section.get('Version') + package_content['Binary'] = package.section.get('Binary') + package_content['Maintainer'] = package.section.get('Maintainer') + if package.section.get('Uploaders'): + package_content['Maintainer'] += ', ' + package.section.get('Uploaders') + package_content['Architecture'] = package.section.get('Architecture') + if package.section.get('Description'): + package_content['Short-Description'] = package.section.get('Description').split('\n')[0] + description = '' + list_needs_newline = False + list_started = False + for line in package.section.get('Description').split('\n ')[1:]: + if line == '.': + line = '' + list_needs_newline = False + elif line[0:3] == ' * ' or line[0:3] == ' - ': + # on seeing a new bullet list, make sure there is a newline before it + if list_needs_newline: + description += '\n' + line = '*' + line[2:] + list_needs_newline = False + list_started = True + elif not list_started: + list_needs_newline = True + description += line + '\n' + package_content['Description'] = markdown.markdown(description.decode("utf-8"), tab_length=1) + package_content['Homepage'] = package.section.get('Homepage') + package_content['VCS-Browser'] = package.section.get('VCS-Browser') + package_content['VCS-Repo'] = package.section.get('VCS-Svn') + if not package_content['VCS-Repo']: + package_content['VCS-Repo'] = package.section.get('VCS-Git') + if not package_content['VCS-Repo']: + package_content['VCS-Repo'] = package.section.get('VCS-Bzr') + package_content['Build-Depends'] = package.section.get('Build-Depends') + package_content['Build-Depends-Indep'] = package.section.get('Build-Depends-Indep') + if package_content['Build-Depends-Indep']: + package_content['Build-Depends'] += ", %s" % package_content['Build-Depends-Indep'] + package_content['Depends'] = package.section.get('Depends') + package_content['Recommends'] = package.section.get('Recommends') + package_content['Suggests'] = package.section.get('Suggests') + package_content['Directory'] = package.section.get('Directory') + package_content['Filename'] = package.section.get('Filename') + package_content['Size'] = package.section.get('Size') + if package.section.has_key('Installed-Size'): + package_content['Installed-Size'] = int(package.section.get('Installed-Size')) * 1024 + package_content['Files'] = package.section.get('Files') + + sourcefield = package.section.get('Source') + if sourcefield: + match = source_re.match(sourcefield) + if match: + package_content['Source'], package_content['Source-Version'] = \ + match.group(1), match.group(2) + else: + package_content['Source'], package_content['Source-Version'] = \ + sourcefield, package_content['Version'] + else: + package_content['Source'], package_content['Source-Version'] = \ + package_content['Package'], package_content['Version'] + + if src: + match = dscname_re.search(package_content['Files']) + if not match: + raise Exception('Files section without .dsc file') + dscfile = settings.APT_DIR + '/' + package_content['Directory'] + '/' + match.group(1) + + package_content['Copyright'] = None + try: + package_content['Copyright'] = subprocess.check_output( + ['dscextract', '-f', dscfile, 'debian/copyright']) + except subprocess.CalledProcessError as e: + if e.returncode != 1 and not running_config['DISTONLY']: + raise + + package_content['Debchangelog'] = None + try: + package_content['Debchangelog'] = subprocess.check_output( + ['dscextract', '-f', dscfile, 'debian/changelog']) + except subprocess.CalledProcessError as e: + if e.returncode != 1 and not running_config['DISTONLY']: + raise + + package_content['Changelog'] = None + try: + package_content['Changelog'] = None # TODO + except subprocess.CalledProcessError as e: + if e.returncode != 1 and not running_config['DISTONLY']: + raise + + # check if the package already exists + sql = "SELECT id FROM apt_%s WHERE " % (tbl) + if src: + sql = sql + "(source, srcversion) = (%(Package)s, %(Version)s)" + else: + sql = sql + "(package, version, arch_id) = (%(Package)s, %(Version)s, %(Architecture)s)" + cursor.execute(sql, package_content) + found = cursor.fetchone() + + if found: + package_id = found[0] + if running_config['FORCE']: + if verbose: + print "\t\tPackage %(Package)s already exists, force delete" % package_content + + # delete list entries + if src: + sql = "DELETE FROM apt_sourcelist WHERE source_id = %s" + else: + sql = "DELETE FROM apt_packagelist WHERE package_id = %s" + cursor.execute(sql, (package_id,)) + + # delete files entries + if src: + sql = "DELETE FROM apt_files WHERE source_id = %s" + cursor.execute(sql, (package_id,)) + + sql = "DELETE FROM apt_%s " % (tbl) + "WHERE id = %s" + cursor.execute(sql, (package_id,)) + insert = True + else: + #if verbose: + # print "Package %(Package)s already exists and up to date, continue" % package_content + insert = False + + if insert: + if verbose: + print "\t\tFound new Package %(Package)s (%(Version)s %(Architecture)s), inserting" % package_content + + # isert new package + sql = "INSERT INTO apt_%s " % (tbl) + if src: + sql = sql + "(source, srcversion, \"binary\", architecture, maintainer, homepage, vcs_browser, vcs_repo, build_depends, directory, copyright, debchangelog, changelog) " + sql = sql + "VALUES (%(Package)s, %(Version)s, %(Binary)s, %(Architecture)s, %(Maintainer)s, %(Homepage)s, %(VCS-Browser)s, %(VCS-Repo)s, %(Build-Depends)s, %(Directory)s, %(Copyright)s, %(Debchangelog)s, %(Changelog)s) " + else: + sql = sql + "(package, version, arch_id, source, srcversion, maintainer, short_description, description, homepage, installed_size, filename, depends, recommends, suggests, size) " + sql = sql + "VALUES (%(Package)s, %(Version)s, %(Architecture)s, %(Source)s, %(Source-Version)s, %(Maintainer)s, %(Short-Description)s, %(Description)s, %(Homepage)s, %(Installed-Size)s, %(Filename)s, %(Depends)s, %(Recommends)s, %(Suggests)s, %(Size)s) " + sql = sql + "RETURNING id" + + cursor.execute(sql, package_content) + package_id = cursor.fetchone()[0] + + if src: + # insert files (dsc, orig.tar.*, ...) + sql = "INSERT INTO apt_files (files, md5_hash, size, source_id) VALUES (%s, %s, %s, %s)" + files_lines = package_content['Files'].split('\n') + for f in files_lines: + f = f.strip().split(" ") + file_md5_hash = f[0] + file_size = f[1] + file_name = f[2] + cursor.execute(sql, (file_name, file_md5_hash, file_size, package_id)) + else: + # insert contents + sql = "INSERT INTO apt_packagecontents (package_id, filename) VALUES (%s, %s)" + for filename in contents[package_content['Package']]: + cursor.execute(sql, (package_id, filename)) + + sql = "INSERT INTO apt_%slist " % (tbl) + if src: + sql = sql + "(suite_id, source_id) " + else: + sql = sql + "(suite_id, package_id) " + sql = sql + "VALUES (%s, %s)" + + cursor.execute(sql, (suite_id, package_id)) + + if src: + sql = "UPDATE apt_srcsuite SET last_update = to_timestamp(%s::int) WHERE (release_id, component_id) = (%s, %s)" + args_tpl = (mtime, release, component) + else: + sql = "UPDATE apt_suite SET last_update = to_timestamp(%s::int) WHERE (release_id, component_id, architecture_id) = (%s, %s, %s)" + args_tpl = (mtime, release, component, architecture) + cursor.execute(sql, args_tpl) + transaction.commit() + +if __name__ == '__main__': + main(sys.argv) -- 2.20.1 --a8Wt8u1KmwUX3Y2C-- ^ permalink raw reply [nested|flat] 2+ messages in thread
end of thread, other threads:[~2013-03-26 08:33 UTC | newest] Thread overview: 2+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2013-03-26 08:33 [PATCH] Add APT repository webpages at /repos/apt/ Adrian Vondendriesch <[email protected]> 2013-03-26 08:33 [PATCH] Add APT repository webpages at /repos/apt/ Adrian Vondendriesch <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox