diff -ruP mailman-2.1.18/bin/blow_away_htdig mailman-2.1.18p/bin/blow_away_htdig --- mailman-2.1.18/bin/blow_away_htdig 1969-12-31 16:00:00.000000000 -0800 +++ mailman-2.1.18p/bin/blow_away_htdig 2014-05-03 12:22:36.835376950 -0700 @@ -0,0 +1,139 @@ +#! @PYTHON@ +# +# Copyright (C) 2002 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +"""Blow away the per list htdig files. + +This script is for when you: + a. decide to stop using Mailman-htdig integration + b. move from local to remote htdig or vice-versa + c. are updgrading to a version of htdig which has an incompatible + index/db file format + d. want to force the creation of new per-list htdig conf files + and the list indexes. + +You really want to stop Mailman operating while you are running this. For +instance, shutdown the MTA delivering mail to Mailman and remove Mailman's +crontab. + +Usage: %(program)s [-v] [-h] [i] [listnames] + +Where: + --verbose / -v + print each list as htdig is run for it + + --help / -h + print this message and exit + + --config/ -c + delete htdig search indices for the lists + and recreate the htdig conf file for the lists + + --indices / -i + only delete htdig search indices for the lists + leave the htdig conf file in place. This is overridden by + the -c option + + listnames + Optionally, only runs htdig for the named lists. Without + this, all archivable lists are processed. + +""" + +# this code was derived from the nightly_gzip cron script + +import sys +import os +from stat import * +import time +from stat import * +import getopt +import paths +from Mailman import MailList +from Mailman import Utils +from Mailman import mm_cfg +from Mailman.Archiver import HyperArch +from Mailman.i18n import _ + +program = sys.argv[0] + +def usage(code, msg=''): + print >> sys.stderr, _( __doc__) + if msg: + print msg + sys.exit(code) + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], 'vhic', + ['verbose', 'help', 'indices', 'config']) + except getopt.error, msg: + usage(1, msg) + + # defaults + verbose = 0 + indices_only = 0 + redo_configs = 0 + for opt, arg in opts: + if opt in ('-h', '--help'): + usage(0) + elif opt in ('-v', '--verbose'): + verbose = 1 + elif opt in ('-c', '--config'): + redo_configs = 1 + indices_only = 0 + elif opt in ('-i', '--indices') and not redo_configs: + indices_only = 1 + + # limit to the specified lists? + if args: + listnames = args + else: + listnames = Utils.list_names() + + # make sure htdig use is off for the moment in this process + mm_cfg.USE_HTDIG = 0 + + # process all the specified lists + for name in listnames: + mlist = MailList.MailList(name, lock=0) + if not mlist.archive: + continue + archive = HyperArch.HyperArchive(mlist) + if verbose: + if redo_configs: + print _('Blowing away all htdig indices of list %(name)s') + print _('Rebuilding htdig config of list %(name)s') + elif indices_only: + print _('Blowing away all htdig indices of list %(name)s') + else: + print _('Blowing away all htdig stuff of list %(name)s') + archive.remove_htdig(indices_only) + if redo_configs: + archive.setup_htdig() + archive.write_TOC() + +if __name__ == '__main__' and \ + mm_cfg.USE_HTDIG and \ + mm_cfg.ARCHIVE_TO_MBOX in (0, 2): + # we're only going to run this if messages are archived to the internal + # archiver and we are using htdig to provide archive search + omask = os.umask(002) + try: + main() + finally: + os.umask(omask) diff -ruP mailman-2.1.18/bin/check_perms mailman-2.1.18p/bin/check_perms --- mailman-2.1.18/bin/check_perms 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/bin/check_perms 2014-05-03 12:22:36.835376950 -0700 @@ -55,6 +55,11 @@ # Gotta check the archives/private/*/database/* files +DBDIRLIST = ['database'] +if mm_cfg.HTDIG_STRICT_FILE_PERM: + DBDIRLIST.append('htdig') + + try: True, False except NameError: @@ -137,7 +142,7 @@ private = mm_cfg.PRIVATE_ARCHIVE_FILE_DIR if path == private or ( os.path.commonprefix((path, private)) == private - and os.path.split(path)[1] == 'database'): + and os.path.split(path)[1] in DBDIRLIST): # then... targetperms = PRIVATEPERMS elif (os.path.commonprefix((path, mm_cfg.QUEUE_DIR)) @@ -249,23 +254,29 @@ # or executable otherwise those files will be accessible when the archives # are public. That may not be a horrible breach, but let's close this off # anyway. + # ditto archives/private/listname/htdig for dir in os.listdir(mm_cfg.PRIVATE_ARCHIVE_FILE_DIR): if dir.endswith('.mbox'): checkmboxfile(dir) - dbdir = os.path.join(mm_cfg.PRIVATE_ARCHIVE_FILE_DIR, dir, 'database') - try: - mode = statmode(dbdir) - except OSError, e: - if e.errno not in (errno.ENOENT, errno.ENOTDIR): raise - continue - if mode & S_IRWXO: - STATE.ERRORS += 1 - print _('%(dbdir)s "other" perms must be 000'), - if STATE.FIX: - print _('(fixing)') - os.chmod(dbdir, mode & ~S_IRWXO) - else: - print + dbdirlist = [os.path.join(mm_cfg.PRIVATE_ARCHIVE_FILE_DIR, dir, + 'database')] + if mm_cfg.HTDIG_STRICT_FILE_PERM: + dbdirlist.append(os.path.join(mm_cfg.PRIVATE_ARCHIVE_FILE_DIR, + dir, 'htdig')) + for dbdir in dbdirlist: + try: + mode = statmode(dbdir) + except OSError, e: + if e.errno not in (errno.ENOENT, errno.ENOTDIR): raise + continue + if mode & S_IRWXO: + STATE.ERRORS += 1 + print _('%(dbdir)s "other" perms must be 000'), + if STATE.FIX: + print _('(fixing)') + os.chmod(dbdir, mode & ~S_IRWXO) + else: + print def checkcgi(): cgidir = os.path.join(mm_cfg.EXEC_PREFIX, 'cgi-bin') diff -ruP mailman-2.1.18/bin/Makefile.in mailman-2.1.18p/bin/Makefile.in --- mailman-2.1.18/bin/Makefile.in 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/bin/Makefile.in 2014-05-03 12:22:36.835376950 -0700 @@ -49,7 +49,7 @@ list_admins genaliases change_pw mailmanctl qrunner inject \ unshunt fix_url.py convert.py transcheck b4b5-archfix \ list_owners msgfmt.py show_qfiles discard rb-archfix \ - reset_pw.py export.py + blow_away_htdig reset_pw.py export.py BUILDDIR= ../build/bin diff -ruP mailman-2.1.18/configure mailman-2.1.18p/configure --- mailman-2.1.18/configure 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/configure 2014-05-03 12:22:36.835376950 -0700 @@ -4508,6 +4508,7 @@ SCRIPTS="build/bin/add_members:bin/add_members \ build/bin/arch:bin/arch \ +build/bin/blow_away_htdig:bin/blow_away_htdig \ build/bin/change_pw:bin/change_pw \ build/bin/check_db:bin/check_db \ build/bin/check_perms:bin/check_perms \ @@ -4555,6 +4556,10 @@ build/cron/gate_news:cron/gate_news \ build/cron/mailpasswds:cron/mailpasswds \ build/cron/nightly_gzip:cron/nightly_gzip \ +build/cron/nightly_htdig:cron/nightly_htdig \ +build/cron/remote_nightly_htdig:cron/remote_nightly_htdig \ +build/cron/remote_nightly_htdig_noshare:cron/remote_nightly_htdig_noshare \ +build/cron/remote_nightly_htdig.pl:cron/remote_nightly_htdig.pl \ build/cron/senddigests:cron/senddigests \ " diff -ruP mailman-2.1.18/configure.in mailman-2.1.18p/configure.in --- mailman-2.1.18/configure.in 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/configure.in 2014-05-03 12:22:36.835376950 -0700 @@ -671,6 +671,7 @@ AC_DEFUN([MM_SCRIPTS], [dnl bin/add_members \ bin/arch \ +bin/blow_away_htdig \ bin/change_pw \ bin/check_db \ bin/check_perms \ @@ -718,6 +719,10 @@ cron/gate_news \ cron/mailpasswds \ cron/nightly_gzip \ +cron/nightly_htdig \ +cron/remote_nightly_htdig \ +cron/remote_nightly_htdig_noshare \ +cron/remote_nightly_htdig.pl \ cron/senddigests \ ]) diff -ruP mailman-2.1.18/cron/crontab.in.in mailman-2.1.18p/cron/crontab.in.in --- mailman-2.1.18/cron/crontab.in.in 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/cron/crontab.in.in 2014-05-03 12:22:36.835376950 -0700 @@ -18,6 +18,11 @@ # or want to exclusively use a callback strategy instead of polling. 0,5,10,15,20,25,30,35,40,45,50,55 * * * * @PYTHON@ -S @prefix@/cron/gate_news # +# At 2:19am every night, regenerate htdig search files. Only +# turn this on if the internal archiver is used and htdig +# use enabled in mm_cfg.py with USE_HTDIG +19 2 * * * @PYTHON@ -S @prefix@/cron/nightly_htdig +# # At 3:27am every night, regenerate the gzip'd archive file. Only # turn this on if the internal archiver is used and # GZIP_ARCHIVE_TXT_FILES is false in mm_cfg.py diff -ruP mailman-2.1.18/cron/Makefile.in mailman-2.1.18p/cron/Makefile.in --- mailman-2.1.18/cron/Makefile.in 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/cron/Makefile.in 2014-05-03 12:22:36.835376950 -0700 @@ -42,7 +42,9 @@ SHELL= /bin/sh PROGRAMS= checkdbs mailpasswds senddigests gate_news \ - nightly_gzip bumpdigests disabled cull_bad_shunt + nightly_gzip bumpdigests disabled cull_bad_shunt \ + nightly_htdig remote_nightly_htdig \ + remote_nightly_htdig_noshare remote_nightly_htdig.pl FILES= crontab.in BUILDDIR= ../build/cron diff -ruP mailman-2.1.18/cron/nightly_htdig mailman-2.1.18p/cron/nightly_htdig --- mailman-2.1.18/cron/nightly_htdig 1969-12-31 16:00:00.000000000 -0800 +++ mailman-2.1.18p/cron/nightly_htdig 2014-05-03 12:22:36.839376950 -0700 @@ -0,0 +1,155 @@ +#! @PYTHON@ +# +# Copyright (C) 2002 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +"""Re-generate the htdig archive search files. + +This script should normally be run nightly from cron. When run from the +command line, the following usage is understood: + +Usage: %(program)s [-v] [-h] [listnames] + +Where: + --verbose / -v + print each list as htdig is run for it + + listnames + Optionally, only runs htdig for the named lists. Without + this, all archivable lists are processed. + + --help / -h + print this message and exit + +""" + +# this code was derived from the nightly_gzip cron script + +import sys +import os +from stat import * +import time +from types import * +import getopt +import paths +import errno +from Mailman import MailList +from Mailman import Utils +from Mailman import mm_cfg +from Mailman.Archiver import HyperArch +from Mailman.i18n import _ + +program = sys.argv[0] + +def usage(code, msg=''): + print >> sys.stderr, _( __doc__) + if msg: + print msg + sys.exit(code) + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], 'vh', ['verbose', 'help']) + except getopt.error, msg: + usage(1, msg) + + # defaults + verbose = 0 + for opt, arg in opts: + if opt in ('-h', '--help'): + usage(0) + elif opt in ('-v', '--verbose'): + verbose = 1 + + # limit to the specified lists? + if args: + listnames = args + else: + listnames = Utils.list_names() + + # process all the specified lists + for name in listnames: + mlist = MailList.MailList(name, lock=0) + if not mlist.archive: + continue + archive_dir = mlist.archive_dir() + try: + os.listdir(archive_dir) + except os.error: + # has the list received any messages? if not, + # last_post_time will + # be zero, so it's not really a bogus archive dir. + if mlist.last_post_time > 0: + print _( + 'List %(name)s has a bogus archive dir: %(archive_dir)s') + continue + + # check htdig has been set up for this list and skip it if not + list_htdig_dir = os.path.join(archive_dir, 'htdig') + if not os.path.exists(list_htdig_dir): + if verbose: + print _('Skipping htdig for list; no htdig setup: %(name)s') + continue + + # check if there have been any archive files created since we + # last ran htdig and skip list if not. well actually we only + # test if the archive volume directories mod times have + # changed + recent_posts = None + rundig_run_file = os.path.join(list_htdig_dir, 'rundig_last_run') + archive = HyperArch.HyperArchive(mlist) + try: + last_rundig_time = os.stat(rundig_run_file)[ST_MTIME] + except OSError, e: + if e.errno <> errno.ENOENT: raise + open(rundig_run_file, 'w').close() + recent_posts = 1 + else: + for volume in archive.archives: + archive_name = os.path.join(archive_dir, volume) + last_archive_change = os.stat(archive_name)[ST_MTIME] + if last_archive_change > last_rundig_time: + recent_posts = 1 + break + if not recent_posts: + if verbose: + print _('Skipping htdig for list; no recent posts: %(name)s') + continue + + # ok, so running htdig is worthwhile + if verbose: + print _("htdig'ing archive of list: %(name)s") + htdig_conf_file = os.path.join(list_htdig_dir, name + '.conf') + cmd = '%s -c %s' % (mm_cfg.HTDIG_RUNDIG_PATH, htdig_conf_file) + status = (os.system(cmd) >> 8) & 0xff + if status: + print _('rundig failed for list %(name)s, exit code: %(status)s') + else: + os.utime(rundig_run_file, None) + archive.write_TOC() + +if __name__ == '__main__' and \ + mm_cfg.USE_HTDIG and \ + mm_cfg.ARCHIVE_TO_MBOX in (0, 2) and \ + os.path.exists(mm_cfg.HTDIG_RUNDIG_PATH): + # we're only going to run the nightly rundig if messages are + # archived to the internal archiver, we are using htdig to provide + # archive search and we know where rundig is. + omask = os.umask(002) + try: + main() + finally: + os.umask(omask) diff -ruP mailman-2.1.18/cron/remote_nightly_htdig mailman-2.1.18p/cron/remote_nightly_htdig --- mailman-2.1.18/cron/remote_nightly_htdig 1969-12-31 16:00:00.000000000 -0800 +++ mailman-2.1.18p/cron/remote_nightly_htdig 2014-05-03 12:22:36.839376950 -0700 @@ -0,0 +1,163 @@ +#! @PYTHON@ +# +# Copyright (C) 2002 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +"""Python script to re-generate the htdig archive search files. Read +INSTALL.htdig-mm to determine if you should be running this script. + +This script has to be edited before use to provide a value for the +configuration parameter MAILMAN_PATH. The value should be the path +to Mailman's installation directory as seen by the script. + +This script should normally be run nightly from cron. When run from the +command line, the following usage is understood: + +Usage: %(program)s [-v] [-h] [listnames] + +Where: + --verbose / -v + print each list as htdig is run for it + + --help / -h + print this message and exit + + listnames + Optionally, only runs htdig for the named lists. Without + this, all archivable lists are processed. + +""" + +# this code was derived from the nightly_gzip cron script + +MAILMAN_PATH = '' + +import sys +import os +from stat import * +import time +import getopt +import string +from types import * +import errno + +import urllib, urlparse +import paths + +program = sys.argv[0] + +def usage(code, msg=''): + print >> sys.stderr, _( __doc__) + if msg: + print msg + sys.exit(code) + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], 'vhm', ['verbose', 'help']) + except getopt.error, msg: + usage(1, msg) + # defaults + verbose = 0 + for opt, arg in opts: + if opt in ('-h', '--help'): + usage(0) + elif opt in ('-v', '--verbose'): + verbose = 1 + # limit to the specified lists? + if args: + listnames = map(string.lower, args) + else: + listnames = Utils.list_names() + # process all the specified lists + for name in listnames: + mlist = MailList.MailList(name, lock=0) + if not mlist.archive: + continue + archive_dir = os.path.join(mm_cfg.REMOTE_PRIVATE_ARCHIVE_FILE_DIR, + name + '/') + try: + os.listdir(archive_dir) + except os.error: + # has the list received any messages? if not, last_post_time + # will be zero, so it's not really a bogus archive dir. + if mlist.last_post_time > 0: + print _( + 'List %(name)s has a bogus archive dir: %(archive_dir)s') + continue + # check htdig has been set up for this list and skip it if not + list_htdig_dir = os.path.join(archive_dir, 'htdig') + if not os.path.exists(list_htdig_dir): + if verbose: + print _( + 'Skipping remote htdig for list; nno htdig setup: %(name)s') + continue + # check if there have been any archive files created since we + # last ran htdig and skip list if not. well actually we only + # test if the archive volume directories mod times have changed + recent_posts = None + rundig_run_file = os.path.join(list_htdig_dir, 'rundig_last_run') + archive = HyperArch.HyperArchive(mlist) + try: + last_rundig_time = os.stat(rundig_run_file)[ST_MTIME] + except OSError, e: + if e.errno <> errno.ENOENT: raise + open(rundig_run_file, 'w').close() + recent_posts = 1 + else: + for volume in archive.archives: + archive_name = os.path.join(archive_dir, volume) + last_archive_change = os.stat(archive_name)[ST_MTIME] + if last_archive_change > last_rundig_time: + recent_posts = 1 + break + if not recent_posts: + if verbose: + print _('Skipping htdig for list; no recent posts: %(name)s') + continue + # ok, so running htdig is worthwhile + if verbose: + print _("htdig'ing archive of list: %(name)s") + htdig_conf_file = os.path.join(list_htdig_dir, name + '.conf') + cmd = '%s -c %s' % (mm_cfg.HTDIG_RUNDIG_PATH, htdig_conf_file) + status = (os.system(cmd) >> 8) & 0xff + if status: + print _('rundig failed for list %(name)s, exit code: %(status)s') + else: + os.utime(rundig_run_file, None) + +if __name__ == '__main__' and os.path.exists(MAILMAN_PATH): + # Access the mailman installation + sys.path = [MAILMAN_PATH] + sys.path + from Mailman import MailList + from Mailman import Utils + from Mailman import mm_cfg + from Mailman.Archiver import HyperArch + from Mailman.i18n import _ + if mm_cfg.USE_HTDIG and \ + mm_cfg.ARCHIVE_TO_MBOX in (0, 2) and \ + os.path.exists(mm_cfg.HTDIG_RUNDIG_PATH): + # we're only going to run the nightly rundig if messages are archived to + # the internal archiver, we are using htdig to provide archive search + # and we know where rundig is. + omask = os.umask(002) + try: + main() + finally: + os.umask(omask) +else: + print 'Invalid configuration variables' + print 'Edit this script in accordance with INSTALL.htdig-mm' diff -ruP mailman-2.1.18/cron/remote_nightly_htdig_noshare mailman-2.1.18p/cron/remote_nightly_htdig_noshare --- mailman-2.1.18/cron/remote_nightly_htdig_noshare 1969-12-31 16:00:00.000000000 -0800 +++ mailman-2.1.18p/cron/remote_nightly_htdig_noshare 2014-05-03 12:22:36.839376950 -0700 @@ -0,0 +1,151 @@ +#! @PYTHON@ +# +# Copyright (C) 2002 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +"""Python script to re-generate the htdig archive search files. Read +INSTALL.htdig-mm to determine if you should be running this script. + +This script has to be edited before use to provide values for the +configuration parameters REMOTE_PRIVATE_ARCHIVE_FILE_DIR and +HTDIG_RUNDIG_PATH. The values should be the same as those acquired by +other of Mailman's python code from $prefix/Mailman/Defaults.py or +overridden in $prefix/Mailman/mm_cfg.py. + +This script should normally be run nightly from cron. When run from the +command line, the following usage is understood: + +Usage: %(program)s [-v] [-h] [listnames] + +Where: + --verbose / -v + print each list as htdig is run for it + + --help /-h + print this message and exit + + listnames + Optionally, only runs htdig for the named lists. Without + this, all archivable lists are processed. + +""" + +# this code was derived from the nightly_gzip cron script + +REMOTE_PRIVATE_ARCHIVE_FILE_DIR = '' +HTDIG_RUNDIG_PATH = '' + +import sys +import os +from stat import * +import time +from types import * +import getopt +import urllib +import urlparse +import string +import errno + +program = sys.argv[0] + +def usage(code, msg=''): + print __doc__ % globals() + if msg: + print msg + sys.exit(code) + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], 'vhm', ['verbose', 'help']) + except getopt.error, msg: + usage(1, msg) + + # defaults + verbose = 0 + for opt, arg in opts: + if opt in ('-h', '--help'): + usage(0) + elif opt in ('-v', '--verbose'): + verbose = 1 + # limit to the specified lists? + if args: + listnames = map(string.lower, args) + else: + listnames = filter(lambda m: m[-5:] != '.mbox', + os.listdir(REMOTE_PRIVATE_ARCHIVE_FILE_DIR)) + # process all the specified lists + listnames.sort() + for name in listnames: + archive_dir = os.path.join(REMOTE_PRIVATE_ARCHIVE_FILE_DIR, name) + # check if this list has an archive and skip it if not + if not os.path.exists(archive_dir): + if verbose: + print 'Skipping remote htdig for list', name, 'no archive' + continue + # check htdig has been set up for this list and skip it if not + list_htdig_dir = os.path.join(archive_dir, 'htdig') + if not os.path.exists(list_htdig_dir): + if verbose: + print 'Skipping remote htdig for list', name, 'no htdig setup' + continue + # check if there have been any archive files created since we + # last ran htdig and skip list if not. well actually we only + # test if the archive volume directories mod times have changed + recent_posts = None + rundig_run_file = os.path.join(list_htdig_dir, 'rundig_last_run') + try: + last_rundig_time = os.stat(rundig_run_file)[ST_MTIME] + except OSError, e: + if e.errno <> errno.ENOENT: raise + open(rundig_run_file, 'w').close() + recent_posts = 1 + else: + for volume in os.listdir(archive_dir): + archive_name = os.path.join(archive_dir, volume) + last_archive_change = os.stat(archive_name)[ST_MTIME] + if last_archive_change > last_rundig_time: + recent_posts = 1 + break + if not recent_posts: + if verbose: + print 'Skipping htdig for list', name, 'no recent posts' + continue + # ok, so running htdig is worthwhile + if verbose: + print "htdig'ing archive of list", name + htdig_conf_file = os.path.join(list_htdig_dir, name + '.conf') + cmd = '%s -c %s' % (HTDIG_RUNDIG_PATH, htdig_conf_file) + status = (os.system(cmd) >> 8) & 0xff + if status: + print 'rundig failed for list %s, exit code: %s' % (name, status) + else: + os.utime(rundig_run_file, None) + +if __name__ == '__main__' and \ + os.path.exists(REMOTE_PRIVATE_ARCHIVE_FILE_DIR) and \ + os.path.exists(HTDIG_RUNDIG_PATH): + # we're only going to run the nightly rundig if we have a sensible + # set of configuration variables and we know where rundig is. + omask = os.umask(002) + try: + main() + finally: + os.umask(omask) +else: + print "Invalid configuration variables" + print "Edit this script in accordance with INSTALL.htdig-mm" + + diff -ruP mailman-2.1.18/cron/remote_nightly_htdig.pl mailman-2.1.18p/cron/remote_nightly_htdig.pl --- mailman-2.1.18/cron/remote_nightly_htdig.pl 1969-12-31 16:00:00.000000000 -0800 +++ mailman-2.1.18p/cron/remote_nightly_htdig.pl 2014-05-03 12:22:36.839376950 -0700 @@ -0,0 +1,151 @@ +#! /usr/bin/env perl +# +# Copyright (C) 2002 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# + +# this code was derived from the nightly_gzip cron script + +my $REMOTE_PRIVATE_ARCHIVE_FILE_DIR = ''; +my $HTDIG_RUNDIG_PATH = ''; + +use strict; +use File::Spec; +use File::stat; +use LWP::Simple; +use Getopt::Long; + +my $doc = < \$VERBOSE, + "help" => \$help); + if ($help) { + usage(0); + } + # limit to the specified lists? + my @listnames = (); + if (scalar(@ARGV)) { + @listnames = map lc, @ARGV; + } else { + opendir(DIR, $REMOTE_PRIVATE_ARCHIVE_FILE_DIR); + @listnames = grep { $_ ne '.' and $_ ne '..' and ! /\.mbox$/ } readdir DIR; + closedir(DIR); + } + # process all the specified lists + foreach my $name (@listnames) { + my $archive_dir = File::Spec->catfile($REMOTE_PRIVATE_ARCHIVE_FILE_DIR, $name); + next if (-e not $archive_dir); + # check htdig has been set up for this list and skip it if not + my $list_htdig_dir = File::Spec->catfile($archive_dir, 'htdig'); + if (not -e $list_htdig_dir) { + print "Skipping remote htdig for list $name, no htdig setup\n" if $VERBOSE; + next + } + # check if there have been any archive files created since we + # last ran htdig and skip list if not. well actually we only + # test if the archive volume directories mod times have changed + my $recent_posts = 0; + my $rundig_run_file = File::Spec->catfile($list_htdig_dir, 'rundig_last_run'); + if (-e $rundig_run_file){ + my $last_rundig_time = stat($rundig_run_file)->mtime(); + opendir(DIR, $archive_dir); + my @volumes = grep { $_ ne '.' and $_ ne '..' } readdir DIR; + closedir(DIR); + foreach my $volume (@volumes) { + my $archive_name = File::Spec->catfile($archive_dir, $volume); + my $last_archive_change = stat($archive_name)->mtime(); + if ($last_archive_change > $last_rundig_time) { + $recent_posts = 1; + last; + } + } + } else { + $recent_posts = 1; + } + if (not $recent_posts) { + if ($VERBOSE) { + print "Skipping htdig for list $name, no recent posts\n"; + } + next; + } + # ok, so running htdig is worthwhile + if ($VERBOSE) { + print "htdig'ing archive of list $name\n"; + } + my $htdig_conf_file = File::Spec->catfile($list_htdig_dir, $name.'.conf'); + my @cmd = ($HTDIG_RUNDIG_PATH, '-c', $htdig_conf_file); + my $status = system(@cmd) >> 8 & 0xFF; + if ($status) { + print "rundig failed for list, $name, exit code, $status\n"; + } else { + system(("touch", $rundig_run_file)); + } + } +} + +if (-x $HTDIG_RUNDIG_PATH and + -d $REMOTE_PRIVATE_ARCHIVE_FILE_DIR) { + # we're only going to run the nightly rundig if we have a sensible + # set of configuration variables and we know where rundig is. + $omask = umask; + umask(002); + eval { main() }; + my $res = $@; + umask($omask); + die $res if ($res); +} else { + die "Invalid configuration variables.\nEdit this script in accordance with INSTALL.htdig-mm\n"; +} diff -ruP mailman-2.1.18/INSTALL.htdig-mm mailman-2.1.18p/INSTALL.htdig-mm --- mailman-2.1.18/INSTALL.htdig-mm 1969-12-31 16:00:00.000000000 -0800 +++ mailman-2.1.18p/INSTALL.htdig-mm 2014-05-03 12:22:36.839376950 -0700 @@ -0,0 +1,1564 @@ +Mailman Patch #444884 (Mailman-htdig integration) Installation Details +---------------------------------------------------------------------- + + +The information below is also available in the INSTALL.htdig-mm.html file which +is installed by this patch. + +Table of Contents +---------------- + + +* Patch identification +* Prerequisites +* Current version +* Changes introduced by this patch version + + +* Introduction +* Installing and Building Mailman with this patch +* What is Installed by the Patch +* Configuration of Mailman-htdig Integration +* Health Warning on the packet! +* Starting from Scratch (Again) +* General +* Permissions Considerations +* htdig +* Apache + + +* Local htdig Configuration +* Remote htdig Configuration +* Upgrading an Existing Standard Mailman Installation +* Changing from local to remote htdig or vice versa +* Coping with htdig Upgrades +* Changing the Addressing Scheme of your web_page_url + + +* Operational Information +* Notes and Warnings +* Archive security problems resolved by htdig-2.1.3-0.2 patch +* Private archive security problem prior to htdig-2.1.1-0.2.patch version +* Maintaining archive security with htdig-2.1.1-0.2.patch version and later +* Upgrading to htdig-2.1.1-0.2.patch or later from an earlier patch version +* Redhat 7.1 and 7.2 installations +* Apache/htdig issues + + +* Contributors +* History +* Compatibility +* Changes + + +* Appendices +* Appendix 1 -Technique for htdigging when Mailman's DEFAULT_URL uses the +https scheme + + + + +Patch Identification +---------------- + +Different versions of this patch are available for different versions of +Mailman. There may be different versions of this patch for any given version of +Mailman, typically as a results of MM version specific improvements or +corrections of bugs in the patched code. The names of patch files for this patch +are structured as follows: + + + htdig--.patch[.gz] + + +Thus, for instance, patch file htdig-2.1.4-0.1.patch is patch version 0.1 for +application to MM version 2.1.4 source code. + +The is reset to 0.1 for the first patch version applicable to +each new version of Mailman. + +The .gz suffix, if present, says that the patch file has been compressed using +gzip. + +As a general rule, you should use the highest patch version number for the MM +version you are installing. + +Current Version +---------------- + +The current version of this patch is for Mailman 2.1.10: +Mailman 2.1.10 - htdig-2.1.10-0.1.patch + +Be sure to read the notes in the cCanges section below about the patch version +you are going to use. + +Patches for previous versions of Mailman are frozen at the highest revision +level they reached while those previous versions of MM were current. + +Information about older Mailman and patch versions is given in the history +section below. + +Changes introduced by this patch version +---------------- + +The following changes are introduced by version 0.1 of this patch: + +* Updated patch for MM 2.1.10 compatibility and made change to setup_htdig() + function in HyperArch.py suggested by Mark Sapiro. +* The frequency with which extra languages are being supported by Mailman + exceed my capacity to cope. From htdig-2.1.9-0.1.patch on only the English + language (default) templates are guaranteed to have been patched. The + following files in a language's default template directory should be checked + and if necessary modified per the changes made to the en language templates + after installation of this patch if that other language is used: + + templates//archidxfoot.html + templates//archidxhead.html + templates//archtoc.html + templates//archtocentry.html + templates//archtocnombox.html + templates//article.html + +Prerequisites[ toc ] +---------------- + +A working Htdig installation +---------------- + +You must have a working installation of htdig with htsearch available and +installed on either the machine on which you are running Mailman or on another +machine which has access to Mailman list archives via NFS or some similarly +competent network file sharing scheme. + +Regardless of how you configure things to provide Mailman's Web UI, if its gives +normal operation of the /mailman/private CGI script for providing access to +private list archives, it should also support access to htdig search results via +the /mailman/mmsearch and /mailman/htdig CGI scripts. + +Warning: This patch has been tested with HTdig 3.1.6 and no testing has been +done with the Beta versions of HTdig 3.2 at the time of writing. You may or may +not encounter problems/issues not described here if you use HTdig 3.2 beta or +stable releases. + +Other Mailman patches +---------------- + +Prior to installing this patch you may also need to install the other MM +patches. This will depend on the version of Mailman and the version of this +patch you are dealing with. For version 0.3 of this patch for MM 2.1.3 the +latest version of patch #444879, indexing-2.1.3-x.y.patch , is required. It is +available from: + +* +http://sourceforge.net/tracker/index.php?func=detail&aid=444879&group_id=103& +atid=300103 +* http://www.openinfo.co.uk/mailman/patches/444879/index.html + + +For any other version of this patch details of its prerequisites are in the +version of INSTALL.htdig-mm file which is installed by that patch. + +Introduction +---------------- + +This integration enables use of the htdig (http://www.htdig.org) search engine +for searching mail list archives produced by pipermail, Mailman's built-in +archiver. + +You can use htdig without applying these patches to Mailman but you may find it +awkward to achieve some of the features offered by this patch. + +The main features of the patch are: + +1. per list search facility with a search form on each list's TOC page. +2. maintenance of privacy of private archives. The user has to establish their +credentials via the normal private archive access mechanism before any access +via htdig is allowed. +3. + +a common base URL for both public and private archive access via htsearch +results. This means that htdig indices are unaffected by changing an archive +from private to public and vice versa. All access to archives via htdig is +controlled by wrapped CGI scripts called htdig.py and mmsearch.py. + +Note that Mailman's attachment scrubber creates a problem when it extracts +attachments from messages as they are being archived because it embeds absolute +URLs to what it has extracted in the archived messages. This can only be fixed +by running $prefix/bin/arch to rebuild the list's archive from its mbox file +after changing its archive from private to public or vices versa. This problem +is generic and unrelated to the use of this patch. One way resolving it is by +use the Mailman-MHonArc integration patch #???????? available from +* TBA +* http://www.openinfo.co.uk/mailman/patches/mhonarc/index.html + + +4. a choice of running htdig on the machine running Mailman (aka local htdig) +or running htdig on another machine which has access to Mailman's archives via +NFS or some similarly competent network file sharing scheme (aka remote htdig). +5. cron activated scripts and crontab entry to run htdig regularly to maintain +the per list search indices. +6. automatic creation, deletion and maintenance of htdig configuration files +and such. Beyond installing htdig and telling Mailman where it is via mm_cfg you +do not have to do much other setup. +7. htdig search related web page elements are retrieved from the +$prefix/templates/ directory hierarchy so that site, virtual host, list and +language tailoring of them can be done. + + +Installing and Building Mailman with this patch +---------------- + +Create your Mailman build directory in the normal way. + +You can apply the patch to either a fresh expansion of the Mailman source +distribution or the one you used to build a currently working Mailman +installation. + +Execute the following command in the Mailman build directory: + + + patch -p1 < path-to-htdig-2.m.n-x.y.patch + + +Follow the configure and make procedures for regular Mailman as given in the +$build/INSTALL file. + +Then follow the Mailman-htdig configuration instructions given below. + +What is Installed by the Patch +---------------- +The patch amends: + + +$build/INSTALL + +Adds a reference to this file to the standard installation notes. + +$prefix/bin/check_perms + +To set the permissions for access to $prefix/archive/private//htdig/ +subdirectories to 2770. This prevents access by 'other', as a security measure. + +$prefix/Mailman/Archiver/HyperArch.py + +The changes in this file set up the per list htdig stuff such as config files +and adds the search forms to the list TOC pages. + +$prefix/Mailman/Queue/ArchRunner.py + +The changes in this file rewrite a list's TOC page if, when archiving a new +message for the list, the update time of the list's TOC page are after the last +time that rundig was last run. This is is only of relevance when one of the +remote_nightly_htdig series of cron scripts (see below) is being used. + +The only deficiency with this approach is that if no message is sent to the list +after rundig is run for the list the TOC page is not rewritten to reflect that +rundig was run. + +$prefix/Mailman/Cgi/private.py + +There is a security hole in the released Mailman code via which private.py will +serve files such as a list's archive pipermail.pck and files in the list's +archive database sub-directory. This hole also allows access to the list's +archive htdig sub-directory. Fixes for this are applied. As htdig.py (see below) +is based on private.py the same security fix has been incorporated into it. + +$build/Mailman/Defaults.py.in + +Adds the default configuration variables needed to support the mailman-htdig +integration + +$build/cron/crontab.in.in + +Adds the nightly_htdig cron script to the default crontab + +$build/configure +$build/configure.in +$build/Makefile.in +$build/cron/Makefile.in +$build/src/Makefile.in +$build/bin/Makefile.in + +Changes to configuration and Makefiles used for installing Mailman + + + +The patch adds: + + +$build/INSTALL.htdig-mm and $build/INSTALL.htdig-mm.html + +These contain the material you are reading. + +$prefix/cgi-bin/htdig +$prefix/Mailman/Cgi/htdig.py + +these are a CGI script and its wrapper, which is always on the path of URLs +returned from searches of htdig indices. The script provides secure access to +such URLs in the same way that the $prefix/cgi-bin/private and +$prefix/Mailman/Cgi/private.py. Both htdig.py and private.py ensures private +archives are kept private, applying the same criteria for permitting access. +Additionally, htdig.py delivers material from public archives without demanding +any authentication. + +$prefix/cgi-bin/mmsearch +$prefix/Mailman/Cgi/mmsearch.py + +these are a CGI script and its wrapper. The script acts as a security wrapper +for htdig's htsearch CGI script. It will only run htsearch if the user is +authorized to access a list's archive. it applies the same criteria as +$prefix/Mailman/Cgi/private.py. In the case of local htdig operation, this +script runs htsearch as a sub-process and returns its results. In the case of +remote htdig operation mmsearch runs htsearch on the remote machine via one or +other of the CGI scripts remote_mmsearch and remote-mmsearch. + +$prefix/Mailman/Cgi/remote_mmsearch +$prefix/Mailman/Cgi/remote-mmsearch + +these are companion scripts of mmsearch for use with remote htdig operation. +They are run by mmsearch via HTTP requests, and in turn run htsearch as a sub +process, returning the results it delivers. + +$prefix/bin/blow_away_htdig + +this is a utility script for removing per list htdig data, e.g. the config file +and indices/db files. This is necessary when: +a. ceasing use of the Mailman-htdig integration +b. moving from local to remote htdig or vice-versa +c. upgrading to a version of htdig which has an incompatible index/db file +format +d. changing the addressing scheme (http versus https) in the web_page_url +configuration variable of a list +e. reconstructing per-list htdig configuration files after upgrading to +htdig-2.1.1-0.2.patch or later from an earlier patch version, and prior to +running nightly_htdig + + +$prefix/cron/nightly_htdig +$prefix/cron/remote_nightly_htdig +$prefix/cron/remote_nightly_htdig_noshare +$prefix/cron/remote_nightly_htdig.pl + +These scripts all do the same thing; they can be installed as a cron task and +run regularly to invoke htdig's rundig script to update mailing list search +indices. Only one of these scripts is used, the choice of which depending on +your system configuration. + +nightly_htdig is used where Mailman and htdig run on the same system. + +the remote_... scripts are used where Mailman and htdig live on different +systems. You choose which one suits your needs best: + +remote_nightly_htdig uses the same python files on both systems, that is the +same .py and .pyc files are accessed, and it hence depends on compatible +bytecode between the Mailman system and htdig system. It also accesses Mailman +data files and depends on compatibility of data files contents, for example +pickled Python values. This should work OK if the same version of python is +being run on both systems even where the systems are not heterogeneous, for +example one is Sun/Solaris and the other is PC/Linux. + +remote_nightly_htdig_noshare shares no Python files between the two systems. +While it is still written in Python it acquires information from the file system +using directory listings and stat operations. + +remote_nightly_htdig.pl is a rewrite of remote_nightly_htdig_noshare in Perl. It +is for use where the htdig system does not have Python available on it: in which +case, shame on you. + +$prefix/templates/en/TOC_htsearch.html +$prefix/templates/en/htdig_access_error.html +$prefix/templates/en/htdig_auth_failure.html +$prefix/templates/en/htdig_conf.txt + +These are English language templates special to the htdig integration: + +TOC_htsearch.html +the HTML of the search form that is embedded in a list's archive TOC page. +htdig_access_error.html +HTML page returned by mmsearch.py in the event of an access error for a page +access. +htdig_auth_failure.html +HTML page returned by mmsearch.py in the event of an authentication error for a +page access. +htdig_conf.txt +template for the per-list htdig.conf files generated by the patched code. + + +Configuration of Mailman-htdig Integration +---------------- + +Configuration of the Mailman-htdig integration is carried out on the Mailman +side. While you must have to hand some information about your htdig +installation, you should not have to tinker much with htdig for the integration +to work. + +Most of the configuration of the integration is done by values assigned to +python variables in either $prefix/Mailman/Defaults.py or +$prefix/Mailman/mm_cfg.py. + +If you opt to run htdig on a different machine or under a different HTTP server +to the one running the HTTP server which provides Mailman's Web UI you will also +have to edit whichever of the patch's three htdig related cron scripts you opt +to run (remote_nightly_htdig, remote_nightly_htdig_noshare, or +remote_nightly_htdig.pl) to add a small amount of configuration information. + +Health Warning on the packet! +---------------- + +Be careful when editing configuration information in $prefix/Mailman/mm_cg.py: +the only Mailman config file you should be editing. Check, double check and then +recheck before going ahead. If you get either variable names or their values +wrong a lot of confusion in the operation of both Mailman and htdig can result. + +You (and others supporting you) can spend hours trying to identify problems and +looking for non-existent bugs as a consequence of such editing errors. Expect to +find errors in these instructions; compensate for them and tell me when you do +(r.barrett at openinfo.co.uk). + +Also do read the htdig documentation, release notes etc. This patch integrates a +working htdig with htsearch available. These notes are about Mailman and +integrating it with that working htdig. It is up to you to sort out the htdig +end of things. + +Starting from Scratch (Again) +---------------- + +This is getting ahead of things but some of you may already be asking "What if +I've already been using an older version of this patch and want to start +afresh?", or "I want to change from local to remote htdig or vice versa?" + +In these cases your friend will be the $prefix/bin/blow_away_htdig script. It +removes existing htdig related stuff out of your Mailman installation to the +extent that it was added by this patch and added to by the normal operation of +pipermail and nightly_htdig. With that removed and a revised Mailman +configuration, the patched code will start rebuilding the htdig data. + +But before you get carried away with blow_away_htdig, read the rest of these +notes. + +General +---------------- + +This patch adds a number of default variables to the file +$prefix/Mailman/Defaults.py that affect operation of the Mailman-htdig +integration. These are in addition to the standard Mailman defaults in that +file. If, in the light of what is said below, you decide any of these are +incorrect, you can override them in $prefix/Mailman/mm_cfg.py [NOT IN +Defaults.py! See the comments in Defaults.py for why]. + +By default the Mailman-htdig integration is NOT ENABLED by the installation of +this patch; the default value of the USE_HTDIG variable in Defaults.py turns off +the operation of the integration. You have to actively override that default in +mm_cfg.py to turn on operation of the integration. + +Once a list is created, changing most of these variables will have either no +effect or a bad effect. You will need to run $prefix/bin/blow_away_htdig script +and/or $prefix/bin/arch to rebuild the archive pages if you make significant +changes to the Mailman-htdig integration configuration variables. + +The install process will not overwrite an existing mm_cfg.py file so you can +freely make changes to this file. If you are re-installing a later version of +this patch you may have to change what is already configured in the existing +file and, if necessary, add extra configuration variables to it. + +Most of the Mailman-htdig control variables default to sensible values which you +will not need to change, especially if you are using local htdig. The semantics +of most variables apply to both local and remote htdig operation but with some +the values assigned will depend on whether htdig is viewing things from the same +or a remote machine. + +The first two variables control what is indexed by htdig. The values assigned +are both embedded in the HTML generated by pipermail in the list archives and +added. Changing the values of these variables will mean that all previously +generated HTML pages in list archives will be out of date and you will probably +want to rebuild existing archives using $prefix/bin/arch: + +ARCHIVE_INDEXING_ENABLE + +Defines a string telling htdig that it should look at the following material +when building it indices. + + + Default: ARCHIVE_INDEXING_ENABLE = '' + +ARCHIVE_INDEXING_DISABLE + +Defines a string telling htdig that it not should not look at the following +material when building it indices. + + + Default: ARCHIVE_INDEXING_DISABLE = '' + +USE_HTDIG + +Semantics: 0 - don't use integrated htdig, 1 - use it + +Turns Mailman-htdig integration on or off. + + + Defaults: USE_HTDIG = 0 + + +Notes: +1. when USE_HTDIG is turned on the patched code in Mailman will start adding +htdig stuff for any archiving-enabled mail lists as new posts for eachlist are +handled by Mailman. Until a new post is made after enabling with USE_HTDIG an +existing mail list's archive will not be htdig searchable. When the new post is +handled: +a. the list's personalised htdig config file is created +b. necessary links to the htdig config file are created +c. a search form is added to the TOC page for the list + + +Even with this done, htdig searches only become available when htdig indices are +constructed. This is done when one or other of the patch's htdig related cron +scripts are run (nightly_htdig, remote_nightly_htdig, +remote_nightly_htdig_noshare, or remote_nightly_htdig.pl, depending on how you +configure your system). These can be run from the command line ahead of their +scheduled cron time to get htdig searches operational. + +2. Turning USE_HTDIG off will not remove htdig indices or search forms from +existing archive-enabled lists. It will however stop htdig features from being +added to newly created lists. If you want to eliminate htdig from your existing +lists then use the $prefix/bin/blow_away_htdig script. + + + +HTDIG_FILES_URL + +This is the URL of the directory containing various HTML and Graphics files +installed by htdig; files such as buttonr.gif, buttonl.gif and button1-10.gif. +The URL must end with a '/'. + + + Default: HTDIG_FILES_URL = '/htdig/' + + +The default assumes the HTTP servers providing access to htdig and to Mailman's +web UI are on the same machine and a symbolic link called 'htdig' has been put +into your HTTP server's top level HTML directory which points to the directory +your htdig install has put the actual files into; this link is often to +/usr/share/htdig. This value will depend on your htdig installation decisions +and HTTP server's configuration files (typically /etc/httpd/httpd.conf on a late +model Apache installation) i.e the Alias through which the link to the htdig +files are reached. + +HTDIG_CONF_LINK_DIR + +This is the name of a directory in which links to list specific htdig config +files are placed. + + + Default: HTDIG_CONF_LINK_DIR = os.path.join(VAR_PREFIX, 'archives', 'htdig') + + +The VAR_PREFIX of the default is resolved to an actual file system path when +when Mailman's 'make install' is run. The 'os.path.join' creates a full file +system path by gluing together the three pieces when Mailman is run. This +definition puts the directory alongside the default PUBLIC_ARCHIVE_FILE_DIR and +PRIVATE_ARCHIVE_FILE_DIR. Unless you are changing the value of these variables +you probably do not want to change HTDIG_CONF_LINK_DIR. + +HTDIG_RUNDIG_PATH + +This is the path in your file system to the rundig shell script that is +installed as part of htdig. This tells one or other of the patch's htdig related +cron scripts (nightly_htdig and remote_nightly_htdig) where to find rundig in +order that they can execute it. + + + Default: HTDIG_RUNDIG_PATH = '/usr/local/bin/rundig' + +HTDIG_HTSEARCH_PATH + +This is the file path to the htsearch program in the htdig package. + + + Default: HTDIG_HTSEARCH_PATH = '/usr/local/bin/rundig' + + +This value will depend on your htdig installation decisions. This path is used +by either the mmsearch CGI script (for local htdig) or the +remote_mmsearch/remote-mmsearch CGI script (for remote htdig) to execute +htsearch as a sub-process. + +HTDIG_EXCLUDED_URLS + +See htdig's configuration file documentation. The value of this MM variable is +inserted into per-list htdig.conf files when they are created as the value of an +htdig excluded_urls directive. But if an exclusion in this value would prevent +indexing of URLs for accessing the htdig.py cgi wrapper then that exclusion is +omitted from that per-list htdig.conf file. + + + Default: HTDIG_EXCLUDED_URLS = '/cgi-bin/ .cgi' + + +Note: these are the same as the htdig 3.1.6 default values. + +REMOTE_HTDIG + +Semantics: 0 - htdig runs on local machine, 1 -on remote machine + +Says whether htdig going to be run on the same machine as Mailman or on another +machine. + + + Default: REMOTE_HTDIG = 0 + +REMOTE_PRIVATE_ARCHIVE_FILE_DIR + +Only relevant if REMOTE_HTDIG = 1. It is the file system path to the directory +in which Mailman stores private archives, as seen by the machine running htdig. + + + Default: REMOTE_PRIVATE_ARCHIVE_FILE_DIR = os.path.join(VAR_PREFIX, + 'archives', 'private') + + +The VAR_PREFIX of the default is resolved to an actual file system path when +when Mailman's 'make install' is run. The 'os.path.join' creates a full file +system path by gluing together the three pieces when Mailman is run. If you +assign a value to this in mm_cfg.py, just put the relevant explicit file system +path in. + +REMOTE_MMSEARCH_URL + +Only relevant if REMOTE_HTDIG = 1. It is the URL on the htdig machine through +which whichever of the the remote_mmsearch/remote-mmsearch CGI scripts you have +opted to use can be reached via an HTTP request. + + + Default: REMOTE_MMSEARCH_URL = '/cgi-bin/remote-mmsearch' + +HTDIG_STRICT_FILE_PERM + +Semantics: 0 - 'other' access allowed, 1 - 'other' access denied + +Says whether 'other' has access permissions for per-list +$prefix/private/archives//htdig/ directories. For local htdig +operation such access is not required and is a security hole if allowd. Such +access may be needed if remote htdig is used; see notes on "Apache". +$prefix/bin/check_perms should be run after changing the value of this variable +in mm_cfg.py to update access permissions of existing directories. + + + Defaults: HTDIG_STRICT_FILE_PERM = 1 + + +HTDIG_EXTRAS + +You can assign a string value to this config variable and that string will be +included in all of your site's list specific htdig configuration files when they +are created. The value of the string can be any attribute declarations as +defined at http://www.htdig.org/confindex.html. + +Be cautious in what you do with this. Most sites will not need to use this at +all. But if you have some idiosyncratic htdig installation it might help +overcome problems in integrating with Mailman. If you think you need to use it I +suggest: +a. You try creating a test list without assigning a value to HTDIG_EXTRAS in +$prefix/Mailman/mm_cfg.py +b. Enable archiving for that test list. +c. Send a message to the test list so that its archive is created together with +its htdig configuration file. +d. Review the content of the list's htdig conf file in +$prefix/archives/private//htdig/.conf. +e. You will see where the default value of HTDIG_EXTRAS from +$prefix/Mailman/Defaults.py has been inserted. This value is onlyan htdig +comment and does nothing. +f. Consider whether what you will assign to HTDIG_EXTRAS in +$prefix/Mailman/mm_cfg.py will make sense in the context of the rest of the +htdig conf file's contents. + + + + +Permissions Considerations +---------------- + +htdig +---------------- + +Python scripts added by this patch (nightly_htdig and its relatives) run the +htdig rundig script identified by HTDIG_RUNDIG_PATH to build search indices for +Mailman archives. Code added by this patch generates per-list htdig +configuration files which are passed as a parameter to the rundig script. These +configuration files identify a list specific directory +($prefix/archives/private/lt;listname>/htdig) in which list specific data files +generated by and used by htdig are to be placed. + +However, the rundig script identified by HTDIG_RUNDIG_PATH may attempt to +generate some files in htdig's COMMON_DIR when it is first run by nightly_htdig; +the files concerned are likely to be root2word.db, word2root.db, synonyms.db and +possibly some others generated by htidg's htfuzzy program. The standard rundig +script generates these files selectively if they do not already exist. Depending +on how you have installed htdig and how the rundig script is first run, there +may be a permissions problem when nightly_hdig executes rundig under the mailman +UID if it tries to generate these files. + +You may need to either give the mailman UID write permission over htdig's +COMMON_DIR or, before the nightly_htdig script is first run, run htdig's htfuzzy +executable with a sufficiently privileged UID in the manner that the rundig +script would run htfuzzy, to create any necessary files in COMMON_DIR. + +See htdig's documentation for further information on this topic. + +Apache +---------------- + +When remote_mmsearch or remote-mmsearch scripts are used as part of a remote +htdig strategy you may encounter a file permissions problem. This is because +these scripts, which in turn execute htsearch as a sub-process, will be run with +UID and GID of the remote Apache server. + +By default, the permissions of the per-list +$prefix/private/archives//htdig/ directories only allow access for the +mailman UID and GID and hence the remotely executed htsearch will be unable to +access them. + +If this problem is encounterd, then you will have to use the +HTDIG_STRICT_FILE_PERM configuration variable to say "open up the permissions" +before running $prefix/bin/check_perms. You can then use a RewriteRule or +similar in the Apache server's httpd.conf file to restrict access to +$prefix/private/archives//htdig/ directories via the web server. + +Local htdig Configuration +---------------- + +This configuration is for when you are running Mailman, htdig, the HTTP server +used to provide Mailman's web UI and htdig's htsearch CGI script, on the same +machine. + +You will need to: + +a. If different to the default value, add the definition of HTDIG_RUNDIG_PATH +to file $prefix/Mailman/mm_cfg.py. +b. If different to the default value, add the definition of HTDIG_HTSEARCH_PATH +to file $prefix/Mailman/mm_cfg.py. +c. Add the definition of USE_HTDIG with the value 1 to +$prefix/Mailman/mm_cfg.py. + + + + USE_HTDIG = 1 + + +If necessary you can override the values of any of the other configuration +variables in file $prefix/Mailman/mm_cfg.py. + +In particular you might need to change the HTDIG_FILES_URL variable from its +default. This URL can be just the path i.e. absolute URL on the same server as +that which serves Mailman's Web UI, or a full URL identifying the scheme (http), +server, server port and path, for example +http://mailer.yourdomain.tld:8080/htdig/ + +Remote htdig Configuration +---------------- + +This configuration is for when you are running htdig and an HTTP server +providing access to htsearch via remote_mmsearch or remote-mmsearch on a +different machine to that is running Mailman. + +For this configuration to work, htdig's programs, both those run from command +lines such as rundig and those run via CGI such as htsearch, must be able to see +Mailman archives through NFS. In the examples below we'll assume that +/mnt/mailman-archives on the htdig machine maps to $prefix/mailman/archives on +the Mailman machine. + +You should also arrange for he mailman UID and its GID to be common to both +machines. Remember that when rundig is called on the htdig machine to produce +search indices for each list it will be trying to write those files via NFS in +Mailman's archive area and will thus need to run with an appropriate identity +and permissions. + +The differences between the local and remote configuration are: + +1. configuration values telling htdig where to find files are as viewed from +the remote machine. +2. configuration values giving URLs that refer to htdiggy things have to be as +viewed from the Mailman machine. + + +You will need to: + +1. Add the definition of HTDIG_HTSEARCH_PATH to file $prefix/Mailman/mm_cfg.py. +This is path to htdig's htsearch on the remote machine running htdig. For +example: + + + HTDIG_HTSEARCH_PATH = '/usr/local/bin/htsearch' + +2. Add the definition of HTDIG_RUNDIG_PATH to file $prefix/Mailman/mm_cfg.py. +This is path to rundig on the remote machine running htdig. For example: + + + HTDIG_RUNDIG_PATH = '/usr/local/bin/rundig' + +3. Add the definition of REMOTE_MMSEARCH_URL to file $prefix/Mailman/mm_cfg.py. +This must be a full URL referring to one of Mailman's +remote_mmsearch/remote-mmsearch CGI scripts on the remote htdig machine, as seen +from the Mailman local machine. For example: + + + REMOTE_MMSEARCH_URL = 'http://htdiggy.your.com/cgi-bin/remote-mmsearch' + +4. Add the definition of HTDIG_FILES_URL to file $prefix/Mailman/mm_cfg.py. +This must be a full URL referring to the directory containing htdig files on the +remote htdig machine as seen from the Mailman local machine. This URL must end +with a '/'. For example: + + + HTDIG_FILES_URL = 'http://htdiggy.your.com/htdig/' + +5. Add the definition of REMOTE_PRIVATE_ARCHIVE_FILE_DIR to +$prefix/Mailman/mm_cfg.py. This must be the absolute file system path to the +directory in which Mailman stores private archives as seen by the machine +running htdig. For example: + + + REMOTE_PRIVATE_ARCHIVE_FILE_DIR = '/mnt/mailman-archives/private' + +6. Add the definition of USE_HTDIG with the value 1 to +$prefix/Mailman/mm_cfg.py. + + + USE_HTDIG = 1 + +7. Add the definition of REMOTE_HTDIG with the value 1 to +$prefix/Mailman/mm_cfg.py. + + + REMOTE_HTDIG = 1 + +8. If necessary add the definition of HTDIG_STRICT_FILE_PERM with the value 0 +to $prefix/Mailman/mm_cfg.py. This may be needed it the UID/GID that Apache on +the htdig server will run the remote mmsearch as is not mailman or in the +mailman group. This change will open up a security hole which you may want to +consider plugging; see under the heading "Apache permissions" for more details. + + + HTDIG_STRICT_FILE_PERM = 0 + + + +You have to choose one of the two remote mmsearch scripts found in +$prefix/Mailman/Cgi - remote-mmsearch (a Perl script) and remote_mmsearch (a +Python script) - to use and transfer it to the htdig machine. You need to add +this script to the directory in which the web server on the htdig machines +expects to find CGI scripts. Having transferred the script to you htdig machine +you will need to use a text editor to set the values of four configuration +variables below the heading "Edit the following configuration variables to suit +your installation", namely: + + +MAILTO +this is the default mail address for your installation. +VALID_IP_LIST +this is a list of IP numbers from which the script should accept an HTTP +request. Normally this should be set to the IP number of your machine running +Mailman. If the list is empty the script will accept HTTP requests from any +machine and be vulnerable to the exploit described under the heading "Private +archive security problem prior to htdig-2.1.1-0.2.patch version" above. +HTDIG_CONF_LINK_DIR +this is the file path to the directory in which links to list specific htdig +config files are placed, as viewed from the remote machine running htdig. +HTDIG_HTSEARCH_PATH +this is the file path to the htsearch program in the htdig package as viewed +from the remote machine running htdig. + + +See "What is Installed by the Patch" for an explanation of the differences +between these remote mmsearch scripts which both do the same job: being a +security wrapper around htdig's htsearch program to restrict searching of a +list's archive indexes to users authorised to see the contents of that archive. + +Note: You may need to change the '#!' on the first line of whichever of the +remote-mmsearch (Perl) and remote_mmsearch (Python) scripts you opt for so that +the correct interpreter is used for running the script on the remote htdig +machine. You may also need to verify the supporting packages/modules used by the +selected script are installed on that system. + + +You have to choose one of the three remote_nightly_htdig scripts found in +$prefix/cron - remote_nightly_htdig, remote_nightly_htdig_noshare and +remote_nightly_htdig.pl - and transfer it to the htdig machine. See above under +heading "What is Installed by the Patch" for an explanation of the differences +between these scripts, which all do the same basic job. You should add the +script to the crontab for the mailman UID on the htdig machine. But first you +need to edit the selected script to add some configuration information. What has +to be added depends on which script you opt to use. In each case the variables +concerned are declared near the top of the script and you just have to enter the +appropriate values: + + +remote_nightly_htdig + +you only need to set the value of the python variable MAILMAN_PATH to be the +directory $prefix as seen from the htdig machine. The whole Mailman installation +must be accessible via NFS in order to use this script. + +remote_nightly_htdig_noshare + +you need to copy the values for the following configuration variables from +either $prefix/Mailman/mm_cfg.py or $prefix/Mailman/Defaults.py to the script: +REMOTE_PRIVATE_ARCHIVE_FILE_DIR, HTDIG_RUNDIG_PATH. The variables declared in +remote_nightly_htdig_noshare use the same names. This script only requires that +the archives directory of the Mailman installation be accessible via NFS. + +remote_nightly_htdig.pl + +you need to copy the values for the following configuration variables from +either $prefix/Mailman/mm_cfg.py or $prefix/Mailman/Defaults.py to the script: +REMOTE_PRIVATE_ARCHIVE_FILE_DIR, HTDIG_RUNDIG_PATH. Being a Perl script, the +variables in remote_nightly_htdig.pl use the same names but prefixed with the +'$' character. This script only requires that the archives directory of the +Mailman installation be accessible via NFS. + + + +Note: You may need to change the '#!' on the first line of whichever of these +scripts you opt for so that the correct interpreter is used for running the +script on the remote htdig machine. You may also need to verify the supporting +packages/modules used by the selected script are installed on that system. + +As with the nightly_htdig script when running with local htdig, these scripts +can be run from the command line using the mailman UID in order to get htdig to +construct an initial set of indices. + + +Upgrading an Existing Standard Mailman Installation +---------------- + +1. You will want to suspend operation of Mailman while doing the upgrade. +Consider doing a shutdown of the MTA delivering mail to Mailman and removing +Mailman's crontab. +2. Configure and install as described above. +3. Restart Mailman's crontab and restart your MTA's delivery to Mailman. +4. If your installation already has archives: +a. Send a message to each of your archive-enabled lists. This will stimulate +the setup of the new per list htdig config files in the Mailman archives. +b. Consider rebuilding your existing archives with $prefix/bin/arch. This will +embed the ARCHIVE_INDEXING_ENABLE and ARCHIVE_INDEXING_DISABLE in the +regenerated archive pages and, after nightly_htdig has been run, give improved +search results. +c. Run the nightly_htdig script from the command line to generate an initial +set of per-list htdig search indices. + + +Changing from local to remote htdig or vice versa +---------------- + +1. You will want to suspend operation of Mailman while making this change. +Consider doing a shutdown of the MTA delivering mail to Mailman and removing +Mailman's crontab. +2. Run the $prefix/bin/blow_away_htdig script to remove all existing per list +htdig config files and htdig indices/db files. +3. Configure per the instructions above for the local or remote target. +4. Restart Mailman's crontab and restart your MTA's delivery to Mailman. +5. Send a message to each of your archive-enabled lists. This will stimulate +the set up of the new per list htdig config files in Mailman archives. +6. Run the nightly_htdig script from the command line to generate a new set of +per list htdig search indices. + + +Coping with htdig Upgrades +---------------- + +If you change the version of htdig you run, you may find that the indices built +with the earlier version are not compatible with the newer version of htdig's +programs. In that case do the following: + +1. You will want to suspend operation of Mailman while making this change. +Consider doing a shutdown of the MTA delivering mail to Mailman and removing +Mailman's crontab. +2. Run the $prefix/bin/blow_away_htdig script with the -i flag to remove all +existing per list htdig indices/db files. +3. Restart Mailman's crontab and restart your MTA's delivery to Mailman. +4. Run the nightly_htdig script from the command line to generate new sets of +per-list htdig search indices. + + +Changing the Addressing Scheme of your web_page_url +---------------- + +If you change the addressing scheme of the web_page_url for a list to or from +http then you will need to rebuild the list's htdig configuration file(s) and +the related htdig indices. Do the following: + +1. You may want to suspend operation of Mailman while making this change. +Consider doing a shutdown of the MTA delivering mail to Mailman and removing +Mailman's crontab. +2. Run the $prefix/bin/blow_away_htdig script to remove all existing per list +htdig material for the list(s) concerned. +3. Restart Mailman's crontab and restart your MTA's delivery to Mailman. +4. Send a message to each affected list to provoke reconstruction of the list's +htdig config file(s). +5. Run the nightly_htdig script from the command line to generate new sets of +per list htdig search indices. + + +Operational Information +---------------- + +If you have just turned USE_HTDIG on or just used $prefix/bin/blow_away_htdig +(without the -i flag) there will be no per-list htdig information saved in the +archives. + +When the first post to each archive-enabled list is archived by pipermail, the +per-list htdig config file will be constructed and some directories and links +added to your Mailman archive directories. The htdig search form will be added +to list's TOC page. + +However, until one of the nightly_htdig scripts is run no htdig indices will be +constructed. You can either wait for the script to run as a cron job or run it +(while using the mailman UID) from the command line. + +Notes and Warnings +---------------- + +Archive security problems resolved by htdig-2.1.3-0.2 patch + + +This patch is hopefully the final step in closing security holes in archive +access. + +In version htdig-2.1.3-0.1.patch, htdig.py was rebased on the standard MM +release's private.py which had moved on since the snapshot of it used as the +basis for htdig.py was originally taken. Among other things, htdig.py had been +modified to prevent access to some files in list archive directories such as a +list's archive pipermail.pck and files in the list's archive database +sub-directory. + +This rebasing action re-introduced to htdig.py the security holes, still extant +in private.py despite it being later code, via which private.py would serve +files such as a list's archive pipermail.pck and files in the list's archive +database sub-directory. + +The permissions on these files and directories mean that they are inaccessible +via the web server using /pipermail/ URIs if a list's archive is public. + +Additionally, check_perms is now modified so that the list archive htdig +subdirectory permissions are set to 2770 by default. Prior to +htdig-2.1.1-0.2.patch, this could not be done as the htsearch script, being run +with uid and gid of the Apache server, could then not gain access to files in +the htdig subdirectories. But, since the introduction of the mmsearch script, +which runs with the mailman gid and spawns htsearch, it can. This prevents +accees to the list archive htdig subdirectories via /pipemail/ URI's. Up until +htdig-2.1.3-0.2.patch this could only be achieved by using a RewriteRule or +similar in the Apache server's httpd.conf file. +The only residual problem is that the revised permissions on the archive htdig +subdirectories may cause problems if the remote_mmsearch and remote-mmsearch are +used. This is because they will be run with uid and gid of the Apache server. If +this problem is encounterd, then you will have to manually add read and execute +permissions for 'other' to the archive htdig subdirectories and read permission +their contents, and then use RewriteRule or similar in the Apache server's +httpd.conf file for protection. + +The solution to this problem has been superceded in htdig-2.1.3-0.3.patch as +follows: Introduced the HTDIG_STRICT_FILE_PERM Mailman config variable as part +of dealing with htsearch access to per-list htdig directories permissions issue +when operating with remote htdig. See under the "Apache" heading above. + +Private archive security problem prior to htdig-2.1.1-0.2.patch version +---------------- + +Versions of the Mailman-htdig integration patch installed by versions of this +patch prior to htdig-2.1.1-0.2.patch allow a security exploit which can expose +information, held in the per-list search indexes of private list archives, to +unauthorised users. + +Via the exploit an unauthoized user can submit a search query to htdig's +htsearch CGI program without their having been authenticated as a user allowed +to access the list archive concerned. The results, returned in good faith by +htsearch, will expose some information that the user is not entitled to see. + +However, the security breakdown is not complete. Attempts to follow links +returned by htsearch, which go via the htdig CGI script installed by this patch, +will be blocked if the user is not authorized to access the list archive. + +Maintaining archive security with htdig-2.1.1-0.2.patch version and later +---------------- + +With htdig-2.1.1-0.2.patch and later versions of the patch: + +1. htsearch is no longer used directly via CGI for searching list archives. +2. The symbolic link named by the HTDIG_MAILMAN_LINK configuration variable is +no longer used. Indeed, when upgrading earlier installations this symlink should +be deleted and the configuration variable deleted. Without this symlink, on a +normally configured system, htsearch no longer has the unaided ability to access +the per-list htdig configuration and other list archive associated files. +3. Thus, even if htsearch can be reached via CGI, it cannot undertake a search +of list archives when requested to do so by an HTTP request which seeks to +circumvent list archive security. +4. A new script, $prefix/Mailman/Cgi/mmsearch.py, is now used to search list +archives. This script applies the same user authentication as private.py and +htdig.py. Only if a user is authorised to access a list, does mmsearch use +htdig's htsearch to search a list's archive. In this case, mmsearch provides +htsearch with the information it needs to access the per-list htdig +configuration and other list archive associated files. +5. Where htidg and Mailman are run on the same machine, mmsearch acts as a +security wrapper, runs htsearch as a sub-process and list security is preserved +by this means. +6. Where htdig is run on a different machine to Mailman, mmsearch can perform +user authentication but has problems in acting as a security wrapper for +htsearch. The solution adopted is for one of two companion CGI scripts +(remote-mmsearch written in Perl or remote_mmsearch written in Python) to be +invoked on the remote htdig machine by an HTTP request made by mmsearch on the +Mailman machine. These scripts run htsearch, providing it with the information +it needs to access the per-list htdig configuration and other list archive +associated files. But, such an HTTP request can be made by other means and thus +the the same security exploit we are trying to avoid still exists. The only +protection in the case of remote htdig operation is that the +remote-mmsearch/remote_mmsearch scripts can be configured to operate only on +HTTP requests originating from specified IP numbers. By restricting operation to +requests originating on the Mailman server some semblance of list privacy can be +preserved. + + +Upgrading to htdig-2.1.1-0.2.patch or later from an earlier patch version +---------------- + +If you are upgrading a Mailman installation that has an earlier version of the +the Mailman-htdig integration patch than that installed by htdig-2.1.1-0.2.patch +or later, you need to make some changes to that installation: + +1. You must delete from your file system the symbolic link named by the +HTDIG_MAILMAN_LINK Mailman configuration variable. This link previously gave +htdig programs access to per list htdig configuration files. This is now done by +other means and the symlink allows a security exploit that prejudices the +privacy of list archives. +2. You must delete the HTDIG_MAILMAN_LINK Mailman configuration variable from +the $prefix/Mailman/mm-cfg.py file. + + +These changes are in addition to the normal installation instructions given +below. Having configured and installed the newly patched version of Mailman you +must: + +1. Run the script $prefix/bin/blow_away_htdig with the -c option to rebuild +per-list htdig conf files and delete existing per-list search indexes. +2. Run the $prefix/cron/nightly_htdig script from the command line to rebuild +per-list search indexes using the revised per-list htdig conf files just created +by blow_away_htdig. + + +Redhat 7.1 and 7.2 installations +---------------- + +If you install htdig from the htdig-3.2.0 binary rpm of RH7.1/2 Binary CD 1 of 2 +you also have to install the htdig-web-3.2.0 binary rpm. This may be from RH +7.1/2 Binary CD 2 of 2 or CD 1 of 2 depending on whether you are using actual +CDs or downloaded CD images. + +Apache/htdig issues +---------------- + +htdig's graphics file must be accessible via you web server and the Mailman +configuration variable HTDIG_FILES_URL setup accordingly. Depending on how you +install htdig and Apache you may need to add Alias and/or ScriptAlias directives +to you Apache configuration file to make the htdig components accessible. Check +the Apache and htdig documentation. + +Contributors +---------------- + +Original author and maintainer: +Richard Barrett - +Past bug fixes: +Nigel Metheringham +Stephan Berndts +Testers: +* Mark T. Valites +* Rehan van der Merwe +Suggested Improvements: +Mark Sapiro + +History +---------------- + +Compatibility +---------------- +Version of patch Version of Mailman +htdig-2.1.10-0.1.patch Mailman 2.1.10 +htdig-2.1.9-0.1.patch Mailman 2.1.9 +htdig-2.1.7-0.1.patch Mailman 2.1.7 and 2.1.8 +htdig-2.1.6-0.1.patch Mailman 2.1.6 +htdig-2.1.4-0.1.patch Mailman 2.1.4 +htdig-2.1.3-0.5.patch Mailman 2.1.3 +htdig-2.1.3-0.4.patch Mailman 2.1.3 +htdig-2.1.3-0.3.patch Mailman 2.1.3 +htdig-2.1.3-0.2.patch Mailman 2.1.3 +htdig-2.1.3-0.1.patch Mailman 2.1.3 +htdig-2.1.2-0.4.patch Mailman 2.1.2 +htdig-2.1.2-0.3.patch Mailman 2.1.2 +htdig-2.1.2-0.2.patch Mailman 2.1.2 +htdig-2.1.2-0.1.patch Mailman 2.1.2 +htdig-2.1.1-0.5.patch Mailman 2.1.1 +htdig-2.1.1-0.4.patch Mailman 2.1.1 +htdig-2.1.1-0.3.patch Mailman 2.1.1 +htdig-2.1.1-0.2.patch Mailman 2.1.1 +htdig-2.1.1-0.1.patch Mailman 2.1.1 +htdig-2.1-0.3.patch Mailman 2.1 +htdig-2.1-0.2.patch Mailman 2.1 +htdig-2.1-0.1.patch Mailman 2.1 +htdig-2.1b6-0.1.patch Mailman 2.1b6 +htdig-2.1b5-0.1.patch Mailman 2.1b5 +htdig-2.1b4-0.1.patch Mailman 2.1b4 +htdig-2.1b3-0.3.patch Mailman 2.1b3 +htdig-2.1b3-0.2.patch Mailman 2.1b3 +htdig-2.1b3-0.1.patch Mailman 2.1b3 +htdig-2.1b2-0.1.patch Mailman 2.1b2 +htdig-2.0.13-0.2.patch Mailman 2.0.13 +htdig-2.0.13-0.1.patch Mailman 2.0.13 +htdig-2.0.12-0.1.patch Mailman 2.0.12 +htdig-2.0.11-0.1.patch Mailman 2.0.11 +htdig-2.0.10-0.2.patch Mailman 2.0.10 +htdig-2.0.10-0.1.patch Mailman 2.0.10 +htdig-2.0.9-0.1.patch Mailman 2.0.9 +htdig-2.0.8-0.1.patch Mailman 2.0.8, 2.0.7, 2.0.6 and probably 2.0.3, 2.0.4 +and 2.0.5 + +Changes +---------------- + +htdig-2.1.10-0.1.patch: +1. Updated patch for MM 2.1.9 compatibility. +2. Change to setup_htdig() function in HyperArch.py suggested by + Mark Sapiro to ensure correct permissions set on directory creation. + +htdig-2.1.9-0.1.patch: +1. Updated patch for MM 2.1.9 compatibility. + +htdig-2.1.7-0.1.patch: +1. Updated patch for MM 2.1.7 compatibility. + +htdig-2.1.6-0.1.patch: +1. Updated patch for MM 2.1.6 compatibility. + + Note: the templates in $build/templates//for the following + languages are NOT modified by this patch or by its precursor indexing + patch: ca, eu, sr, sv + + The following files in a language's default template directory should be + modified per the changes made to the en language templates after + installation of this patch if that other language is used ;' + + templates//archidxfoot.html + templates//archidxhead.html + templates//archtoc.html + templates//archtocentry.html + templates//archtocnombox.html + templates//article.html + +htdig-2.1.4-0.1.patch: +1. Updated patch for MM 2.1.4 compatibility. +2. Removed untranslated versions of htdig.html from per-language directories + under $build/templates, with the exception of the default templates/en/ + directory, that were present in previous versions of this patch. + +htdig-2.1.3-0.5.patch: +1. Modified htdig.py and private.py; the security changes introduced by + htdig-2.1.3-0.2 patch to these scripts incorrectly blocked access to the + .mbox/.mbox file. The O.5 revison of the patch corrects + this error. This problem and a suggested fix were pointed out to me in a + private email by Stephan Berndts + +htdig-2.1.3-0.4.patch: +1. Modified htdig.py and introduced htdig.html templates. The changes mean that + if the user is challenged for authentication, when the credentials are + submitted and accepted, the URL requested which led to the challenge is then + presented. + +htdig-2.1.3-0.3.patch: +1. Patch documentation layout revised and simplified. +2. Changes to $prefix/bin/check_perms and $prefix/Mailman/Archiver/HyperArch.py +to improve handling of htdig subdirectory permissions if remote htdig is used. +End result is the same as with prior patch version in the case of local htdig. +3. Introduced the HTDIG_STRICT_FILE_PERM Mailman config variable as part of +dealing with htsearch access to per-list htdig directories permissions issue +when operating with remote htdig. See under the "Apache" heading above. + + +htdig-2.1.3-0.2.patch: +1. This patch is hopefully the final step in closing security holes in archive +access. See the discussion below under the heading "Archive security problems +resolved by htdig-2.1.3-0.2 patch". + + +htdig-2.1.3-0.1.patch: +1. updated patch for MM 2.1.3 compatibility. + + +htdig-2.1.2-0.4.patch: +1. corrected error in mmsearch.py and remote_mmsearch. This caused a problem if +https was being used for accessing the archives as a pattern match to extract +the list name was misused. + + +htdig-2.1.2-0.3.patch: +1. updates HyperArch.py so htdig related code uses quick_maketext() function +instead of the Utils.Maketext() function. + + +htdig-2.1.2-0.2.patch: +1. corrects stupid error inserted in unpublished htdig-2.1.1-0.5.patch and +carried forward into htdig-2.1.2-0.1.patch + + +htdig-2.1.2-0.1.patch: +1. updated patch for MM 2.1.2 compatibility + + +htdig-2.1.1-0.5.patch: +1. with previous version the protototype htdig_conf.txt contained an htdig +exclude_urls directive for /cgi-bin/ and .cgi. If MM is configured so that the +URL for accessing the htdig.py cgi wrapper matches these excluded URLS (for +instance by running ./configure with --with-cgi-ext=".cgi") then nothing gets +indexed by rundig. The revised patch: +a. makes the excluded URL configurable through a MM config variable +HTDIG_EXCLUDED_URLS which defaults to the old hard-wired value. +b. when generating a list-specific htdig.conf file a check is made against +HTDIG_EXCLUDED_URLS and if anything in it would prevent indexing of the URL for +accessing the htdig.py cgi wrapper for that list, it is omitted from the +exclude_urls directive in that htdig.conf file. + + +htdig-2.1.1-0.4.patch: +1. mmsearch.py and its remote kin remote-mmsearch and mm_search were overly +restrictive on the form fields they were willing to accept. Extended the list so +that multi-page search results worked. + + +htdig-2.1.1-0.3.patch: +1. corrects silly error in raising an excpetion in mmsearch.py. This will only +show if there is a problem with mmsearch running the htsearch program. + + +htdig-2.1.1-0.2.patch: +1. This version corrects a security exploit which allowed a URL to obtain an +htsearch results page without the user being authorised to access the list. Any +attempt to follows links on the results page were blocked correctly by +$prefix/Mailman/htdig.py but there was leakage of private information from the +list's search indexes on the page returned by htdig's htsearch CGI program. The +exploit is removed by this patch's revisions. The following sections describe +the problem, the solution and special actions required when updating a Mailman +installation using an earlier version of this patch: +a. Private archive security problem prior to htdig-2.1.1-0.2.patch version. +b. Maintaining private archive security with htdig-2.1.1-0.2.patch version and +later. +c. Upgrading to htdig-2.1.1-0.2.patch or later from an earlier patch version. + +Note that there is no patch revision to deal with this security problem for MM +2.0.13 or earlier and you should seriously consider updating to MM 2.1.x if you +want to implement this security fix. + + +htdig-2.1.1-0.1.patch: +1. No functional change. Applies without offset warnings to MM 2.1.1 + + +htdig-2.1-0.3.patch: +1. corrects errors in the way $prefix/Mailman/htdig.py worked out content type +of file being returned. +2. $prefix/Mailman/htdig.py adopts revised method for establishing the default +URL introduced in 2.1 and as used in $prefix/Mailman/MailList.py +3. removed unecessary setup of variable DEFAULT_URL in cron scripts +$prefix/cron/remote_nightly_htdig_noshare and +$prefix/cron/remote_nightly_htdig.pl +4. Changes references to DEFAULT_URL in this document to DEFAULT_URL_PATTERN. + + +htdig-2.1-0.2.patch: +1. improved content type and security handling in $prefix/Mailman/htdig.py. +Fixes bug with htdig.py and problem of interaction with bug in +$prefix/scripts/driver script (see patch #668685 for more details) + + +htdig-2.1-0.1.patch: +1. Reworked patch for compatibility with MM 2.1. + + +htdig-2.1b6-0.1.patch: +1. Reworked patch for compatibility with MM 2.1b6. + + +htdig-2.1b5-0.1.patch: +1. Reworked patch for compatibility with MM 2.1b5. + + +htdig-2.1b4-0.1.patch: +1. Reworked patch for compatibility with MM 2.1b4. As a consequence, the +remainder of the mailman-htdig integration templates that were strings declared +in Mailman/Archiver/HyperArch.py have been extracted into files under the +templates directory. Edit these with care if you must. + + +htdig-2.1b3-0.3.patch: +1. Removed unecessary code dependency on Python 2.2 file() function + + +htdig-2.1b3-0.2.patch: +1. Removed syntax error in htdig-2.1b3-0.1.patch which showed up as logged +errors in the operation of the ArchRunner qrunner at line 721 of HyperArch.py + + +htdig-2.1b3-0.1.patch: +1. Reworked patch for compatibility with MM 2.1b3 +2. Removed non-English language template files which were acting as +placeholders until someone actually translated them. +3. Removed updateTOC.py and replaced it with an alternate mechanism in a patch +to $prefix/Mailma/Queue/ArchRunner.py to update list TOC page after reindexing +by htdig. This new method is only exercised when the remote_nightly_htdig series +of cron scripts are used. +4. Changes to remote_nightly_htdig series of cron scripts to reflect demise of +updateTOC cgi script. +5. Multiple instances of code hygiene and conformance to MM "standards" +cleanup. +6. Tidied up this documentation. + + +htdig-2.1b2-0.1.patch: +1. reworked patch for compatibility with MM 2.1b2 + + +htdig-2.0.13-0.2.patch: +1. Added license header + + +htdig-2.0.13-0.1.patch: +1. Rebuilt patch to get no-comment application on Mailman 2.0.13 + + +htdig-2.0.12-0.1.patch: +1. Rebuilt patch to get no-comment application on Mailman 2.0.12 +2. Added HTDIG_EXTRAS xonfig variable to allow arbitrary htdig configuration +parameters to be specified for addition to every htdig.conf file created i.e. +site wide additions. + + +htdig-2.0.11-0.1.patch: +1. No substantive change. Simply rebuilt patch to get no-comment application on +Mailman 2.0.11 + + +htdig-2.0.10-0.2.patch: +1. Python 2.2 compatibility fixes to nightly_htdig cron script and its +relatives. Doing import * inside a function removed. +2. Added note on potential problems with htdig and file permissions. + + +htdig-2.0.10-0.1.patch: +1. change in src/Makefile.in to get clean patch application to MM 2.0.10 + + +htdig-2.0.9-0.1.patch: +1. minor cosmetic changes to get clean patch application to MM 2.0.9 + + +htdig-2.0.8-0.1.patch: +1. resolves a problem with the integration of htdig when the web_page_url for a +list, which is usually the same as DEFAULT_URL from either +$prefix/Mailman/Defaults.py or $prefix/Mailman/mm_cfg.py, when it doesn't use +the http addressing scheme. This arises because htdig will only build indices if +the URLs for pages use the http addressing scheme. There is a work-around for +this problem posted in htdig's mail archives - see the copy in Appendix 1 to +this document. +2. This patch revision implements the solution documented in that e-mail. If +non-http URLs are used by the web_page_url of a list an additional htdig +configuration file for use by htsearch is generated. +3. In all other respects the operation of the Mailman-htdig integration remains +unchanged. There is no benefit in upgrading to this revised patch unless you +need to use other than http addressing in your DEFAULT_URL or set other than +http addressing in the web_page_url configuration of any of your lists. +4. If changing to or from a non-http addressing scheme then the per list htdig +config files of the lists affected and their associated htdig indices must be +reconstructed. See the section below entitled "Changing the Addressing Scheme of +your web_page_url" for details of how to do this. + + +htdig-2.0.6-0.3.patch: +1. adds support for remote htdig, that is: running htdig on a different system +to Mailman. +2. enhances the configurability of the integration. Some of the programmed +assumptions made in previous versions are now configurable in mm_cfg.py. The +configuration variables concerned default to the previous fixed values so that +this version is backwards compatible with earlier versions. +3. does some minor cosmetic code changes. +4. extends the associated documentation. + + + + +Appendices +---------------- + +Appendix 1 -Technique for htdigging when Mailman's web_page_url uses the https +scheme + + + +A technique for htdigging when Mailman's web_page_url uses the https +addressing scheme is described in this archived e-mail: +http://www.htdig.org/mail/1999/10/0187.html + +The text of that e-mail is as follows: + +[htdig] Re: Help about htdig indexing https files + +------------------------------------------------------------------------ +Gilles Detillieux (grdetil at scrc.umanitoba.ca) +Wed, 27 Oct 1999 10:18:31 -0500 (CDT) + + +Messages sorted by: [ date ] [ thread ] [ subject ] [ author ] +Next message: Avi Rappoport: "[htdig] indexing SSL (was: Help building +the database)" +Previous message: Gilles Detillieux: "Re: Fw: [htdig] mutiple search +results" +In reply to: Torsten Neuer: "Re: Fw: [htdig] mutiple search results" + +------------------------------------------------------------------------ +According to Edouard DESSIOUX: +> >Currently, htdig will not support URLs that begin with https://, even +> >when using local_urls to bypass the server. A trick that might work +> >would be to index using http:// instead, but use local_urls to point +> >to the directory that contains the contents of the secure server. +> +> I used that, and now, when i use htsearch, it work, except the fact +> that all my URL are http://x.y.z/ instead of https://x.y.z/ +> +> >You'd need to use separate +> >configuration files for digging and searching, and use +> >url_part_aliases in each of these configuration files to rewrite the +> >http:// into https:// in the search results. +> +> This is the part i dont understand, and i would like you to explain. + + +It basically works as a search and replace. One url_part_aliases in the +configuration file used by htdig maps the http://x.y.z/ into some +special code like "*site", and another url_part_aliases in the +configuration file used by htsearch maps the "*site" back into the value +you want, i.e. https://x.y.z/. The substitution is left to right in +htdig, and right to left in htsearch. So, if you use the same config +file for both, or the same setting for both, you get back what you +started with (but saved some space in the database because of the +encoding). However, if you use two separate config files with different +url_part_aliases setting for htdig and htsearch, you can remap parts of +URLs from one substring to another. + + +I hope this makes things clearer. I thought the current description at +http://www.htdig.org/attrs.html#url_part_aliases was already quite +clear. + + + +-- +Gilles R. Detillieux E-mail: +Spinal Cord Research Centre WWW: +http://www.scrc.umanitoba.ca/~grdetil +Dept. Physiology, U. of Manitoba Phone: (204)789-3766 +Winnipeg, MB R3E 3J7 (Canada) Fax: (204)789-3930 +------------------------------------ diff -ruP mailman-2.1.18/INSTALL.htdig-mm.html mailman-2.1.18p/INSTALL.htdig-mm.html --- mailman-2.1.18/INSTALL.htdig-mm.html 1969-12-31 16:00:00.000000000 -0800 +++ mailman-2.1.18p/INSTALL.htdig-mm.html 2014-05-03 12:22:36.839376950 -0700 @@ -0,0 +1,2050 @@ + + + + OpenInfo Web Site + + + + + + +

Mailman Patch #444884 Installation Details

+

+ The information below is also available in the INSTALL.htdig-mm file which is installed by this patch. +

+

Table of Contents

+ +

Patch Identification [ toc ]

+

+ Different versions of this patch are available for different versions of Mailman. There may be different versions of this patch for any given version of Mailman, typically as a results of MM version specific improvements or corrections of bugs in the patched code. The names of patch files for this patch are structured as follows: +

+
+    htdig-<MM-version-no>-<patch-version-no>.patch[.gz]
+

+ Thus, for instance, patch file htdig-2.1.4-0.1.patch is patch version 0.1 for application to MM version 2.1.4 source code. +

+

+ The <patch-version-no> is reset to 0.1 for the first patch version applicable to each new version of Mailman. +

+

+ The .gz suffix, if present, says that the patch file has been compressed using gzip. +

+

+ As a general rule, you should use the highest patch version number for the MM version you are installing. +

+

Current Version [ toc ]

+

+ The current version of this patch is for Mailman 2.1.10: +

+ + + + + + +
+ Mailman 2.1.10 + + - + + htdig-2.1.10-0.1.patch +
+

+ Be sure to read the notes in the changes section below about the patch version you are going to use. +

+

+ Patches for previous versions of Mailman are frozen at the highest revision level they reached while those previous versions of MM were current. +

+

+ Information about older Mailman and patch versions is given in the history section below. +

+

Changes introduced by this patch version [ toc ]

+

+ The following changes are introduced by version 0.1 of this patch: +

+
    +
  • + Updated patch for MM 2.1.10 compatibility +
    • +
  • + Made change to setup_htdig() function in HyperArch.py suggested by Mark Sapiro. +
    • +
+

+ The frequency with which extra languages are being supported by Mailman exceed my capacity to cope. From htdig-2.1.9-0.1.patch on only the English language (default) templates are guaranteed to have been patched. The following files in a language's default template directory should be checked and if necessary modified per the changes made to the en language templates after installation of this patch if that other language is used: +

+
    +
  • templates//archidxfoot.html
    • +
  • templates//archidxhead.html
    • +
  • templates//archtoc.html
    • +
  • templates//archtocentry.html
    • +
  • templates//archtocnombox.html
    • +
  • templates//article.html
    • +
+

Prerequisites [ toc ]

A working Htdig installation

+

+ You must have a working installation of htdig with htsearch available and installed on either the machine on which you are running Mailman or on another machine which has access to Mailman list archives via NFS or some similarly competent network file sharing scheme. +

+

+ Regardless of how you configure things to provide Mailman's Web UI, if its gives normal operation of the /mailman/private CGI script for providing access to private list archives, it should also support access to htdig search results via the /mailman/mmsearch and /mailman/htdig CGI scripts. +

+

+ Warning: This patch has been tested with HTdig 3.1.6 and no testing has been done with the Beta versions of HTdig 3.2 at the time of writing. You may or may not encounter problems/issues not described here if you use HTdig 3.2 beta or stable releases. +

+

Other Mailman patches

+

+ Prior to installing this patch you may also need to install the other MM patches. This will depend on the version of Mailman and the version of this patch you are dealing with. For version 0.3 of this patch for MM 2.1.3 the latest version of patch #444879, + + indexing-2.1.3-x.y.patch + + , is required. It is available from: +

+ +

+ For any other version of this patch details of its prerequisites are in the version of INSTALL.htdig-mm file which is installed by that patch. +

+

Introduction [ toc ]

+

+ This integration enables use of the htdig (http://www.htdig.org) search engine for searching mail list archives produced by pipermail, Mailman's built-in archiver. +

+

+ You can use htdig without applying these patches to Mailman but you may find it awkward to achieve some of the features offered by this patch. +

+

+ The main features of the patch are: +

+
    +
  1. + per list search facility with a search form on each list's TOC page. +
    2. +
  2. + maintenance of privacy of private archives. The user has to establish their credentials via the normal private archive access mechanism before any access via htdig is allowed. +
    3. +
  3. +

    + a common base URL for both public and private archive access via htsearch results. This means that htdig indices are unaffected by changing an archive from private to public and vice versa. All access to archives via htdig is controlled by wrapped CGI scripts called htdig.py and mmsearch.py. +

    +

    + Note that Mailman's attachment scrubber creates a problem when it extracts attachments from messages as they are being archived because it embeds absolute URLs to what it has extracted in the archived messages. This can only be fixed by running $prefix/bin/arch to rebuild the list's archive from its mbox file after changing its archive from private to public or vices versa. This problem is generic and unrelated to the use of this patch. One way resolving it is by use the Mailman-MHonArc integration patch #???????? available from +

    + +
    4. +
  4. + a choice of running htdig on the machine running Mailman (aka local htdig) or running htdig on another machine which has access to Mailman's archives via NFS or some similarly competent network file sharing scheme (aka remote htdig). +
    5. +
  5. + cron activated scripts and crontab entry to run htdig regularly to maintain the per list search indices. +
    6. +
  6. + automatic creation, deletion and maintenance of htdig configuration files and such. Beyond installing htdig and telling Mailman where it is via mm_cfg you do not have to do much other setup. +
    7. +
  7. + htdig search related web page elements are retrieved from the $prefix/templates/ directory hierarchy so that site, virtual host, list and language tailoring of them can be done. +
    8. +
+

Installing and Building Mailman with this patch [ toc ]

+

+ Create your Mailman build directory in the normal way. +

+

+ You can apply the patch to either a fresh expansion of the Mailman source distribution or the one you used to build a currently working Mailman installation. +

+

+ Execute the following command in the Mailman build directory: +

+
+    patch -p1 < path-to-htdig-2.m.n-x.y.patch
+

+ Follow the configure and make procedures for regular Mailman as given in the $build/INSTALL file. +

+

+ Then follow the Mailman-htdig configuration instructions given below. +

+

What is Installed by the Patch [ toc ]

The patch amends:

+
+
+ $build/INSTALL +
+
+

+ Adds a reference to this file to the standard installation notes. +

+
+
+ $prefix/bin/check_perms +
+
+

+ To set the permissions for access to $prefix/archive/private/<listname>/htdig/ subdirectories to 2770. This prevents access by 'other', as a security measure. +

+
+
+ $prefix/Mailman/Archiver/HyperArch.py +
+
+

+ The changes in this file set up the per list htdig stuff such as config files and adds the search forms to the list TOC pages. +

+
+
+ $prefix/Mailman/Queue/ArchRunner.py +
+
+

+ The changes in this file rewrite a list's TOC page if, when archiving a new message for the list, the update time of the list's TOC page are after the last time that rundig was last run. This is is only of relevance when one of the remote_nightly_htdig series of cron scripts (see below) is being used. +

+

+ The only deficiency with this approach is that if no message is sent to the list after rundig is run for the list the TOC page is not rewritten to reflect that rundig was run. +

+
+
+ $prefix/Mailman/Cgi/private.py +
+
+

+ There is a security hole in the released Mailman code via which private.py will serve files such as a list's archive pipermail.pck and files in the list's archive database sub-directory. This hole also allows access to the list's archive htdig sub-directory. Fixes for this are applied. As htdig.py (see below) is based on private.py the same security fix has been incorporated into it. +

+
+
+ $build/Mailman/Defaults.py.in +
+
+

+ Adds the default configuration variables needed to support the mailman-htdig integration +

+
+
+ $build/cron/crontab.in.in +
+
+

+ Adds the nightly_htdig cron script to the default crontab +

+
+
+ $build/configure +
+
+ $build/configure.in +
+
+ $build/Makefile.in +
+
+ $build/cron/Makefile.in +
+
+ $build/src/Makefile.in +
+
+ $build/bin/Makefile.in +
+
+

+ Changes to configuration and Makefiles used for installing Mailman +

+
+
+

The patch adds:

+
+
+ $build/INSTALL.htdig-mm and $build/INSTALL.htdig-mm.html +
+
+

+ These contain the material you are reading. +

+
+
+ $prefix/cgi-bin/htdig +
+
+ $prefix/Mailman/Cgi/htdig.py +
+
+

+ these are a CGI script and its wrapper, which is always on the path of URLs returned from searches of htdig indices. The script provides secure access to such URLs in the same way that the $prefix/cgi-bin/private and $prefix/Mailman/Cgi/private.py. Both htdig.py and private.py ensures private archives are kept private, applying the same criteria for permitting access. Additionally, htdig.py delivers material from public archives without demanding any authentication. +

+
+
+ $prefix/cgi-bin/mmsearch +
+
+ $prefix/Mailman/Cgi/mmsearch.py +
+
+

+ these are a CGI script and its wrapper. The script acts as a security wrapper for htdig's htsearch CGI script. It will only run htsearch if the user is authorized to access a list's archive. it applies the same criteria as $prefix/Mailman/Cgi/private.py. In the case of local htdig operation, this script runs htsearch as a sub-process and returns its results. In the case of remote htdig operation mmsearch runs htsearch on the remote machine via one or other of the CGI scripts remote_mmsearch and remote-mmsearch. +

+
+
+ $prefix/Mailman/Cgi/remote_mmsearch +
+
+ $prefix/Mailman/Cgi/remote-mmsearch +
+
+

+ these are companion scripts of mmsearch for use with remote htdig operation. They are run by mmsearch via HTTP requests, and in turn run htsearch as a sub process, returning the results it delivers. +

+
+
+ $prefix/bin/blow_away_htdig +
+
+

+ this is a utility script for removing per list htdig data, e.g. the config file and indices/db files. This is necessary when: +

+
    +
  1. + ceasing use of the Mailman-htdig integration +
    2. +
  2. + moving from local to remote htdig or vice-versa +
    3. +
  3. + upgrading to a version of htdig which has an incompatible index/db file format +
    4. +
  4. + changing the addressing scheme (http versus https) in the web_page_url configuration variable of a list +
    5. +
  5. + reconstructing per-list htdig configuration files after upgrading to htdig-2.1.1-0.2.patch or later from an earlier patch version, and prior to running nightly_htdig +
    6. +
+
+
+ $prefix/cron/nightly_htdig +
+
+ $prefix/cron/remote_nightly_htdig +
+
+ $prefix/cron/remote_nightly_htdig_noshare +
+
+ $prefix/cron/remote_nightly_htdig.pl +
+
+

+ These scripts all do the same thing; they can be installed as a cron task and run regularly to invoke htdig's rundig script to update mailing list search indices. Only one of these scripts is used, the choice of which depending on your system configuration. +

+

+ nightly_htdig is used where Mailman and htdig run on the same system. +

+

+ the remote_... scripts are used where Mailman and htdig live on different systems. You choose which one suits your needs best: +

+

+ remote_nightly_htdig uses the same python files on both systems, that is the same .py and .pyc files are accessed, and it hence depends on compatible bytecode between the Mailman system and htdig system. It also accesses Mailman data files and depends on compatibility of data files contents, for example pickled Python values. This should work OK if the same version of python is being run on both systems even where the systems are not heterogeneous, for example one is Sun/Solaris and the other is PC/Linux. +

+

+ remote_nightly_htdig_noshare shares no Python files between the two systems. While it is still written in Python it acquires information from the file system using directory listings and stat operations. +

+

+ remote_nightly_htdig.pl is a rewrite of remote_nightly_htdig_noshare in Perl. It is for use where the htdig system does not have Python available on it: in which case, shame on you. +

+
+
+ $prefix/templates/en/TOC_htsearch.html +
+
+ $prefix/templates/en/htdig_access_error.html +
+
+ $prefix/templates/en/htdig_auth_failure.html +
+
+ $prefix/templates/en/htdig_conf.txt +
+
+

+ These are English language templates special to the htdig integration: +

+
+
+ TOC_htsearch.html +
+
+ the HTML of the search form that is embedded in a list's archive TOC page. +
+
+ htdig_access_error.html +
+
+ HTML page returned by mmsearch.py in the event of an access error for a page access. +
+
+ htdig_auth_failure.html +
+
+ HTML page returned by mmsearch.py in the event of an authentication error for a page access. +
+
+ htdig_conf.txt +
+
+ template for the per-list htdig.conf files generated by the patched code. +
+
+
+
+

Configuration of Mailman-htdig Integration [ toc ]

+

+ Configuration of the Mailman-htdig integration is carried out on the Mailman side. While you must have to hand some information about your htdig installation, you should not have to tinker much with htdig for the integration to work. +

+

+ Most of the configuration of the integration is done by values assigned to python variables in either $prefix/Mailman/Defaults.py or $prefix/Mailman/mm_cfg.py. +

+

+ If you opt to run htdig on a different machine or under a different HTTP server to the one running the HTTP server which provides Mailman's Web UI you will also have to edit whichever of the patch's three htdig related cron scripts you opt to run (remote_nightly_htdig, remote_nightly_htdig_noshare, or remote_nightly_htdig.pl) to add a small amount of configuration information. +

+

Health Warning on the packet! [ toc ]

+

+ Be careful when editing configuration information in $prefix/Mailman/mm_cg.py: the only Mailman config file you should be editing. Check, double check and then recheck before going ahead. If you get either variable names or their values wrong a lot of confusion in the operation of both Mailman and htdig can result. +

+

+ You (and others supporting you) can spend hours trying to identify problems and looking for non-existent bugs as a consequence of such editing errors. Expect to find errors in these instructions; compensate for them and tell me when you do (r.barrett at openinfo.co.uk). +

+

+ Also do read the htdig documentation, release notes etc. This patch integrates a working htdig with htsearch available. These notes are about Mailman and integrating it with that working htdig. It is up to you to sort out the htdig end of things. +

+

Starting from Scratch (Again) [ toc ]

+

+ This is getting ahead of things but some of you may already be asking "What if I've already been using an older version of this patch and want to start afresh?", or "I want to change from local to remote htdig or vice versa?" +

+

+ In these cases your friend will be the $prefix/bin/blow_away_htdig script. It removes existing htdig related stuff out of your Mailman installation to the extent that it was added by this patch and added to by the normal operation of pipermail and nightly_htdig. With that removed and a revised Mailman configuration, the patched code will start rebuilding the htdig data. +

+

+ But before you get carried away with blow_away_htdig, read the rest of these notes. +

+

General [ toc ]

+

+ This patch adds a number of default variables to the file $prefix/Mailman/Defaults.py that affect operation of the Mailman-htdig integration. These are in addition to the standard Mailman defaults in that file. If, in the light of what is said below, you decide any of these are incorrect, you can override them in $prefix/Mailman/mm_cfg.py [NOT IN Defaults.py! See the comments in Defaults.py for why]. +

+

+ By default the Mailman-htdig integration is NOT ENABLED by the installation of this patch; the default value of the USE_HTDIG variable in Defaults.py turns off the operation of the integration. You have to actively override that default in mm_cfg.py to turn on operation of the integration. +

+

+ Once a list is created, changing most of these variables will have either no effect or a bad effect. You will need to run $prefix/bin/blow_away_htdig script and/or $prefix/bin/arch to rebuild the archive pages if you make significant changes to the Mailman-htdig integration configuration variables. +

+

+ The install process will not overwrite an existing mm_cfg.py file so you can freely make changes to this file. If you are re-installing a later version of this patch you may have to change what is already configured in the existing file and, if necessary, add extra configuration variables to it. +

+

+ Most of the Mailman-htdig control variables default to sensible values which you will not need to change, especially if you are using local htdig. The semantics of most variables apply to both local and remote htdig operation but with some the values assigned will depend on whether htdig is viewing things from the same or a remote machine. +

+

+ The first two variables control what is indexed by htdig. The values assigned are both embedded in the HTML generated by pipermail in the list archives and added. Changing the values of these variables will mean that all previously generated HTML pages in list archives will be out of date and you will probably want to rebuild existing archives using $prefix/bin/arch: +

+
+
+ ARCHIVE_INDEXING_ENABLE +
+
+

+ Defines a string telling htdig that it should look at the following material when building it indices. +

+
+    Default: ARCHIVE_INDEXING_ENABLE = '<!--/htdig_noindex-->'
+
+
+ ARCHIVE_INDEXING_DISABLE +
+
+

+ Defines a string telling htdig that it not should not look at the following material when building it indices. +

+
+    Default: ARCHIVE_INDEXING_DISABLE = '<!--htdig_noindex-->'
+
+
+ USE_HTDIG +
+
+

+ Semantics: 0 - don't use integrated htdig, 1 - use it +

+

+ Turns Mailman-htdig integration on or off. +

+
+    Defaults: USE_HTDIG = 0
+
+

+ Notes: +

+
    +
  1. +

    + when USE_HTDIG is turned on the patched code in Mailman will start adding htdig stuff for any archiving-enabled mail lists as new posts for eachlist are handled by Mailman. Until a new post is made after enabling with USE_HTDIG an existing mail list's archive will not be htdig searchable. When the new post is handled: +

    +
      +
    1. + the list's personalised htdig config file is created +
      2. +
    2. + necessary links to the htdig config file are created +
      3. +
    3. + a search form is added to the TOC page for the list +
      4. +
    +

    + Even with this done, htdig searches only become available when htdig indices are constructed. This is done when one or other of the patch's htdig related cron scripts are run (nightly_htdig, remote_nightly_htdig, remote_nightly_htdig_noshare, or remote_nightly_htdig.pl, depending on how you configure your system). These can be run from the command line ahead of their scheduled cron time to get htdig searches operational. +

    +
    2. +
  2. +

    + Turning USE_HTDIG off will not remove htdig indices or search forms from existing archive-enabled lists. It will however stop htdig features from being added to newly created lists. If you want to eliminate htdig from your existing lists then use the $prefix/bin/blow_away_htdig script. +

    +
    3. +
+
+
+ HTDIG_FILES_URL +
+
+

+ This is the URL of the directory containing various HTML and Graphics files installed by htdig; files such as buttonr.gif, buttonl.gif and button1-10.gif. The URL must end with a '/'. +

+
+    Default: HTDIG_FILES_URL = '/htdig/'
+

+ The default assumes the HTTP servers providing access to htdig and to Mailman's web UI are on the same machine and a symbolic link called 'htdig' has been put into your HTTP server's top level HTML directory which points to the directory your htdig install has put the actual files into; this link is often to /usr/share/htdig. This value will depend on your htdig installation decisions and HTTP server's configuration files (typically /etc/httpd/httpd.conf on a late model Apache installation) i.e the Alias through which the link to the htdig files are reached. +

+
+
+ HTDIG_CONF_LINK_DIR +
+
+

+ This is the name of a directory in which links to list specific htdig config files are placed. +

+
+    Default: HTDIG_CONF_LINK_DIR = os.path.join(VAR_PREFIX, 'archives', 'htdig')
+

+ The VAR_PREFIX of the default is resolved to an actual file system path when when Mailman's 'make install' is run. The 'os.path.join' creates a full file system path by gluing together the three pieces when Mailman is run. This definition puts the directory alongside the default PUBLIC_ARCHIVE_FILE_DIR and PRIVATE_ARCHIVE_FILE_DIR. Unless you are changing the value of these variables you probably do not want to change HTDIG_CONF_LINK_DIR. +

+
+
+ HTDIG_RUNDIG_PATH +
+
+

+ This is the path in your file system to the rundig shell script that is installed as part of htdig. This tells one or other of the patch's htdig related cron scripts (nightly_htdig and remote_nightly_htdig) where to find rundig in order that they can execute it. +

+
+    Default: HTDIG_RUNDIG_PATH = '/usr/local/bin/rundig'
+
+
+ HTDIG_HTSEARCH_PATH +
+
+

+ This is the file path to the htsearch program in the htdig package. +

+
+    Default: HTDIG_HTSEARCH_PATH = '/usr/local/bin/rundig'
+

+ This value will depend on your htdig installation decisions. This path is used by either the mmsearch CGI script (for local htdig) or the remote_mmsearch/remote-mmsearch CGI script (for remote htdig) to execute htsearch as a sub-process. +

+
+
+ HTDIG_EXCLUDED_URLS +
+
+

+ See htdig's configuration file documentation. The value of this MM variable is inserted into per-list htdig.conf files when they are created as the value of an htdig excluded_urls directive. But if an exclusion in this value would prevent indexing of URLs for accessing the htdig.py cgi wrapper then that exclusion is omitted from that per-list htdig.conf file. +

+
+    Default: HTDIG_EXCLUDED_URLS = '/cgi-bin/ .cgi'
+
+

+ Note: these are the same as the htdig 3.1.6 default values. +

+
+
+ REMOTE_HTDIG +
+
+

+ Semantics: 0 - htdig runs on local machine, 1 -on remote machine +

+

+ Says whether htdig going to be run on the same machine as Mailman or on another machine. +

+
+    Default: REMOTE_HTDIG = 0
+
+
+ REMOTE_PRIVATE_ARCHIVE_FILE_DIR +
+
+

+ Only relevant if REMOTE_HTDIG = 1. It is the file system path to the directory in which Mailman stores private archives, as seen by the machine running htdig. +

+
+    Default: REMOTE_PRIVATE_ARCHIVE_FILE_DIR = os.path.join(VAR_PREFIX, 
+                                              'archives', 'private')
+

+ The VAR_PREFIX of the default is resolved to an actual file system path when when Mailman's 'make install' is run. The 'os.path.join' creates a full file system path by gluing together the three pieces when Mailman is run. If you assign a value to this in mm_cfg.py, just put the relevant explicit file system path in. +

+
+
+ REMOTE_MMSEARCH_URL +
+
+

+ Only relevant if REMOTE_HTDIG = 1. It is the URL on the htdig machine through which whichever of the the remote_mmsearch/remote-mmsearch CGI scripts you have opted to use can be reached via an HTTP request. +

+
+    Default: REMOTE_MMSEARCH_URL = '/cgi-bin/remote-mmsearch'
+
+
+ HTDIG_STRICT_FILE_PERM +
+
+

+ Semantics: 0 - 'other' access allowed, 1 - 'other' access denied +

+

+ Says whether 'other' has access permissions for per-list $prefix/private/archives/<listname>/htdig/ directories. For local htdig operation such access is not required and is a security hole if allowd. Such access may be needed if remote htdig is used; see notes on "Apache". $prefix/bin/check_perms should be run after changing the value of this variable in mm_cfg.py to update access permissions of existing directories. +

+
+    Defaults: HTDIG_STRICT_FILE_PERM = 1
+
+
+
+ HTDIG_EXTRAS +
+
+

+ You can assign a string value to this config variable and that string will be included in all of your site's list specific htdig configuration files when they are created. The value of the string can be any attribute declarations as defined at http://www.htdig.org/confindex.html. +

+

+ Be cautious in what you do with this. Most sites will not need to use this at all. But if you have some idiosyncratic htdig installation it might help overcome problems in integrating with Mailman. If you think you need to use it I suggest: +

+
    +
  1. + You try creating a test list without assigning a value to HTDIG_EXTRAS in $prefix/Mailman/mm_cfg.py +
    2. +
  2. + Enable archiving for that test list. +
    3. +
  3. + Send a message to the test list so that its archive is created together with its htdig configuration file. +
    4. +
  4. + Review the content of the list's htdig conf file in $prefix/archives/private/<listname>/htdig/<listname>.conf. +
    5. +
  5. + You will see where the default value of HTDIG_EXTRAS from $prefix/Mailman/Defaults.py has been inserted. This value is onlyan htdig comment and does nothing. +
    6. +
  6. + Consider whether what you will assign to HTDIG_EXTRAS in $prefix/Mailman/mm_cfg.py will make sense in the context of the rest of the htdig conf file's contents. +
    7. +
+
+
+

Permissions Considerations [ toc ]

htdig [ toc ]

+

+ Python scripts added by this patch (nightly_htdig and its relatives) run the htdig rundig script identified by HTDIG_RUNDIG_PATH to build search indices for Mailman archives. Code added by this patch generates per-list htdig configuration files which are passed as a parameter to the rundig script. These configuration files identify a list specific directory ($prefix/archives/private/lt;listname>/htdig) in which list specific data files generated by and used by htdig are to be placed. +

+

+ However, the rundig script identified by HTDIG_RUNDIG_PATH may attempt to generate some files in htdig's COMMON_DIR when it is first run by nightly_htdig; the files concerned are likely to be root2word.db, word2root.db, synonyms.db and possibly some others generated by htidg's htfuzzy program. The standard rundig script generates these files selectively if they do not already exist. Depending on how you have installed htdig and how the rundig script is first run, there may be a permissions problem when nightly_hdig executes rundig under the mailman UID if it tries to generate these files. +

+

+ You may need to either give the mailman UID write permission over htdig's COMMON_DIR or, before the nightly_htdig script is first run, run htdig's htfuzzy executable with a sufficiently privileged UID in the manner that the rundig script would run htfuzzy, to create any necessary files in COMMON_DIR. +

+

+ See htdig's documentation for further information on this topic. +

+

Apache [ toc ]

+

+ When remote_mmsearch or remote-mmsearch scripts are used as part of a remote htdig strategy you may encounter a file permissions problem. This is because these scripts, which in turn execute htsearch as a sub-process, will be run with UID and GID of the remote Apache server. +

+

+ By default, the permissions of the per-list $prefix/private/archives/<listname>/htdig/ directories only allow access for the mailman UID and GID and hence the remotely executed htsearch will be unable to access them. +

+

+ If this problem is encounterd, then you will have to use the HTDIG_STRICT_FILE_PERM configuration variable to say "open up the permissions" before running $prefix/bin/check_perms. You can then use a RewriteRule or similar in the Apache server's httpd.conf file to restrict access to $prefix/private/archives/<listname>/htdig/ directories via the web server. +

+

Local htdig Configuration [ toc ]

+

+ This configuration is for when you are running Mailman, htdig, the HTTP server used to provide Mailman's web UI and htdig's htsearch CGI script, on the same machine. +

+

+ You will need to: +

+
    +
  1. + If different to the default value, add the definition of HTDIG_RUNDIG_PATH to file $prefix/Mailman/mm_cfg.py. +
    2. +
  2. + If different to the default value, add the definition of HTDIG_HTSEARCH_PATH to file $prefix/Mailman/mm_cfg.py. +
    3. +
  3. + Add the definition of USE_HTDIG with the value 1 to $prefix/Mailman/mm_cfg.py. +
    4. +
+
+        USE_HTDIG = 1
+

+ If necessary you can override the values of any of the other configuration variables in file $prefix/Mailman/mm_cfg.py. +

+

+ In particular you might need to change the HTDIG_FILES_URL variable from its default. This URL can be just the path i.e. absolute URL on the same server as that which serves Mailman's Web UI, or a full URL identifying the scheme (http), server, server port and path, for example http://mailer.yourdomain.tld:8080/htdig/ +

+

Remote htdig Configuration [ toc ]

+

+ This configuration is for when you are running htdig and an HTTP server providing access to htsearch via remote_mmsearch or remote-mmsearch on a different machine to that is running Mailman. +

+

+ For this configuration to work, htdig's programs, both those run from command lines such as rundig and those run via CGI such as htsearch, must be able to see Mailman archives through NFS. In the examples below we'll assume that /mnt/mailman-archives on the htdig machine maps to $prefix/mailman/archives on the Mailman machine. +

+

+ You should also arrange for he mailman UID and its GID to be common to both machines. Remember that when rundig is called on the htdig machine to produce search indices for each list it will be trying to write those files via NFS in Mailman's archive area and will thus need to run with an appropriate identity and permissions. +

+

+ The differences between the local and remote configuration are: +

+
    +
  1. + configuration values telling htdig where to find files are as viewed from the remote machine. +
    2. +
  2. + configuration values giving URLs that refer to htdiggy things have to be as viewed from the Mailman machine. +
    3. +
+

+ You will need to: +

+
    +
  1. + Add the definition of HTDIG_HTSEARCH_PATH to file $prefix/Mailman/mm_cfg.py. This is path to htdig's htsearch on the remote machine running htdig. For example: 
    +    HTDIG_HTSEARCH_PATH = '/usr/local/bin/htsearch'
    +
    2. +
  2. + Add the definition of HTDIG_RUNDIG_PATH to file $prefix/Mailman/mm_cfg.py. This is path to rundig on the remote machine running htdig. For example: 
    +    HTDIG_RUNDIG_PATH = '/usr/local/bin/rundig'
    +
    3. +
  3. + Add the definition of REMOTE_MMSEARCH_URL to file $prefix/Mailman/mm_cfg.py. This must be a full URL referring to one of Mailman's remote_mmsearch/remote-mmsearch CGI scripts on the remote htdig machine, as seen from the Mailman local machine. For example: 
    +    REMOTE_MMSEARCH_URL = 'http://htdiggy.your.com/cgi-bin/remote-mmsearch'
    +
    4. +
  4. + Add the definition of HTDIG_FILES_URL to file $prefix/Mailman/mm_cfg.py. This must be a full URL referring to the directory containing htdig files on the remote htdig machine as seen from the Mailman local machine. This URL must end with a '/'. For example:
    +    HTDIG_FILES_URL = 'http://htdiggy.your.com/htdig/'
    +
    5. +
  5. + Add the definition of REMOTE_PRIVATE_ARCHIVE_FILE_DIR to $prefix/Mailman/mm_cfg.py. This must be the absolute file system path to the directory in which Mailman stores private archives as seen by the machine running htdig. For example: 
    +    REMOTE_PRIVATE_ARCHIVE_FILE_DIR = '/mnt/mailman-archives/private'
    +
    6. +
  6. + Add the definition of USE_HTDIG with the value 1 to $prefix/Mailman/mm_cfg.py. 
    +    USE_HTDIG = 1
    +
    7. +
  7. + Add the definition of REMOTE_HTDIG with the value 1 to $prefix/Mailman/mm_cfg.py. 
    +    REMOTE_HTDIG = 1
    +
    8. +
  8. + If necessary add the definition of HTDIG_STRICT_FILE_PERM with the value 0 to $prefix/Mailman/mm_cfg.py. This may be needed it the UID/GID that Apache on the htdig server will run the remote mmsearch as is not mailman or in the mailman group. This change will open up a security hole which you may want to consider plugging; see under the heading "Apache permissions" for more details.
    +    HTDIG_STRICT_FILE_PERM = 0
    +
    9. +
+

+ You have to choose one of the two remote mmsearch scripts found in $prefix/Mailman/Cgi - remote-mmsearch (a Perl script) and remote_mmsearch (a Python script) - to use and transfer it to the htdig machine. You need to add this script to the directory in which the web server on the htdig machines expects to find CGI scripts. Having transferred the script to you htdig machine you will need to use a text editor to set the values of four configuration variables below the heading "Edit the following configuration variables to suit your installation", namely: +

+
+
+
+ MAILTO +
+
+ this is the default mail address for your installation. +
+
+ VALID_IP_LIST +
+
+ this is a list of IP numbers from which the script should accept an HTTP request. Normally this should be set to the IP number of your machine running Mailman. If the list is empty the script will accept HTTP requests from any machine and be vulnerable to the exploit described under the heading "Private archive security problem prior to htdig-2.1.1-0.2.patch version" above. +
+
+ HTDIG_CONF_LINK_DIR +
+
+ this is the file path to the directory in which links to list specific htdig config files are placed, as viewed from the remote machine running htdig. +
+
+ HTDIG_HTSEARCH_PATH +
+
+ this is the file path to the htsearch program in the htdig package as viewed from the remote machine running htdig. +
+
+

+ See "What is Installed by the Patch" for an explanation of the differences between these remote mmsearch scripts which both do the same job: being a security wrapper around htdig's htsearch program to restrict searching of a list's archive indexes to users authorised to see the contents of that archive. +

+

+ Note: You may need to change the '#!' on the first line of whichever of the remote-mmsearch (Perl) and remote_mmsearch (Python) scripts you opt for so that the correct interpreter is used for running the script on the remote htdig machine. You may also need to verify the supporting packages/modules used by the selected script are installed on that system. +

+
+

+ You have to choose one of the three remote_nightly_htdig scripts found in $prefix/cron - remote_nightly_htdig, remote_nightly_htdig_noshare and remote_nightly_htdig.pl - and transfer it to the htdig machine. See above under heading "What is Installed by the Patch" for an explanation of the differences between these scripts, which all do the same basic job. You should add the script to the crontab for the mailman UID on the htdig machine. But first you need to edit the selected script to add some configuration information. What has to be added depends on which script you opt to use. In each case the variables concerned are declared near the top of the script and you just have to enter the appropriate values: +

+
+
+
+ remote_nightly_htdig +
+
+

+ you only need to set the value of the python variable MAILMAN_PATH to be the directory $prefix as seen from the htdig machine. The whole Mailman installation must be accessible via NFS in order to use this script. +

+
+
+ remote_nightly_htdig_noshare +
+
+

+ you need to copy the values for the following configuration variables from either $prefix/Mailman/mm_cfg.py or $prefix/Mailman/Defaults.py to the script: REMOTE_PRIVATE_ARCHIVE_FILE_DIR, HTDIG_RUNDIG_PATH. The variables declared in remote_nightly_htdig_noshare use the same names. This script only requires that the archives directory of the Mailman installation be accessible via NFS. +

+
+
+ remote_nightly_htdig.pl +
+
+

+ you need to copy the values for the following configuration variables from either $prefix/Mailman/mm_cfg.py or $prefix/Mailman/Defaults.py to the script: REMOTE_PRIVATE_ARCHIVE_FILE_DIR, HTDIG_RUNDIG_PATH. Being a Perl script, the variables in remote_nightly_htdig.pl use the same names but prefixed with the '$' character. This script only requires that the archives directory of the Mailman installation be accessible via NFS. +

+
+
+

+ Note: You may need to change the '#!' on the first line of whichever of these scripts you opt for so that the correct interpreter is used for running the script on the remote htdig machine. You may also need to verify the supporting packages/modules used by the selected script are installed on that system. +

+

+ As with the nightly_htdig script when running with local htdig, these scripts can be run from the command line using the mailman UID in order to get htdig to construct an initial set of indices. +

+
+

Upgrading an Existing Standard Mailman Installation [ toc ]

+
    +
  1. + You will want to suspend operation of Mailman while doing the upgrade. Consider doing a shutdown of the MTA delivering mail to Mailman and removing Mailman's crontab. +
    2. +
  2. + Configure and install as described above. +
    3. +
  3. + Restart Mailman's crontab and restart your MTA's delivery to Mailman. +
    4. +
  4. + If your installation already has archives: +
      +
    1. + Send a message to each of your archive-enabled lists. This will stimulate the setup of the new per list htdig config files in the Mailman archives. +
      2. +
    2. + Consider rebuilding your existing archives with $prefix/bin/arch. This will embed the ARCHIVE_INDEXING_ENABLE and ARCHIVE_INDEXING_DISABLE in the regenerated archive pages and, after nightly_htdig has been run, give improved search results. +
      3. +
    3. + Run the nightly_htdig script from the command line to generate an initial set of per-list htdig search indices. +
      4. +
    +
    5. +
+

Changing from local to remote htdig or vice versa [ toc ]

+
    +
  1. + You will want to suspend operation of Mailman while making this change. Consider doing a shutdown of the MTA delivering mail to Mailman and removing Mailman's crontab. +
    2. +
  2. + Run the $prefix/bin/blow_away_htdig script to remove all existing per list htdig config files and htdig indices/db files. +
    3. +
  3. + Configure per the instructions above for the local or remote target. +
    4. +
  4. + Restart Mailman's crontab and restart your MTA's delivery to Mailman. +
    5. +
  5. + Send a message to each of your archive-enabled lists. This will stimulate the set up of the new per list htdig config files in Mailman archives. +
    6. +
  6. + Run the nightly_htdig script from the command line to generate a new set of per list htdig search indices. +
    7. +
+

Coping with htdig Upgrades [ toc ]

+

+ If you change the version of htdig you run, you may find that the indices built with the earlier version are not compatible with the newer version of htdig's programs. In that case do the following: +

+
    +
  1. + You will want to suspend operation of Mailman while making this change. Consider doing a shutdown of the MTA delivering mail to Mailman and removing Mailman's crontab. +
    2. +
  2. + Run the $prefix/bin/blow_away_htdig script with the -i flag to remove all existing per list htdig indices/db files. +
    3. +
  3. + Restart Mailman's crontab and restart your MTA's delivery to Mailman. +
    4. +
  4. + Run the nightly_htdig script from the command line to generate new sets of per-list htdig search indices. +
    5. +
+

Changing the Addressing Scheme of your web_page_url [ toc ]

+

+ If you change the addressing scheme of the web_page_url for a list to or from http then you will need to rebuild the list's htdig configuration file(s) and the related htdig indices. Do the following: +

+
    +
  1. + You may want to suspend operation of Mailman while making this change. Consider doing a shutdown of the MTA delivering mail to Mailman and removing Mailman's crontab. +
    2. +
  2. + Run the $prefix/bin/blow_away_htdig script to remove all existing per list htdig material for the list(s) concerned. +
    3. +
  3. + Restart Mailman's crontab and restart your MTA's delivery to Mailman. +
    4. +
  4. + Send a message to each affected list to provoke reconstruction of the list's htdig config file(s). +
    5. +
  5. + Run the nightly_htdig script from the command line to generate new sets of per list htdig search indices. +
    6. +
+

Operational Information [ toc ]

+

+ If you have just turned USE_HTDIG on or just used $prefix/bin/blow_away_htdig (without the -i flag) there will be no per-list htdig information saved in the archives. +

+

+ When the first post to each archive-enabled list is archived by pipermail, the per-list htdig config file will be constructed and some directories and links added to your Mailman archive directories. The htdig search form will be added to list's TOC page. +

+

+ However, until one of the nightly_htdig scripts is run no htdig indices will be constructed. You can either wait for the script to run as a cron job or run it (while using the mailman UID) from the command line. +

+

Notes and Warnings [ toc ]

Archive security problems resolved by htdig-2.1.3-0.2 patch

+

+ This patch is hopefully the final step in closing security holes in archive access. +

+

+ In version htdig-2.1.3-0.1.patch, htdig.py was rebased on the standard MM release's private.py which had moved on since the snapshot of it used as the basis for htdig.py was originally taken. Among other things, htdig.py had been modified to prevent access to some files in list archive directories such as a list's archive pipermail.pck and files in the list's archive database sub-directory. +

+

+ This rebasing action re-introduced to htdig.py the security holes, still extant in private.py despite it being later code, via which private.py would serve files such as a list's archive pipermail.pck and files in the list's archive database sub-directory. +

+

+ The permissions on these files and directories mean that they are inaccessible via the web server using /pipermail/ URIs if a list's archive is public. +

+

+ Additionally, check_perms is now modified so that the list archive htdig subdirectory permissions are set to 2770 by default. Prior to htdig-2.1.1-0.2.patch, this could not be done as the htsearch script, being run with uid and gid of the Apache server, could then not gain access to files in the htdig subdirectories. But, since the introduction of the mmsearch script, which runs with the mailman gid and spawns htsearch, it can. This prevents accees to the list archive htdig subdirectories via /pipemail/ URI's. Up until htdig-2.1.3-0.2.patch this could only be achieved by using a RewriteRule or similar in the Apache server's httpd.conf file. +

+ + The only residual problem is that the revised permissions on the archive htdig subdirectories may cause problems if the remote_mmsearch and remote-mmsearch are used. This is because they will be run with uid and gid of the Apache server. If this problem is encounterd, then you will have to manually add read and execute permissions for 'other' to the archive htdig subdirectories and read permission their contents, and then use RewriteRule or similar in the Apache server's httpd.conf file for protection. + +

+ The solution to this problem has been superceded in htdig-2.1.3-0.3.patch as follows: Introduced the HTDIG_STRICT_FILE_PERM Mailman config variable as part of dealing with htsearch access to per-list htdig directories permissions issue when operating with remote htdig. See under the "Apache" heading above. +

+

Private archive security problem prior to htdig-2.1.1-0.2.patch version [ toc ]

+

+ Versions of the Mailman-htdig integration patch installed by versions of this patch prior to htdig-2.1.1-0.2.patch allow a security exploit which can expose information, held in the per-list search indexes of private list archives, to unauthorised users. +

+

+ Via the exploit an unauthoized user can submit a search query to htdig's htsearch CGI program without their having been authenticated as a user allowed to access the list archive concerned. The results, returned in good faith by htsearch, will expose some information that the user is not entitled to see. +

+

+ However, the security breakdown is not complete. Attempts to follow links returned by htsearch, which go via the htdig CGI script installed by this patch, will be blocked if the user is not authorized to access the list archive. +

+

Maintaining archive security with htdig-2.1.1-0.2.patch version and later [ toc ]

+

+ With htdig-2.1.1-0.2.patch and later versions of the patch: +

+
    +
  1. + htsearch is no longer used directly via CGI for searching list archives. +
    2. +
  2. + The symbolic link named by the HTDIG_MAILMAN_LINK configuration variable is no longer used. Indeed, when upgrading earlier installations this symlink should be deleted and the configuration variable deleted. Without this symlink, on a normally configured system, htsearch no longer has the unaided ability to access the per-list htdig configuration and other list archive associated files. +
    3. +
  3. + Thus, even if htsearch can be reached via CGI, it cannot undertake a search of list archives when requested to do so by an HTTP request which seeks to circumvent list archive security. +
    4. +
  4. + A new script, $prefix/Mailman/Cgi/mmsearch.py, is now used to search list archives. This script applies the same user authentication as private.py and htdig.py. Only if a user is authorised to access a list, does mmsearch use htdig's htsearch to search a list's archive. In this case, mmsearch provides htsearch with the information it needs to access the per-list htdig configuration and other list archive associated files. +
    5. +
  5. + Where htidg and Mailman are run on the same machine, mmsearch acts as a security wrapper, runs htsearch as a sub-process and list security is preserved by this means. +
    6. +
  6. + Where htdig is run on a different machine to Mailman, mmsearch can perform user authentication but has problems in acting as a security wrapper for htsearch. The solution adopted is for one of two companion CGI scripts (remote-mmsearch written in Perl or remote_mmsearch written in Python) to be invoked on the remote htdig machine by an HTTP request made by mmsearch on the Mailman machine. These scripts run htsearch, providing it with the information it needs to access the per-list htdig configuration and other list archive associated files. But, such an HTTP request can be made by other means and thus the the same security exploit we are trying to avoid still exists. The only protection in the case of remote htdig operation is that the remote-mmsearch/remote_mmsearch scripts can be configured to operate only on HTTP requests originating from specified IP numbers. By restricting operation to requests originating on the Mailman server some semblance of list privacy can be preserved. +
    7. +
+

Upgrading to htdig-2.1.1-0.2.patch or later from an earlier patch version [ toc ]

+

+ If you are upgrading a Mailman installation that has an earlier version of the the Mailman-htdig integration patch than that installed by htdig-2.1.1-0.2.patch or later, you need to make some changes to that installation: +

+
    +
  1. + You must delete from your file system the symbolic link named by the HTDIG_MAILMAN_LINK Mailman configuration variable. This link previously gave htdig programs access to per list htdig configuration files. This is now done by other means and the symlink allows a security exploit that prejudices the privacy of list archives. +
    2. +
  2. + You must delete the HTDIG_MAILMAN_LINK Mailman configuration variable from the $prefix/Mailman/mm-cfg.py file. +
    3. +
+

+ These changes are in addition to the normal installation instructions given below. Having configured and installed the newly patched version of Mailman you must: +

+
    +
  1. + Run the script $prefix/bin/blow_away_htdig with the -c option to rebuild per-list htdig conf files and delete existing per-list search indexes. +
    2. +
  2. + Run the $prefix/cron/nightly_htdig script from the command line to rebuild per-list search indexes using the revised per-list htdig conf files just created by blow_away_htdig. +
    3. +
+

Redhat 7.1 and 7.2 installations [ toc ]

+

+ If you install htdig from the htdig-3.2.0 binary rpm of RH7.1/2 Binary CD 1 of 2 you also have to install the htdig-web-3.2.0 binary rpm. This may be from RH 7.1/2 Binary CD 2 of 2 or CD 1 of 2 depending on whether you are using actual CDs or downloaded CD images. +

+

Apache/htdig issues [ toc ]

+

+ htdig's graphics file must be accessible via you web server and the Mailman configuration variable HTDIG_FILES_URL setup accordingly. Depending on how you install htdig and Apache you may need to add Alias and/or ScriptAlias directives to you Apache configuration file to make the htdig components accessible. Check the Apache and htdig documentation. +

+

Contributors [ toc ]

+
+
+ Original author and maintainer: +
+
+ Richard Barrett - <r.barrett at openinfo.co.uk> +
+
+ Past bug fixes: +
+
+
    +
  • + Nigel Metheringham <nigel.metheringham at vdata.co.uk> +
    • +
  • + Stephan Berndts <stb-mm at spline.de> +
    • +
+
+
+ Testers: +
+
+
    +
  • + Mark T. Valites <valites at geneseo.edu> +
    • +
  • + Rehan van der Merwe <rehan at nha.co.za> +
    • +
+
+
+ Suggested Improvements: +
+
+
    +
  • + Mark Sapiro <msapiro at msapiro.net> +
    • +
+
+
+

History [ toc ]

Compatibility [ toc ]

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Version of patch + + Version of Mailman +
+ htdig-2.1.10-0.1.patch + + Mailman 2.1.10 +
+ htdig-2.1.9-0.1.patch + + Mailman 2.1.9 +
+ htdig-2.1.7-0.1.patch + + Mailman 2.1.7 and 2.1.8 +
+ htdig-2.1.6-0.1.patch + + Mailman 2.1.6 +
+ htdig-2.1.4-0.1.patch + + Mailman 2.1.4 +
+ htdig-2.1.3-0.5.patch + + Mailman 2.1.3 +
+ htdig-2.1.3-0.4.patch + + Mailman 2.1.3 +
+ htdig-2.1.3-0.3.patch + + Mailman 2.1.3 +
+ htdig-2.1.3-0.2.patch + + Mailman 2.1.3 +
+ htdig-2.1.3-0.1.patch + + Mailman 2.1.3 +
+ htdig-2.1.2-0.4.patch + + Mailman 2.1.2 +
+ htdig-2.1.2-0.3.patch + + Mailman 2.1.2 +
+ htdig-2.1.2-0.2.patch + + Mailman 2.1.2 +
+ htdig-2.1.2-0.1.patch + + Mailman 2.1.2 +
+ htdig-2.1.1-0.5.patch + + Mailman 2.1.1 +
+ htdig-2.1.1-0.4.patch + + Mailman 2.1.1 +
+ htdig-2.1.1-0.3.patch + + Mailman 2.1.1 +
+ htdig-2.1.1-0.2.patch + + Mailman 2.1.1 +
+ htdig-2.1.1-0.1.patch + + Mailman 2.1.1 +
+ htdig-2.1-0.3.patch + + Mailman 2.1 +
+ htdig-2.1-0.2.patch + + Mailman 2.1 +
+ htdig-2.1-0.1.patch + + Mailman 2.1 +
+ htdig-2.1b6-0.1.patch + + Mailman 2.1b6 +
+ htdig-2.1b5-0.1.patch + + Mailman 2.1b5 +
+ htdig-2.1b4-0.1.patch + + Mailman 2.1b4 +
+ htdig-2.1b3-0.3.patch + + Mailman 2.1b3 +
+ htdig-2.1b3-0.2.patch + + Mailman 2.1b3 +
+ htdig-2.1b3-0.1.patch + + Mailman 2.1b3 +
+ htdig-2.1b2-0.1.patch + + Mailman 2.1b2 +
+ htdig-2.0.13-0.2.patch + + Mailman 2.0.13 +
+ htdig-2.0.13-0.1.patch + + Mailman 2.0.13 +
+ htdig-2.0.12-0.1.patch + + Mailman 2.0.12 +
+ htdig-2.0.11-0.1.patch + + Mailman 2.0.11 +
+ htdig-2.0.10-0.2.patch + + Mailman 2.0.10 +
+ htdig-2.0.10-0.1.patch + + Mailman 2.0.10 +
+ htdig-2.0.9-0.1.patch + + Mailman 2.0.9 +
+ htdig-2.0.8-0.1.patch + + Mailman 2.0.8, 2.0.7, 2.0.6 and probably 2.0.3, 2.0.4 and 2.0.5 +
+

Changes [ toc ]

+
+
+ htdig-2.1.10-0.1.patch: +
+
+
    +
  1. + updated patch for MM 2.1.10 compatibility. +
    2. +
  2. + Change to setup_htdig() function in HyperArch.py suggested by Mark Sapiro to ensure correct permissions set on directory creation. +
    3. +
+
+ +
+ htdig-2.1.9-0.1.patch: +
+
+
    +
  1. + updated patch for MM 2.1.9 compatibility. +
    2. +
+
+
+ htdig-2.1.7-0.1.patch: +
+
+
    +
  1. + updated patch for MM 2.1.7 compatibility. +
    2. +
+
+
+ htdig-2.1.6-0.1.patch: +
+
+
    +
  1. + updated patch for MM 2.1.6 compatibility. +
    2. +
  2. + Note: the templates in $build/templates/<lang>/for the following languages are NOT modified by this patch or by its precursor indexing patch: ca, eu, sr, sv
    + The following files in a language's default template directory should be modified per the changes made to the en language templates after installation of this patch if that other language is used ;' +
      +
    • templates/<lang>/archidxfoot.html
      • +
    • templates/<lang>/archidxhead.html
      • +
    • templates/<lang>/archtoc.html
      • +
    • templates/<lang>/archtocentry.html
      • +
    • templates/<lang>/archtocnombox.html
      • +
    • templates/<lang>/article.html
      • +
    +
    3. +
+
+
+ htdig-2.1.4-0.1.patch: +
+
+
    +
  1. + updated patch for MM 2.1.4 compatibility. +
    2. +
  2. + removed untranslated versions of htdig.html from per-language directories under $build/templates, with the exception of the default templates/en/ directory, that were present in previous versions of this patch. +
    3. +
+
+
+ htdig-2.1.3-0.5.patch: +
+
+
    +
  1. + Modified htdig.py and private.py; the security changes introduced by htdig-2.1.3-0.2 patch to these scripts incorrectly blocked access to the <listname>.mbox/<listname>.mbox file. The O.5 revison of the patch corrects this error. This problem and a suggested fix were pointed out to me in a private email by Stephan Berndts <stb-mm at spline.de> +
    2. +
+
+
+ htdig-2.1.3-0.4.patch: +
+
+
    +
  1. + Modified htdig.py and introduced htdig.html templates. The changes mean that if the user is challenged for authentication, when the credentials are submitted and accepted, the URL requested which led to the challenge is then presented. +
    2. +
+
+
+ htdig-2.1.3-0.3.patch: +
+
+
    +
  1. + Patch documentation layout revised and simplified. +
    2. +
  2. + Changes to $prefix/bin/check_perms and $prefix/Mailman/Archiver/HyperArch.py to improve handling of htdig subdirectory permissions if remote htdig is used. End result is the same as with prior patch version in the case of local htdig. +
    3. +
  3. + Introduced the HTDIG_STRICT_FILE_PERM Mailman config variable as part of dealing with htsearch access to per-list htdig directories permissions issue when operating with remote htdig. See under the "Apache" heading above. +
    4. +
+
+
+ htdig-2.1.3-0.2.patch: +
+
+
    +
  1. + This patch is hopefully the final step in closing security holes in archive access. See the discussion below under the heading "Archive security problems resolved by htdig-2.1.3-0.2 patch". +
    2. +
+
+
+ htdig-2.1.3-0.1.patch: +
+
+
    +
  1. + updated patch for MM 2.1.3 compatibility. +
    2. +
+
+
+ htdig-2.1.2-0.4.patch: +
+
+
    +
  1. + corrected error in mmsearch.py and remote_mmsearch. This caused a problem if https was being used for accessing the archives as a pattern match to extract the list name was misused. +
    2. +
+
+
+ htdig-2.1.2-0.3.patch: +
+
+
    +
  1. + updates HyperArch.py so htdig related code uses quick_maketext() function instead of the Utils.Maketext() function. +
    2. +
+
+
+ htdig-2.1.2-0.2.patch: +
+
+
    +
  1. + corrects stupid error inserted in unpublished htdig-2.1.1-0.5.patch and carried forward into htdig-2.1.2-0.1.patch +
    2. +
+
+
+ htdig-2.1.2-0.1.patch: +
+
+
    +
  1. + updated patch for MM 2.1.2 compatibility +
    2. +
+
+
+ htdig-2.1.1-0.5.patch: +
+
+
    +
  1. + with previous version the protototype htdig_conf.txt contained an htdig exclude_urls directive for /cgi-bin/ and .cgi. If MM is configured so that the URL for accessing the htdig.py cgi wrapper matches these excluded URLS (for instance by running ./configure with --with-cgi-ext=".cgi") then nothing gets indexed by rundig. The revised patch: +
      +
    1. + makes the excluded URL configurable through a MM config variable HTDIG_EXCLUDED_URLS which defaults to the old hard-wired value. +
      2. +
    2. + when generating a list-specific htdig.conf file a check is made against HTDIG_EXCLUDED_URLS and if anything in it would prevent indexing of the URL for accessing the htdig.py cgi wrapper for that list, it is omitted from the exclude_urls directive in that htdig.conf file. +
      3. +
    +
    2. +
+
+
+ htdig-2.1.1-0.4.patch: +
+
+
    +
  1. + mmsearch.py and its remote kin remote-mmsearch and mm_search were overly restrictive on the form fields they were willing to accept. Extended the list so that multi-page search results worked. +
    2. +
+
+
+ htdig-2.1.1-0.3.patch: +
+
+
    +
  1. + corrects silly error in raising an excpetion in mmsearch.py. This will only show if there is a problem with mmsearch running the htsearch program. +
    2. +
+
+
+ htdig-2.1.1-0.2.patch: +
+
+
    +
  1. + This version corrects a security exploit which allowed a URL to obtain an htsearch results page without the user being authorised to access the list. Any attempt to follows links on the results page were blocked correctly by $prefix/Mailman/htdig.py but there was leakage of private information from the list's search indexes on the page returned by htdig's htsearch CGI program. The exploit is removed by this patch's revisions. The following sections describe the problem, the solution and special actions required when updating a Mailman installation using an earlier version of this patch: +
      +
    1. + Private archive security problem prior to htdig-2.1.1-0.2.patch version. +
      2. +
    2. + Maintaining private archive security with htdig-2.1.1-0.2.patch version and later. +
      3. +
    3. + Upgrading to htdig-2.1.1-0.2.patch or later from an earlier patch version. +
      4. +
    + Note that there is no patch revision to deal with this security problem for MM 2.0.13 or earlier and you should seriously consider updating to MM 2.1.x if you want to implement this security fix. +
    2. +
+
+
+ htdig-2.1.1-0.1.patch: +
+
+
    +
  1. + No functional change. Applies without offset warnings to MM 2.1.1 +
    2. +
+
+
+ htdig-2.1-0.3.patch: +
+
+
    +
  1. + corrects errors in the way $prefix/Mailman/htdig.py worked out content type of file being returned. +
    2. +
  2. + $prefix/Mailman/htdig.py adopts revised method for establishing the default URL introduced in 2.1 and as used in $prefix/Mailman/MailList.py +
    3. +
  3. + removed unecessary setup of variable DEFAULT_URL in cron scripts $prefix/cron/remote_nightly_htdig_noshare and $prefix/cron/remote_nightly_htdig.pl +
    4. +
  4. + Changes references to DEFAULT_URL in this document to DEFAULT_URL_PATTERN. +
    5. +
+
+
+ htdig-2.1-0.2.patch: +
+
+
    +
  1. + improved content type and security handling in $prefix/Mailman/htdig.py. Fixes bug with htdig.py and problem of interaction with bug in $prefix/scripts/driver script (see patch #668685 for more details) +
    2. +
+
+
+ htdig-2.1-0.1.patch: +
+
+
    +
  1. + Reworked patch for compatibility with MM 2.1. +
    2. +
+
+
+ htdig-2.1b6-0.1.patch: +
+
+
    +
  1. + Reworked patch for compatibility with MM 2.1b6. +
    2. +
+
+
+ htdig-2.1b5-0.1.patch: +
+
+
    +
  1. + Reworked patch for compatibility with MM 2.1b5. +
    2. +
+
+
+ htdig-2.1b4-0.1.patch: +
+
+
    +
  1. + Reworked patch for compatibility with MM 2.1b4. As a consequence, the remainder of the mailman-htdig integration templates that were strings declared in Mailman/Archiver/HyperArch.py have been extracted into files under the templates directory. Edit these with care if you must. +
    2. +
+
+
+ htdig-2.1b3-0.3.patch: +
+
+
    +
  1. + Removed unecessary code dependency on Python 2.2 file() function +
    2. +
+
+
+ htdig-2.1b3-0.2.patch: +
+
+
    +
  1. + Removed syntax error in htdig-2.1b3-0.1.patch which showed up as logged errors in the operation of the ArchRunner qrunner at line 721 of HyperArch.py +
    2. +
+
+
+ htdig-2.1b3-0.1.patch: +
+
+
    +
  1. + Reworked patch for compatibility with MM 2.1b3 +
    2. +
  2. + Removed non-English language template files which were acting as placeholders until someone actually translated them. +
    3. +
  3. + Removed updateTOC.py and replaced it with an alternate mechanism in a patch to $prefix/Mailma/Queue/ArchRunner.py to update list TOC page after reindexing by htdig. This new method is only exercised when the remote_nightly_htdig series of cron scripts are used. +
    4. +
  4. + Changes to remote_nightly_htdig series of cron scripts to reflect demise of updateTOC cgi script. +
    5. +
  5. + Multiple instances of code hygiene and conformance to MM "standards" cleanup. +
    6. +
  6. + Tidied up this documentation. +
    7. +
+
+
+ htdig-2.1b2-0.1.patch: +
+
+
    +
  1. + reworked patch for compatibility with MM 2.1b2 +
    2. +
+
+
+ htdig-2.0.13-0.2.patch: +
+
+
    +
  1. + Added license header +
    2. +
+
+
+ htdig-2.0.13-0.1.patch: +
+
+
    +
  1. + Rebuilt patch to get no-comment application on Mailman 2.0.13 +
    2. +
+
+
+ htdig-2.0.12-0.1.patch: +
+
+
    +
  1. + Rebuilt patch to get no-comment application on Mailman 2.0.12 +
    2. +
  2. + Added HTDIG_EXTRAS xonfig variable to allow arbitrary htdig configuration parameters to be specified for addition to every htdig.conf file created i.e. site wide additions. +
    3. +
+
+
+ htdig-2.0.11-0.1.patch: +
+
+
    +
  1. + No substantive change. Simply rebuilt patch to get no-comment application on Mailman 2.0.11 +
    2. +
+
+
+ htdig-2.0.10-0.2.patch: +
+
+
    +
  1. + Python 2.2 compatibility fixes to nightly_htdig cron script and its relatives. Doing import * inside a function removed. +
    2. +
  2. + Added note on potential problems with htdig and file permissions. +
    3. +
+
+
+ htdig-2.0.10-0.1.patch: +
+
+
    +
  1. + change in src/Makefile.in to get clean patch application to MM 2.0.10 +
    2. +
+
+
+ htdig-2.0.9-0.1.patch: +
+
+
    +
  1. + minor cosmetic changes to get clean patch application to MM 2.0.9 +
    2. +
+
+
+ htdig-2.0.8-0.1.patch: +
+
+
    +
  1. + resolves a problem with the integration of htdig when the web_page_url for a list, which is usually the same as DEFAULT_URL from either $prefix/Mailman/Defaults.py or $prefix/Mailman/mm_cfg.py, when it doesn't use the http addressing scheme. This arises because htdig will only build indices if the URLs for pages use the http addressing scheme. There is a work-around for this problem posted in htdig's mail archives - see the copy in Appendix 1 to this document. +
    2. +
  2. + This patch revision implements the solution documented in that e-mail. If non-http URLs are used by the web_page_url of a list an additional htdig configuration file for use by htsearch is generated. +
    3. +
  3. + In all other respects the operation of the Mailman-htdig integration remains unchanged. There is no benefit in upgrading to this revised patch unless you need to use other than http addressing in your DEFAULT_URL or set other than http addressing in the web_page_url configuration of any of your lists. +
    4. +
  4. + If changing to or from a non-http addressing scheme then the per list htdig config files of the lists affected and their associated htdig indices must be reconstructed. See the section below entitled "Changing the Addressing Scheme of your web_page_url" for details of how to do this. +
    5. +
+
+
+ htdig-2.0.6-0.3.patch: +
+
+
    +
  1. + adds support for remote htdig, that is: running htdig on a different system to Mailman. +
    2. +
  2. + enhances the configurability of the integration. Some of the programmed assumptions made in previous versions are now configurable in mm_cfg.py. The configuration variables concerned default to the previous fixed values so that this version is backwards compatible with earlier versions. +
    3. +
  3. + does some minor cosmetic code changes. +
    4. +
  4. + extends the associated documentation. +
    5. +
+
+
+

Appendices [ toc ]

Appendix 1 -Technique for htdigging when Mailman's web_page_url uses the https scheme

+A technique for htdigging when Mailman's web_page_url uses the https 
+addressing scheme is described in this archived e-mail: 
+http://www.htdig.org/mail/1999/10/0187.html
+
+The text of that e-mail is as follows:
+
+[htdig] Re: Help about htdig indexing https files
+
+------------------------------------------------------------------------
+Gilles Detillieux (grdetil at scrc.umanitoba.ca)
+Wed, 27 Oct 1999 10:18:31 -0500 (CDT) 
+
+
+Messages sorted by: [ date ] [ thread ] [ subject ] [ author ] 
+Next message: Avi Rappoport: "[htdig] indexing SSL (was: Help building 
+the database)" 
+Previous message: Gilles Detillieux: "Re: Fw: [htdig] mutiple search 
+results" 
+In reply to: Torsten Neuer: "Re: Fw: [htdig] mutiple search results" 
+
+------------------------------------------------------------------------
+According to Edouard DESSIOUX: 
+> >Currently, htdig will not support URLs that begin with https://, even
+> >when using local_urls to bypass the server. A trick that might work 
+> >would be to index using http:// instead, but use local_urls to point 
+> >to the directory that contains the contents of the secure server. 
+> 
+> I used that, and now, when i use htsearch, it work, except the fact 
+> that all my URL are http://x.y.z/ instead of https://x.y.z/ 
+> 
+> >You'd need to use separate 
+> >configuration files for digging and searching, and use 
+> >url_part_aliases in each of these configuration files to rewrite the 
+> >http:// into https:// in the search results. 
+> 
+> This is the part i dont understand, and i would like you to explain. 
+
+
+It basically works as a search and replace. One url_part_aliases in the 
+configuration file used by htdig maps the http://x.y.z/ into some 
+special code like "*site", and another url_part_aliases in the 
+configuration file used by htsearch maps the "*site" back into the value 
+you want, i.e. https://x.y.z/. The substitution is left to right in 
+htdig, and right to left in htsearch. So, if you use the same config 
+file for both, or the same setting for both, you get back what you 
+started with (but saved some space in the database because of the 
+encoding). However, if you use two separate config files with different 
+url_part_aliases setting for htdig and htsearch, you can remap parts of 
+URLs from one substring to another. 
+
+
+I hope this makes things clearer. I thought the current description at 
+http://www.htdig.org/attrs.html#url_part_aliases was already quite 
+clear. 
+
+
+
+-- 
+Gilles R. Detillieux              E-mail: <grdetil@scrc.umanitoba.ca>
+Spinal Cord Research Centre       WWW:    
+http://www.scrc.umanitoba.ca/~grdetil
+Dept. Physiology, U. of Manitoba  Phone:  (204)789-3766
+Winnipeg, MB  R3E 3J7  (Canada)   Fax:    (204)789-3930
+------------------------------------
+ + diff -ruP mailman-2.1.18/Mailman/Archiver/HyperArch.py mailman-2.1.18p/Mailman/Archiver/HyperArch.py --- mailman-2.1.18/Mailman/Archiver/HyperArch.py 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/Mailman/Archiver/HyperArch.py 2014-05-03 12:22:36.839376950 -0700 @@ -30,10 +30,14 @@ import sys import re +import string import errno import urllib +import urlparse import time import os +from stat import * +import errno import types import HyperDatabase import pipermail @@ -490,6 +494,8 @@ d['listurl'] = self._mlist.GetScriptURL('listinfo', absolute=1) d['listname'] = self._mlist.real_name d['encoding'] = '' + d["indexing_enable"] = mm_cfg.ARCHIVE_INDEXING_ENABLE + d["indexing_disable"] = mm_cfg.ARCHIVE_INDEXING_DISABLE finally: i18n.set_translation(otrans) @@ -659,6 +665,9 @@ self.lang = maillist.preferred_language self.charset = Utils.GetCharSet(maillist.preferred_language) + if mm_cfg.USE_HTDIG: + self.setup_htdig() + if hasattr(self.maillist,'archive_volume_frequency'): if self.maillist.archive_volume_frequency == 0: self.ARCHIVE_PERIOD='year' @@ -701,6 +710,8 @@ "listinfo": mlist.GetScriptURL('listinfo', absolute=1), "version": self.version, "listname": html_quote(mlist.real_name, self.lang), + "indexing_enable": mm_cfg.ARCHIVE_INDEXING_ENABLE, + "indexing_disable": mm_cfg.ARCHIVE_INDEXING_DISABLE, } i = {"thread": _("thread"), "subject": _("subject"), @@ -737,6 +748,8 @@ "firstdate": quotetime(self.firstdate), "lastdate": quotetime(self.lastdate), "size": self.size, + "indexing_enable": mm_cfg.ARCHIVE_INDEXING_ENABLE, + "indexing_disable": mm_cfg.ARCHIVE_INDEXING_DISABLE, } i = {"thread": _("thread"), "subject": _("subject"), @@ -771,6 +784,9 @@ "fullarch": '../%s.mbox/%s.mbox' % (listname, listname), "size": sizeof(mbox, mlist.preferred_language), 'meta': '', + "indexing_enable": mm_cfg.ARCHIVE_INDEXING_ENABLE, + "indexing_disable": mm_cfg.ARCHIVE_INDEXING_DISABLE, + "htsearch": '', } # Avoid i18n side-effects otrans = i18n.get_translation() @@ -791,6 +807,30 @@ d["archive_listing_end"] = quick_maketext( 'archlistend.html', mlist=mlist) + if mm_cfg.USE_HTDIG: + list_htdig_dir = os.path.join(self.maillist.archive_dir(), + 'htdig') + rundig_file = os.path.join(list_htdig_dir, 'rundig_last_run') + try: + last_rundig_mtime = os.stat(rundig_file)[ST_MTIME] + lastrun = time.strftime("%A, %d %b %Y %H:%M:%S %Z", + time.localtime(last_rundig_mtime)) + except OSError, e: + if e.errno <> errno.ENOENT: raise + lastrun = '[has yet to be built for this new list]' + h = {"listname": self.maillist.internal_name(), + "mmsearchcgi": mlist.GetScriptURL('mmsearch', absolute=1), + "lastrun": lastrun, + "htsearchconf": '', + } + conf_name_search = self.maillist.internal_name() + \ + '.htsearch.conf' + conf_file_search = os.path.join(list_htdig_dir, + conf_name_search) + if os.path.exists(conf_file_search): + h['htsearchconf'] = '.htsearch' + d["htsearch"] = quick_maketext('TOC_htsearch.html', dict=h, + mlist=self.maillist) accum = [] for a in self.archives: @@ -837,10 +877,141 @@ 'archtocentry.html', {'archive': arch, 'archivelabel': self.volNameToDesc(arch), - 'textlink': textlink + 'textlink': textlink, + 'indexing_enable': mm_cfg.ARCHIVE_INDEXING_ENABLE, + 'indexing_disable': mm_cfg.ARCHIVE_INDEXING_DISABLE, }, mlist=self.maillist) + def remove_htdig(self, indices_only): + list_htdig_dir = os.path.join(self.maillist.archive_dir(), 'htdig') + if not os.path.exists(list_htdig_dir): + return + conf_name_dig = self.maillist.internal_name() + '.conf' + conf_file_dig = os.path.join(list_htdig_dir, conf_name_dig) + conf_name_search = self.maillist.internal_name() + '.htsearch.conf' + conf_file_search = os.path.join(list_htdig_dir, conf_name_search) + dual_conf_files = None + if os.path.exists(conf_file_search): + dual_conf_files = 1 + if indices_only: + cfd = open(conf_file_dig, 'r') + conf_data_dig = cfd.readlines() + cfd.close() + if dual_conf_files: + cfd = open(conf_file_search, 'r') + conf_data_search = cfd.readlines() + cfd.close() + os.system('rm -rf ' + list_htdig_dir + '/*') + cfd = open(conf_file_dig, 'w') + cfd.writelines(conf_data_dig) + cfd.close() + if dual_conf_files: + cfd = open(conf_file_search, 'w') + cfd.writelines(conf_data_search) + cfd.close() + else: + os.system('rm -rf ' + list_htdig_dir) + conf_file_link_dig = os.path.join(mm_cfg.HTDIG_CONF_LINK_DIR, conf_name_dig) + os.unlink(conf_file_link_dig) + if dual_conf_files: + conf_file_link_search = os.path.join(mm_cfg.HTDIG_CONF_LINK_DIR, conf_name_search) + os.unlink(conf_file_link_search) + + def setup_htdig(self): + listname = self.maillist.internal_name() + # we want to make a directory to put the mail list's htdig stuff in + list_htdig_dir = os.path.join(self.maillist.archive_dir(), 'htdig') + hperm = 02775 + if mm_cfg.HTDIG_STRICT_FILE_PERM: + hperm = 02770 + # but we bug out if this has already been done + try: + omask = os.umask(2) + try: + os.mkdir(list_htdig_dir, hperm) + finally: + os.umask(omask) + except OSError, e: + if e.errno <> errno.EEXIST: raise + return + # assemble the mapping for characterising the htdig config + htdigfiles = mm_cfg.HTDIG_FILES_URL + if mm_cfg.HTDIG_FILES_URL[-1] == '/': + htdigfile = htdigfiles[:-1] + d = {'databases': list_htdig_dir, + "filepath": self.maillist.archive_dir() + '/', + "maintainer": Utils.get_site_email(), + "indexing_enable": mm_cfg.ARCHIVE_INDEXING_ENABLE, + "indexing_disable": mm_cfg.ARCHIVE_INDEXING_DISABLE, + "htdig_url": htdigfiles, + "htdig_extras": mm_cfg.HTDIG_EXTRAS, + "mmsearchcgi": self.maillist.GetScriptURL('mmsearch', absolute=1), + } + # we need to changes paths to be relative to file system of + # remote machine if we are not running htdig on mailman machine + if mm_cfg.REMOTE_HTDIG: + d['filepath'] = os.path.join( + mm_cfg.REMOTE_PRIVATE_ARCHIVE_FILE_DIR, + listname + '/') + d['databases'] = os.path.join(d['filepath'], 'htdig') + # now the URL through which htdig access to the pipermail data will go + starturl_dig = self.maillist.GetScriptURL('htdig') + '/' + starturl_search = starturl_dig + # we need to know if the addressing scheme for the URL as htdig cannot + # cope with other than http (https for instance) when building indices + # we'll need different conf files for htdig and htsearch in that case + dual_conf_files = None + urlbits = urlparse.urlparse(starturl_dig) + if urlbits[0] != 'http': + urlbits = ('http',) + urlbits[1:] + starturl_dig = urlparse.urlunparse(urlbits) + dual_conf_files = 1 + # check that the excluded_urls wil not exclude the htdig cgi url + excluded_urls = [] + for exc in mm_cfg.HTDIG_EXCLUDED_URLS.split(): + if starturl_dig.find(exc) == -1: + excluded_urls.append(exc) + d['excluded_urls'] = string.join(excluded_urls) + # create htdig config files. we may need one for digging and another + # for searching if the addressing scheme is https these config files + # are slightly different we'll put the files in the directory we just + # created above + conf_name_dig = listname + '.conf' + d['url_part_aliases'] = starturl_dig + " *mm-htdig*" + d['starturl'] = starturl_dig + d['urlpath'] = starturl_dig + conf_file_dig = os.path.join(list_htdig_dir, conf_name_dig) + fd = open(conf_file_dig, 'w') + fd.write(quick_maketext('htdig_conf.txt', dict=d, mlist=self.maillist)) + fd.close() + # we need symlinks so that htdig will be able to find the config files + conf_file_link_dig = os.path.join(mm_cfg.HTDIG_CONF_LINK_DIR, + conf_name_dig) + try: + os.unlink(conf_file_link_dig) + except OSError, e: + if e.errno <> errno.ENOENT: raise + os.symlink(conf_file_dig, conf_file_link_dig) + # make the second conf file and link to it for htsearch if necessary + if dual_conf_files: + conf_name_search = listname + '.htsearch.conf' + d['url_part_aliases'] = starturl_search + " *mm-htdig*" + d['starturl'] = starturl_search + d['urlpath'] = starturl_search + conf_file_search = os.path.join(list_htdig_dir, conf_name_search) + fd = open(conf_file_search, 'w') + fd.write(quick_maketext('htdig_conf.txt', dict=d, + mlist=self.maillist)) + fd.close() + conf_file_link_search = os.path.join(mm_cfg.HTDIG_CONF_LINK_DIR, + conf_name_search) + try: + os.unlink(conf_file_link_search) + except OSError, e: + if e.errno <> errno.ENOENT: raise + os.symlink(conf_file_search, conf_file_link_search) + def GetArchLock(self): if self._lock_file: return 1 diff -ruP mailman-2.1.18/Mailman/Cgi/htdig.py mailman-2.1.18p/Mailman/Cgi/htdig.py --- mailman-2.1.18/Mailman/Cgi/htdig.py 1969-12-31 16:00:00.000000000 -0800 +++ mailman-2.1.18p/Mailman/Cgi/htdig.py 2014-05-03 12:22:36.843376950 -0700 @@ -0,0 +1,197 @@ +# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +"""Provide a password-interface wrapper around private archives. +""" + +import os +import sys +import cgi +import mimetypes +import re +import string + +from Mailman import mm_cfg +from Mailman import Utils +from Mailman import MailList +from Mailman import Errors +from Mailman import i18n +from Mailman.htmlformat import * +from Mailman.Logging.Syslog import syslog + +# Set up i18n. Until we know which list is being requested, we use the +# server's default. +_ = i18n._ +i18n.set_language(mm_cfg.DEFAULT_SERVER_LANGUAGE) + +SLASH = '/' + + + +def true_path(path): + "Ensure that the path is safe by removing .." + # Workaround for path traverse vulnerability. Unsuccessful attempts will + # be logged in logs/error. + parts = [x for x in path.split(SLASH) if x not in ('.', '..')] + return SLASH.join(parts)[1:] + + + +def guess_type(url, strict): + if hasattr(mimetypes, 'common_types'): + return mimetypes.guess_type(url, strict) + return mimetypes.guess_type(url) + + + +def main(): + doc = Document() + doc.set_language(mm_cfg.DEFAULT_SERVER_LANGUAGE) + + parts = Utils.GetPathPieces() + if not parts: + doc.SetTitle(_("Sought (htdig) Archive Error")) + doc.AddItem(Header(3, _("You must specify a list."))) + print doc.Format() + return + + path = os.environ.get('PATH_INFO') + # BAW: This needs to be converted to the Site module abstraction + true_filename = os.path.join( + mm_cfg.PRIVATE_ARCHIVE_FILE_DIR, + true_path(path)) + + listname = parts[0].lower() + mboxfile = '' + if len(parts) == 2: + mboxfile = parts[1] + + # See if it's the list's mbox file is being requested + if listname.endswith('.mbox') and mboxfile.endswith('.mbox') and \ + listname[:-5] == mboxfile[:-5]: + listname = listname[:-5] + else: + mboxfile = '' + + # If it's a directory, we have to append index.html in this script. We + # must also check for a gzipped file, because the text archives are + # usually stored in compressed form. + wasdir = 0 + if os.path.isdir(true_filename): + true_filename = true_filename + '/index.html' + wasdir = 1 + if not os.path.exists(true_filename) and \ + os.path.exists(true_filename + '.gz'): + true_filename = true_filename + '.gz' + + try: + mlist = MailList.MailList(listname, lock=0) + except Errors.MMListError, e: + # Avoid cross-site scripting attacks + safelistname = Utils.websafe(listname) + msg = _('No such list %(safelistname)s') + doc.SetTitle(_("Sought (htdig) Archive Error - %(msg)s")) + doc.AddItem(Header(2, msg)) + print doc.Format() + syslog('error', 'No such list "%s": %s\n', listname, e) + return + + # If the path is not: + # .mbox/.mbox + # it should conform to: + # //... + # / + # and any filenames starting with '.' are also not allowed + num_parts = len(parts) + if not mboxfile and num_parts > 1 and \ + (parts[1] in ('database', 'htdig') or \ + (num_parts == 2 and \ + not re.compile(r'\.(html|txt|txt\.gz)$').search(true_filename) \ + ) or parts[-1][0] == '.' \ + ): + msg = _('Invalid archive file requested') + doc.SetTitle(msg) + doc.AddItem(Header(2, msg)) + print doc.Format() + syslog('error', 'Invalid archive file requested: %s', true_filename) + return + + i18n.set_language(mlist.preferred_language) + doc.set_language(mlist.preferred_language) + + if mlist.archive_private: + message = '' + cgidata = cgi.FieldStorage() + username = cgidata.getvalue('username', '') + password = cgidata.getvalue('password', '') + + if not mlist.WebAuthenticate((mm_cfg.AuthUser, + mm_cfg.AuthListModerator, + mm_cfg.AuthListAdmin, + mm_cfg.AuthSiteAdmin), + password, username): + if cgidata.has_key('submit'): + # This is a re-authorization attempt + message = Bold(FontSize('+1', _('Authorization failed.'))).Format() + # Output the password form + charset = Utils.GetCharSet(mlist.preferred_language) + print 'Content-type: text/html; charset=' + charset + '\n\n' + while path and path[0] == '/': + path=path[1:] # Remove leading /'s + # Bring them back to the same point when they login + actionURL = mlist.GetScriptURL('htdig', absolute=1) + mbox_extra_bit = '' + if mboxfile: + mbox_extra_bit = '.mbox' + if len(parts) > 1: + actionURL = actionURL + mbox_extra_bit + '/' + string.join(parts[1:], '/') + if wasdir: + actionURL = actionURL + '/index.html' + print Utils.maketext( + 'htdig.html', + {'action' : actionURL, + 'realname': mlist.real_name, + 'message' : message, + }, mlist=mlist) + return + lang = mlist.getMemberLanguage(username) + i18n.set_language(lang) + doc.set_language(lang) + + # Authorization confirmed... output the desired file + try: + ctype, enc = guess_type(path, strict=0) + if ctype is None: + ctype = 'text/html' + if mboxfile: + f = open(os.path.join(mlist.archive_dir() + '.mbox', + mlist.internal_name() + '.mbox')) + ctype = 'text/plain' + elif true_filename.endswith('.gz'): + import gzip + f = gzip.open(true_filename, 'r') + else: + f = open(true_filename, 'r') + except IOError: + msg = _('Sought (htdig) archive file not found') + doc.SetTitle(msg) + doc.AddItem(Header(2, msg)) + print doc.Format() + syslog('error', 'Sought (htdig) archive file not found: %s', true_filename) + else: + print 'Content-type: %s\n' % ctype + sys.stdout.write(f.read()) + f.close() diff -ruP mailman-2.1.18/Mailman/Cgi/mmsearch.py mailman-2.1.18p/Mailman/Cgi/mmsearch.py --- mailman-2.1.18/Mailman/Cgi/mmsearch.py 1969-12-31 16:00:00.000000000 -0800 +++ mailman-2.1.18p/Mailman/Cgi/mmsearch.py 2014-05-03 12:22:36.843376950 -0700 @@ -0,0 +1,229 @@ +# Copyright (C) 2003-2011 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +"""Provide an authentication wrapper around htdig's htsearch. + +In order to search a list's archive the HTTP request must present a valid +cookie authorizing access to the list's archives for private archives. +This cookie must be obtained by the same process as the user must +adopt for accessing the archive via the private.py script. + +The basic premise is that htsearch cannot reach a list's htdig conf file +unless this script enhances the environment in which htsearch runs to +provide access to a list's htdig conf file. + +If the use authenticates OK for a private archive, or the archive is public, +this script: + +a. in the case of local htdig, this script enhances the environment with +'CONFIG-DIR' pointing to MM's list htdig conf files, and runs htsearch. + +b. in the case of remote htdig, this script performs a HTTP request to a +companion CGI script running on the htdig machine. This request can of +course be forged easily. The only protection available is the the companion +script can optionaaly check the IP number from which requests to it are made +and reject them if they do not come from the MM server. +""" + +import sys +import os +import cgi +import re +import exceptions +import types +import httplib +import urllib +import urlparse + +from subprocess import Popen, PIPE + +from Mailman import mm_cfg +from Mailman import Utils +from Mailman import MailList +from Mailman import Errors +from Mailman import i18n +from Mailman.htmlformat import * +from Mailman.Logging.Syslog import syslog + + +# Set up i18n. Until we know which list is being requested, we use the +# server's default. +_ = i18n._ +i18n.set_language(mm_cfg.DEFAULT_SERVER_LANGUAGE) + +errors = {'cgi': _('CGI problem.'), + 'info': _('Path info.'), + 'list': _('The requested list cannot be accessed.'), + 'htsearch': _('search failed'), + 'auth': Utils.maketext('htdig_auth_failure.html', dict=None, raw=0) + } + +class _search_exception(exceptions.Exception): + + def __init__(self, listname, reason, detail): + self.listname = Utils.websafe(listname) + self.reason = reason + self.detail = Utils.websafe(detail) + + def __str__(self): + return 'listname: %s, reason: %s, detail: %s' % (self.listname, self.reason, self.detail) + + +def true_path(path): + "Ensure that the path is safe by removing .." + path = path.replace("../", "") + path = path.replace("./", "") + return path[1:] + +def make_inserts(listname): + urlbase = mm_cfg.DEFAULT_URL or \ + mm_cfg.DEFAULT_URL_PATTERN % mm_cfg.DEFAULT_URL_HOST + return { + 'mailto': Utils.get_site_email(), + 'listinfo_link': urlbase + '/listinfo/' + listname, + 'referer': Utils.websafe( + os.environ.get('HTTP_REFERER', _('Referer not known'))), + 'uri': Utils.websafe( + os.environ.get('REQUEST_URI', _('URI not known'))), + } + +def error_quit(why): + d = make_inserts(why.listname) + d['error'] = errors[why.reason] + ' ' + why.detail + charset = Utils.GetCharSet(mm_cfg.DEFAULT_SERVER_LANGUAGE) + if why.reason == 'list': + stuff = 'Status: 404 Not Found\n' + elif why.reason == 'auth': + stuff = 'Status: 401 Unauthorized\n' + else: + stuff = '' + stuff += 'Content-type: text/html; charset=' + charset + '\n\n' + stuff += Utils.maketext('htdig_access_error.html', dict=d, + lang=mm_cfg.DEFAULT_SERVER_LANGUAGE) + print stuff + syslog('error', "htsearch for list: %s, cause: %s, detail: %s" % \ + (why.listname, why.reason, why.detail)) + sys.exit(0) + +_required_fields = ('method', + 'format', + 'sort', + 'config', + ) + +_allowed_fields = {'method': 0, + 'format': 0, + 'sort': 0, + 'config': 1, + 'words': 0, + 'submit': 0, + 'restrict': 0, + 'exclude': 0, + 'page': 0, + } + +def check_params(listname, fs, detail): + detail = detail + ' fields: ' + ','.join(fs.keys()) + for fieldname in _required_fields: + if not fs.has_key(fieldname): + raise _search_exception(listname, 'cgi', '-5-' + detail) + fieldhash = {} + for fieldname in fs.keys(): + if not _allowed_fields.has_key(fieldname): + raise _search_exception(listname, 'cgi', '-6- ' + detail) + if type(fs[fieldname]) is types.ListType: + raise _search_exception(listname, 'cgi', '-8- ' + detail) + fieldhash[fieldname] = fs[fieldname].value + return urllib.urlencode(fieldhash) + +def main(): + try: + try: + request_method = os.environ['REQUEST_METHOD'] + form = cgi.FieldStorage() + except: + raise _search_exception('', 'cgi', 'No list -1-') + list_info = Utils.GetPathPieces() + if not list_info or len(list_info) != 1: + raise _search_exception('', 'info', 'No list -2-') + path_listname = list_info[0].lower() + # Check right number of expected fields and get them + # url encoded + params = check_params(path_listname, form, 'Field count -4-') + # Extract the listname from the form + listname = form['config'].value + if listname.endswith('.htsearch'): + listname = listname[:-len('.htsearch')] + # Access the list OK? + if path_listname != listname: + raise _search_exception(listname, 'list', '%s:%s' % (path_listname, listname)) + try: + mlist = MailList.MailList(listname, lock=0) + except: + raise _search_exception(listname, 'list', '-11-') + # We only need to authorize the user if it's a private archive + if mlist.archive_private: + if not mlist.WebAuthenticate((mm_cfg.AuthUser, + mm_cfg.AuthListModerator, + mm_cfg.AuthListAdmin, + mm_cfg.AuthSiteAdmin), + '', ''): + raise _search_exception(listname, 'auth', '-10-') + if mm_cfg.REMOTE_HTDIG: + # We are going to do an HTTP request to the server that is + # running htsearch and then return the response. + headers = {"Content-type": "application/x-www-form-urlencoded", + "Accept": "text/plain, text/html"} + urlbits = urlparse.urlsplit(mm_cfg.REMOTE_MMSEARCH_URL) + hostbit = urlbits[1] + uribit = urlbits[2] + '/' + listname + conn = httplib.HTTPConnection(hostbit) + conn.request("POST", uribit, params, headers) + response = conn.getresponse() + if response.status == 200: + ctype = response.getheader('Content-type', 'text/html') + print "Content-type: %s\n\n" % ctype + print response.read() + conn.close() + else: + syslog('error', "remote htsearch failed, url: %s, response code: %d" % \ + (mm_cfg.REMOTE_MMSEARCH_URL, response.status)) + conn.close() + raise _search_exception(listname, 'cgi', + ' -9- %d' % response.status) + else: + # We are going to execute htsearch and return its response. + # We need to let htsearch get at the list specific htdig + # conf file + os.environ['CONFIG_DIR'] = mm_cfg.HTDIG_CONF_LINK_DIR + cmd = "%s" % mm_cfg.HTDIG_HTSEARCH_PATH + child = Popen(cmd, stdin=PIPE, stdout=PIPE) + if request_method == 'POST': + os.environ['CONTENT_LENGTH'] = str(len(params)) + response = child.communicate(params)[0] + else: + response = child.communicate()[0] + exitstatus = child.returncode + exitstatus = (exitstatus >> 8) & 0xff + if exitstatus: + syslog('error', "htsearch for list: %s, existatus: %s" % \ + (listname, str(exitstatus))) + if not response: + raise _search_exception(listname, 'htsearch', '-12-') + print response + except _search_exception, e: + error_quit(e) + sys.exit(0) diff -ruP mailman-2.1.18/Mailman/Cgi/private.py mailman-2.1.18p/Mailman/Cgi/private.py --- mailman-2.1.18/Mailman/Cgi/private.py 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/Mailman/Cgi/private.py 2014-05-03 12:22:36.843376950 -0700 @@ -21,6 +21,7 @@ import sys import cgi import mimetypes +import re from Mailman import mm_cfg from Mailman import Utils @@ -81,7 +82,7 @@ listname = parts[0].lower() mboxfile = '' - if len(parts) > 1: + if len(parts) == 2: mboxfile = parts[1] # See if it's the list's mbox file is being requested @@ -114,6 +115,26 @@ syslog('error', 'private: No such list "%s": %s\n', listname, e) return + # If the path is not: + # .mbox/.mbox + # it should conform to: + # //... + # / + # and any filenames starting with '.' are also not allowed + num_parts = len(parts) + if not mboxfile and num_parts > 1 and \ + (parts[1] in ('database', 'htdig') or \ + (num_parts == 2 and \ + not re.compile(r'\.(html|txt|txt\.gz)$').search(true_filename) \ + ) or parts[-1][0] == '.' \ + ): + msg = _('Invalid archive file requested') + doc.SetTitle(msg) + doc.AddItem(Header(2, msg)) + print doc.Format() + syslog('error', 'Invalid archive file requested: %s', true_filename) + return + i18n.set_language(mlist.preferred_language) doc.set_language(mlist.preferred_language) diff -ruP mailman-2.1.18/Mailman/Cgi/remote_mmsearch mailman-2.1.18p/Mailman/Cgi/remote_mmsearch --- mailman-2.1.18/Mailman/Cgi/remote_mmsearch 1969-12-31 16:00:00.000000000 -0800 +++ mailman-2.1.18p/Mailman/Cgi/remote_mmsearch 2014-05-03 12:22:36.843376950 -0700 @@ -0,0 +1,235 @@ +#! /usr/local/bin/python + +# Copyright (C) 2003-2011 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +"""Provide an authentication wrapper around htdig's htsearch when +it is running on a different machine to Mailman. + +This relies on htsearch not normally being able to access per list htdig +conf files for list archives without the intervention of this script, +which inserts a 'CONFIG_DIR' environment variable to enable that acccess. + +The security available with this script is limited as it cannot +consult the user authentication information held by Mailman. Instead +we rely on the mmsearch CGI script on the Mailman server doing that and +then passing the request to this script using an HTTP request. + +The only protection against malicious request to this script is to restrict +this script to responding only when an HTTP request originates from a machine +with a particular IP number: e.g. we only allow requests from our 'trusted' +MM server. +""" + +# Edit the following configuration variables to suit your installation +# +# For example: +# +#MAILTO = 'mailman@mailman.yourdomain.com' +#VALID_IP_LIST = ['192.168.1.111'] +#HTDIG_CONF_LINK_DIR = '/mailman/run/archives/htdig' +#HTDIG_HTSEARCH_PATH = '/opt/www/htdig/bin/htsearch' + +MAILTO = '' +VALID_IP_LIST = [] +HTDIG_CONF_LINK_DIR = '' +HTDIG_HTSEARCH_PATH = '' + +# End of things for you to edit + +import sys +import os +import cgi +import re +import exceptions +import types +import httplib +import urllib +import urlparse + +from subprocess import Popen, PIPE + +errors = {'cgi': 'CGI problem.', + 'info': 'Path info.', + 'list': 'The requested list cannot be accessed.', + 'htsearch': 'htearch failed', + 'auth': 'Authentication failure.', + } + +class _search_exception(exceptions.Exception): + + def __init__(self, listname, reason, detail): + self.listname = cgi.escape(listname) + self.reason = reason + self.detail = cgi.escape(detail) + + def __str__(self): + return 'listname: %s, reason: %s' % (self.listname, self.reason) + + +def true_path(path): + "Ensure that the path is safe by removing .." + path = path.replace("../", "") + path = path.replace("./", "") + return path[1:] + +def GetPathPieces(envar='PATH_INFO'): + path = os.environ.get(envar) + if path: + return [p for p in path.split('/') if p] + return None + +def make_inserts(listname): + return { + 'mailto': MAILTO, + 'listname': listname, + 'referer': cgi.escape( + os.environ.get('HTTP_REFERER', 'Referer not known')), + 'uri': cgi.escape( + os.environ.get('REQUEST_URI', 'URI not known')), + } + +def error_quit(why): + d = make_inserts(why.listname) + d['error'] = errors[why.reason] + ' ' + why.detail + print """\ +Content-type: text/html + + + + htdig Archives Access Failure + + +

htdig Archives Access Failure

+%(error)s +

+ Searching the archives of list %(listname)s failed. +

+

+ If this problem persists then please e-mail the following information to the +%(mailto)s: +

+
+    %(referer)s
+    %(uri)s
+
+
+ + +""" % d + sys.exit(0) + +_required_fields = ('method', + 'format', + 'sort', + 'config', + ) + +_allowed_fields = {'method': 0, + 'format': 0, + 'sort': 0, + 'config': 1, + 'words': 0, + 'submit': 0, + 'restrict': 0, + 'exclude': 0, + 'page': 0, + } + +def check_params(listname, fs, detail): + detail = detail + ' fields: ' + ','.join(fs.keys()) + for fieldname in _required_fields: + if not fs.has_key(fieldname): + raise _search_exception(listname, 'cgi', '-5-' + detail) + fieldhash = {} + for fieldname in fs.keys(): + if not _allowed_fields.has_key(fieldname): + raise _search_exception(listname, 'cgi', '-6- ' + detail) + if type(fs[fieldname]) is types.ListType: + raise _search_exception(listname, 'cgi', '-8- ' + detail) + fieldhash[fieldname] = fs[fieldname].value + return urllib.urlencode(fieldhash) + +def main(): + try: + try: + request_method = os.environ['REQUEST_METHOD'] + form = cgi.FieldStorage() + except: + raise _search_exception('', 'cgi', 'No list -1-') + list_info = GetPathPieces() + if not list_info or len(list_info) != 1: + raise _search_exception('', 'info', 'No list -2-') + path_listname = list_info[0].lower() + # Check right number of expected fields and get them + # url encoded + params = check_params(path_listname, form, 'Field count -4-') + # Extract the listname from the form + listname = form['config'].value + if listname.endswith('.htsearch'): + listname = listname[:-len('.htsearch')] + # Access the list OK? + if path_listname != listname: + raise _search_exception(listname, 'list', '%s:%s' % \ + (path_listname, listname)) + # We check to origin IP of the request if restrictions are specified + if len(VALID_IP_LIST): + try: + request_bits = os.environ['REMOTE_ADDR'].split('.') + for ip in VALID_IP_LIST: + valid_bits = ip.split('.') + for i in range(4): + if request_bits[i] != valid_bits[i]: + break + else: + break + else: + raise _search_exception(listname, 'auth', ' -10- %s' % \ + os.environ['REMOTE_ADDR']) + except _search_exception: + raise + except: + raise _search_exception(listname, 'auth', ' -12-') + # We are going to call execute htsearch and return its response. + # We need to let htsearch get at the list specific htdig + # conf file + if request_method == 'POST': + os.environ['CONTENT_LENGTH'] = str(len(params)) + else: + raise _search_exception(listname, 'auth', ' -11- method') + os.environ['CONFIG_DIR'] = HTDIG_CONF_LINK_DIR + cmd = HTDIG_HTSEARCH_PATH + child = Popen(cmd, stdin=PIPE, stdout=PIPE) + response = child.communicate(params)[0] + exitstatus = child.returncode + exitstatus = (exitstatus >> 8) & 0xff + if exitstatus: + raise _search_exception(listname, 'htsearch', ' -12- exit: %d' \ + % existatus, ) + if not response: + raise _search_exception(listname, 'htsearch', ' -13-') + print response + except _search_exception, e: + error_quit(e) + sys.exit(0) + +if __name__ == '__main__' and \ + MAILTO and \ + os.path.isdir(HTDIG_CONF_LINK_DIR) and \ + os.access(HTDIG_HTSEARCH_PATH, os.X_OK): + main() +else: + error_quit(_search_exception('', 'cgi', '-14- misconfigured')) diff -ruP mailman-2.1.18/Mailman/Cgi/remote-mmsearch mailman-2.1.18p/Mailman/Cgi/remote-mmsearch --- mailman-2.1.18/Mailman/Cgi/remote-mmsearch 1969-12-31 16:00:00.000000000 -0800 +++ mailman-2.1.18p/Mailman/Cgi/remote-mmsearch 2014-05-03 12:22:36.843376950 -0700 @@ -0,0 +1,237 @@ +#! /usr/bin/perl + +# Copyright (C) 2003-2011 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +my $doc = < 'CGI problem.', + 'info' => 'Path info.', + 'list' => 'The requested list cannot be accesssed.', + 'htsearch' => 'htearch failed', + 'auth' => 'Authentication failure.', + ); + +sub true_path { + my $path = shift; + $path =~ s/\.\.\///g; + $path =~ s/\.\///g; + return substr $path, 1; +} + +sub GetPathPieces { + my $path = $ENV{'PATH_INFO'}; + if ($path) { + my @pathbits = (); + foreach my $bit (split /\//, $path ) { + push (@pathbits, $bit) if $bit; + } + return @pathbits if scalar @pathbits; + } + return (); +} + +my @required_fields = ('method', + 'format', + 'sort', + 'config', + ); + +my %allowed_fields = ('method' => 0, + 'format' => 0, + 'sort' => 0, + 'config' => 1, + 'words' => 0, + 'submit' => 0, + 'restrict' => 0, + 'exclude' => 0, + 'page' => 0, + ); + +sub error_quit { + my ($listname, $reason, $detail) = @_; + my $fault = $errors{$reason} . " $detail"; + my $mailto = $MAILTO; + my $referer = $ENV{'HTTP_REFERER'} ; # 'Referer not known' + my $uri = $ENV{'REQUEST_URI'}; # 'URI not known' + print < + + htdig Archives Access Failure + + +

htdig Archives Access Failure

+$fault +

+ Searching the archives of list CGI->escapeHTML($listname) failed. +

+

+ If this problem persists then please e-mail the following information to the +$mailto: +

+
+    CGI->escapeHTML($referer)
+    CGI->escapeHTML($uri)
+
+
+ + +EOT + exit(0); +} + +sub check_params { + my ($listname, $query, $detail) = @_; + my @fieldnames = $query->param(); + $detail = "$detail fields: " . join(',', @fieldnames); + foreach my $fieldname (@required_fields) { + if (! $query->param($fieldname)) { + error_quit($listname, 'cgi', " -5- $detail"); + } + } + my @urlencoded = (); + foreach my $fieldname (@fieldnames) { + if (! exists $allowed_fields{$fieldname}) { + error_quit($listname, 'cgi', "-6- $detail|$fieldname"); + } + my $fc = $query->param($fieldname); + if (ref(\$fc) ne 'SCALAR') { + error_quit($listname , 'cgi', "-8- $detail|$fieldname"); + } + push @urlencoded, "$fieldname=" . uri_escape($fc); + } + return join '&', @urlencoded; +} + +my $htaccess = '.htaccess'; +my $htaccess_len = length($htaccess); + +sub doit { + my $request_method = $ENV{'REQUEST_METHOD'}; + my $query = CGI::new(); + my @list_info = GetPathPieces(); + if (scalar(@list_info) != 1) { + error_quit('', 'info', '-2-'); + } + my $path_listname = lc $list_info[0]; + my $encoded_params = check_params($path_listname, $query, 'Field count -4-'); + my $listname; + my $cfg = $query->param('config'); + if (substr($cfg, -$htaccess_len) eq $htaccess) { + $listname = substr $cfg, 0, length($cfg) - $htaccess_len; + } elsif ($cfg) + { + $listname = $cfg; + + } else { + error_quit('', 'list', '-3-'); + } + if ($path_listname ne $listname) { + error_quit($listname, 'list', "$path_listname:$listname"); + } + if (scalar @VALID_IP_LIST) { + my $remote_addr = $ENV{'REMOTE_ADDR'}; + my @request_bits = split /\./, $remote_addr; + my $got_a_match = 0; + my $ip; + foreach $ip (@VALID_IP_LIST) { + @valid_bits = split /\./, $ip; + my $mismatch = 0; + for (my $i=0; $i<4; $i++) { + if ($request_bits[$i] ne $valid_bits[$i]) { + $mismatch = 1; + last; + } + } + if (! $mismatch) { + $got_a_match = 1; + last; + } + } + if (! $got_a_match) { + error_quit($listname, 'auth', "-10- $remote_addr"); + } + } + $ENV{'CONFIG_DIR'} = $HTDIG_CONF_LINK_DIR; + my $cmd = $HTDIG_HTSEARCH_PATH; + if ($request_method eq 'POST') { + $ENV{'CONTENT_LENGTH'} = length $encoded_params; + } else { + error_quit($listname, 'auth', "-11- method"); + } + my $child_pid = open2(\*Readchild, \*Writechild, $cmd); + print Writechild $encoded_params; + close Writechild; + my $response = ''; + while () { + $response .= $_; + } + my $rc = waitpid $child_pid, 0; + my $exitstatus = ($? >> 8) & 0xff; + if ($exitstatus) { + error_quit($listname, 'htsearch', "-12- exit: $existstatus"); + } + if (! $response) { + error_quit($listname, 'htsearch', "-13-"); + } + print $response; + exit 0; +} + +if ($MAILTO and -d $HTDIG_CONF_LINK_DIR and -x $HTDIG_HTSEARCH_PATH) { + doit(); +} else { + error_quit('', 'cgi', '-14- misconfigured'); +} diff -ruP mailman-2.1.18/Mailman/Defaults.py.in mailman-2.1.18p/Mailman/Defaults.py.in --- mailman-2.1.18/Mailman/Defaults.py.in 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/Mailman/Defaults.py.in 2014-05-03 12:22:36.843376950 -0700 @@ -1561,6 +1561,84 @@ # Import a bunch of version numbers from Version import * +# Strings for wrapping html stuff we do not want a search engine to +# pay attention to in the pipermail archives. Of course the search engine +# must be able to interpret such strings. +#ARCHIVE_INDEXING_ENABLE = '' # no htdig integration default +#ARCHIVE_INDEXING_DISABLE = '' # no htdig integration default +# For example, you could insert the following into your mm_cfg if you +# were using htdig for searching archives. They are default values for +# htdig config attributes noindex_end and noindex_start respectively +#ARCHIVE_INDEXING_ENABLE = '' +#ARCHIVE_INDEXING_DISABLE = '' +# You can also cater for controlling multiple search engines that +# recognise different tags to control what they should add to their +# indexes by putting the tags recognised by each search engine +# into the strings. For instance: +#ARCHIVE_INDEXING_ENABLE = '\n' +#ARCHIVE_INDEXING_DISABLE = '\n' + +ARCHIVE_INDEXING_ENABLE = '' +ARCHIVE_INDEXING_DISABLE = '' +# htdig integration parameters +# if you set USE_HTDIG then you must also set HTDIG_HTSEARCH_PATH +# and HTDIG_RUNDIG_PATH to suit your htdig installation. They must point to +# the htsearch program and rundig script in your file system, for instance: +# HTDIG_HTSEARCH_PATH = '/usr/bin/htsearch' +# HTDIG_RUNDIG_PATH = '/usr/bin/rundig' +USE_HTDIG = 0 # 0 - don't use integrated htdig, 1 - use it +HTDIG_FILES_URL = '/htdig/' +HTDIG_CONF_LINK_DIR = os.path.join(VAR_PREFIX, 'archives', 'htdig') +HTDIG_HTSEARCH_PATH = '/usr/local/bin/htsearch' +HTDIG_RUNDIG_PATH = '/usr/local/bin/rundig' + +# the value of this string is assigned as the value of an htdig +# 'excluded_urls:' configuration directive when per-list htdig.conf +# files are generated. Conflicts can arise depending on the values you +# assign to DEFAULT_URL_PATTERN and CGIEXT and list names themselves. +# The code attempts to avoid this problem when generating each per-list +# htdig.conf and will, if necessary, modify the value obtained from +# HTDIG_EXCLUDED_URLS on a list-by-list basis. +HTDIG_EXCLUDED_URLS = '/cgi-bin/ .cgi' + +# you can use the HTDIG_EXTRAS parameter to add arbitrary htdig +# configuration attributes to per list htdig config files. The string +# value you specify is inserted verbatim at the top of each htdig conf +# file when it is generated. The default value does nothing. Make sure +# you understand what you are doing before you fool with this facility. +HTDIG_EXTRAS = """\ +# start of extra site specific htdig configuration attributes +# +# replace these line with your htdig config attribute declarations +# as defined at http://www.htdig.org/confindex.html +# +# end of extra site specific htdig configuration attributes +""" + +# remote htdig support parameters for mailman-htdig integration +# provides support for running htdig on a different machine from the one +# running mailman but one having NFS access to the installation directory +# of the Mailman package. +# set REMOTE_HTDIG if you are running htdig on a different machine to +# Mailman. Has no effect unless you also set REMOTE_HTDIG +# REMOTE_PRIVATE_ARCHIVE_FILE_DIR is the absolute path to the directory in +# which Mailman stores private archives as seen by the machine running htdig. +# It should resolve to the same directory as PRIVATE_ARCHIVE_FILE_DIR when +# viewed from the remote system. +# REMOTE_MMSEARCH_URL is the URL for the mmsearch's companion CGI script +# script on the remote htdig machine. This should reach one of either +# remote_mmsearch or remote-mmsearch +# HTDIG_STRICT_FILE_PERM says whether 'other' does not have permission to +# access per-list htdig directory (True) or not (False). False may be +# appropriate if REMOTE_HTDIG is True and the UID and GID the remote Apache +# server runs CGI scripts as is not the/in the mailman user/group. +# Weaker security but prevents permissions problems for +# remote_mmsearch/remote-mmseach +REMOTE_HTDIG = 0 # 0 - htdig runs on Mailman machine, 1 - runs on remote machine +REMOTE_PRIVATE_ARCHIVE_FILE_DIR = os.path.join(VAR_PREFIX, 'archives', 'private') +REMOTE_MMSEARCH_URL = '/cgi-bin/remote_mmsearch' +HTDIG_STRICT_FILE_PERM = 1 + # Vgg: Language descriptions and charsets dictionary, any new supported # language must have a corresponding entry here. Key is the name of the # directories that hold the localized texts. Data are tuples with first diff -ruP mailman-2.1.18/Mailman/Queue/ArchRunner.py mailman-2.1.18p/Mailman/Queue/ArchRunner.py --- mailman-2.1.18/Mailman/Queue/ArchRunner.py 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/Mailman/Queue/ArchRunner.py 2014-05-03 12:22:36.843376950 -0700 @@ -17,17 +17,33 @@ """Archive queue runner.""" import time +import errno +import os +from stat import * from email.Utils import parsedate_tz, mktime_tz, formatdate from Mailman import mm_cfg from Mailman import LockFile from Mailman.Queue.Runner import Runner +from Mailman import MailList +from Mailman import Utils +from Mailman.Archiver import HyperArch +# Part of the Mailman-htdig integration. +# This controls how often _doperiodic() will try to deal with the +# consequences of a remote machine having reindexed mail archives +# and hence the need for the MM machine to update affected lists +# TOC pages to reflect the datetime when the htdigging was done. +CHECK_REMOTE_RUNDIG_EFFECTS = 10 class ArchRunner(Runner): QDIR = mm_cfg.ARCHQUEUE_DIR + def __init__(self, slice=None, numslices=1): + Runner.__init__(self, slice, numslices) + self._periodic_htdig_check = CHECK_REMOTE_RUNDIG_EFFECTS + def _dispose(self, mlist, msg, msgdata): # Support clobber_date, i.e. setting the date in the archive to the # received date, not the (potentially bogus) Date: header of the @@ -74,3 +90,35 @@ mlist.Save() finally: mlist.Unlock() + + def _doperiodic(self): + """Do some processing `every once in a while'. + + If the mailman-htdig archiving is being used then we want to ensure + that the TOC page for each list has been updated since the last time + htdigging of the list was done. This is only necessary if we are + running htdig on a different machine to Mailman. + + """ + + if mm_cfg.USE_HTDIG and mm_cfg.REMOTE_HTDIG: + self._periodic_htdig_check -= 1 + if self._periodic_htdig_check <= 0: + self._periodic_htdig_check = CHECK_REMOTE_RUNDIG_EFFECTS + listnames = Utils.list_names() + for name in listnames: + mlist = MailList.MailList(name, lock=0) + if not mlist.last_post_time > 0: continue + arch_dir = mlist.archive_dir() + rundig_run_file = os.path.join(arch_dir, 'htdig', + 'rundig_last_run') + toc_file = os.path.join(arch_dir, 'index.html') + try: + last_rundig_time = os.stat(rundig_run_file)[ST_MTIME] + last_toc_time = os.stat(toc_file)[ST_MTIME] + except OSError, e: + if e.errno <> errno.ENOENT: raise + else: + if last_rundig_time > last_toc_time: + HyperArch.HyperArchive(mlist).write_TOC() + diff -ruP mailman-2.1.18/Makefile.in mailman-2.1.18p/Makefile.in --- mailman-2.1.18/Makefile.in 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/Makefile.in 2014-05-03 12:22:36.843376950 -0700 @@ -43,7 +43,7 @@ VAR_DIRS= \ logs archives lists locks data spam qfiles \ - archives/private archives/public + archives/private archives/public archives/htdig ARCH_INDEP_DIRS= \ bin templates scripts cron pythonlib \ diff -ruP mailman-2.1.18/README.NOINDEXtags mailman-2.1.18p/README.NOINDEXtags --- mailman-2.1.18/README.NOINDEXtags 1969-12-31 16:00:00.000000000 -0800 +++ mailman-2.1.18p/README.NOINDEXtags 2014-05-03 12:22:36.843376950 -0700 @@ -0,0 +1,29 @@ +If you are defining values for the ARCHIVE_INDEXING_ENABLE and +ARCHIVE_INDEXING_DISABLE configuration attributes in MM_cfg.py you +may want to try and control the indexing activities of multiple search +engines that you let access your mail archives. + +At the time of writing this, the problem you face is that there is no standard +tag defined to exert partial control over search engine indexing of a page. By +this I mean a way of telling the search engine to index only a specified part of +the page content. There is no formal or de facto standard equivalent to the +robot property on the HTML 4.0 META tag e.g. +, +which gives whole page control with most search engines. + +However, you should be able to put multiple start and stop indexing tags in the +values you assign to the ARCHIVE_INDEXING_ENABLE/DISABLE strings in mm_cfg.py. + +For example, some writers on the web suggest using and tags +because they are recognised and honoured by a number of search engines. + +The defaults recognised by htdig are actually HTML comments of the form + and . + +You could combine these in your mm_cfg.py file as follows: + +ARCHIVE_INDEXING_ENABLE = '\n' +ARCHIVE_INDEXING_DISABLE = '\n' + +Most browsers and search engines should be happy with the results of this as, in +general, they will ignore tags they do not understand and act on those they do. diff -ruP mailman-2.1.18/src/Makefile.in mailman-2.1.18p/src/Makefile.in --- mailman-2.1.18/src/Makefile.in 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/src/Makefile.in 2014-05-03 12:22:36.843376950 -0700 @@ -73,7 +73,8 @@ # Fixed definitions CGI_PROGS= admindb admin confirm create edithtml listinfo options \ - private rmlist roster subscribe + private rmlist roster subscribe \ + htdig mmsearch COMMONOBJS= common.o vsnprintf.o @@ -81,7 +82,7 @@ #ALIAS_PROGS= addaliases -SUID_CGI_PROGS= private +SUID_CGI_PROGS= private htdig mmsearch SUID_MAIL_PROGS= diff -ruP mailman-2.1.18/templates/cs/article.html mailman-2.1.18p/templates/cs/article.html --- mailman-2.1.18/templates/cs/article.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/cs/article.html 2014-05-03 12:22:36.843376950 -0700 @@ -17,6 +17,7 @@

%(subject_html)s

+%(indexing_disable)s %(author_html)s %(email_html)s @@ -33,9 +34,11 @@
+%(indexing_enable)s %(body)s +%(indexing_disable)s

    @@ -50,4 +53,5 @@
Dal informace o konferenci %(listname)s
+%(indexing_enable)s diff -ruP mailman-2.1.18/templates/da/archidxfoot.html mailman-2.1.18p/templates/da/archidxfoot.html --- mailman-2.1.18/templates/da/archidxfoot.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/da/archidxfoot.html 2014-05-03 12:22:36.843376950 -0700 @@ -1,4 +1,5 @@ +%(indexing_disable)s

Dato for nyeste meddelelse: %(lastdate)s
@@ -16,6 +17,7 @@

Dette arkiv blev genereret af +%(indexing_enable)s Pipermail %(version)s. diff -ruP mailman-2.1.18/templates/da/archidxhead.html mailman-2.1.18p/templates/da/archidxhead.html --- mailman-2.1.18/templates/da/archidxhead.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/da/archidxhead.html 2014-05-03 12:22:36.843376950 -0700 @@ -6,6 +6,7 @@ %(encoding)s +%(indexing_disable)s

Arkivet for %(archive)s sorteret efter %(archtype)s

    @@ -21,4 +22,5 @@

    Startdato: %(firstdate)s
    Slutdato: %(lastdate)s
    Meddelelser: %(size)s

    +%(indexing_enable)s

      diff -ruP mailman-2.1.18/templates/da/archtocentry.html mailman-2.1.18p/templates/da/archtocentry.html --- mailman-2.1.18/templates/da/archtocentry.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/da/archtocentry.html 2014-05-03 12:22:36.843376950 -0700 @@ -5,7 +5,9 @@ [ Tråd ] [ Emne ] [ Afsender ] +%(indexing_enable)s [ Dato ] +%(indexing_disable)s %(textlink)s diff -ruP mailman-2.1.18/templates/da/archtoc.html mailman-2.1.18p/templates/da/archtoc.html --- mailman-2.1.18/templates/da/archtoc.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/da/archtoc.html 2014-05-03 12:22:36.843376950 -0700 @@ -6,15 +6,18 @@ %(meta)s +%(indexing_disable)s

      %(listname)s arkiv

      Du kan se mere information om denne liste eller du kan downloade hele arkivet (%(size)s).

      + %(htsearch)s %(noarchive_msg)s %(archive_listing_start)s %(archive_listing)s %(archive_listing_end)s +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/da/article.html mailman-2.1.18p/templates/da/article.html --- mailman-2.1.18/templates/da/article.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/da/article.html 2014-05-03 12:22:36.843376950 -0700 @@ -17,6 +17,7 @@

      %(subject_html)s

      +%(indexing_disable)s %(author_html)s %(email_html)s @@ -33,9 +34,11 @@
    +%(indexing_enable)s %(body)s +%(indexing_disable)s

      @@ -51,4 +54,5 @@
      Mere information om maillisten %(listname)s.
      +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/de/article.html mailman-2.1.18p/templates/de/article.html --- mailman-2.1.18/templates/de/article.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/de/article.html 2014-05-03 12:22:36.843376950 -0700 @@ -17,6 +17,7 @@

      %(subject_html)s

      +%(indexing_disable)s %(author_html)s
    +%(indexing_enable)s %(body)s +%(indexing_disable)s

      @@ -51,4 +54,5 @@
    Mehr Informationen über die Mailingliste %(listname)s
    +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/en/archidxfoot.html mailman-2.1.18p/templates/en/archidxfoot.html --- mailman-2.1.18/templates/en/archidxfoot.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/en/archidxfoot.html 2014-05-03 12:22:36.843376950 -0700 @@ -1,4 +1,5 @@ -
+%(indexing_disable)s +

Last message date: %(lastdate)s
@@ -17,5 +18,6 @@

This archive was generated by Pipermail %(version)s. +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/en/archidxhead.html mailman-2.1.18p/templates/en/archidxhead.html --- mailman-2.1.18/templates/en/archidxhead.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/en/archidxhead.html 2014-05-03 12:22:36.843376950 -0700 @@ -6,7 +6,8 @@ %(encoding)s - +%(indexing_disable)s +

%(archive)s Archives by %(archtype)s

  • Messages sorted by: @@ -22,3 +23,4 @@ Ending: %(lastdate)s
    Messages: %(size)s

      +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/en/archtocentry.html mailman-2.1.18p/templates/en/archtocentry.html --- mailman-2.1.18/templates/en/archtocentry.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/en/archtocentry.html 2014-05-03 12:22:36.843376950 -0700 @@ -5,7 +5,9 @@ [ Thread ] [ Subject ] [ Author ] +%(indexing_enable)s [ Date ] +%(indexing_disable)s %(textlink)s diff -ruP mailman-2.1.18/templates/en/archtoc.html mailman-2.1.18p/templates/en/archtoc.html --- mailman-2.1.18/templates/en/archtoc.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/en/archtoc.html 2014-05-03 12:22:36.843376950 -0700 @@ -6,15 +6,18 @@ %(meta)s +%(indexing_disable)s

      The %(listname)s Archives

      You can get more information about this list or you can download the full raw archive (%(size)s).

      + %(htsearch)s %(noarchive_msg)s %(archive_listing_start)s %(archive_listing)s %(archive_listing_end)s +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/en/archtocnombox.html mailman-2.1.18p/templates/en/archtocnombox.html --- mailman-2.1.18/templates/en/archtocnombox.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/en/archtocnombox.html 2014-05-03 12:22:36.843376950 -0700 @@ -6,13 +6,16 @@ %(meta)s +%(indexing_disable)s

      The %(listname)s Archives

      You can get more information about this list.

      + %(htsearch)s %(noarchive_msg)s %(archive_listing_start)s %(archive_listing)s %(archive_listing_end)s +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/en/article.html mailman-2.1.18p/templates/en/article.html --- mailman-2.1.18/templates/en/article.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/en/article.html 2014-05-03 12:22:36.843376950 -0700 @@ -17,6 +17,7 @@

      %(subject_html)s

      +%(indexing_disable)s %(author_html)s %(email_html)s @@ -33,9 +34,11 @@
    +%(indexing_enable)s %(body)s +%(indexing_disable)s

      @@ -52,4 +55,5 @@
      More information about the %(listname)s mailing list
      +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/en/htdig_access_error.html mailman-2.1.18p/templates/en/htdig_access_error.html --- mailman-2.1.18/templates/en/htdig_access_error.html 1969-12-31 16:00:00.000000000 -0800 +++ mailman-2.1.18p/templates/en/htdig_access_error.html 2014-05-03 12:22:36.843376950 -0700 @@ -0,0 +1,22 @@ + + + htdig Archives Access Failure + + +

      htdig Archives Access Failure

      +%(error)s +

      + If you want to make another attempt to access a list archive then go via the + list users information page. +

      +

      + If this problem persists then please e-mail the following information to the +%(mailto)s: +

      +
      +    %(referer)s
+    %(uri)s
+
      +
      + + diff -ruP mailman-2.1.18/templates/en/htdig_auth_failure.html mailman-2.1.18p/templates/en/htdig_auth_failure.html --- mailman-2.1.18/templates/en/htdig_auth_failure.html 1969-12-31 16:00:00.000000000 -0800 +++ mailman-2.1.18p/templates/en/htdig_auth_failure.html 2014-05-03 12:22:36.847376950 -0700 @@ -0,0 +1,22 @@ +

      + You are not authorised to access the URL referenced. +

      +

      + This access failure may be due to: +

      +
        +
      1. + If cookies are disabled in your browser then your attempt to + authenticate yourself for access to the desired list will have + been compromised. You should enable cookies in your browser and + try again. +
        2. +
      2. + You have not attempted to authenticate yourself and are trying + to access private data. +
        3. +
      3. + An earlier attempt to authenticate yourself for access to private + data failed. +
        4. +
      diff -ruP mailman-2.1.18/templates/en/htdig_conf.txt mailman-2.1.18p/templates/en/htdig_conf.txt --- mailman-2.1.18/templates/en/htdig_conf.txt 1969-12-31 16:00:00.000000000 -0800 +++ mailman-2.1.18p/templates/en/htdig_conf.txt 2014-05-03 12:22:36.847376950 -0700 @@ -0,0 +1,53 @@ +# There is nothing to language translate in this template which is for the +# Mailman-htdig integration +# +# This is taken from the example config file for ht://Dig, with most comments excised +# See the htdig.conf from the distribution you have installed +# +# This is the template for the per mailing list htddig.conf files +# +%(htdig_extras)s +database_dir: %(databases)s +start_url: %(starturl)s +limit_urls_to: ${start_url} +local_urls: %(urlpath)s=%(filepath)s +local_urls_only: true +url_part_aliases: %(url_part_aliases)s +script_name: %(mmsearchcgi)s +noindex_end: %(indexing_enable)s +noindex_start: %(indexing_disable)s +exclude_urls: %(excluded_urls)s +bad_extensions: .wav .gz .z .sit .au .zip .tar .hqx .exe .com .gif \ + .jpg .jpeg .aiff .class .map .ram .tgz .bin .rpm .mpg .mov .avi +maintainer: %(maintainer)s +max_head_length: 10000 +max_doc_size: 200000 +no_excerpt_show_top: true +search_algorithm: exact:1 synonyms:0.5 endings:0.1 +template_map: Long long ${common_dir}/long.html \ + Short short ${common_dir}/short.html +template_name: short +next_page_text: next +no_next_page_text: +prev_page_text: prev +no_prev_page_text: +page_number_text: '1' \ + '2' \ + '3' \ + '4' \ + '5' \ + '6' \ + '7' \ + '8' \ + '9' \ + '10' +no_page_number_text: '1' \ + '2' \ + '3' \ + '4' \ + '5' \ + '6' \ + '7' \ + '8' \ + '9' \ + '10' diff -ruP mailman-2.1.18/templates/en/htdig.html mailman-2.1.18p/templates/en/htdig.html --- mailman-2.1.18/templates/en/htdig.html 1969-12-31 16:00:00.000000000 -0800 +++ mailman-2.1.18p/templates/en/htdig.html 2014-05-03 12:22:36.847376950 -0700 @@ -0,0 +1,43 @@ + + + %(realname)s Private Archives Authentication + + +
      +%(message)s + + + + + + + + + + + + + + + +
      + %(realname)s Private + Archives Authentication +
      Email address:
      Password:
      +
      +

      Important: From this point on, you + must have cookies enabled in your browser, otherwise no + administrative changes will take effect. + +

      Session cookies are used in Mailman's + administrative interface so that you don't need to + re-authenticate with every administrative operation. This + cookie will expire automatically when you exit your browser, or + you can explicitly expire the cookie by hitting the + Logout link under Other Administrative + Activities (which you'll see once you successfully log in). +

      + + diff -ruP mailman-2.1.18/templates/en/TOC_htsearch.html mailman-2.1.18p/templates/en/TOC_htsearch.html --- mailman-2.1.18/templates/en/TOC_htsearch.html 1969-12-31 16:00:00.000000000 -0800 +++ mailman-2.1.18p/templates/en/TOC_htsearch.html 2014-05-03 12:22:36.847376950 -0700 @@ -0,0 +1,40 @@ +

      + To search this archive fill in the following form: +

      +

      +

      + + Match: + Format: + Sort by: + + + +
      + Search: + + +
      +

      +

      + Note:The archive search index was last rebuilt at + %(lastrun)s. Any postings after that will not be found by + a search. Index rebuild is usally done once every 24 hours for + this list. You can use a "View by date" link below to access + more recent postings. +

      diff -ruP mailman-2.1.18/templates/es/article.html mailman-2.1.18p/templates/es/article.html --- mailman-2.1.18/templates/es/article.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/es/article.html 2014-05-03 12:22:36.847376950 -0700 @@ -17,6 +17,7 @@

      %(subject_html)s

      +%(indexing_disable)s %(author_html)s %(email_html)s @@ -33,9 +34,11 @@
    +%(indexing_enable)s %(body)s +%(indexing_disable)s

      @@ -51,4 +54,5 @@
      Ms informacin sobre la lista de distribucin %(listname)s
      +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/et/article.html mailman-2.1.18p/templates/et/article.html --- mailman-2.1.18/templates/et/article.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/et/article.html 2014-05-03 12:22:36.847376950 -0700 @@ -18,6 +18,7 @@

      %(subject_html)s

      +%(indexing_disable)s %(author_html)s
    +%(indexing_enable)s %(body)s +%(indexing_disable)s

      @@ -53,4 +56,5 @@
    %(listname)s info
    +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/fi/article.html mailman-2.1.18p/templates/fi/article.html --- mailman-2.1.18/templates/fi/article.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/fi/article.html 2014-05-03 12:22:36.847376950 -0700 @@ -19,6 +19,7 @@

    %(subject_html)s

    +%(indexing_disable)s %(author_html)s
+%(indexing_enable)s %(body)s +%(indexing_disable)s

    @@ -54,5 +57,6 @@
    Listietoja postituslistasta %(listname)s
    +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/fr/archidxfoot.html mailman-2.1.18p/templates/fr/archidxfoot.html --- mailman-2.1.18/templates/fr/archidxfoot.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/fr/archidxfoot.html 2014-05-03 12:22:36.847376950 -0700 @@ -1,3 +1,4 @@ +%(indexing_disable)s

Date du dernier message: @@ -16,5 +17,6 @@

Archive générée par Pipermail %(version)s. +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/fr/archidxhead.html mailman-2.1.18p/templates/fr/archidxhead.html --- mailman-2.1.18/templates/fr/archidxhead.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/fr/archidxhead.html 2014-05-03 12:22:36.847376950 -0700 @@ -6,6 +6,7 @@ %(encoding)s +%(indexing_disable)s

%(archive)s Archives par %(archtype)s

    @@ -22,3 +23,4 @@ Fin : %(lastdate)s
    Messages : %(size)s

      +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/fr/archtocentry.html mailman-2.1.18p/templates/fr/archtocentry.html --- mailman-2.1.18/templates/fr/archtocentry.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/fr/archtocentry.html 2014-05-03 12:22:36.847376950 -0700 @@ -5,7 +5,9 @@ [ Enfilade ] [ Sujet ] [ Auteur ] +%(indexing_enable)s [ Date ] +%(indexing_disable)s %(textlink)s diff -ruP mailman-2.1.18/templates/fr/archtoc.html mailman-2.1.18p/templates/fr/archtoc.html --- mailman-2.1.18/templates/fr/archtoc.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/fr/archtoc.html 2014-05-03 12:22:36.847376950 -0700 @@ -6,14 +6,17 @@ %(meta)s +%(indexing_disable)s

      Les Archives de %(listname)s

      vous pouvez obtenir plus d' informations à propos de cette liste ou vous pouvez  télécharger les archives complètes (%(size)s).

      + %(htsearch)s %(noarchive_msg)s %(archive_listing_start)s %(archive_listing)s %(archive_listing_end)s +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/fr/article.html mailman-2.1.18p/templates/fr/article.html --- mailman-2.1.18/templates/fr/article.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/fr/article.html 2014-05-03 12:22:36.847376950 -0700 @@ -18,6 +18,7 @@

      %(subject_html)s

      +%(indexing_disable)s %(author_html)s
    +%(indexing_enable)s %(body)s +%(indexing_disable)s

      @@ -52,6 +55,7 @@
    Plus d'informations sur la liste de diffusion %(listname)s
    +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/hr/archidxfoot.html mailman-2.1.18p/templates/hr/archidxfoot.html --- mailman-2.1.18/templates/hr/archidxfoot.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/hr/archidxfoot.html 2014-05-03 12:22:36.847376950 -0700 @@ -1,3 +1,4 @@ +%(indexing_disable)s

Datum zadnje poruke: @@ -17,5 +18,6 @@

Ova arhiva je generirana sa Pipermail %(version)s. +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/hr/archidxhead.html mailman-2.1.18p/templates/hr/archidxhead.html --- mailman-2.1.18/templates/hr/archidxhead.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/hr/archidxhead.html 2014-05-03 12:22:36.847376950 -0700 @@ -6,6 +6,7 @@ %(encoding)s +%(indexing_disable)s

%(archive)s Arhiva od %(archtype)s

    @@ -22,3 +23,4 @@ Kraj: %(lastdate)s
    Poruke: %(size)s

      +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/hr/archtoc.html mailman-2.1.18p/templates/hr/archtoc.html --- mailman-2.1.18/templates/hr/archtoc.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/hr/archtoc.html 2014-05-03 12:22:36.847376950 -0700 @@ -6,15 +6,18 @@ %(meta)s +%(indexing_disable)s

      %(listname)s Arhiva

      Moete dobiti vie informacija o ovoj listi ili moete downloadati cijelu arhivu (%(size)s).

      + %(htsearch)s %(noarchive_msg)s %(archive_listing_start)s %(archive_listing)s %(archive_listing_end)s +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/hr/article.html mailman-2.1.18p/templates/hr/article.html --- mailman-2.1.18/templates/hr/article.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/hr/article.html 2014-05-03 12:22:36.847376950 -0700 @@ -17,6 +17,7 @@

      %(subject_html)s

      +%(indexing_disable)s %(author_html)s %(email_html)s @@ -33,9 +34,11 @@
    +%(indexing_enable)s %(body)s +%(indexing_disable)s

      @@ -52,4 +55,5 @@
      Vie informacija o %(listname)s mailing listi
      +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/hu/archidxfoot.html mailman-2.1.18p/templates/hu/archidxfoot.html --- mailman-2.1.18/templates/hu/archidxfoot.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/hu/archidxfoot.html 2014-05-03 12:22:36.847376950 -0700 @@ -1,3 +1,4 @@ +%(indexing_disable)s

    Utols levl idpontja: @@ -15,5 +16,6 @@

    Az archvum a Pipermail %(version)s verzjval kszlt. +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/hu/archidxhead.html mailman-2.1.18p/templates/hu/archidxhead.html --- mailman-2.1.18/templates/hu/archidxhead.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/hu/archidxhead.html 2014-05-03 12:22:36.847376950 -0700 @@ -6,6 +6,7 @@ %(encoding)s +%(indexing_disable)s

    %(archive)si Archvum %(archtype)s szerint rendezve

      @@ -21,3 +22,5 @@ Utols idpont: %(lastdate)s
      zenetek: %(size)s

        +%(indexing_enable)s + diff -ruP mailman-2.1.18/templates/hu/archtocentry.html mailman-2.1.18p/templates/hu/archtocentry.html --- mailman-2.1.18/templates/hu/archtocentry.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/hu/archtocentry.html 2014-05-03 12:22:36.847376950 -0700 @@ -5,7 +5,9 @@ [ Tma ] [ Trgy ] [ Szerz ] +%(indexing_enable)s [ Dtum ] +%(indexing_disable)s %(textlink)s diff -ruP mailman-2.1.18/templates/hu/archtoc.html mailman-2.1.18p/templates/hu/archtoc.html --- mailman-2.1.18/templates/hu/archtoc.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/hu/archtoc.html 2014-05-03 12:22:36.847376950 -0700 @@ -6,15 +6,18 @@ %(meta)s +%(indexing_disable)s

        A(z) %(listname)s Archvum

        Tovbbi informcikat a listrl itt olvashatsz vagy letltheted a teljes nyers archvumt (%(size)s).

        + %(htsearch)s %(noarchive_msg)s %(archive_listing_start)s %(archive_listing)s %(archive_listing_end)s +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/hu/article.html mailman-2.1.18p/templates/hu/article.html --- mailman-2.1.18/templates/hu/article.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/hu/article.html 2014-05-03 12:22:36.847376950 -0700 @@ -18,6 +18,7 @@

        %(subject_html)s

        +%(indexing_disable)s %(author_html)s %(email_html)s @@ -34,9 +35,11 @@
      +%(indexing_enable)s %(body)s +%(indexing_disable)s

        @@ -51,4 +54,5 @@
      Tovbbi informcik a(z) %(listname)s levelezlistrl
      +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/it/archidxfoot.html mailman-2.1.18p/templates/it/archidxfoot.html --- mailman-2.1.18/templates/it/archidxfoot.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/it/archidxfoot.html 2014-05-03 12:22:36.847376950 -0700 @@ -1,3 +1,4 @@ +%(indexing_disable)s

    Data dell'ultimo messaggio: @@ -17,5 +18,6 @@

    Questo archivio è stato generato da Pipermail %(version)s. +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/it/archidxhead.html mailman-2.1.18p/templates/it/archidxhead.html --- mailman-2.1.18/templates/it/archidxhead.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/it/archidxhead.html 2014-05-03 12:22:36.847376950 -0700 @@ -6,6 +6,7 @@ %(encoding)s +%(indexing_disable)s

    Archivio %(archive)s ordinato per %(archtype)s

      @@ -22,3 +23,4 @@ Data fine: %(lastdate)s
      Messaggi: %(size)s

        +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/it/archtocentry.html mailman-2.1.18p/templates/it/archtocentry.html --- mailman-2.1.18/templates/it/archtocentry.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/it/archtocentry.html 2014-05-03 12:22:36.847376950 -0700 @@ -5,7 +5,9 @@ [ Thread ] [ Soggetto ] [ Autore ] +%(indexing_enable)s [ Data ] +%(indexing_disable)s %(textlink)s diff -ruP mailman-2.1.18/templates/it/archtoc.html mailman-2.1.18p/templates/it/archtoc.html --- mailman-2.1.18/templates/it/archtoc.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/it/archtoc.html 2014-05-03 12:22:36.847376950 -0700 @@ -6,15 +6,18 @@ %(meta)s +%(indexing_disable)s

        Gli archivi della lista %(listname)s

        Puoi ottenere ulteriori informazioni su questa lista o puoi scaricare l'intero archivio grezzo (%(size)s).

        + %(htsearch)s %(noarchive_msg)s %(archive_listing_start)s %(archive_listing)s %(archive_listing_end)s +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/it/archtocnombox.html mailman-2.1.18p/templates/it/archtocnombox.html --- mailman-2.1.18/templates/it/archtocnombox.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/it/archtocnombox.html 2014-05-03 12:22:36.847376950 -0700 @@ -6,13 +6,16 @@ %(meta)s +%(indexing_disable)s

        Gli archivi di %(listname)s

        Maggiori informazioni su questa lista.

        + %(htsearch)s %(noarchive_msg)s %(archive_listing_start)s %(archive_listing)s %(archive_listing_end)s +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/it/article.html mailman-2.1.18p/templates/it/article.html --- mailman-2.1.18/templates/it/article.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/it/article.html 2014-05-03 12:22:36.847376950 -0700 @@ -17,6 +17,7 @@

        %(subject_html)s

        +%(indexing_disable)s %(author_html)s %(email_html)s @@ -33,9 +34,11 @@
      +%(indexing_enable)s %(body)s +%(indexing_disable)s

        @@ -51,4 +54,5 @@
        Maggiori informazioni sulla lista %(listname)s
        +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/ja/archidxfoot.html mailman-2.1.18p/templates/ja/archidxfoot.html --- mailman-2.1.18/templates/ja/archidxfoot.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/ja/archidxfoot.html 2014-05-03 12:22:36.847376950 -0700 @@ -1,3 +1,4 @@ +%(indexing_disable)s

      ǿ: @@ -17,5 +18,6 @@

      ¸ˤ Pipermail %(version)s Ǻޤ. +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/ja/archidxhead.html mailman-2.1.18p/templates/ja/archidxhead.html --- mailman-2.1.18/templates/ja/archidxhead.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/ja/archidxhead.html 2014-05-03 12:22:36.847376950 -0700 @@ -6,6 +6,7 @@ %(encoding)s +%(indexing_disable)s

      %(archive)s ¸ %(archtype)s

        @@ -22,3 +23,4 @@ ǽ: %(lastdate)s
        : %(size)s

          +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/ja/archtocentry.html mailman-2.1.18p/templates/ja/archtocentry.html --- mailman-2.1.18/templates/ja/archtocentry.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/ja/archtocentry.html 2014-05-03 12:22:36.847376950 -0700 @@ -5,7 +5,9 @@ [ å ] [ ̾ ] [ ȯ ] +%(indexing_enable)s [ ] +%(indexing_disable)s %(textlink)s diff -ruP mailman-2.1.18/templates/ja/archtoc.html mailman-2.1.18p/templates/ja/archtoc.html --- mailman-2.1.18/templates/ja/archtoc.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/ja/archtoc.html 2014-05-03 12:22:36.847376950 -0700 @@ -6,15 +6,18 @@ %(meta)s +%(indexing_disable)s

          %(listname)s ¸

          ꥹȤΰ * Υ᡼mboxǥ (%(size)s).

          + %(htsearch)s %(noarchive_msg)s %(archive_listing_start)s %(archive_listing)s %(archive_listing_end)s +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/ja/article.html mailman-2.1.18p/templates/ja/article.html --- mailman-2.1.18/templates/ja/article.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/ja/article.html 2014-05-03 12:22:36.851376950 -0700 @@ -17,6 +17,7 @@

          %(subject_html)s

          +%(indexing_disable)s %(author_html)s %(email_html)s @@ -33,9 +34,11 @@
        +%(indexing_enable)s %(body)s +%(indexing_disable)s

          @@ -52,4 +55,5 @@
          %(listname)s ᡼󥰥ꥹȤΰ
          +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/ko/article.html mailman-2.1.18p/templates/ko/article.html --- mailman-2.1.18/templates/ko/article.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/ko/article.html 2014-05-03 12:22:36.851376950 -0700 @@ -18,6 +18,7 @@

          %(subject_html)s

          +%(indexing_disable)s %(author_html)s
        +%(indexing_enable)s %(body)s +%(indexing_disable)s

        +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/lt/archidxfoot.html mailman-2.1.18p/templates/lt/archidxfoot.html --- mailman-2.1.18/templates/lt/archidxfoot.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/lt/archidxfoot.html 2014-05-03 12:22:36.851376950 -0700 @@ -1,3 +1,4 @@ +%(indexing_disable)s

      Paskutiniojo laiko data: @@ -16,5 +17,6 @@

      Archyv generavo Pipermail %(version)s. +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/lt/archidxhead.html mailman-2.1.18p/templates/lt/archidxhead.html --- mailman-2.1.18/templates/lt/archidxhead.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/lt/archidxhead.html 2014-05-03 12:22:36.851376950 -0700 @@ -6,6 +6,7 @@ %(encoding)s +%(indexing_disable)s

      %(archive)s Archyvas %(archtype)s

        @@ -21,3 +22,5 @@ Paskutinysis: %(lastdate)s
        Laik: %(size)s

          +%(indexing_enable)s + diff -ruP mailman-2.1.18/templates/lt/archtocentry.html mailman-2.1.18p/templates/lt/archtocentry.html --- mailman-2.1.18/templates/lt/archtocentry.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/lt/archtocentry.html 2014-05-03 12:22:36.851376950 -0700 @@ -5,7 +5,9 @@ [ Gija ] [ Tema ] [ Autorius ] +%(indexing_enable)s [ Data ] +%(indexing_disable)s %(textlink)s diff -ruP mailman-2.1.18/templates/lt/archtoc.html mailman-2.1.18p/templates/lt/archtoc.html --- mailman-2.1.18/templates/lt/archtoc.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/lt/archtoc.html 2014-05-03 12:22:36.851376950 -0700 @@ -6,15 +6,18 @@ %(meta)s +%(indexing_disable)s

          Forumo %(listname)s archyvai

          Paspaud ia galite suinoti daugiau apie forum; ia - atsisisti vis forumo archyv. (%(size)s).

          + %(htsearch)s %(noarchive_msg)s %(archive_listing_start)s %(archive_listing)s %(archive_listing_end)s +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/lt/article.html mailman-2.1.18p/templates/lt/article.html --- mailman-2.1.18/templates/lt/article.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/lt/article.html 2014-05-03 12:22:36.851376950 -0700 @@ -18,6 +18,7 @@

          %(subject_html)s

          +%(indexing_disable)s %(author_html)s
        +%(indexing_enable)s %(body)s +%(indexing_disable)s

        +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/nl/article.html mailman-2.1.18p/templates/nl/article.html --- mailman-2.1.18/templates/nl/article.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/nl/article.html 2014-05-03 12:22:36.851376950 -0700 @@ -17,6 +17,7 @@

        %(subject_html)s

        +%(indexing_disable)s %(author_html)s %(email_html)s @@ -33,9 +34,11 @@
      +%(indexing_enable)s %(body)s +%(indexing_disable)s

        @@ -52,4 +55,5 @@
        Meer informatie over de %(listname)s maillijst
        +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/no/archidxfoot.html mailman-2.1.18p/templates/no/archidxfoot.html --- mailman-2.1.18/templates/no/archidxfoot.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/no/archidxfoot.html 2014-05-03 12:22:36.851376950 -0700 @@ -1,3 +1,4 @@ +%(indexing_disable)s

      Dato på nyeste melding: @@ -17,5 +18,6 @@

      Dette arkivet ble generert av Pipermail %(version)s. +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/no/archidxhead.html mailman-2.1.18p/templates/no/archidxhead.html --- mailman-2.1.18/templates/no/archidxhead.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/no/archidxhead.html 2014-05-03 12:22:36.851376950 -0700 @@ -6,6 +6,7 @@ %(encoding)s +%(indexing_disable)s

      Arkivet for %(archive)s sortert på %(archtype)s

        @@ -22,3 +23,5 @@ Sluttdato: %(lastdate)s
        Meldinger: %(size)s

          +%(indexing_enable)s + diff -ruP mailman-2.1.18/templates/no/archtocentry.html mailman-2.1.18p/templates/no/archtocentry.html --- mailman-2.1.18/templates/no/archtocentry.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/no/archtocentry.html 2014-05-03 12:22:36.851376950 -0700 @@ -5,7 +5,9 @@ [ Tråd ] [ Tittel ] [ Forfatter ] +%(indexing_enable)s [ Dato ] +%(indexing_disable)s %(textlink)s diff -ruP mailman-2.1.18/templates/no/archtoc.html mailman-2.1.18p/templates/no/archtoc.html --- mailman-2.1.18/templates/no/archtoc.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/no/archtoc.html 2014-05-03 12:22:36.851376950 -0700 @@ -6,15 +6,18 @@ %(meta)s +%(indexing_disable)s

          %(listname)s arkivet

          Du kan se mer informasjon om denne listen eller du kan laste ned hele arkivet (%(size)s).

          + %(htsearch)s %(noarchive_msg)s %(archive_listing_start)s %(archive_listing)s %(archive_listing_end)s +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/no/archtocnombox.html mailman-2.1.18p/templates/no/archtocnombox.html --- mailman-2.1.18/templates/no/archtocnombox.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/no/archtocnombox.html 2014-05-03 12:22:36.851376950 -0700 @@ -6,13 +6,16 @@ %(meta)s +%(indexing_disable)s

          %(listname)s arkivet

          Du kan se mer informasjon om denne listen.

          + %(htsearch)s %(noarchive_msg)s %(archive_listing_start)s %(archive_listing)s %(archive_listing_end)s +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/no/article.html mailman-2.1.18p/templates/no/article.html --- mailman-2.1.18/templates/no/article.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/no/article.html 2014-05-03 12:22:36.851376950 -0700 @@ -17,6 +17,7 @@

          %(subject_html)s

          +%(indexing_disable)s %(author_html)s %(email_html)s @@ -33,9 +34,11 @@
        +%(indexing_enable)s %(body)s +%(indexing_disable)s

          @@ -51,4 +54,5 @@
          Mer informasjon om epostlisten %(listname)s.
          +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/pl/archidxfoot.html mailman-2.1.18p/templates/pl/archidxfoot.html --- mailman-2.1.18/templates/pl/archidxfoot.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/pl/archidxfoot.html 2014-05-03 12:22:36.851376950 -0700 @@ -1,3 +1,4 @@ +%(indexing_disable)s

        Data ostatniej wiadomoci: @@ -17,5 +18,6 @@

        Archiwum zostao wygenerowane przez program Pipermail %(version)s. +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/pl/archidxhead.html mailman-2.1.18p/templates/pl/archidxhead.html --- mailman-2.1.18/templates/pl/archidxhead.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/pl/archidxhead.html 2014-05-03 12:22:36.851376950 -0700 @@ -6,6 +6,7 @@ %(encoding)s +%(indexing_disable)s

        Archiwum za %(archive)s posortowane wedug %(archtype)s

          @@ -22,3 +23,4 @@ Koniec: %(lastdate)s
          Liczba wiadomoci: %(size)s

            +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/pl/archtocentry.html mailman-2.1.18p/templates/pl/archtocentry.html --- mailman-2.1.18/templates/pl/archtocentry.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/pl/archtocentry.html 2014-05-03 12:22:36.851376950 -0700 @@ -5,7 +5,9 @@ [ Wtku ] [ Tematu ] [ Autora ] +%(indexing_enable)s [ Daty ] +%(indexing_disable)s %(textlink)s diff -ruP mailman-2.1.18/templates/pl/archtoc.html mailman-2.1.18p/templates/pl/archtoc.html --- mailman-2.1.18/templates/pl/archtoc.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/pl/archtoc.html 2014-05-03 12:22:36.851376950 -0700 @@ -6,15 +6,18 @@ %(meta)s +%(indexing_disable)s

            Archiwum listy %(listname)s

            Odwied stron informacyjn tej listy, albo pobierz cae archiwum (%(size)s).

            + %(htsearch)s %(noarchive_msg)s %(archive_listing_start)s %(archive_listing)s %(archive_listing_end)s +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/pl/archtocnombox.html mailman-2.1.18p/templates/pl/archtocnombox.html --- mailman-2.1.18/templates/pl/archtocnombox.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/pl/archtocnombox.html 2014-05-03 12:22:36.851376950 -0700 @@ -6,13 +6,16 @@ %(meta)s +%(indexing_disable)s

            Archiwum listy %(listname)s

            Odwied stron informacyjn tej listy.

            + %(htsearch)s %(noarchive_msg)s %(archive_listing_start)s %(archive_listing)s %(archive_listing_end)s +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/pl/article.html mailman-2.1.18p/templates/pl/article.html --- mailman-2.1.18/templates/pl/article.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/pl/article.html 2014-05-03 12:22:36.851376950 -0700 @@ -16,6 +16,7 @@ %(next)s +%(indexing_disable)s

            %(subject_html)s

            %(author_html)s
          +%(indexing_enable)s %(body)s +%(indexing_disable)s

            @@ -52,4 +55,5 @@
            Wicej informacji o licie %(listname)s
            +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/pt/archidxfoot.html mailman-2.1.18p/templates/pt/archidxfoot.html --- mailman-2.1.18/templates/pt/archidxfoot.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/pt/archidxfoot.html 2014-05-03 12:22:36.851376950 -0700 @@ -1,3 +1,4 @@ +%(indexing_disable)s

          Data da ltima mensagem: @@ -17,5 +18,6 @@

          Este arquivo foi criado pelo Pipermail %(version)s. +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/pt/archidxhead.html mailman-2.1.18p/templates/pt/archidxhead.html --- mailman-2.1.18/templates/pt/archidxhead.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/pt/archidxhead.html 2014-05-03 12:22:36.851376950 -0700 @@ -6,6 +6,7 @@ %(encoding)s +%(indexing_disable)s

          %(archive)s Arquivos por %(archtype)s

            @@ -22,3 +23,4 @@ Fim: %(lastdate)s
            Mensagens: %(size)s

              +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/pt/archtocentry.html mailman-2.1.18p/templates/pt/archtocentry.html --- mailman-2.1.18/templates/pt/archtocentry.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/pt/archtocentry.html 2014-05-03 12:22:36.851376950 -0700 @@ -5,7 +5,9 @@ [ Tpico ] [ Assunto ] [ Autor ] +%(indexing_enable)s [ Data ] +%(indexing_disable)s %(textlink)s diff -ruP mailman-2.1.18/templates/pt/archtoc.html mailman-2.1.18p/templates/pt/archtoc.html --- mailman-2.1.18/templates/pt/archtoc.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/pt/archtoc.html 2014-05-03 12:22:36.851376950 -0700 @@ -6,6 +6,7 @@ %(meta)s +%(indexing_disable)s

              Arquivos de %(listname)s

              Pode ter mais informao sobre esta lista @@ -13,9 +14,11 @@ no processado (%(size)s).

              + %(htsearch)s %(noarchive_msg)s %(archive_listing_start)s %(archive_listing)s %(archive_listing_end)s +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/pt/article.html mailman-2.1.18p/templates/pt/article.html --- mailman-2.1.18/templates/pt/article.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/pt/article.html 2014-05-03 12:22:36.851376950 -0700 @@ -16,6 +16,7 @@ %(next)s +%(indexing_disable)s

              %(subject_html)s

              %(author_html)s
            +%(indexing_enable)s %(body)s +%(indexing_disable)s

            +%(indexing_enable)s %(body)s +%(indexing_disable)s

              @@ -52,4 +55,5 @@
              Mais detalhes sobre a lista de discussão %(listname)s
              +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/ro/archidxfoot.html mailman-2.1.18p/templates/ro/archidxfoot.html --- mailman-2.1.18/templates/ro/archidxfoot.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/ro/archidxfoot.html 2014-05-03 12:22:36.851376950 -0700 @@ -1,3 +1,4 @@ +%(indexing_disable)s

            Data ultimului mesaj: @@ -16,5 +17,6 @@

            Aceast arhiv a fost generat de Pipermail %(version)s. +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/ro/archidxhead.html mailman-2.1.18p/templates/ro/archidxhead.html --- mailman-2.1.18/templates/ro/archidxhead.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/ro/archidxhead.html 2014-05-03 12:22:36.851376950 -0700 @@ -7,6 +7,7 @@ %(encoding)s +%(indexing_disable)s

            Arhivele %(archive)s dup %(archtype)s

              @@ -22,3 +23,4 @@ Se termin la: %(lastdate)s
              Mesaje: %(size)s

                +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/ro/archtocentry.html mailman-2.1.18p/templates/ro/archtocentry.html --- mailman-2.1.18/templates/ro/archtocentry.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/ro/archtocentry.html 2014-05-03 12:22:36.851376950 -0700 @@ -5,7 +5,9 @@ [ Fir ] [ Subiect ] [ Autor ] +%(indexing_enable)s [ Dat ] +%(indexing_disable)s %(textlink)s diff -ruP mailman-2.1.18/templates/ro/archtoc.html mailman-2.1.18p/templates/ro/archtoc.html --- mailman-2.1.18/templates/ro/archtoc.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/ro/archtoc.html 2014-05-03 12:22:36.851376950 -0700 @@ -7,15 +7,18 @@ %(meta)s +%(indexing_disable)s

                Arhivele listei de discuii %(listname)s

                Putei obine mai multe detalii despre aceast list sau putei descrca ntreaga arhiv n form brut (%(size)s).

                + %(htsearch)s %(noarchive_msg)s %(archive_listing_start)s %(archive_listing)s %(archive_listing_end)s +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/ro/article.html mailman-2.1.18p/templates/ro/article.html --- mailman-2.1.18/templates/ro/article.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/ro/article.html 2014-05-03 12:22:36.851376950 -0700 @@ -16,6 +16,7 @@ %(next)s +%(indexing_disable)s

                %(subject_html)s

                %(author_html)s
              +%(indexing_enable)s %(body)s +%(indexing_disable)s

                @@ -51,4 +54,5 @@
                Mai multe informaii despre lista de discuii %(listname)s
                +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/ru/archidxfoot.html mailman-2.1.18p/templates/ru/archidxfoot.html --- mailman-2.1.18/templates/ru/archidxfoot.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/ru/archidxfoot.html 2014-05-03 12:22:36.851376950 -0700 @@ -1,3 +1,4 @@ +%(indexing_disable)s

              : @@ -15,5 +16,6 @@

              Pipermail %(version)s. +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/ru/archidxhead.html mailman-2.1.18p/templates/ru/archidxhead.html --- mailman-2.1.18/templates/ru/archidxhead.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/ru/archidxhead.html 2014-05-03 12:22:36.851376950 -0700 @@ -6,6 +6,7 @@ %(encoding)s +%(indexing_disable)s

              %(archive)s, %(archtype)s

                @@ -21,3 +22,4 @@ : %(lastdate)s
                : %(size)s

                  +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/ru/archtocentry.html mailman-2.1.18p/templates/ru/archtocentry.html --- mailman-2.1.18/templates/ru/archtocentry.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/ru/archtocentry.html 2014-05-03 12:22:36.851376950 -0700 @@ -5,7 +5,9 @@ [ ] [ ] [ ] +%(indexing_enable)s [ ] +%(indexing_disable)s %(textlink)s diff -ruP mailman-2.1.18/templates/ru/archtoc.html mailman-2.1.18p/templates/ru/archtoc.html --- mailman-2.1.18/templates/ru/archtoc.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/ru/archtoc.html 2014-05-03 12:22:36.851376950 -0700 @@ -6,14 +6,17 @@ %(meta)s +%(indexing_disable)s

                  %(listname)s

                  . mbox (%(size)s).

                  + %(htsearch)s %(noarchive_msg)s %(archive_listing_start)s %(archive_listing)s %(archive_listing_end)s +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/ru/article.html mailman-2.1.18p/templates/ru/article.html --- mailman-2.1.18/templates/ru/article.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/ru/article.html 2014-05-03 12:22:36.851376950 -0700 @@ -17,6 +17,7 @@

                  %(subject_html)s

                  +%(indexing_disable)s %(author_html)s %(email_html)s @@ -33,9 +34,11 @@
                +%(indexing_enable)s %(body)s +%(indexing_disable)s

                  @@ -50,4 +53,5 @@
                %(listname)s
                +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/sl/archidxfoot.html mailman-2.1.18p/templates/sl/archidxfoot.html --- mailman-2.1.18/templates/sl/archidxfoot.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/sl/archidxfoot.html 2014-05-03 12:22:36.851376950 -0700 @@ -1,3 +1,4 @@ +%(indexing_disable)s

              Datum zadnjega sporoila: @@ -17,5 +18,6 @@

              Ta arhiv je bil ustvarjen s programom Pipermail %(version)s. +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/sl/archidxhead.html mailman-2.1.18p/templates/sl/archidxhead.html --- mailman-2.1.18/templates/sl/archidxhead.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/sl/archidxhead.html 2014-05-03 12:22:36.851376950 -0700 @@ -6,6 +6,7 @@ %(encoding)s +%(indexing_disable)s

              Arhivi %(archive)s glede na %(archtype)s

                @@ -22,3 +23,4 @@ Konec: %(lastdate)s
                Sporoil: %(size)s

                  +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/sl/archtocentry.html mailman-2.1.18p/templates/sl/archtocentry.html --- mailman-2.1.18/templates/sl/archtocentry.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/sl/archtocentry.html 2014-05-03 12:22:36.851376950 -0700 @@ -5,7 +5,9 @@ [ Tema ] [ Zadeva ] [ Avtor ] +%(indexing_enable)s [ Datum ] +%(indexing_disable)s %(textlink)s diff -ruP mailman-2.1.18/templates/sl/archtoc.html mailman-2.1.18p/templates/sl/archtoc.html --- mailman-2.1.18/templates/sl/archtoc.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/sl/archtoc.html 2014-05-03 12:22:36.851376950 -0700 @@ -6,15 +6,18 @@ %(meta)s +%(indexing_disable)s

                  Arhivi za seznam %(listname)s

                  Ogledate si lahko dodatne informacije o tem seznamu ali pa prenesete celoten raw arhiv (%(size)s).

                  + %(htsearch)s %(noarchive_msg)s %(archive_listing_start)s %(archive_listing)s %(archive_listing_end)s +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/sl/article.html mailman-2.1.18p/templates/sl/article.html --- mailman-2.1.18/templates/sl/article.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/sl/article.html 2014-05-03 12:22:36.851376950 -0700 @@ -17,6 +17,7 @@

                  %(subject_html)s

                  +%(indexing_disable)s %(author_html)s %(email_html)s @@ -33,9 +34,11 @@
                +%(indexing_enable)s %(body)s +%(indexing_disable)s

                  @@ -52,4 +55,5 @@
                  Dodatne informacije o seznamu %(listname)s
                  +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/tr/archidxfoot.html mailman-2.1.18p/templates/tr/archidxfoot.html --- mailman-2.1.18/templates/tr/archidxfoot.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/tr/archidxfoot.html 2014-05-03 12:22:36.851376950 -0700 @@ -1,3 +1,4 @@ +%(indexing_disable)s

                Son mesaj tarihi: @@ -17,6 +18,7 @@

                Bu ariv Pipermail %(version)s ile oluturulmutur. +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/tr/archidxhead.html mailman-2.1.18p/templates/tr/archidxhead.html --- mailman-2.1.18/templates/tr/archidxhead.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/tr/archidxhead.html 2014-05-03 12:22:36.851376950 -0700 @@ -6,6 +6,7 @@ %(encoding)s +%(indexing_disable)s

                %(archtype)s sralamas ile %(archive)s Arivleri

                  @@ -22,4 +23,5 @@ Biti tarihi: %(lastdate)s
                  Mesaj says: %(size)s

                    +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/tr/archtocentry.html mailman-2.1.18p/templates/tr/archtocentry.html --- mailman-2.1.18/templates/tr/archtocentry.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/tr/archtocentry.html 2014-05-03 12:22:36.851376950 -0700 @@ -5,7 +5,9 @@ [ Thread ] [ Konu ] [ Yazar ] +%(indexing_enable)s [ Tarih ] +%(indexing_disable)s %(textlink)s diff -ruP mailman-2.1.18/templates/tr/archtoc.html mailman-2.1.18p/templates/tr/archtoc.html --- mailman-2.1.18/templates/tr/archtoc.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/tr/archtoc.html 2014-05-03 12:22:36.851376950 -0700 @@ -6,16 +6,19 @@ %(meta)s +%(indexing_disable)s

                    %(listname)s Arivleri

                    Bu listeyle ilgili daha fazla bilgi alabilirsiniz veya tm arivi indirebilirsiniz (%(size)s).

                    + %(htsearch)s %(noarchive_msg)s %(archive_listing_start)s %(archive_listing)s %(archive_listing_end)s +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/tr/article.html mailman-2.1.18p/templates/tr/article.html --- mailman-2.1.18/templates/tr/article.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/tr/article.html 2014-05-03 12:22:36.851376950 -0700 @@ -17,6 +17,7 @@

                    %(subject_html)s

                    +%(indexing_disable)s %(author_html)s %(email_html)s @@ -33,9 +34,11 @@
                  +%(indexing_enable)s %(body)s +%(indexing_disable)s

                    @@ -52,5 +55,6 @@
                    %(listname)s mesaj listesiyle ilgili daha fazla bilgi
                    +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/uk/archidxfoot.html mailman-2.1.18p/templates/uk/archidxfoot.html --- mailman-2.1.18/templates/uk/archidxfoot.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/uk/archidxfoot.html 2014-05-03 12:22:36.851376950 -0700 @@ -1,3 +1,4 @@ +%(indexing_disable)s

                  Дата останнього повідомлення: @@ -16,5 +17,6 @@

                  Цей архів було створено програмою Pipermail %(version)s. +%(indexing_enable)s diff -ruP mailman-2.1.18/templates/uk/archidxhead.html mailman-2.1.18p/templates/uk/archidxhead.html --- mailman-2.1.18/templates/uk/archidxhead.html 2014-05-03 10:37:22.000000000 -0700 +++ mailman-2.1.18p/templates/uk/archidxhead.html 2014-05-03 12:22:36.851376950 -0700 @@ -6,6 +6,7 @@ %(encoding)s +%(indexing_disable)s

                  Том %(archive)s, повідомлення впорядковані %(archtype)s