Revised scripts

[David Relson]

Greetings,

I've made a pass (or two) over the utility scripts (bf_compact,
bf_copy, bf_resize, and bf_tar).  They've been cleaned up, had some
minor defects fixed, and made more consistent.  AFAIK, they all
work as well as (or better than) before.  However there's always the
possibility of inadvertent breakages.  

I've attached copies of the scripts as they now stand.  If you use
them, please give these new versions a test drive and let me know how
it goes.

Thanks.

David

-------------- next part --------------
#!/bin/sh

#  bf_compact source_dir [wordlist_name...]
#
#    use to compact wordlist.db
#    replaces original directory with new one
#    renames original directory with '.old' extension

# $Id: bf_compact,v 1.22 2005/05/08 02:41:04 relson Exp $

set -e # die on errors

if [ -z "$1" ] ; then
    echo usage: bf_compact source_dir \[wordlist_name...\]
    exit 1
fi

# extract home directory
BOGOHOME="$1"
shift

if [ ! -d "$BOGOHOME" ] ; then
    echo $BOGOHOME must be a directory, not a file
    exit 1
fi

# strip trailing slashes
while true; do
    case "$BOGOHOME" in
	*/) BOGOHOME=${BOGOHOME%/} ;;
	*)  break ;;
    esac
done

# find wordlists
if [ -n "$1" ] ; then
    FILES="$@"
else
    FILES=`ls "$BOGOHOME" | grep .db$`
fi

TEMP="bf_compact.$$"

mkdir "$TEMP" || {
    echo "Cannot create directory $TEMP. Abort."
    exit 1
}

# copy Berkeley DB configuration if present
if test -f "$BOGOHOME"/DB_CONFIG ; then
    cp -p "$BOGOHOME"/DB_CONFIG "$TEMP"/
fi

# determine transactions
if test "`find "$BOGOHOME/" -name "log.??????????" -print`" != "" ; then
    TXN=yes
fi

# reload files
for FILE in $FILES ; do
    bogoutil -d "$BOGOHOME"/"$FILE" | bogoutil --db-transaction=no -l $TEMP/"$FILE"
done

if [ -n "$TXN" ] ; then
    #create database environment files
    bogofilter -e -C -d $TEMP --db-transaction=yes < /dev/null
fi

# remove $BOGOHOME.old so we don't move the new backup *into* it
# rather than renaming the backup to it.
rm -rf "$BOGOHOME.old"
mv "$BOGOHOME" "$BOGOHOME.old"
mv  $TEMP      "$BOGOHOME"
-------------- next part --------------
#!/bin/sh

#  bf_copy [-c] source_dir dest_dir
#
#    use to copy wordlist.db and related files
#    from one directory to another

# $Id: bf_copy,v 1.18 2005/05/08 02:41:39 relson Exp $

set -e # die on errors

COMPACT=0
while test "$1" ; do
    case "$1" in
	-c) COMPACT=1 ;;
	--) shift ; break ;;
	-*) echo "unknown option $1" >&2 ; exit 1 ;;
	*) break;
    esac
    shift
done

if [ $# -ne 2 ] ; then
    echo usage: bf_copy \[-c\] source_dir dest_dir
    echo "   use -c to copy active logs, not all"
    exit 1
fi

SRC="$1"
DST="$2"

# flush mempools
bogoutil --db-checkpoint="$SRC" || :

mkdir "$DST"

TMP=bfc.$$.unneeded
rm -f $TMP
trap "rm -rf $TMP \"$DST\"" 0
if test $COMPACT -eq 1 ; then
  # don't copy unneeded logs
  bogoutil --db-list-logfiles="$SRC" >$TMP
else
  : >$TMP
fi

# XXX FIXME - use Berkeley DB environment probing here
LOGS=`ls "$SRC"/log.* 2>/dev/null | grep -v -F -f $TMP || :`
if test "$LOGS" ; then cp -p $LOGS "$DST" ; fi
if test -f "$SRC"/DB_CONFIG ; then cp -p "$SRC"/DB_CONFIG "$DST" ; fi

for FILE in "$SRC"/*.db ; do
    SIZE=`bogoutil --db-print-pagesize="$FILE"`
    dd bs=$SIZE if=$FILE of="$DST/"`basename "$FILE"`
done

if test "$LOGS" ; then bogoutil --db-recover="$DST" ; fi
rm -f $TMP
trap - 0
-------------- next part --------------
#! /bin/sh

# (C) 2004 Matthias Andree, GNU GPL v2
#
# This script, bf_resize, tries to guess useful sizes for the lock
# environment.

# Usage: bf_resize [DIR]
#   DIR defaults to . and is the path of a bogofilter Berkeley database
#   environment.
#
# This script reads all databases in that directory, calculates a lock
# size and writes it to DB_CONFIG, then runs recovery to make the change
# effective.

set -e

BOGOHOME=${1:-.}

if [ ! -d "$BOGOHOME" ] ; then
    echo $BOGOHOME must be a directory, not a file
    exit 1
fi

# obtain list of database files
PAGES=0
DATABASES=`bogofilter -QQ -d "$BOGOHOME" | grep '^wordlist' | cut -f3 -d,`
if [ ! "$dbs" ] ; then
    echo >&2 "No database files in \"$BOGOHOME\" found"
    exit 1
fi

# count pages in database files
for DB in $DATABASES ; do
    COUNT=`bogoutil --db-print-leafpage-count="$DB"`
    let PAGES="$PAGES + $COUNT"
done

# be generous, double count
let PAGES="$PAGES + $PAGES"

# and update or create DB_CONFIG
: >> "$BOGOHOME"/DB_CONFIG
(
  set +e
  egrep -v '^set_lk_max_(locks|objects)' "$BOGOHOME"/DB_CONFIG >"$BOGOHOME"/DB_CONFIG.new
)
if [ $? -ge 2 ] ; then exit 1 ; fi
echo >>"$BOGOHOME"/DB_CONFIG.new set_lk_max_locks $PAGES
echo >>"$BOGOHOME"/DB_CONFIG.new set_lk_max_objects $PAGES
mv "$BOGOHOME"/DB_CONFIG.new "$BOGOHOME"/DB_CONFIG

# make change effective by recreating the environment
bogoutil --db-recover="$BOGOHOME"
-------------- next part --------------
#! /bin/sh

# This file dumps a bogofilter to standard output in POSIX-tar format.
#
# Requires: pax
#
# (C) 2004 by Matthias Andree
# GNU GPL v2.

# $Id: bf_tar,v 1.18 2005/05/08 02:41:09 relson Exp $

set -e

REMOVEBEF=0
REMOVEAFT=0
while [ "$1" ] ; do
    case "$1" in
	-r) REMOVEAFT=1 ;;
	-R) REMOVEBEF=1 ;;
	--) shift ; break ;;
	-*) echo >&2 "`basename $0`: unknown option $1" ; exit 1 ;;
	*) break;
    esac
    shift
done

if [ $# -ne 1 ] ; then
    echo >&2 "Usage: `basename $0` [options] bogodir > outfile.tar"
    echo >&2 "   or: `basename $0` [options] bogodir | gzip -c >outfile.tar.gz"
    echo >&2 'Options are:'
    echo >&2 ' -r - remove inactive log files after archiving'
    echo >&2 ' -R - remove inactive log files before archiving (use with caution)'
    exit 1
fi

BOGOHOME=${1:-.}

if [ ! -d "$BOGOHOME" ] ; then
    echo $BOGOHOME must be a directory, not a file
    exit 1
fi

nukelogs() {
    bogoutil --db-prune="$BOGOHOME"
}

# remove if requested
if [ $REMOVEBEF -eq 1 ] ; then
    nukelogs
else
    bogoutil --db-checkpoint="$BOGOHOME"
fi

# database first, if it's newer than the log, it's not recoverable!
# pax options: -w: write archive, -v: verbosely, -x ustar: choose tar
# format.
(
  bogofilter -QQ -d "$BOGOHOME" | grep '^wordlist ' | cut -f3 -d,
  bogoutil --db-list-logfiles="$BOGOHOME" all absolute
) | pax -w -v -x ustar

# remove if requested
if [ $REMOVEAFT -eq 1 ] ; then nukelogs ; fi