#!/bin/sh -x
#
# From: Greg Louis <glouis@dynamicro.on.ca>
# Date: Sun, 29 Aug 2004 08:26:05 -0400
# Subject: Re: large bogofilter installations
# 
# The concept could be tested: take a wordlist with tokens from 20,000 or
# so messages, make two copies.  To one copy add a hundred randomly
# chosen messages three times each.  To the other, unregister a hundred
# randomly chosen messages three times each.  This is far more extreme
# corruption than should ever be encountered in practice.  Then measure
# fp and fn on five thousand more messages (all these numbers should be
# half spam and half nonspam) using each of the three wordlists, and see               
# if the difference is significant.  I'd guess the multiple            
# unregistrations might hurt a bit but the multiple registrations will be
# hardly noticeable.

if [ -n "$1" ] ; then
    INST="$1"
else
    INST=`date +%H%M`
fi
DATE=`date +%m%d`.$INST

DIR=$DATE.d
FILE=$DATE.out
[ ! -d $DIR ] && mkdir $DIR

cp $0 $DIR
(echo "######## $DATE ########" ; echo ) > $FILE

scramble()
{
	perl \
	-e' srand ( time() ^ ($$ + ($$ << 15)) );' \
	-e' foreach $key (<>) {' \
	-e'     $shuf{$key} = rand;' \
	-e' }' \
	-e' foreach $key (sort { $shuf{$b} <=> $shuf{$a} } keys %shuf ) {' \
	-e'     print $key;' \
	-e' }'
}

# Assumes ham is in files $DIR/*.h/*
#    and spam is in files $DIR/*.s/*
MSG_SRC="$HOME/Archive/2004/"

if [ ! -f $DIR/ham.all ] ; then
    find $MSG_SRC -type f | grep h/ | sort > $DIR/ham.all
    find $MSG_SRC -type f | grep s/ | sort > $DIR/spam.all

    # select 10000 each ham and spam for building wordlists
    # select 5000 each ham and spam for scoring test

    scramble < $DIR/ham.all > tmp
    head -10000 < tmp > $DIR/ham.10000
    head -15000 < tmp | tail -5000 > $DIR/ham.5000

    scramble < $DIR/spam.all > tmp
    head -10000 < tmp > $DIR/spam.10000
    head -15000 < tmp | tail -5000 > $DIR/spam.5000

    # randomly chose 100 messages for 3x registration
    cat $DIR/ham.10000  $DIR/ham.5000  | scramble | head -50 > $DIR/ham.50.n
    cat $DIR/spam.10000 $DIR/spam.5000 | scramble | head -50 > $DIR/spam.50.s

    # randomly chose 100 messages for 3x unregistration
    cat $DIR/ham.10000  $DIR/ham.5000  | scramble | head -50 > $DIR/ham.50.N
    cat $DIR/spam.10000 $DIR/spam.5000 | scramble | head -50 > $DIR/spam.50.S
fi

# create 2 copies of a wordlist with 10,000 ham and 10,000 spam
if [ ! -d $DIR/orig ] ; then 
    mkdir $DIR/orig $DIR/reg $DIR/unreg
    bogofilter -d $DIR/orig -n -b < $DIR/ham.10000
    bogofilter -d $DIR/orig -s -b < $DIR/spam.10000
    cp -p $DIR/orig/wordlist.db $DIR/reg
    cp -p $DIR/orig/wordlist.db $DIR/unreg
fi

for R in 1x 2x 3x ; do
    echo "#### $R ####" | tee -a $FILE
    # register 50 ham and 50 spam
    cat $DIR/ham.50.n  | bogofilter -d $DIR/reg -n -b
    cat $DIR/spam.50.s | bogofilter -d $DIR/reg -s -b
    
    # unregister 50 ham and 50 spam
    cat $DIR/ham.50.N  | bogofilter -d $DIR/unreg -N -b
    cat $DIR/spam.50.S | bogofilter -d $DIR/unreg -S -b

    # score 5000 ham and 5000 spam with each wordlist
    cat $DIR/ham.5000 $DIR/spam.5000 | bogofilter -v -C -b -d $DIR/orig  > $DIR/orig.out
    cat $DIR/ham.5000 $DIR/spam.5000 | bogofilter -v -C -b -d $DIR/reg   > $DIR/reg.out
    cat $DIR/ham.5000 $DIR/spam.5000 | bogofilter -v -C -b -d $DIR/unreg > $DIR/unreg.out

    # report correct and incorrect results
    printf "%-5s %5s %4s %4s %4s %4s\n"  " " "CNT" "HH" "FP" "FN" "SS" | tee -a $FILE
    for D in orig reg unreg ; do
	HH=`egrep -c "h/.*N" $DIR/$D.out`
	FP=`egrep -c "h/.*Y" $DIR/$D.out`
	FN=`egrep -c "s/.*N" $DIR/$D.out`
	SS=`egrep -c "s/.*Y" $DIR/$D.out`
	CN=`bogoutil -w $DIR/$D/wordlist.db .MSG_COUNT | tail -1 | awk '{print $2}'`
	printf "%-5s %5d %5d %3d %4d %5d\n"  $D $CN $HH $FP $FN $SS | tee -a $FILE
    done
    echo | tee -a $FILE
done

cp -p $FILE $DIR/$FILE
ls -l $DIR/$FILE
