fixed --max-fmeasure not computing max-fmeasure; added testing scripts

icsiboost/src/icsiboost.c

@@ -2987,15 +2987,15 @@ int main(int argc, char** argv)
 			weakclassifier_t* classifier = vector_get(classifiers, iteration);
 			double error=compute_classification_error(classifiers, examples, iteration, sum_of_weights, classes->length); // compute error rate and update weights
 			if(use_max_fmeasure) error = compute_max_fmeasure(examples, fmeasure_class_id, NULL, NULL, NULL);
-            if(display_maxclass_error) error = compute_max_error(examples, classes->length);
+            else if(display_maxclass_error) error = compute_max_error(examples, classes->length);
 			double dev_error=NAN;
 			//double dev_error_monoclass=NAN;
 			double threshold = NAN;
 			if(dev_examples!=NULL)
 			{
 				dev_error = compute_test_error(classifiers, dev_examples, iteration, classes->length); // compute error rate on test
 				if(use_max_fmeasure) dev_error = compute_max_fmeasure(dev_examples, fmeasure_class_id, &threshold, NULL, NULL);
-				if(display_maxclass_error) dev_error= compute_max_error(dev_examples, classes->length);
+                else if(display_maxclass_error) dev_error= compute_max_error(dev_examples, classes->length);
 				if(optimal_iterations!=0 && ((!use_max_fmeasure && dev_error<minimum_test_error) || (use_max_fmeasure && dev_error>minimum_test_error)))
 				{
 					minimum_test_error=dev_error;
@@ -3008,8 +3008,8 @@ int main(int argc, char** argv)
 			if(test_examples!=NULL)
 			{
 				test_error = compute_test_error(classifiers, test_examples, iteration, classes->length); // compute error rate on test
-				if (display_maxclass_error) test_error = compute_max_error(test_examples, classes->length);
 				if(use_max_fmeasure) test_error = compute_max_fmeasure(test_examples, fmeasure_class_id, &threshold, NULL, NULL);
+                else if (display_maxclass_error) test_error = compute_max_error(test_examples, classes->length);
 			}
 			if(dev_examples==NULL && test_examples!=NULL)
 			{
@@ -3133,7 +3133,7 @@ int main(int argc, char** argv)
 		double train_recall = NAN;
 		double train_precision = NAN;
 		if(use_max_fmeasure) error = compute_max_fmeasure(examples, fmeasure_class_id, NULL, &train_recall, &train_precision);
-        if(display_maxclass_error) error = compute_max_error(examples, classes->length);
+        else if(display_maxclass_error) error = compute_max_error(examples, classes->length);
 		double dev_recall = NAN;
 		double dev_precision = NAN;
 		double dev_error=NAN;
@@ -3143,7 +3143,7 @@ int main(int argc, char** argv)
 		{
 			dev_error = compute_test_error(classifiers, dev_examples, iteration, classes->length); // compute error rate on test
 			if(use_max_fmeasure) dev_error = compute_max_fmeasure(dev_examples, fmeasure_class_id, &threshold, &dev_recall, &dev_precision);
-			if (display_maxclass_error) dev_error = compute_max_error(dev_examples, classes->length);
+            else if (display_maxclass_error) dev_error = compute_max_error(dev_examples, classes->length);
 			if(optimal_iterations!=0 && ((!use_max_fmeasure && dev_error<minimum_test_error) || (use_max_fmeasure && dev_error>minimum_test_error)))
 			{
 				minimum_test_error=dev_error;
@@ -3159,8 +3159,8 @@ int main(int argc, char** argv)
 		if(test_examples!=NULL)
 		{
 			test_error = compute_test_error(classifiers, test_examples, iteration, classes->length); // compute error rate on test
-			if (display_maxclass_error) test_error = compute_max_error(test_examples, classes->length);
 			if(use_max_fmeasure) test_error = compute_max_fmeasure(test_examples, fmeasure_class_id, &test_threshold, &test_recall, &test_precision);
+            else if (display_maxclass_error) test_error = compute_max_error(test_examples, classes->length);
 		}
 		if(dev_examples==NULL && test_examples!=NULL)
 		{

icsiboost/src/test_all_releases.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# don't forget to put the right release number
+# you should run this script in a directory of its own
+mkdir -p test_log
+for i in `seq 172`
+do
+    ./test_release.sh r$i 2>&1 | tee test_log/r$i.log
+    echo r$i $? > test_log/r$i.exit
+done

icsiboost/src/test_analysis.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+cat test_log/*.test | grep ^r | LANG=C sort > all.test
+cat test_log/*.train | grep ^r | LANG=C sort > all.train
+cat test_log/*.perf | LANG=C sort > all.perf
+join all.perf all.train | join - all.test | sort -k2 -tr -n > all.analysis
+cut -d " " -f 1,2,4,8,27,31 all.analysis | tr " " ";" > all.csv
+

icsiboost/src/test_release.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+if [ $# != 1 ]
+then
+    echo "USAGE: $0 release" >&2
+    exit 1
+fi
+
+export release=$1
+rm -rf adult.shyp adult.test.out test_source
+mkdir -p test_log
+
+echo "==== checkout ===="
+svn checkout -r $release http://icsiboost.googlecode.com/svn/trunk/ test_source || exit 1
+pushd test_source/icsiboost/
+echo "==== configure ===="
+./configure || exit 2
+echo "==== compile ===="
+make || exit 3
+popd
+echo "==== training ===="
+format="$release %E %e %S %U %P %M %t %K %D %p %X %Z %F %R %W %c %w %I %O %r %s %k %x"
+/usr/bin/time -f "$format" -o test_log/$release.train test_source/icsiboost/src/icsiboost -S adult -n 100 || exit 4
+echo "==== testing ===="
+/usr/bin/time -f "$format" -o test_log/$release.test test_source/icsiboost/src/icsiboost -S adult -C < adult.test > adult.test.out || exit 5
+echo "==== test error rate ===="
+cat adult.test.out | awk -vr=$release '{if($1 == 1 && $3 < $4 || $2 == 1 && $3 > $4){e++}}END{print r, e/NR}' | tee test_log/$release.perf || exit 6
+
+rm -rf adult.shyp adult.test.out test_source
+