5 #include "getopts/getopts.h"
19 int main(
int argc,
char **argv ) {
25 opts.addOption(
"v",
"version" ,
"Display version info" ,
false );
26 opts.addOption(
"r",
"xml-req" ,
"(option 1) Test request file in XML" ,
true );
27 opts.addOption(
"o",
"model" ,
"(option 2) Serialized model file" ,
true );
28 opts.addOption(
"p",
"points" ,
"(option 2) TAB-delimited file with points" ,
true );
29 opts.addOption(
"" ,
"calc-matrix" ,
"Calculate confusion matrix" ,
false );
30 opts.addOption(
"t",
"threshold" ,
"Confusion matrix threshold" ,
true );
31 opts.addOption(
"" ,
"ignore-abs" ,
"Ignore absences for the confusion matrix" ,
false );
32 opts.addOption(
"" ,
"calc-roc" ,
"Calculate ROC curve" ,
false );
33 opts.addOption(
"n",
"resolution" ,
"Number of points in the ROC curve" ,
true );
34 opts.addOption(
"b",
"num-background",
"Number of background points for the ROC curve when there are no absences",
true );
35 opts.addOption(
"e",
"max-omission" ,
"Calculate ROC partial area ratio given the maximum omission",
true );
36 opts.addOption(
"" ,
"abs-background",
"Use absences as background in ROC curve" ,
false );
37 opts.addOption(
"s",
"result" ,
"File to store test result in XML" ,
true );
38 opts.addOption(
"",
"log-level" ,
"Set the log level (debug, warn, info, error)",
true );
39 opts.addOption(
"",
"log-file" ,
"Log file" ,
true );
40 opts.addOption(
"" ,
"prog-file" ,
"File to store test progress" ,
true );
41 opts.addOption(
"c",
"config-file" ,
"Configuration file for openModeller" ,
true );
43 std::string log_level(
"info");
44 std::string request_file;
45 std::string model_file;
46 std::string points_file;
47 bool calc_matrix =
false;
48 std::string threshold_string(
"");
50 bool ignore_abs =
false;
51 bool calc_roc =
false;
52 std::string resolution_string(
"");
54 std::string num_background_string(
"");
55 int num_background = -1;
56 std::string max_omission_string(
"");
57 double max_omission = 1.0;
58 bool abs_background =
false;
59 std::string result_file;
61 std::string progress_file;
62 std::string config_file;
64 if ( ! opts.parse( argc, argv ) ) {
66 opts.showHelp( argv[0] );
74 while ( ( option = opts.cycle() ) >= 0 ) {
79 printf(
"om_test %s\n", om.
getVersion().c_str() );
80 printf(
"This is free software; see the source for copying conditions. There is NO\n");
81 printf(
"warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n");
85 request_file = opts.getArgs( option );
88 model_file = opts.getArgs( option );
91 points_file = opts.getArgs( option );
97 threshold_string = opts.getArgs( option );
106 resolution_string = opts.getArgs( option );
109 num_background_string = opts.getArgs( option );
112 max_omission_string = opts.getArgs( option );
115 abs_background =
true;
118 result_file = opts.getArgs( option );
121 log_level = opts.getArgs( option );
124 log_file = opts.getArgs( option );
127 progress_file = opts.getArgs( option );
130 config_file = opts.getArgs( option );
140 if ( ! progress_file.empty() ) {
156 if ( ! log_file.empty() ) {
167 if ( ! config_file.empty() ) {
174 if ( ! request_file.empty() ) {
176 if ( ! model_file.empty() ) {
178 Log::instance()->
warn(
"Model file parameter will be ignored (using XML request instead)\n" );
180 if ( ! points_file.empty() ) {
182 printf(
"Points file parameter will be ignored (using XML request instead)\n");
186 printf(
"Parameter to calculate ROC curve will be ignored (when using XML request you should specify it in the XML)\n");
190 printf(
"Parameter to calculate confusion matrix will be ignored (when using XML request you should specify it in the XML)\n");
193 else if ( ( ! model_file.empty() ) && ! points_file.empty() ) {
196 if ( ! threshold_string.empty() ) {
198 threshold = atof( threshold_string.c_str() );
202 if ( ! resolution_string.empty() ) {
204 resolution = atoi( resolution_string.c_str() );
208 if ( ! num_background_string.empty() ) {
210 num_background = atoi( num_background_string.c_str() );
214 if ( ! max_omission_string.empty() ) {
216 max_omission = atof( max_omission_string.c_str() );
221 printf(
"Please specify either a test request file in XML or a serialized model and a TAB-delimited file with the points to be tested\n");
224 if ( ! progress_file.empty() ) {
233 if ( ! calc_matrix ) {
235 if ( ! threshold_string.empty() ) {
237 Log::instance()->
warn(
"Ignoring threshold - option only available with confusion matrix\n" );
241 Log::instance()->
warn(
"Ignoring ignore-abs - option only available with confusion matrix\n" );
247 if ( ! resolution_string.empty() ) {
249 Log::instance()->
warn(
"Ignoring resolution - option only available with ROC curve\n" );
251 if ( ! max_omission_string.empty() ) {
253 Log::instance()->
warn(
"Ignoring maximum omission - option only available with ROC curve\n" );
255 if ( abs_background ) {
257 Log::instance()->
warn(
"Ignoring abs-background - option only available with ROC curve\n" );
259 if ( ! num_background_string.empty() ) {
261 Log::instance()->
warn(
"Ignoring number of background points - option only available with ROC curve\n" );
276 if ( ! request_file.empty() ) {
284 sampler =
createSampler( input->getSubsection(
"Sampler" ) );
292 ConfigurationPtr matrix_param = statistics_param->getSubsection(
"ConfusionMatrix" );
298 int ignore_absences_int = matrix_param->getAttributeAsInt(
"IgnoreAbsences", 0 );
300 if ( ignore_absences_int > 0 ) {
318 resolution = roc_param->getAttributeAsInt(
"Resolution", -1 );
320 num_background = roc_param->getAttributeAsInt(
"BackgroundPoints", -1 );
322 max_omission = roc_param->getAttributeAsDouble(
"MaxOmission", 1.0 );
324 int use_absences_as_background_int = roc_param->getAttributeAsInt(
"UseAbsencesAsBackground", 0 );
326 if ( use_absences_as_background_int > 0 ) {
328 abs_background =
true;
353 Log::instance()->
debug(
"Loading training sampler to get layers, label and spatial reference\n" );
361 OccurrencesPtr training_presences = training_sampler->getPresences();
363 std::string label( training_presences->label() );
364 std::string spatial_ref( training_presences->coordSystem() );
366 Log::instance()->
debug(
"Loading test points %s %s\n", label.c_str(), spatial_ref.c_str() );
379 if ( ! alg->done() ) {
381 Log::instance()->
error(
"No model could be found as part of the specified algorithm. Aborting.\n");
384 if ( ! progress_file.empty() ) {
397 int num_presences = sampler->numPresence();
398 int num_absences = sampler->numAbsence();
403 if ( calc_matrix && ( num_presences || num_absences ) ) {
405 if ( threshold < 0.0 ) {
412 matrix.
reset( threshold, ignore_abs );
414 matrix.
calculate( alg->getModel(), sampler );
421 if ( calc_roc && num_presences ) {
425 if ( abs_background ) {
431 if ( num_background > 0 ) {
433 roc_curve.
initialize( resolution, num_background );
441 roc_curve.
calculate( alg->getModel(), sampler );
444 if ( calc_matrix && ! num_presences ) {
446 Log::instance()->
warn(
"No presence points - ROC curve and omission error won't be calculated\n" );
449 if ( calc_matrix && ! num_absences ) {
451 Log::instance()->
warn(
"No absence points - commission error won't be calculated\n" );
455 if ( calc_roc && ! num_presences ) {
462 if ( num_presences || num_absences ) {
468 if ( num_presences ) {
470 int omissions = matrix.
getValue(0.0, 1.0);
471 int total = omissions + matrix.
getValue(1.0, 1.0);
476 if ( num_absences ) {
478 int commissions = matrix.
getValue(1.0, 0.0);
479 int total = commissions + matrix.
getValue(0.0, 0.0);
487 if ( num_presences ) {
491 if ( max_omission < 1.0 ) {
500 bool no_statistics =
true;
502 if ( calc_matrix && matrix.
ready() ) {
506 output->addSubsection( cm_config );
508 no_statistics =
false;
511 if ( calc_roc && roc_curve.
ready() ) {
515 output->addSubsection( roc_config );
517 no_statistics =
false;
525 std::ostringstream test_output;
532 if ( ! result_file.empty() ) {
534 ofstream file( result_file.c_str() );
535 file << test_output.str();
541 std::cout << test_output.str().c_str() << endl << flush;
545 if ( ! progress_file.empty() ) {
551 catch ( runtime_error e ) {
554 if ( ! progress_file.empty() ) {
560 printf(
"om_test aborted: %s\n", e.what() );
static void loadConfig(const std::string configFile)
void reset(Scalar predictionThreshold=CONF_MATRIX_DEFAULT_THRESHOLD, bool ignoreAbsences=false)
void initialize(int resolution=ROC_DEFAULT_RESOLUTION)
void warn(const char *format,...)
'Warn' level.
static ConfigurationPtr readXml(char const *filename)
static AlgorithmPtr newAlgorithm(std::string const id)
double getAccuracy() const
static Log * instance()
Returns the instance pointer, creating the object on the first call.
int getValue(Scalar predictionValue, Scalar actualValue) const
static OccurrencesFactory & instance()
Log::Level getLogLevel(std::string level)
virtual OccurrencesPtr getAbsences(const char *groupId)
void setLevel(Level level)
double getCommissionError() const
SamplerPtr createSampler(const EnvironmentPtr &env, const OccurrencesPtr &presence, const OccurrencesPtr &absence)
void error(const char *format,...)
'Error' level.
void setupExternalResources()
virtual OccurrencesPtr getPresences(const char *groupId)
void calculate(const EnvironmentPtr &env, const Model &model, const OccurrencesPtr &presences, const OccurrencesPtr &absences=OccurrencesPtr())
int main(int argc, char **argv)
Main code.
void set(Level level, std::string fileName, char const *pref="")
static void writeXml(const ConstConfigurationPtr &config, char const *fileaname)
static int searchDefaultDirs()
void setLowestTrainingThreshold(const Model &model, const SamplerPtr &sampler)
double getOmissionError() const
double getThreshold() const
#define ROC_DEFAULT_RESOLUTION
void info(const char *format,...)
'Info' level.
void calculate(const Model &model, const SamplerPtr &sampler)
ConfigurationPtr getConfiguration() const
ConfigurationPtr getConfiguration() const
#define CONF_MATRIX_DEFAULT_THRESHOLD
void debug(const char *format,...)
'Debug' level.
double getPartialAreaRatio(double e=1.0)
void progressFileCallback(float progress, void *progressData)
OccurrencesReader * create(const char *source, const char *coordSystem)