60 if ( ! parameters.
retrieve(
"Sampler", samplerPtr ) ) {
68 if ( ! parameters.
retrieve(
"Algorithm", algorithmPtr ) ) {
80 info[
"Sampler"] =
"samplerPtr";
81 info[
"Algorithm"] =
"algorithmPtr";
82 info[
"PropTrain"] =
"double";
88 info[
"Accuracy"] =
"double";
89 info[
"Mean"] =
"double";
90 info[
"Variance"] =
"double";
91 info[
"Deviation"] =
"double";
92 info[
"Estimate"] =
"double";
93 info[
"Bias"] =
"double";
99 info[
"Accuracy without layer"] =
"double";
107 params_.retrieve(
"Sampler", samplerPtr );
110 params_.retrieve(
"Algorithm", algorithmPtr );
114 if ( ! params_.retrieve(
"PropTrain", propTrain ) ) {
120 if ( ! samplerPtr->getEnvironment() ) {
122 std::string msg =
"Sampler has no environment.\n";
129 int num_layers = samplerPtr->numIndependent();
131 if ( num_layers < 2 ) {
133 std::string msg =
"Jackknife needs at least 2 layers.\n";
144 splitSampler( samplerPtr, &training_sampler, &testing_sampler, propTrain );
147 AlgorithmPtr algorithm_ptr = algorithmPtr->getFreshCopy();
149 algorithm_ptr->createModel( training_sampler );
153 conf_matrix.
calculate( algorithm_ptr->getModel(), testing_sampler );
155 double out_param = conf_matrix.
getAccuracy() * 100;
159 std::multimap<double, int> out_params;
162 double variance = 0.0;
163 double std_deviation = 0.0;
164 double jackknife_estimate = 0.0;
165 double jackknife_bias = 0.0;
173 if ( training_sampler->numPresence() ) {
175 training_presences = training_sampler->getPresences()->clone();
178 if ( training_sampler->numAbsence() ) {
180 training_absences = training_sampler->getAbsences()->clone();
183 if ( testing_sampler->numPresence() ) {
185 testing_presences = testing_sampler->getPresences()->clone();
188 if ( testing_sampler->numAbsence() ) {
190 testing_absences = testing_sampler->getAbsences()->clone();
193 for (
int i = 0; i < num_layers; ++i ) {
198 EnvironmentPtr new_environment = samplerPtr->getEnvironment()->clone();
203 new_environment->removeLayer( i );
206 if ( training_presences ) {
208 training_presences->setEnvironment( new_environment );
211 if ( training_absences ) {
213 training_absences->setEnvironment( new_environment );
216 if ( testing_presences ) {
218 testing_presences->setEnvironment( new_environment );
221 if ( testing_absences ) {
223 testing_absences->setEnvironment( new_environment );
230 AlgorithmPtr new_algorithm = algorithmPtr->getFreshCopy();
232 new_algorithm->createModel( new_training_sampler );
240 if ( new_algorithm->needNormalization() && ! new_testing_sampler->isNormalized() ) {
244 Normalizer * normalizer = new_algorithm->getNormalizer();
249 new_testing_sampler->normalize( normalizer );
253 Log::instance()->
error(
"Jackknife algorithm requires normalization but did not specify any normalizer\n");
260 conf_matrix.
calculate( new_algorithm->getModel(), new_testing_sampler );
262 double myaccuracy = conf_matrix.
getAccuracy() * 100;
266 out_params.insert( std::pair<double, int>( myaccuracy, i ) );
268 result.
store(
"Accuracy without layer", myaccuracy );
270 result_by_layer_[samplerPtr->getEnvironment()->getLayerPath(i)] = result;
301 std::multimap<double, int>::const_iterator it = out_params.begin();
302 std::multimap<double, int>::const_iterator end = out_params.end();
303 for ( ; it != end; ++it ) {
305 Log::instance()->
debug(
"Accuracy without layer %d: %.2f%% (%s)\n", (*it).second, (*it).first, (environment_ptr->getLayerPath( (*it).second )).c_str() );
306 variance += ((*it).first - mean)*((*it).first - mean);
311 variance /= num_layers;
313 variance /= (num_layers - 1);
317 std_deviation = sqrt(variance);
321 jackknife_bias = (num_layers - 1)*(mean - out_param);
323 jackknife_estimate = out_param - jackknife_bias;
329 params_.store(
"Accuracy", out_param );
330 params_.store(
"Mean", mean );
331 params_.store(
"Variance", variance );
332 params_.store(
"Deviation", std_deviation );
333 params_.store(
"Estimate", jackknife_estimate );
334 params_.store(
"Bias", jackknife_bias );
void reset(Scalar predictionThreshold=CONF_MATRIX_DEFAULT_THRESHOLD, bool ignoreAbsences=false)
void getLayersetResultSpec(stringMap &info)
double getAccuracy() const
static Log * instance()
Returns the instance pointer, creating the object on the first call.
std::map< string, string > stringMap
SamplerPtr createSampler(const EnvironmentPtr &env, const OccurrencesPtr &presence, const OccurrencesPtr &absence)
void error(const char *format,...)
'Error' level.
bool retrieve(const PreMultiContainerKeyT &obj_key, ObjectT &obj_reference) const
void store(const PreMultiContainerKeyT &obj_key, const ObjectT &obj_reference)
void getLayerResultSpec(stringMap &info)
void calculate(const EnvironmentPtr &env, const Model &model, const OccurrencesPtr &presences, const OccurrencesPtr &absences=OccurrencesPtr())
void splitSampler(const SamplerPtr &orig, SamplerPtr *train, SamplerPtr *test, double propTrain)
bool checkParameters(const PreParameters ¶meters) const
void info(const char *format,...)
'Info' level.
AlgParamMetadata parameters[NUM_PARAM]
void getAcceptedParameters(stringMap &info)
void debug(const char *format,...)
'Debug' level.