source: java/ecj/cecj/eval/TDLImprovingEvaluator.java @ 1304

/*
  Copyright 2009 by Marcin Szubert
  Licensed under the Academic Free License version 3.0
 */

package cecj.eval;

import ec.EvolutionState;
import ec.Individual;
import ec.util.Parameter;

/**
 * Wrapper for <code>CoevolutionaryEvaluator</code> performing an additional learning phase before
 * actual evaluation.
 * 
 * The only role of this class is improving each individual of the population by running a specific
 * temporal difference learning (TDL) algorithm before the evaluation. Since the exact
 * implementation of this algorithm depends on the problem, evaluator delegates the learning task to
 * the provided <code>TDLImprover</code> interface realization. At the beginning of evolutionary
 * process individuals may require some preparation for running TDL. For this reason, appropriate
 * interface methods are invoked before the first evaluation. Clearly, not every problem can be
 * approached by reinforcement learning paradigm so this class has also a limited scope of
 * applicability.
 * 
 * Note that this evaluator realizes the Coevolutionary Reinforcement Learning idea.
 * 
 * @author Marcin Szubert
 * @see TDLImprover
 * 
 */
public class TDLImprovingEvaluator extends CoevolutionaryEvaluator {

        private static final String P_INNER_EVALUATOR = "inner-evaluator";
        private static final String P_TDL_IMPROVER = "tdl-improver";
        private static final String P_TDL_FREQUENCY = "tdl-frequency";

        private CoevolutionaryEvaluator innerEvaluator;
        private TDLImprover temporalDifferenceImprover;
        private boolean firstEvaluation = true;
        private int tdlFrequency;

        @Override
        public void setup(EvolutionState state, Parameter base) {
                super.setup(state, base);

                Parameter innerEvaluatorParam = base.push(P_INNER_EVALUATOR);
                innerEvaluator = (CoevolutionaryEvaluator) (state.parameters.getInstanceForParameter(
                                innerEvaluatorParam, null, CoevolutionaryEvaluator.class));
                innerEvaluator.setup(state, innerEvaluatorParam);

                Parameter tdlImproverParam = base.push(P_TDL_IMPROVER);
                temporalDifferenceImprover = (TDLImprover) (state.parameters.getInstanceForParameter(
                                tdlImproverParam, null, TDLImprover.class));
                temporalDifferenceImprover.setup(state, tdlImproverParam);

                Parameter tdlImprovingFrequency = base.push(P_TDL_FREQUENCY);
                tdlFrequency = state.parameters.getIntWithDefault(tdlImprovingFrequency, null, 1);
        }

        @Override
        public void evaluatePopulation(EvolutionState state) {
                if (firstEvaluation) {
                        for (int subpop = 0; subpop < numSubpopulations; subpop++) {
                                Individual[] inds = state.population.subpops[subpop].individuals;
                                for (Individual ind : inds) {
                                        temporalDifferenceImprover.prepareForImproving(state, ind);
                                }
                        }
                        firstEvaluation = false;
                }

                if ((state.generation % tdlFrequency) == 0) {
                        for (int subpop = 0; subpop < numSubpopulations; subpop++) {
                                Individual[] inds = state.population.subpops[subpop].individuals;
                                for (Individual ind : inds) {
                                        temporalDifferenceImprover.improve(state, ind);
                                }
                        }
                }

                innerEvaluator.evaluatePopulation(state);
        }
}