Commit of MK branch revs. 393 and 396. Updates to stats, clustering plus all new CBNEA.

src/eva2/gui/Plot.java

@@ -39,14 +39,14 @@ import javax.swing.JFrame;
 import javax.swing.JOptionPane;
 import javax.swing.JPanel;
 
-
 import com.sun.image.codec.jpeg.JPEGCodec;
 import com.sun.image.codec.jpeg.JPEGImageEncoder;
 
 import eva2.EvAInfo;
+import eva2.server.go.individuals.AbstractEAIndividual;
+import eva2.server.go.populations.Population;
 import eva2.tools.chart2d.DPointSet;
 import eva2.tools.tool.BasicResourceLoader;
-import eva2.server.go.populations.Population;
 /*==========================================================================*
  * CLASS DECLARATION
  *==========================================================================*/
@@ -287,10 +287,22 @@ public class Plot implements PlotInterface, Serializable {
 	 */
     public void drawPopulation(String prefix, Population pop) {
 		for (int i=0; i<pop.size(); i++) {
-			getFunctionArea().drawIcon(1, prefix+" "+pop.getEAIndividual(i).getFitness(0), pop.getEAIndividual(i).getDoublePosition(), 2);
+			drawIndividual(1, 2, prefix, pop.getEAIndividual(i));
 		}
     }
     
+	/**
+	 * Draw an individual to the Plot instance. It is annotated with the
+	 * given prefix and its fitness.
+	 * 
+	 * @param prefix
+	 * @param pop
+	 * @see FunctionArea.drawIcon
+	 */
+    public void drawIndividual(int iconType, int graphID, String prefix, AbstractEAIndividual indy) {
+    	getFunctionArea().drawIcon(iconType, prefix+" "+indy.getFitness(0), indy.getDoublePosition(), graphID);
+    }
+    
 	public void setPreferredSize(Dimension prefSize) {
 		if (m_Frame != null) {
 			m_Frame.setPreferredSize(prefSize);

src/eva2/server/go/operators/cluster/ClusteringDensityBased.java

@@ -6,6 +6,7 @@ import eva2.server.go.individuals.AbstractEAIndividual;
 import eva2.server.go.operators.distancemetric.InterfaceDistanceMetric;
 import eva2.server.go.operators.distancemetric.PhenotypeMetric;
 import eva2.server.go.populations.Population;
+import eva2.tools.Pair;
 
 
 /** The DBSCAN method. As far as I recall this is an hierachical
@@ -76,14 +77,7 @@ public class ClusteringDensityBased implements InterfaceClustering, java.io.Seri
         return (Object) new ClusteringDensityBased(this);
     }
 
-    /** This method allows you to search for clusters in a given population. The method
+    public Population[] cluster(Population pop, Population referencePop) {
-     * returns Number of populations. The first population contains all individuals that
-     * could not be associated with any cluster and may be empty.
-     * All other populations group individuals into clusters.
-     * @param pop       The population of individuals that is to be clustered.
-     * @return Population[]
-     */
-    public Population[] cluster(Population pop) {
         ConnectionMatrix    = new boolean[pop.size()][pop.size()];
         Clustered           = new boolean[pop.size()];
         AbstractEAIndividual   tmpIndy1, tmpIndy2;
@@ -151,34 +145,62 @@ public class ClusteringDensityBased implements InterfaceClustering, java.io.Seri
      * @param species2  The second species.
      * @return True if species converge, else False.
      */
-    public boolean mergingSpecies(Population species1, Population species2) {
+    public boolean mergingSpecies(Population species1, Population species2, Population referencePop) {
         if (m_TestConvergingSpeciesOnBestOnly) {
-            if (this.m_Metric.distance(species1.getBestEAIndividual(), species2.getBestEAIndividual()) < this.m_ClusterDistance) return true;
+        	double specDist = this.m_Metric.distance(species1.getBestEAIndividual(), species2.getBestEAIndividual());
-            else return false;
+//        	System.out.println("Dist between species is " + specDist);
+            return (specDist < this.m_ClusterDistance);
         } else {
             Population tmpPop = new Population(species1.size()+species2.size());
             tmpPop.addPopulation(species1);
             tmpPop.addPopulation(species2);
-            if (this.cluster(tmpPop).length <= 2) return true;
+            if (this.cluster(tmpPop, referencePop).length <= 2) return true;
             else return false;
         }
     }
 
-    /** This method decides if a unclustered individual belongs to an already established species.
+//    /** This method decides if a unclustered individual belongs to an already established species.
-     * @param indy          A unclustered individual.
+//     * @param indy          A unclustered individual.
-     * @param species       A species.
+//     * @param species       A species.
-     * @return True or False.
+//     * @return True or False.
+//     */
+//    public boolean belongsToSpecies(AbstractEAIndividual indy, Population species, Population pop) {
+//        if (this.m_TestConvergingSpeciesOnBestOnly) {
+//            if (this.m_Metric.distance(indy, species.getBestEAIndividual()) < this.m_ClusterDistance) return true;
+//            else return false;
+//        } else {
+//            Population tmpPop = (Population)species.clone();
+//            tmpPop.add(indy);
+//            if (this.cluster(tmpPop)[0].size() == 0) return true;
+//            else return false;
+//        }
+//    }
+    
+    /**
+     * Try to associate a set of loners with a given set of species. Return a list
+     * of indices assigning loner i with species j for all loners. If no species can
+     * be associated, -1 is returned as individual entry.
+     * Note that the last cluster threshold is used which may have depended on the last
+     * generation.
+     * 
+     * @param loners
+     * @param species
+     * @return associative list matching loners to species.
      */
-    public boolean belongsToSpecies(AbstractEAIndividual indy, Population species) {
+    public int[] associateLoners(Population loners, Population[] species, Population referencePop) {
-        if (this.m_TestConvergingSpeciesOnBestOnly) {
+    	int[] res = new int[loners.size()];
-            if (this.m_Metric.distance(indy, species.getBestEAIndividual()) < this.m_ClusterDistance) return true;
+    	for (int l=0; l<loners.size(); l++) {
-            else return false;
+    		double minDist = -1;
-        } else {
+    		res[l]=-1;
-            Population tmpPop = (Population)species.clone();
+    		for (int spI=0; spI<species.length; spI++) {  // O(species.length^2)
-            tmpPop.add(indy);
+    			Pair<Integer,Double> iDist = Population.getClosestFarthestIndy(loners.getEAIndividual(l), species[spI], m_Metric, true);
-            if (this.cluster(tmpPop)[0].size() == 0) return true;
+    			if (iDist.tail() < m_ClusterDistance) { // its close enough to be added
-            else return false;
+    				// set SP ID only if its the closest species which is still below cluster distance
+    				if (minDist<0 || (iDist.tail() < minDist)) res[l]=spI;
     			}
+    		} // end for all species
+    	} // end for all loners
+    	return res;
     }
     
 /**********************************************************************************************************************
@@ -236,6 +258,10 @@ public class ClusteringDensityBased implements InterfaceClustering, java.io.Seri
         return "Set the minimum group size for the DBSCAN method.";
     }
 
+	public String initClustering(Population pop) {
+		return null;
+	}
+
 
 //    /** For debuggy only
 //     * @param plot TopoPlot

src/eva2/server/go/operators/cluster/ClusteringKMeans.java

@@ -1,5 +1,7 @@
 package eva2.server.go.operators.cluster;
 
+import java.util.Arrays;
+
 import eva2.gui.Chart2DDPointIconCircle;
 import eva2.gui.Chart2DDPointIconText;
 import eva2.gui.GraphPointSet;
@@ -59,7 +61,7 @@ public class ClusteringKMeans implements InterfaceClustering, java.io.Serializab
      * @param pop       The population of individuals that is to be clustered.
      * @return Population[]
      */
-    public Population[] cluster(Population pop) {
+    public Population[] cluster(Population pop, Population referencePop) {
         double[][] data     = this.extractClusterDataFrom(pop);
         if (!(this.m_ReuseC) || (this.m_C == null)) {
             this.m_C            = new double[this.m_K][];
@@ -278,19 +280,26 @@ public class ClusteringKMeans implements InterfaceClustering, java.io.Serializab
      * @param species2  The second species.
      * @return True if species converge, else False.
      */
-    public boolean mergingSpecies(Population species1, Population species2) {
+    public boolean mergingSpecies(Population species1, Population species2, Population referencePop) {
         // @todo i could use the BIC metric from X-means to calculate this
         return false;
     }
 
-    /** This method decides if a unclustered individual belongs to an already established species.
+//    /** This method decides if a unclustered individual belongs to an already established species.
-     * @param indy          A unclustered individual.
+//     * @param indy          A unclustered individual.
-     * @param species       A species.
+//     * @param species       A species.
-     * @return True or False.
+//     * @return True or False.
-     */
+//     */
-    public boolean belongsToSpecies(AbstractEAIndividual indy, Population species) {
+//    public boolean belongsToSpecies(AbstractEAIndividual indy, Population species, Population pop) {
-        // @todo perhaps the same as in convergingSpecies
+//        // @todo perhaps the same as in convergingSpecies
-        return false;
+//        return false;
+//    }
+    
+	public int[] associateLoners(Population loners, Population[] species, Population referencePop) {
+		int[] res=new int[loners.size()];
+		System.err.println("Warning, associateLoners not implemented for " + this.getClass());
+		Arrays.fill(res, -1);
+		return res;
 	}
 
     /** This method allows you to recieve the c centroids
@@ -313,7 +322,7 @@ public class ClusteringKMeans implements InterfaceClustering, java.io.Serializab
         f1.setProblemDimension(2);
         f1.setEAIndividual(new ESIndividualDoubleData());
         f1.initPopulation(pop);
-        ckm.cluster(pop);
+        ckm.cluster(pop, (Population)null);
 
     }
 
@@ -374,4 +383,8 @@ public class ClusteringKMeans implements InterfaceClustering, java.io.Serializab
     public String reuseCTipText() {
         return "Toggel reuse of previously found cluster centroids.";
     }
+
+	public String initClustering(Population pop) {
+		return null;
+	}
 }

src/eva2/server/go/operators/cluster/ClusteringNearestBetter.java

@@ -0,0 +1,401 @@
+package eva2.server.go.operators.cluster;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.Vector;
+
+import eva2.gui.BeanInspector;
+import eva2.gui.GenericObjectEditor;
+import eva2.server.go.individuals.AbstractEAIndividual;
+import eva2.server.go.individuals.AbstractEAIndividualComparator;
+import eva2.server.go.operators.distancemetric.InterfaceDistanceMetric;
+import eva2.server.go.operators.distancemetric.PhenotypeMetric;
+import eva2.server.go.populations.Population;
+import eva2.tools.Pair;
+
+/**
+ * Hierarchical clustering after Preuss et al., "Counteracting Genetic Drift and Disruptive Recombination 
+ * in (mu+,lambda)-EA on Multimodal Fitness Landscapes", GECCO '05.
+ * 
+ * A tree is produced by assigning each individual the closest individual with better fitness.
+ * Connections with a distance above a certain threshold are cut. After that, each interconnected subtree forms a cluster.
+ * In the paper, the threshold is deduced as 2*d_p for d_p: the mean distance in the population.  
+ * 
+ * @author mkron
+ *
+ */
+public class ClusteringNearestBetter implements InterfaceClustering, Serializable {
+	private static final long serialVersionUID = 1L;
+	private InterfaceDistanceMetric     	metric            = new PhenotypeMetric();
+    private double                      	absoluteDistThreshold = 0.5;
+    private boolean 						thresholdMultipleOfMeanDist = true;
+    private double 							meanDistFactor = 2.; // recommended setting
+    private double							currentMeanDistance = -1.;
+    private int                         	minimumGroupSize  = 3;
+	private boolean 						testConvergingSpeciesOnBestOnly = true; // if two species are tested for convergence, only the best indies may be compared regarding the distance threshold 
+
+	private int[]                 			uplink;
+    private double[]						uplinkDist;
+    private AbstractEAIndividualComparator comparator = new AbstractEAIndividualComparator();
+    private Vector<Integer>[]				children;
+	private static final String initializedForKey = "initializedClustNearestBetterOnHash";
+	private static final String initializedRefData = "initializedClustNearestBetterData";
+	
+	private static boolean TRACE = true;
+
+	public ClusteringNearestBetter() {
+	}
+
+    public ClusteringNearestBetter(ClusteringNearestBetter o) {
+    	this.metric = o.metric;
+    	this.absoluteDistThreshold = o.absoluteDistThreshold;
+    	this.thresholdMultipleOfMeanDist = o.thresholdMultipleOfMeanDist;
+    	this.meanDistFactor = o.meanDistFactor;
+    	this.currentMeanDistance = o.currentMeanDistance;
+    	this.minimumGroupSize = o.minimumGroupSize;
+    	this.comparator = (AbstractEAIndividualComparator)o.comparator.clone();
+    	this.testConvergingSpeciesOnBestOnly = o.testConvergingSpeciesOnBestOnly;
+	}
+    
+    public void hideHideable() {
+    	setAdaptiveThreshold(isAdaptiveThreshold());
+    }
+
+	/** This method allows you to make a deep clone of
+     * the object
+     * @return the deep clone
+     */
+    public Object clone() {
+        return (Object) new ClusteringNearestBetter(this);
+    }
+	
+    /**
+     * Try to associate a set of loners with a given set of species. Return a list
+     * of indices assigning loner i with species j for all loners. If no species can
+     * be associated, -1 is returned as individual entry.
+     * Note that the last cluster threshold is used which may have depended on the last
+     * generation.
+     * 
+     * @param loners
+     * @param species
+     * @return associative list matching loners to species.
+     */
+    public int[] associateLoners(Population loners, Population[] species, Population referenceSet) {
+//    	Pair<Integer,Double>[][] closestPerSpecList = new Pair[loners.size()][species.length];
+    	int[] res = new int[loners.size()];
+    	getRefData(referenceSet, loners);
+    	for (int l=0; l<loners.size(); l++) { // for each loner: search closest better indy for each species.
+    		int nearestBetterSpeciesID=-1; 
+			double nearestBetterDist=-1;
+    		
+    		for (int spI=0; spI<species.length; spI++) { // loop species
+				boolean lonerIndyIsBest = (comparator.compare(loners.getEAIndividual(l), species[spI].getBestEAIndividual())<=0);
+				if (lonerIndyIsBest) { // if the loner is the best, check the distance to the best indy within the species
+					double curDist = metric.distance(loners.getEAIndividual(l), species[spI].getBestEAIndividual()); 
+						//Population.getClosestFarthestIndy(loners.getEAIndividual(l), species[spI], metric, true).tail();
+					if (nearestBetterDist<0 || (curDist < nearestBetterDist)) {
+//						System.out.println("Loner is better " + loners.getEAIndividual(l) + " than best " + species[spI].getBestEAIndividual() + ", dist is "+curDist);
+						nearestBetterSpeciesID=spI;
+						nearestBetterDist = curDist;
+					}
+				} else {
+					for (int i=0; i<species[spI].size(); i++) { //loop indies in species
+						double curDist = metric.distance(loners.getEAIndividual(l), species[spI].getEAIndividual(i));
+						boolean specIndyIsBetter = (comparator.compare(species[spI].getEAIndividual(i), loners.getEAIndividual(l))<0);
+						if (specIndyIsBetter && (nearestBetterDist<0 || (curDist < nearestBetterDist))) {
+							// if the found indy species is better than the loner, it is a possible cluster. 
+							// store the closest possible cluster.    					
+							//    					nearestBetterIndyID = i;
+							nearestBetterSpeciesID=spI;
+							nearestBetterDist = curDist;
+						}					
+					} 
+				}
+//    			if (comparator.compare(species[spI].getEAIndividual(closestID), loners.getEAIndividual(l))<0) {
+//
+//    				if (closestClustDist<0 || (closestDist < closestClustDist)) {
+//    					closestClustDist = closestDist;
+//    					closestClustID = spI;
+//    				}
+//    			}
+    		} // end loop species
+    		if (nearestBetterDist < currentDistThreshold()) {
+//    			System.out.println("dist is " + nearestBetterDist + ", assigning spec " + nearestBetterSpeciesID);
+    			res[l]=nearestBetterSpeciesID;
+    		} else res[l]=-1;
+    	} // end for all loners
+    	return res;
+    }
+    
+//	public boolean belongsToSpecies(AbstractEAIndividual indy,
+//			Population species, Population pop) {
+//		// this sucks since every time the full clustering must be performed...
+//		return false;
+////		if (thresholdMultipleOfMeanDist) currentMeanDistance = pop.getPopulationMeasures(metric)[0];
+////		ArrayList<AbstractEAIndividual> sorted = pop.getSorted(comparator);
+////		for (int i=sorted.size()-1; i>=1; i--) {  // start with worst indies
+////			if (sorted.get(i).getIndyID()==indy.getIndyID()) { // found the desired indy.
+////				int uplink=-1; double uplinkDist = -1;
+////				for (int j=i-1; j>=0; j--) { // search nearest better indy
+////					double curDist = metric.distance(sorted.get(i), sorted.get(j));
+////					if (uplinkDist<0 || (curDist < uplinkDist)) {
+////						uplink = j;
+////						uplinkDist = curDist;
+////					}					
+////				}
+////				// if it belongs to species spec and the distance is below threshold, be happy and return true
+////				if (uplink==-1) { // it is the best individual?
+////					return false;
+////				}
+////				if (uplinkDist > currentDistThreshold()) return false;
+////				else {
+////					return (species.isMemberByID(pop.getEAIndividual(uplink)));
+////				}
+////			}
+////		}
+////		// size <= 1?
+////		return false;
+//	}
+
+    /**
+     * Perform one clustering step to measure the mean distance to the
+     * nearest better individual (only if used).
+     */
+    public String initClustering(Population pop) {
+    	if (this.isAdaptiveThreshold()) {
+	    	ArrayList<AbstractEAIndividual> sorted = pop.getSorted(comparator);
+	    	if (uplink==null || (uplink.length!=pop.size())) uplink = new int[pop.size()]; // parent index of all indys
+	    	if (uplinkDist==null || (uplinkDist.length!=pop.size())) uplinkDist = new double[pop.size()]; // parent distance for all indys
+	    	if (children==null || (children.length!=pop.size())) children = new Vector[pop.size()]; // list of children for all indies
+	    	else if (children.length==pop.size()) for (int i=0; i<pop.size(); i++) children[i]=null;
+	    	currentMeanDistance = createClusterTreeFromSortedPop(sorted);
+	    	if (TRACE) pop.putData(initializedForKey, pop.hashCode());
+	    	pop.putData(initializedRefData, currentMeanDistance);
+	    	return initializedRefData;
+    	} else return null;
+    }
+    
+	public Population[] cluster(Population pop, Population referenceSet) {
+		if (pop.isEmpty()) return new Population[]{pop.cloneWithoutInds()};
+		ArrayList<AbstractEAIndividual> sorted = pop.getSorted(comparator);
+		if (uplink==null || (uplink.length!=pop.size())) uplink = new int[pop.size()]; // parent index of all indys
+		if (uplinkDist==null || (uplinkDist.length!=pop.size())) uplinkDist = new double[pop.size()]; // parent distance for all indys
+		if (children==null || (children.length!=pop.size())) children = new Vector[pop.size()]; // list of children for all indies
+		else if (children.length==pop.size()) for (int i=0; i<pop.size(); i++) children[i]=null;
+ 		
+		if (TRACE) {
+			System.out.println("Current pop measures: " + BeanInspector.toString(pop.getPopulationMeasures(metric)[0]));
+			System.out.println("Current threshold: " + currentDistThreshold());
+		}
+		if (isAdaptiveThreshold()) { // test if there was a valid initialization step
+			if (!getRefData(referenceSet, pop)) currentMeanDistance=createClusterTreeFromSortedPop(sorted);
+			else createClusterTreeFromSortedPop(sorted);
+		} else createClusterTreeFromSortedPop(sorted);
+		
+		// now go through indies starting with best. 
+		// Add all children which are closer than threshold and recursively their children to a cluster.
+		// Mark them as clustered and start with the next best unclustered.
+		int current = 0; // top indy is first
+		boolean[] clustered = new boolean[pop.size()];
+		LinkedList<Population> allClusters = new LinkedList<Population>();
+		while (current<sorted.size()) {
+			Population currentClust = pop.cloneWithoutInds();
+			currentClust.add(sorted.get(current));
+			clustered[current]=true;
+			addChildren(current, clustered, sorted, currentClust);
+			// currentClust now recursively contains all children - the cluster is complete 
+			// now jump to the next best unclustered indy
+			allClusters.add(currentClust);
+			while (current<sorted.size() && (clustered[current])) current++;
+		}
+		
+		ArrayList<Population> finalClusts = new ArrayList<Population>(allClusters.size());
+		finalClusts.add(pop.cloneWithoutInds()); 
+		for (Population clust : allClusters) {
+			if (clust.size()<minimumGroupSize) { // add to loner population
+				finalClusts.get(0).addPopulation(clust);
+			} else { // add to cluster list
+				finalClusts.add(clust);
+			}
+		}
+		Population[] finalArr = new Population[finalClusts.size()]; 
+		return finalClusts.toArray(finalArr);
+	}
+
+	/**
+	 * Get the reference data from a population instance that should have been initialized.
+	 * If the reference set is null, the backup is treated as reference set.
+	 * 
+	 * @param referenceSet
+	 * @param backup
+	 */
+	private boolean getRefData(Population referenceSet, Population backup) {
+		if (referenceSet==null) referenceSet=backup;
+		Double refDat = (Double)referenceSet.getData(initializedRefData);
+		if (refDat!=null) {
+			if (TRACE) { // check hash
+				Integer hash=(Integer)referenceSet.getData(initializedForKey);
+				if ((hash==null) || (hash!=referenceSet.hashCode())) {
+					System.err.println("Warning, missing initialization before clustering for ClusteringNearestBetter!");
+					return false;
+				}
+			}
+			currentMeanDistance = refDat.doubleValue();
+			return true;
+		} else {
+			System.err.println("Warning, missing reference data - forgot reference set initialization? " + this.getClass());
+			return false;
+		}
+	}
+
+	private double createClusterTreeFromSortedPop(ArrayList<AbstractEAIndividual> sorted) {
+		double edgeLengthSum=0; int edgeCnt = 0;
+		for (int i=sorted.size()-1; i>=1; i--) {  // start with worst indies
+			// search for closest indy which is better
+			uplink[i]=-1;
+			uplinkDist[i] = -1;
+			for (int j=i-1; j>=0; j--) { // look at all which are better
+				// if the j-th indy is closer, reset the index
+				double curDist = metric.distance(sorted.get(i), sorted.get(j));
+				if (uplinkDist[i]<0 || (curDist < uplinkDist[i])) {
+					uplink[i] = j;
+					uplinkDist[i] = curDist;
+				}
+			}
+			// the closest best for indy i is now known. connect them in the graph.
+			if (children[uplink[i]]==null) children[uplink[i]]=new Vector<Integer>();
+			children[uplink[i]].add(i);
+			edgeLengthSum+=uplinkDist[i];
+			edgeCnt++;
+		}
+//		currentMeanDistance = pop.getPopulationMeasures(metric)[0];
+		return edgeLengthSum/((double)edgeCnt); // the average edge length
+	}
+
+	/**
+	 * Add the next layer of children to the clustered population.
+	 * 
+	 * @param current
+	 * @param clustered
+	 * @param sorted
+	 * @param currentClust
+	 */
+	private void addChildren(int current, boolean[] clustered, ArrayList<AbstractEAIndividual> sorted, Population currentClust) {
+		if (children[current]!=null && (children[current].size()>0)) {
+			for (int i=0; i<children[current].size(); i++) {
+				if ((!clustered[children[current].get(i)]) && (uplinkDist[children[current].get(i)] < currentDistThreshold())) {
+					// the first child is not clustered yet and below distance threshold.
+					// so add it to the cluster, mark it, and proceed recursively.
+					currentClust.add(sorted.get(children[current].get(i)));
+					clustered[children[current].get(i)]=true;
+					addChildren(children[current].get(i), clustered, sorted, currentClust);
+				}
+			}
+		} else {
+			// nothing more to do
+		}
+	}
+
+	private double currentDistThreshold() {
+		if (thresholdMultipleOfMeanDist) return meanDistFactor*currentMeanDistance;
+		else return absoluteDistThreshold;
+	}
+
+    /** This method allows you to decide if two species converge.
+     * @param species1  The first species.
+     * @param species2  The second species.
+     * @return True if species converge, else False.
+     */
+    public boolean mergingSpecies(Population species1, Population species2, Population referenceSet) {
+    	getRefData(referenceSet, species1);
+        if (testConvergingSpeciesOnBestOnly) {
+            if (this.metric.distance(species1.getBestEAIndividual(), species2.getBestEAIndividual()) < this.currentDistThreshold()) return true;
+            else return false;
+        } else {
+            Population tmpPop = new Population(species1.size()+species2.size());
+            tmpPop.addPopulation(species1);
+            tmpPop.addPopulation(species2);
+            if (this.cluster(tmpPop, referenceSet).length <= 2) return true;
+            else return false;
+        }
+    }
+    
+    public String globalInfo() { 
+    	return "A tree is produced by assigning each individual the closest individual with better fitness. Connections with a distance above a certain threshold are cut. After that, each interconnected subtree forms a cluster.";
+    }
+
+    public String metricTipText() {
+    	return "The metric to use during clustering.";
+    }
+    public InterfaceDistanceMetric getMetric() {
+		return metric;
+	}
+	public void setMetric(InterfaceDistanceMetric metric) {
+		this.metric = metric;
+	}
+
+	public String distThresholdTipText() {
+		return "In the non-adaptive case the absolute threshold below which clusters are connected.";
+	}
+	public double getDistThreshold() {
+		return absoluteDistThreshold;
+	}
+	public void setDistThreshold(double distThreshold) {
+		this.absoluteDistThreshold = distThreshold;
+	}
+
+	public String minimumGroupSizeTipText() {
+		return "Minimum group size that makes an own cluster.";
+	}
+	public int getMinimumGroupSize() {
+		return minimumGroupSize;
+	}
+	public void setMinimumGroupSize(int minimumGroupSize) {
+		this.minimumGroupSize = minimumGroupSize;
+	}
+
+	public String comparatorTipText() {
+		return "Define the comparator by which the population is sorted before clustering.";
+	}
+	public AbstractEAIndividualComparator getComparator() {
+		return comparator;
+	}
+//	public void setComparator(AbstractEAIndividualComparator comparator) {
+//		this.comparator = comparator;
+//	}
+	
+	public String adaptiveThresholdTipText() {
+		return "Activate adaptive threshold which is calculated from mean distance in the population and a constant factor.";
+	}
+    public boolean isAdaptiveThreshold() {
+		return thresholdMultipleOfMeanDist;
+	}
+	public void setAdaptiveThreshold(boolean thresholdMultipleOfMeanDist) {
+		this.thresholdMultipleOfMeanDist = thresholdMultipleOfMeanDist;
+		GenericObjectEditor.setHideProperty(this.getClass(), "meanDistFactor", !thresholdMultipleOfMeanDist);
+		GenericObjectEditor.setHideProperty(this.getClass(), "distThreshold", thresholdMultipleOfMeanDist);
+	}
+
+	public String meanDistFactorTipText() {
+		return "Factor producing the distance threshold from population mean distance.";
+	}
+	public double getMeanDistFactor() {
+		return meanDistFactor;
+	}
+	public void setMeanDistFactor(double meanDistFactor) {
+		this.meanDistFactor = meanDistFactor;
+	}	
+
+	public String testConvergingSpeciesOnBestOnlyTipText() {
+		return "Only the best individuals may be compared when testing whether to merge two species."; 
+	}
+    public boolean isTestConvergingSpeciesOnBestOnly() {
+		return testConvergingSpeciesOnBestOnly;
+	}
+	public void SetTestConvergingSpeciesOnBestOnly(
+			boolean testConvergingSpeciesOnBestOnly) {
+		this.testConvergingSpeciesOnBestOnly = testConvergingSpeciesOnBestOnly;
+	}
+
+}

src/eva2/server/go/operators/cluster/ClusteringXMeans.java

@@ -1,5 +1,7 @@
 package eva2.server.go.operators.cluster;
 
+import java.util.Arrays;
+
 import eva2.gui.Chart2DDPointIconCircle;
 import eva2.gui.Chart2DDPointIconText;
 import eva2.gui.GraphPointSet;
@@ -53,7 +55,7 @@ public class ClusteringXMeans implements InterfaceClustering, java.io.Serializab
      * @param pop       The population of individuals that is to be clustered.
      * @return Population[]
      */
-    public Population[] cluster(Population pop) {
+    public Population[] cluster(Population pop, Population referencePop) {
         ClusteringKMeans    kmeans      = new ClusteringKMeans();
         Population[][]      tmpResults  = new Population[this.m_MaxK][];
         double[][][]        tmpC        = new double[this.m_MaxK][][];
@@ -68,7 +70,7 @@ public class ClusteringXMeans implements InterfaceClustering, java.io.Serializab
         for (int i = 1; i < this.m_MaxK; i++) {
             kmeans.setUseSearchSpace(this.m_UseSearchSpace);
             kmeans.setK(i+1);
-            tmpResults[i]   = kmeans.cluster(pop);
+            tmpResults[i]   = kmeans.cluster(pop, (Population)null);
             tmpC[i]         = kmeans.getC();
         }
 
@@ -221,19 +223,26 @@ public class ClusteringXMeans implements InterfaceClustering, java.io.Serializab
      * @param species2  The second species.
      * @return True if species converge, else False.
      */
-    public boolean mergingSpecies(Population species1, Population species2) {
+    public boolean mergingSpecies(Population species1, Population species2, Population referencePop) {
         // @todo i could use the BIC metric from X-means to calculate this
         return false;
     }
 
-    /** This method decides if a unclustered individual belongs to an already established species.
+//    /** This method decides if a unclustered individual belongs to an already established species.
-     * @param indy          A unclustered individual.
+//     * @param indy          A unclustered individual.
-     * @param species       A species.
+//     * @param species       A species.
-     * @return True or False.
+//     * @return True or False.
-     */
+//     */
-    public boolean belongsToSpecies(AbstractEAIndividual indy, Population species) {
+//    public boolean belongsToSpecies(AbstractEAIndividual indy, Population species, Population pop) {
-        // @todo perhaps the same as in convergingSpecies
+//        // @todo perhaps the same as in convergingSpecies
-        return false;
+//        return false;
+//    }
+
+	public int[] associateLoners(Population loners, Population[] species, Population referencePop) {
+		int[] res=new int[loners.size()];
+		System.err.println("Warning, associateLoners not implemented for " + this.getClass());
+		Arrays.fill(res, -1);
+		return res;
 	}
 	
     /** This method allows you to recieve the c centroids
@@ -322,7 +331,7 @@ public class ClusteringXMeans implements InterfaceClustering, java.io.Serializab
         } else {
             f1.initPopulation(pop);
         }
-        ckm.cluster(pop);
+        ckm.cluster(pop, (Population)null);
 
     }
 
@@ -370,4 +379,8 @@ public class ClusteringXMeans implements InterfaceClustering, java.io.Serializab
     public String useSearchSpaceTipText() {
         return "Toggel between search/objective space distance.";
     }
+
+	public String initClustering(Population pop) {
+		return null;
+	}
 }

src/eva2/server/go/operators/cluster/InterfaceClustering.java

@@ -33,19 +33,46 @@ public interface InterfaceClustering {
      * @param pop       The population of individuals that is to be clustered.
      * @return Population[]
      */
-    public Population[] cluster(Population pop);
+    public Population[] cluster(Population pop, Population referenceSet);
 
     /** This method allows you to decide if two species are to be merged regarding this clustering algorithm.
      * @param species1  The first species.
      * @param species2  The second species.
      * @return True if species converge, else False.
      */
-    public boolean mergingSpecies(Population species1, Population species2);
+    public boolean mergingSpecies(Population species1, Population species2, Population referenceSet);
 
-    /** This method decides if an unclustered individual belongs to an already established species.
+    /**
+     * Do some pre-calculations on a population for clustering. If additional population data
+     * is set, return the associated key, otherwise null.
+     * 
+     * @param pop
+     */
+    public String initClustering(Population pop);
+
+    /**
+     * This method decides if an unclustered individual belongs to an already established species.
+     * For some clustering methods this can only be decided in reference to the complete population.
+     * 
      * @param indy          A unclustered individual.
      * @param species       A species.
+     * @param pop			The complete population as a reference.
      * @return True or False.
      */
-    public boolean belongsToSpecies(AbstractEAIndividual indy, Population species);
+    //Removed since for some clustering methods its not feasible to associate loners sequentially. Instead, a whole set of
+    // lone individuals can now be associated to a given set of clusters
+    //public boolean belongsToSpecies(AbstractEAIndividual indy, Population species);
+    
+    /**
+     * Try to associate a set of loners with a given set of species. Return a list
+     * of indices assigning loner i with species j for all loners. If no species can
+     * be associated, -1 is returned as individual entry.
+     * Note that the last cluster threshold is used which may have depended on the last
+     * generation.
+     * 
+     * @param loners
+     * @param species
+     * @return associative list matching loners to species.
+     */
+    public int[] associateLoners(Population loners, Population[] species, Population referenceSet);
 }

src/eva2/server/go/operators/distancemetric/IndividualDataMetric.java

@@ -5,6 +5,7 @@ import java.io.Serializable;
 import eva2.server.go.individuals.AbstractEAIndividual;
 import eva2.server.go.individuals.InterfaceDataTypeDouble;
 import eva2.server.go.strategies.ParticleSwarmOptimization;
+import eva2.tools.EVAERROR;
 
 /**
  * Define a metric on data stored within individuals, such as the personal best position
@@ -45,7 +46,12 @@ public class IndividualDataMetric implements InterfaceDistanceMetric, Serializab
 					double[][] range2 = ((InterfaceDataTypeDouble)indy2).getDoubleRange();
 					return EuclideanMetric.normedEuclideanDistance((double[])data1, range1, (double[])data2, range2);
 				} else return EuclideanMetric.euclideanDistance((double[])data1, (double[])data2);
-			} else throw new RuntimeException("Error, invalid key data, double array required by " + this.getClass().getName());
+			} else {
+				EVAERROR.errorMsgOnce("Error, invalid key data, double array required by " + this.getClass().getName());
+				EVAERROR.errorMsgOnce("Using PhenotypeMetric as Backup...");
+				return (new PhenotypeMetric().distance(indy1, indy2));
+//				throw new RuntimeException("Invalid data key, double array required by " + this.getClass().getName());
+			}
 		}
 	}
 

src/eva2/server/go/operators/fitnessmodifier/FitnessAdaptiveClustering.java

@@ -42,9 +42,9 @@ public class FitnessAdaptiveClustering implements java.io.Serializable, Interfac
                 // also note that if all individual achieve equal fitness the sum will be zero
                 result[i] = data[i][x] -min + 0.1;
             }
-
+            this.m_ClusteringAlgorithm.initClustering(population);
             // Now search for clusters
-            Population[]    ClusterResult = this.m_ClusteringAlgorithm.cluster(population);
+            Population[]    ClusterResult = this.m_ClusteringAlgorithm.cluster(population, population);
             Population      cluster;
             for (int i = 1; i < ClusterResult.length; i++) {
                 cluster = ClusterResult[i];

src/eva2/server/go/operators/migration/MOClusteringSeparation.java

@@ -136,7 +136,7 @@ public class MOClusteringSeparation implements InterfaceMigration, java.io.Seria
 
         // first set the K to the K-Means
         this.m_KMeans.setK(islands.length);
-        this.m_KMeans.cluster(toCluster);
+        this.m_KMeans.cluster(toCluster, (Population)null);
         double[][] c = this.m_KMeans.getC();
         newIPOP = this.m_KMeans.cluster(collector, c);
 

src/eva2/server/go/operators/migration/MOXMeansSeparation.java

@@ -131,7 +131,7 @@ public class MOXMeansSeparation implements InterfaceMigration, java.io.Serializa
 
         // first set the K to the K-Means
         this.m_XMeans.setMaxK(islands.length);
-        this.m_XMeans.cluster(toCluster);
+        this.m_XMeans.cluster(toCluster, (Population)null);
         double[][] c = this.m_XMeans.getC();
         //@todo Hier muss ich mal denk machen und weniger click...
         newIPOP = this.m_XMeans.cluster(collector, c);

src/eva2/server/go/operators/postprocess/PostProcess.java

@@ -173,7 +173,8 @@ public class PostProcess {
         //cluster the undifferentiated population
     	Population result = new Population(10);
     	result.setSameParams(pop);
-    	Population[] clusters = clustering.cluster(pop);
+    	clustering.initClustering(pop);
+    	Population[] clusters = clustering.cluster(pop, null);
     	if (TRACE) {
     		System.out.println("found " + clusters.length + " clusters!");
     		int sum=0;
@@ -423,7 +424,7 @@ public class PostProcess {
 		
 		ppRunnable = new OptimizerRunnable(OptimizerFactory.makeParams(nms, pop, problem, 0, term), true);
 		// as nms creates a new population and has already evaluated them, send a signal to stats
-		ppRunnable.getStats().createNextGenerationPerformed(nms.getPopulation(), null);
+		ppRunnable.getStats().createNextGenerationPerformed(nms.getPopulation(), nms, null);
 		
 		runPP();
 
@@ -464,7 +465,7 @@ public class PostProcess {
 		pop.SetFunctionCalls(baseEvals);
 		
 		ppRunnable = new OptimizerRunnable(cmaParams, true);
-		ppRunnable.getStats().createNextGenerationPerformed(cmaParams.getOptimizer().getPopulation(), null);
+		ppRunnable.getStats().createNextGenerationPerformed(cmaParams.getOptimizer().getPopulation(), cmaParams.getOptimizer(), null);
 		
 		runPP();
 		pop.clear();

src/eva2/server/go/populations/Population.java

@@ -69,6 +69,7 @@ public class Population extends ArrayList implements PopulationInterface, Clonea
     // a sorted queue (for efficiency)
     transient private ArrayList<AbstractEAIndividual> sortedArr = null;
     private int lastFitCrit = -1;
+	private transient static boolean TRACE = false;
     
     // remember when the last evaluation was performed
 //	private Pair<Integer,Integer> evaluationTimeHashes = null;
@@ -76,6 +77,7 @@ public class Population extends ArrayList implements PopulationInterface, Clonea
 //	private int evaluationTimeModCount = -1;
 
     public Population() {
+    	if (TRACE) System.err.println("TRACING POP");
     }
     
     /**
@@ -86,10 +88,17 @@ public class Population extends ArrayList implements PopulationInterface, Clonea
      */
     public Population(int initialCapacity) {
     	super(initialCapacity);
+    	if (TRACE) System.err.println("TRACING POP");
     	setTargetSize(initialCapacity);
     }
 
+    /**
+     * Clones parameters, individuals, history and archive. 
+     * 
+     * @param population
+     */
     public Population(Population population) {
+    	if (TRACE) System.err.println("TRACING POP");
         setSameParams(population);
         for (int i = 0; i < population.size(); i++) {
             if (population.get(i) != null)
@@ -98,9 +107,48 @@ public class Population extends ArrayList implements PopulationInterface, Clonea
         copyHistAndArchive(population);
     }
         
+    public static Population makePopFromList(List<AbstractEAIndividual> indies) {
+		Population pop = new Population(indies.size());
+    	if (TRACE) System.err.println("TRACING POP");
+		pop.setTargetSize(indies.size());
+		for (AbstractEAIndividual indy : indies) {
+			pop.add(indy);
+		}
+		return pop;
+	}
+
+    /**
+     * Create a new population from a solution set by merging
+     * both current population and solutions from the set.
+     *  
+     * @param allSolutions
+     */
+	public Population(InterfaceSolutionSet allSolutions) {
+		this(allSolutions.getCurrentPopulation().size()+allSolutions.getSolutions().size());
+    	if (TRACE) System.err.println("TRACING POP");
+		addPopulation(allSolutions.getCurrentPopulation());
+		HashMap<Long, Integer> checkCols = new HashMap<Long, Integer>(size());
+		for (int i=0; i<size(); i++) {
+			checkCols.put(getEAIndividual(i).getIndyID(), 1);
+		}
+		Population sols = allSolutions.getSolutions();
+		for (int i=0; i<sols.size();i++) {
+//			addPopulation(allSolutions.getSolutions());
+			if (!checkCols.containsKey(sols.getEAIndividual(i).getIndyID())) add(sols.getEAIndividual(i));
+		}
+	}
+
 	public void copyHistAndArchive(Population population) {
     	if (population.m_Archive != null) this.m_Archive = (Population)population.m_Archive.clone();
-    	if (population.m_History != null) this.m_History = (ArrayList)population.m_History.clone();
+    	if (population.m_History != null) this.m_History = (ArrayList<AbstractEAIndividual>)population.m_History.clone();
+    	if (population.additionalPopData!=null) {
+    		this.additionalPopData = (HashMap<String, Object>)additionalPopData.clone();
+    		if (population.additionalPopData.size()>0) {
+    			for (String key : population.additionalPopData.keySet()) {
+					additionalPopData.put(key, population.additionalPopData.get(key));
+				}
+    		}
+    	}
     }
     
     /**
@@ -173,7 +221,7 @@ public class Population extends ArrayList implements PopulationInterface, Clonea
      * have been inited by a problem
      */
     public void init() {
-        this.m_History = new ArrayList();
+        this.m_History = new ArrayList<AbstractEAIndividual>();
         this.m_Generation       = 0;
         this.m_FunctionCalls    = 0;
 //    	evaluationTimeHashes = null;
@@ -302,8 +350,8 @@ public class Population extends ArrayList implements PopulationInterface, Clonea
      */
     protected void firePropertyChangedEvent(String name) {
         if (listeners != null) {
-        	for (Iterator iterator = listeners.iterator(); iterator.hasNext();) {
+        	for (Iterator<InterfacePopulationChangedEventListener> iterator = listeners.iterator(); iterator.hasNext();) {
-				InterfacePopulationChangedEventListener listener = (InterfacePopulationChangedEventListener) iterator.next();
+				InterfacePopulationChangedEventListener listener = iterator.next();
 				if (listener!=null) listener.registerPopulationStateChanged(this, name);
 			}
         }
@@ -390,8 +438,8 @@ public class Population extends ArrayList implements PopulationInterface, Clonea
      * Note: After this operation the target population size may be exceeded.
      * @param pop   The population that is to be added.
      */
-    public void addPopulation(Population pop) {
+    public Population addPopulation(Population pop) {
-        if (pop == null) return;
+        if (pop != null) {
         	for (int i = 0; i < pop.size(); i++) {
         		AbstractEAIndividual indy = (AbstractEAIndividual)pop.get(i);
         		if (indy != null) {
@@ -399,6 +447,75 @@ public class Population extends ArrayList implements PopulationInterface, Clonea
         		}
         	}
         }
+        return this;
+    }
+    
+    /** 
+     * Fill the population up to the given size with random elements
+     * from the given population.
+     * 
+     * @param upTo 	target size of the population
+     * @param fromPop   The population that is to be added.
+     * 
+     */
+    public boolean fillWithRandom(int upTo, Population fromPop) {
+    	if (upTo <= this.size()) return true;
+    	else if (fromPop==null || (fromPop.size()<1)) return false;
+    	else {
+    		int[] perm = RNG.randomPerm(fromPop.size());
+    		int i=0;
+    		while ((i<perm.length) && (this.size()<upTo)) { // until instance is filled or no more indys can be selected
+    			AbstractEAIndividual indy = (AbstractEAIndividual)fromPop.get(perm[i]);
+//				System.out.println("checking " + indy.getStringRepresentation());
+    			if ((indy != null) && (!containsByPosition(indy))) {
+    				this.add(indy);
+    			} else {
+//    				System.out.println(indy.getStringRepresentation() + " was contained!");
+    			}
+    			i++;
+        	}
+    		if (size()==upTo) return true;
+    		else return false;
+        }
+    }
+    
+    /**
+     * Find a list of pairs of indices within the population at which individuals with equal
+     * positions can be found.
+     * @return
+     */
+    public List<Pair<Integer,Integer>> findSamePositions() {
+    	ArrayList<Pair<Integer,Integer>> dupes = new ArrayList<Pair<Integer,Integer>>();
+    	for (int i=0; i<size()-1; i++) {
+    		int nextIndex = indexByPosition(i+1, getEAIndividual(i));
+    		if (nextIndex >= 0) dupes.add(new Pair<Integer,Integer>(i, nextIndex));
+    	}
+    	return dupes;
+    }
+    
+    /**
+     * Return true if an individual with equal position is contained within the population.
+     * 
+     * @param indy
+     * @return
+     */
+    public boolean containsByPosition(AbstractEAIndividual indy) {
+		return indexByPosition(0,indy)>=0;
+	}
+
+    /**
+     * Return the index of the first individual which has an equal position or -1.
+     * 
+     * @param startIndex the first index to start the search
+     * @param indy
+     * @return the index of the first individual which has an equal position or -1
+     */
+	public int indexByPosition(int startIndex, AbstractEAIndividual indy) {
+		for (int i=startIndex; i<size(); i++) {
+			if (Arrays.equals(AbstractEAIndividual.getDoublePositionShallow(indy), AbstractEAIndividual.getDoublePositionShallow(getEAIndividual(i)))) return i;
+		}
+		return -1;
+	}
 
 	/**
      * Resets the fitnes to the maximum possible value for the given individual.
@@ -799,6 +916,7 @@ public class Population extends ArrayList implements PopulationInterface, Clonea
      * @return
      */
     public ArrayList<AbstractEAIndividual> getSorted(AbstractEAIndividualComparator comp) {
+    	if (super.size()==0) return new ArrayList<AbstractEAIndividual>();
 		PriorityQueue<AbstractEAIndividual> sQueue = new PriorityQueue<AbstractEAIndividual>(super.size(), comp);
 		for (int i = 0; i < super.size(); i++) {
 			AbstractEAIndividual indy = getEAIndividual(i);
@@ -1132,7 +1250,8 @@ public class Population extends ArrayList implements PopulationInterface, Clonea
     }
 
     /**
-	 * ArrayList does not increase the modCount in set. Why???
+	 * ArrayList does not increase the modCount in set. Maybe because it is not
+	 * seen as "structural change"?
      */
     public Object set(int index, Object element) {
     	Object prev = super.set(index, element);
@@ -1250,10 +1369,32 @@ public class Population extends ArrayList implements PopulationInterface, Clonea
      * @return the average, minimal and maximal mean distance of individuals in an array of three
      */
     public double[] getPopulationMeasures(InterfaceDistanceMetric metric) {
+//    	Integer lastMeasuresModCount = (Integer)getData(lastPopMeasuresFlagKey);
+//    	if (lastMeasuresModCount!=null && (lastMeasuresModCount==modCount)) {
+//    		double[] measures = (double[])getData(lastPopMeasuresDatKey);
+//            if (TRACE ) {
+//            	double[] verify = calcPopulationMeasures(metric);
+//            	if (Mathematics.dist(measures, verify, 2)>1e-10) {
+//            		System.out.println(getStringRepresentation());
+//            		System.err.println("Warning, invalid measures!!!");
+//            		
+//            	}
+//            }
+//    		return measures;
+//    	} 
+    	double[] res = calcPopulationMeasures(metric);
+//        putData(lastPopMeasuresFlagKey, new Integer(modCount));
+//        putData(lastPopMeasuresDatKey, res);
+//        System.out.println(getStringRepresentation());
+//        System.out.println("0-1-dist: " + BeanInspector.toString(metric.distance((AbstractEAIndividual)this.get(0), (AbstractEAIndividual)this.get(1))));
+        return res;
+    }
+
+	private double[] calcPopulationMeasures(InterfaceDistanceMetric metric) {
 		double d;
     	double[] res = new double[3];
     	
-    	double distSum = 0.;
+    	double meanDist = 0.;
     	double maxDist = Double.MIN_VALUE;
     	double minDist = Double.MAX_VALUE;
     	
@@ -1262,19 +1403,18 @@ public class Population extends ArrayList implements PopulationInterface, Clonea
         		if (metric == null) d = EuclideanMetric.euclideanDistance(AbstractEAIndividual.getDoublePositionShallow(getEAIndividual(i)), 
                 		AbstractEAIndividual.getDoublePositionShallow(getEAIndividual(j)));
         		else d = metric.distance((AbstractEAIndividual)this.get(i), (AbstractEAIndividual)this.get(j));
-                distSum += d;
+                meanDist += d;
                 if (d < minDist) minDist = d;
                 if (d > maxDist) maxDist = d;
             }
         }
         res[1] = minDist;
         res[2] = maxDist;
-        if (this.size() > 1) res[0] = distSum / (this.size() * (this.size()-1) / 2);
+        if (this.size() > 1) res[0] = meanDist / (this.size() * (this.size()-1) / 2);
         else { // only one indy?
         	res[1]=0;
         	res[2]=0;
         }
-//        System.out.println("0-1-dist: " + BeanInspector.toString(metric.distance((AbstractEAIndividual)this.get(0), (AbstractEAIndividual)this.get(1))));
 		return res;
 	}
     
@@ -1343,6 +1483,29 @@ public class Population extends ArrayList implements PopulationInterface, Clonea
 		 return new Pair<Integer,Double>(sel,dist);
 	 }
 	 
+	 /**
+	  * Search for the closest (farthest) individual to the given individual.
+	  * Return a Pair of the individuals index and distance.
+	  * If the population is empty, a Pair of (-1,-1) is returned.
+	  * 
+	  * @param pos
+	  * @param pop
+	  * @param closestOrFarthest if true, the closest individual is retrieved, otherwise the farthest
+	  * @return 
+	  */
+	 public static Pair<Integer,Double> getClosestFarthestIndy(AbstractEAIndividual refIndy, Population pop, InterfaceDistanceMetric metric, boolean closestOrFarthest) {
+		 double dist = -1.;
+		 int sel=-1;
+		 for (int i = 0; i < pop.size(); i++) {
+			 double curDist = metric.distance(refIndy, pop.getEAIndividual(i));
+			 if ((dist<0) 	|| (!closestOrFarthest && (dist < curDist)) 
+					 		|| (closestOrFarthest && (dist > curDist))) {
+				 dist = curDist;
+				 sel = i;
+			 }
+		 }
+		 return new Pair<Integer,Double>(sel,dist);
+	 }
 	/**
 	 * Calculate the average position of the population.
 	 * 
@@ -1574,6 +1737,36 @@ public class Population extends ArrayList implements PopulationInterface, Clonea
 //		if (evaluationTimeHashes == null) return false;
 //		else return ((hashes.head().equals(evaluationTimeHashes.head())) && (hashes.tail().equals(evaluationTimeHashes.tail())) && (evaluationTimeModCount == modCount));
 //	}
+	/**
+	 * Add the population data of a given population to this instance.
+	 * Note that collisions are not checked!
+	 */
+	public void addDataFromPopulation(Population pop) {
+		if (pop.additionalPopData!=null) {
+			Set<String> keys = pop.additionalPopData.keySet();
+			for (String key : keys) {
+				Object earlierDat = this.getData(key);
+				if (earlierDat!=null && !(earlierDat.equals(pop.getData(key)))) System.err.println("Warning: Population already contained data keyed by " + key + ", overwriting data " + earlierDat + " with " + pop.getData(key));
+				this.putData(key, pop.getData(key));
+			}
+		}
+	}
+
+	/**
+	 * Returns true if the given individual is member of this population,
+	 * where the comparison is done by individual ID.
+	 * 
+	 * @param indy
+	 * @return
+	 */
+	public boolean isMemberByID(AbstractEAIndividual indy) {
+		for (int i=0; i<size(); i++) {
+			if (getEAIndividual(i).getIndyID()==indy.getIndyID()) {
+				return true;
+			}
+		}
+		return false;
+	}
 
 	/**
 	 * Return true if the targeted size of the population has been reached.

src/eva2/server/go/strategies/DifferentialEvolution.java

@@ -250,8 +250,8 @@ public class DifferentialEvolution implements InterfaceOptimizer, java.io.Serial
             indy = (AbstractEAIndividual)(pop.getEAIndividual(parentIndex)).getClone();
             esIndy = (InterfaceDataTypeDouble)indy;
         } catch (java.lang.ClassCastException e) {
-            EVAERROR.errorMsgOnce("Differential Evolution currently requires InterfaceESIndividual as basic data type!");
+            throw new RuntimeException("Differential Evolution currently requires InterfaceESIndividual as basic data type!");
-            return (AbstractEAIndividual)((AbstractEAIndividual)pop.get(RNG.randomInt(0, pop.size()-1))).getClone();
+//            return (AbstractEAIndividual)((AbstractEAIndividual)pop.get(RNG.randomInt(0, pop.size()-1))).getClone();
         }
         double[] nX, vX, oX;
         oX = esIndy.getDoubleData();

src/eva2/server/go/strategies/EvolutionStrategies.java

@@ -239,7 +239,7 @@ public class EvolutionStrategies implements InterfaceOptimizer, java.io.Serializ
      */
     protected Population getReplacePop(Population nextGeneration) {
     	if (forceOrigPopSize && (origPopSize > 0) && (origPopSize < nextGeneration.size())) {
-    		// this is especially for CBN: 
+    		// this is especially for CBN: earlier selection to immediately reduce the size of mu+lambda to lambda
     		this.m_EnvironmentSelection.prepareSelection(nextGeneration);
     		Population tmpPop = (Population)nextGeneration.clone();
     		nextGeneration.clear();
@@ -368,8 +368,9 @@ public class EvolutionStrategies implements InterfaceOptimizer, java.io.Serializ
         	System.err.println("Invalid mu/lambda ratio! Setting mu=lambda="+m_Mu);
         	this.m_Lambda = this.m_Mu;
         }
-        if (this.m_UsePlusStrategy) this.m_Population.setTargetSize(this.m_Mu + this.m_Lambda);
+//        if (this.m_UsePlusStrategy) this.m_Population.setTargetSize(this.m_Mu + this.m_Lambda);
-        else this.m_Population.setTargetSize(this.m_Lambda);
+//        else this.m_Population.setTargetSize(this.m_Lambda);
+		this.m_Population.setTargetSize(this.m_Lambda);
         origPopSize=m_Population.getTargetSize();
     }
 

src/eva2/server/go/strategies/InterfaceSpeciesAware.java

@@ -0,0 +1,32 @@
+package eva2.server.go.strategies;
+
+import eva2.server.go.populations.Population;
+
+/**
+ * An interface for optimizers which are to be notified in case of species
+ * based optimization; namely merging and split events.
+ * 
+ * @author mkron
+ *
+ */
+public interface InterfaceSpeciesAware {
+	// these can be used to tag a population as explorer or local search population.
+	public final static String populationTagKey="specAwarePopulationTag";
+	public final static Integer explorerPopTag=23; 
+	public final static Integer localPopTag=42; 
+	
+	/**
+	 * Two species have been merged to the first one.
+	 * @param p1
+	 * @param p2
+	 */
+	public void mergeToFirstPopulationEvent(Population p1, Population p2);
+	
+	/**
+	 * Notify that a split has occurred separating p2 from p1.
+	 * 
+	 * @param p1
+	 * @param p2
+	 */
+	public void splitFromFirst(Population p1, Population p2);
+}

src/eva2/server/go/strategies/ParticleSwarmOptimization.java

@@ -1201,7 +1201,7 @@ public class ParticleSwarmOptimization implements InterfaceOptimizer, java.io.Se
 	}
 
 	public void optimize() {
-
+//		System.out.println(">>> " + m_Population.getStringRepresentation());
 		startOptimize();
 
 		// Update the individuals
@@ -1216,7 +1216,7 @@ public class ParticleSwarmOptimization implements InterfaceOptimizer, java.io.Se
 		// log the best individual of the population
 		logBestIndividual();
 
-//		System.out.println(">>> " + m_Population.getBestEAIndividual().getStringRepresentation());
+//		System.out.println("<<< " + m_Population.getStringRepresentation());
 		
 //		if (doLocalSearch && (m_Population.getGeneration()%localSearchGens==0)) {
 ////			System.out.println("Local search at gen "+m_Population.getGeneration());

src/eva2/server/modules/Processor.java

@@ -343,6 +343,7 @@ public class Processor extends Thread implements InterfaceProcessor, InterfacePo
     		if (this.goParams.getOptimizer() instanceof InterfaceAdditionalPopulationInformer) informerList.add(this.goParams.getOptimizer());
     		m_Statistics.createNextGenerationPerformed(
     				(PopulationInterface)this.goParams.getOptimizer().getPopulation(), 
+    				this.goParams.getOptimizer(),
     				informerList);
     		if (m_ListenerModule != null) {
     			m_ListenerModule.updateProgress(

src/eva2/server/stat/AbstractStatistics.java

@@ -14,8 +14,10 @@ import eva2.server.go.IndividualInterface;
 import eva2.server.go.PopulationInterface;
 import eva2.server.go.individuals.AbstractEAIndividual;
 import eva2.server.go.operators.distancemetric.InterfaceDistanceMetric;
+import eva2.server.go.populations.InterfaceSolutionSet;
 import eva2.server.go.populations.Population;
 import eva2.server.go.problems.InterfaceAdditionalPopulationInformer;
+import eva2.server.go.strategies.InterfaceOptimizer;
 import eva2.tools.Mathematics;
 import eva2.tools.Pair;
 
@@ -77,7 +79,7 @@ public abstract class AbstractStatistics implements InterfaceTextListener, Inter
 	
 	private ArrayList<InterfaceTextListener> textListeners;
 	private List<InterfaceAdditionalPopulationInformer> lastInformerList = null;
-	private PopulationInterface lastPop = null;
+	private PopulationInterface lastSols = null;
 
 	public AbstractStatistics() {
 		firstPlot = true;
@@ -173,7 +175,7 @@ public abstract class AbstractStatistics implements InterfaceTextListener, Inter
 		currentBestFeasibleFit=null;
 		bestOfRunFeasibleIndy = null;
 		lastInformerList = null;
-		lastPop = null;
+		lastSols = null;
 		runIterCnt = 0;
     	if (printRunIntroVerbosity()) printToTextListener("\n****** Multirun "+runNumber);
     	if (params != null) {
@@ -578,8 +580,7 @@ public abstract class AbstractStatistics implements InterfaceTextListener, Inter
 	 * @param pop
 	 */
 	private void updateLastAdditionalInfo() {
-		// TODO Auto-generated method stub
+		Double[] lastVals = appendAdditionalInfo(lastInformerList,  lastSols, null);
-		Double[] lastVals = appendAdditionalInfo(lastInformerList, lastPop, null);
 		lastAdditionalInfoSums = updateAdditionalInfo(lastAdditionalInfoSums, lastVals);
 	}
 	
@@ -602,9 +603,8 @@ public abstract class AbstractStatistics implements InterfaceTextListener, Inter
 	 *
 	 */
 	public synchronized void createNextGenerationPerformed(PopulationInterface
-			pop, List<InterfaceAdditionalPopulationInformer> informerList) {
+			pop, InterfaceOptimizer opt, List<InterfaceAdditionalPopulationInformer> informerList) {
 		lastInformerList  = informerList;
-		lastPop  = pop;
 		if (firstPlot) {
 			initPlots(m_StatsParams.getPlotDescriptions());
 //			if (doTextOutput()) printToTextListener(getOutputHeader(informer, pop)+'\n');
@@ -676,7 +676,7 @@ public abstract class AbstractStatistics implements InterfaceTextListener, Inter
 				meanCollection.add(means);
 			} else {
 				if (meanCollection.size()<=runIterCnt) {// bad case!
-					// may happen for dynamic pop-sizes, e.g. in Tribe, when runs do not necessarily send the
+					// may happen for dynamic pop-sizes, e.g. in Tribes, when runs do not necessarily send the
 					// "generation performed" event the same number of times. 
 					// thus: dont do an update for events that are "too late"
 					means = null;
@@ -686,8 +686,10 @@ public abstract class AbstractStatistics implements InterfaceTextListener, Inter
 		}
 //		meanCollection.set(pop.getGenerations()-1, means);
 		
+		lastSols  = (opt!=null) ? new Population(opt.getAllSolutions()) : pop;
 		if (doTextOutput()) {
-			Pair<String,Double[]> addInfo = getOutputLine(informerList, pop);
+			Pair<String,Double[]> addInfo = getOutputLine(informerList, lastSols);
+
 			if (printLineByVerbosity(runIterCnt)) printToTextListener(addInfo.head()+'\n');
 //			updateAdditionalInfo(addInfo.tail());
 			if (addInfo.tail()!=null) {

src/eva2/server/stat/InterfaceStatistics.java

@@ -17,6 +17,7 @@ import java.util.List;
 import eva2.server.go.IndividualInterface;
 import eva2.server.go.PopulationInterface;
 import eva2.server.go.problems.InterfaceAdditionalPopulationInformer;
+import eva2.server.go.strategies.InterfaceOptimizer;
 /*==========================================================================*
  * INTERFACE DECLARATION
  *==========================================================================*/
@@ -35,7 +36,7 @@ public interface InterfaceStatistics {
 	public void addTextListener(InterfaceTextListener listener);
 	public boolean removeTextListener(InterfaceTextListener listener);
 	public void printToTextListener(String s);
-	public void createNextGenerationPerformed(PopulationInterface Pop, List<InterfaceAdditionalPopulationInformer> informerList);
+	public void createNextGenerationPerformed(PopulationInterface Pop, InterfaceOptimizer opt, List<InterfaceAdditionalPopulationInformer> informerList);
 	public void createNextGenerationPerformed(double[] bestfit,double[] worstfit,int calls);
 	public InterfaceStatisticsParameter getStatisticsParameter(); // called from moduleadapter
 	public IndividualInterface getRunBestSolution(); // return the best fitness of the last run (may not be equal to the last population)

src/eva2/server/stat/StatisticsDummy.java

@@ -6,6 +6,7 @@ import eva2.server.go.IndividualInterface;
 import eva2.server.go.PopulationInterface;
 import eva2.server.go.individuals.AbstractEAIndividual;
 import eva2.server.go.problems.InterfaceAdditionalPopulationInformer;
+import eva2.server.go.strategies.InterfaceOptimizer;
 
 /**
  * This may be given to a Processor if no further stats are required. It speeds up
@@ -36,7 +37,7 @@ public class StatisticsDummy implements InterfaceStatistics, InterfaceTextListen
 		System.err.println("addTextListener not provided!");
 	}
 
-	public void createNextGenerationPerformed(PopulationInterface pop,
+	public void createNextGenerationPerformed(PopulationInterface pop, InterfaceOptimizer opt,
 			List<InterfaceAdditionalPopulationInformer> informerList) {
 		bestCurrentIndividual = (AbstractEAIndividual)pop.getBestIndividual();
 		if ((bestIndividualAllover == null) || (AbstractStatistics.secondIsBetter(bestIndividualAllover, bestCurrentIndividual))) {

src/eva2/server/stat/StatisticsWithGUI.java

@@ -247,23 +247,12 @@ public class StatisticsWithGUI extends AbstractStatistics implements Serializabl
 			int subGraph=0;
 			if (doPlotCurrentBest) plotFitnessPoint(0, subGraph++, functionCalls, currentBestFit[0]);
 			if (doPlotRunBest) plotFitnessPoint(0, subGraph++, functionCalls, bestOfRunIndy.getFitness()[0]);
-			
+			if (doPlotWorst) plotFitnessPoint(0, subGraph++ , functionCalls, currentWorstFit[0]);
-			if (doPlotWorst) {// schlechteste Fitness plotten
-				if (currentWorstFit == null) {
-					System.err.println("m_WorstFitness==null in plotStatisticsPerformed");
-					return;
-				}
-				plotFitnessPoint(0, subGraph++ , functionCalls, currentWorstFit[0]);
-			}
 			if (doPlotAvgDist) plotFitnessPoint(0, subGraph++, functionCalls, avgPopDist);
 			if (doPlotMaxPopDist) plotFitnessPoint(0, subGraph++, functionCalls, maxPopDist);
+			if (doPlotCurBestFeasible && currentBestFeasibleFit!=null) plotFitnessPoint(0, subGraph++, functionCalls, currentBestFeasibleFit[0]);
+			if (doPlotRunBestFeasible && bestOfRunFeasibleIndy!=null) plotFitnessPoint(0, subGraph++, functionCalls, bestOfRunFeasibleIndy.getFitness()[0]);
 
-			if (doPlotCurBestFeasible && currentBestFeasibleFit!=null) {
-				plotFitnessPoint(0, subGraph++, functionCalls, currentBestFeasibleFit[0]);
-			}
-			if (doPlotRunBestFeasible && bestOfRunFeasibleIndy!=null) {
-				plotFitnessPoint(0, subGraph++, functionCalls, bestOfRunFeasibleIndy.getFitness()[0]);
-			}
 		}
 	}