Land cover areas serialized to XML for improved performance.

debug_config.properties

 BASE_DIR=C:/Users/Bart/git/plumv2
 
 YIELD_DIR=C:/Users/Bart/Documents/PhD/LURG/rcp60
+YIELD_DIR_TOP=rcp60
 
 OUTPUT_DIR = C:/Users/Bart/git/plumv2/output
 
@@ -9,18 +10,18 @@ WOOD_AND_CARBON_DIR = C:/Users/Bart/Documents/PhD/Carbon and wood yields/forestr
 BASE_YEAR=2010
 START_TIMESTEP=0
 TIMESTEP_SIZE=1
-END_TIMESTEP=0
+END_TIMESTEP=10
 
-IS_CALIBRATION_RUN=true
+IS_CALIBRATION_RUN=false
 END_FIRST_STAGE_CALIBRATION=0
 
 GENERATE_NEW_YIELD_CLUSTERS=false
 
 YIELD_FILENAME=yield.out
 
-DEBUG_LIMIT_COUNTRIES=false
-DEBUG_COUNTRY_NAME=United States of America
-GAMS_COUNTRY_TO_SAVE=United States of America
+DEBUG_LIMIT_COUNTRIES=true
+DEBUG_COUNTRY_NAME=United Kingdom
+GAMS_COUNTRY_TO_SAVE=United Kingdom
 
 INIT_WOOD_PRICE=0.4
 INIT_WOOD_STOCK=1000

src/ac/ed/lurg/ModelConfig.java

@@ -276,6 +276,7 @@ public class ModelConfig {
 	public static final String CALIB_DIR = IS_CALIBRATION_RUN ? OUTPUT_DIR : getProperty("CALIB_DIR", OUTPUT_DIR);
 	public static final int END_FIRST_STAGE_CALIBRATION = getIntProperty("END_FIRST_STAGE_CALIBRATION", 10);
 	public static final String SERIALIZED_LAND_USE_FILE = CALIB_DIR + File.separator +  "landUseRaster.ser";
+	public static final String SERIALIZED_LAND_COVER_FILE = CALIB_DIR + File.separator +  "landCoverData.xml";
 	public static final String SERIALIZED_CROP_USAGE_FILE = CALIB_DIR + File.separator +  "countryCropUsages.ser";
 	public static final String SERIALIZED_INTERNATIONAL_MARKET_FILE = CALIB_DIR + File.separator +  "internationalMarket.ser";
 	public static final boolean MARKET_ADJ_PRICE = getBooleanProperty("MARKET_ADJ_PRICE", true);

src/ac/ed/lurg/ModelMain.java

@@ -8,6 +8,7 @@ import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
 import ac.ed.lurg.carbon.CarbonFluxRasterSet;
@@ -41,7 +42,9 @@ import ac.ed.lurg.landuse.IrrigiationCostReader;
 import ac.ed.lurg.landuse.LccKey;
 import ac.ed.lurg.landuse.LandCoverItem;
 import ac.ed.lurg.landuse.LandCoverReader;
+import ac.ed.lurg.landuse.LandCoverTile;
 import ac.ed.lurg.landuse.LandUseItem;
+import ac.ed.lurg.landuse.LandUseSerializer;
 import ac.ed.lurg.landuse.LandTileReader;
 import ac.ed.lurg.landuse.MaxCropAreaReader;
 import ac.ed.lurg.landuse.ProtectedAreasReader;
@@ -477,7 +480,7 @@ public class ModelMain {
 			}
 			outputWaterAvailablity(timestep, currentIrrigationData);  // uses the year directory structure created above
 		}
-
+		
 		if (ModelConfig.IS_CALIBRATION_RUN) {
 			if (timestep.isFinalTimestep()) {
 				serializeLandUse(landUseRaster);
@@ -614,17 +617,28 @@ public class ModelMain {
 	}
 
 	private void serializeLandUse(RasterSet<LandUseItem> landUseRaster) {
+		RasterSet<LandUseItem> rasterToSerialise = new RasterSet<LandUseItem>(desiredProjection);
+		for (Map.Entry<RasterKey, LandUseItem> entry : landUseRaster.entrySet()) {
+			LandUseItem newLuItem = new LandUseItem(entry.getValue());
+			newLuItem.overwriteLandCoverAreas(new HashMap<LandCoverType, LandCoverTile>()); // Clear data
+			rasterToSerialise.put(entry.getKey(), newLuItem);
+		}
 		try {
 			LogWriter.println("Starting serializing LandUse to " + ModelConfig.SERIALIZED_LAND_USE_FILE);
 			FileOutputStream fileOut = new FileOutputStream(ModelConfig.SERIALIZED_LAND_USE_FILE);
 			ObjectOutputStream out = new ObjectOutputStream(fileOut);
-			out.writeObject(landUseRaster);
+			out.writeObject(rasterToSerialise);
 			out.close();
 			fileOut.close();
 			LogWriter.println("Serialized data is saved");
 		} catch (IOException i) {
 			i.printStackTrace();
 		}
+
+		LogWriter.println("Starting serializing LandCover to " + ModelConfig.SERIALIZED_LAND_COVER_FILE);
+		LandUseSerializer luSer = new LandUseSerializer();
+		luSer.serialiseLandUse(globalLandUseRaster);
+		LogWriter.println("LandCover data is saved");
 	}
 
 	@SuppressWarnings("unchecked")
@@ -637,6 +651,8 @@ public class ModelMain {
 			in.close();
 			fileIn.close();
 			LogWriter.println("Deserialized " + ModelConfig.SERIALIZED_LAND_USE_FILE);
+			LandUseSerializer luSer = new LandUseSerializer();
+			luSer.deserializeLandUse(initLU);
 			return initLU;
 		} catch (IOException i) {
 			LogWriter.printlnError("Problem deserializing " + ModelConfig.SERIALIZED_LAND_USE_FILE);

src/ac/ed/lurg/carbon/CarbonFluxReader.java

@@ -68,7 +68,7 @@ public class CarbonFluxReader {
 			protected void setData(RasterKey key, CarbonFluxItem item, Map<String, Double> rowValues) {
 
 				Double[] fluxes = getArrayFromRowValues(rowValues);
-				item.calcConversionCarbonFlux(lccKey, fluxes, landUseRaster.get(key).getLandUseTiles(), timestep);
+				item.calcConversionCarbonFlux(lccKey, fluxes, landUseRaster.get(key).getLandCoverTiles(), timestep);
 			}			
 		};
 	
@@ -81,7 +81,7 @@ public class CarbonFluxReader {
 			protected void setData(RasterKey key, CarbonFluxItem item, Map<String, Double> rowValues) {
 
 				Double[] fluxes = getArrayFromRowValues(rowValues);
-				item.calcEcosystemCarbonFlux(lcType, fluxes, landUseRaster.get(key).getLandUseTiles(), timestep);
+				item.calcEcosystemCarbonFlux(lcType, fluxes, landUseRaster.get(key).getLandCoverTiles(), timestep);
 			}			
 		};
 	

src/ac/ed/lurg/forestry/WoodYieldReader.java

@@ -50,8 +50,8 @@ public class WoodYieldReader {
 					}	
 				}
 				
-				item.calcYieldData(yieldMap, landUseRaster.get(key).getLandUseTiles(), timestep);
-				item.calcRotationData(yieldMap, landUseRaster.get(key).getLandUseTiles(), timestep, woodPrice);
+				item.calcYieldData(yieldMap, landUseRaster.get(key).getLandCoverTiles(), timestep);
+				item.calcRotationData(yieldMap, landUseRaster.get(key).getLandCoverTiles(), timestep, woodPrice);
 			}			
 		};
 	
@@ -78,7 +78,7 @@ public class WoodYieldReader {
 					}	
 				}
 				
-				item.calcYieldData(yieldMap, landUseRaster.get(key).getLandUseTiles(), timestep);
+				item.calcYieldData(yieldMap, landUseRaster.get(key).getLandCoverTiles(), timestep);
 			}			
 		};
 		

src/ac/ed/lurg/landuse/Intensity.java

@@ -58,6 +58,10 @@ public class Intensity implements Serializable {
 		return maxIrrigRate * irrigationIntensity;
 	}
 
+	public double getMaxIrrigRate() {
+		return maxIrrigRate;
+	}
+
 	public Intensity(Intensity from, Intensity to, double factor) {
 		this(
 			Interpolator.interpolate(from.fertiliserIntensity, to.fertiliserIntensity, factor), 

src/ac/ed/lurg/landuse/LandCoverTile.java

@@ -57,6 +57,10 @@ public class LandCoverTile implements Serializable, InterpolatingRasterItem<Land
 		}
 	}
 	
+	public double getProtectedArea(int age) {
+		return protectedAreas[age];
+	}
+	
 	public void setProtectedArea(int age, double area) {
 		protectedAreas[age] = area;
 	}
@@ -89,6 +93,10 @@ public class LandCoverTile implements Serializable, InterpolatingRasterItem<Land
 		}
 	}
 	
+	public double getUnavailableArea() {
+		return unavailableArea;
+	}
+	
 	public void addUnavailableArea(double d) {
 		unavailableArea += d;
 	}

src/ac/ed/lurg/landuse/LandUseItem.java

@@ -46,6 +46,11 @@ public class LandUseItem implements InterpolatingRasterItem<LandUseItem>, Serial
 		protectedFraction = (luItemToCopy.protectedFraction);
 	}
 
+	public void overwriteLandCoverAreas(Map<LandCoverType, LandCoverTile> landCoverAreas) {
+		this.landCoverAreas = new HashMap<LandCoverType, LandCoverTile>();
+		this.landCoverAreas.putAll(landCoverAreas);
+	}
+
 	public void addLandCoverTiles(LandCoverItem landCover) {		
 		double protectableFraction = 0;
 		for (LandCoverType lcType : LandCoverType.getProtectibleTypes()) {
@@ -184,7 +189,7 @@ public class LandUseItem implements InterpolatingRasterItem<LandUseItem>, Serial
 		}
 	}
 	
-	public Map<LandCoverType, LandCoverTile> getLandUseTiles() {
+	public Map<LandCoverType, LandCoverTile> getLandCoverTiles() {
 		return landCoverAreas;
 	}
 	
@@ -311,6 +316,10 @@ public class LandUseItem implements InterpolatingRasterItem<LandUseItem>, Serial
 			protectedFraction = 1.0;		
 	}
 	
+	public Map<CropType, Intensity> getIntensityMap() {
+		return intensityMap;
+	}
+	
 	public Intensity getIntensity(CropType crop) {
 		return intensityMap.get(crop);
 	}
@@ -430,6 +439,10 @@ public class LandUseItem implements InterpolatingRasterItem<LandUseItem>, Serial
 		}
 		return totalFract;
 	}
+	
+	public Map<CropType, Double> getCropFractionMap() {
+		return cropFractions;
+	}
 
 	public void setCropFraction(CropType c, double areaFract) {
 		cropFractions.put(c, areaFract);

src/ac/ed/lurg/landuse/LandUseSerializer.java

+package ac.ed.lurg.landuse;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.UnsupportedEncodingException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.xml.stream.FactoryConfigurationError;
+import javax.xml.stream.XMLOutputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamWriter;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+
+import ac.ed.lurg.ModelConfig;
+import ac.ed.lurg.types.CropType;
+import ac.ed.lurg.types.LandCoverType;
+import ac.ed.lurg.utils.LogWriter;
+import ac.sac.raster.RasterKey;
+import ac.sac.raster.RasterSet;
+
+public class LandUseSerializer {
+	
+	XMLStreamWriter xmlWriter;
+	
+	public void LandUseSerialiser() {}
+
+	public void serialiseLandUse(RasterSet<LandUseItem> luRaster) {
+		
+		try {
+
+		OutputStream outStream = new FileOutputStream(new File(ModelConfig.SERIALIZED_LAND_COVER_FILE));
+		
+		xmlWriter = XMLOutputFactory.newInstance().createXMLStreamWriter(
+				new OutputStreamWriter(outStream, "utf-8"));
+		
+		xmlWriter.writeStartDocument("UTF-8", "1.0");
+		xmlWriter.writeStartElement("LandUseItems");
+
+		int counter = 1;
+		for (Map.Entry<RasterKey, LandUseItem> entry : luRaster.entrySet()) {
+			RasterKey key = entry.getKey();
+			LandUseItem luItem = entry.getValue();
+			
+			xmlWriter.writeStartElement("LandUseItem");
+			xmlWriter.writeAttribute("id", Integer.toString(counter));
+			writeElement("col", key.getCol());
+			writeElement("row", key.getRow());
+			
+			//writeIntensityData(luItem.getIntensityMap());
+			//writeCropData(luItem.getCropFractionMap());
+			writeLandCoverData(luItem.getLandCoverTiles());
+			//writeElement("protectedFraction", luItem.getProtectedFraction());
+			
+			xmlWriter.writeEndElement(); // LandUseItem
+			xmlWriter.flush();
+			counter++;
+			//if (counter > 1) break;
+		}
+
+		xmlWriter.writeEndElement(); // LandUseItems
+		xmlWriter.writeEndDocument();
+		
+		xmlWriter.flush();
+		xmlWriter.close();
+		
+		} catch (FileNotFoundException e) {
+			LogWriter.print(e);
+		} catch (XMLStreamException e) {
+			LogWriter.print(e);
+		} catch (UnsupportedEncodingException e) {
+			LogWriter.print(e);
+		} catch (FactoryConfigurationError e) {
+			LogWriter.print(e.getException());
+		}
+	}
+	
+	private void writeIntensityData(Map<CropType, Intensity> intensityMap) {
+		try {
+			xmlWriter.writeStartElement("intensity");
+			for (Map.Entry<CropType, Intensity> entry : intensityMap.entrySet()) {
+				CropType crop = entry.getKey();
+				Intensity inten = entry.getValue();
+				if (inten == null) {
+					continue;
+				}
+				xmlWriter.writeStartElement(crop.getGamsName());
+				writeElement("fertiliserIntensity", inten.getFertiliserIntensity());
+				writeElement("irrigationIntensity", inten.getIrrigationIntensity());
+				writeElement("otherIntensity", inten.getOtherIntensity());
+				writeElement("yield", inten.getYield());
+				writeElement("unitEnergy", inten.getUnitEnergy());
+				writeElement("maxIrrigRate", inten.getMaxIrrigRate());
+				xmlWriter.writeEndElement();
+			}
+			xmlWriter.writeEndElement();
+
+
+		} catch (XMLStreamException e) {
+			LogWriter.print(e);
+		} 
+
+	}
+	
+	private void writeCropData(Map<CropType, Double> cropFractMap) {
+		try {
+			xmlWriter.writeStartElement("cropFractions");
+			for (Map.Entry<CropType, Double> entry : cropFractMap.entrySet()) {
+				CropType crop = entry.getKey();
+				Double fract = entry.getValue();
+				if (fract == null) {
+					continue;
+				}
+				writeElement(crop.getGamsName(), fract.doubleValue());
+
+			}
+			xmlWriter.writeEndElement();
+
+		} catch (XMLStreamException e) {
+			LogWriter.print(e);
+		} 
+
+	}
+	
+	private void writeLandCoverData(Map<LandCoverType, LandCoverTile> lcData) {
+		try {
+			xmlWriter.writeStartElement("landCover");
+			for (Map.Entry<LandCoverType, LandCoverTile> entry : lcData.entrySet()) {
+				LandCoverType lcType = entry.getKey();
+				LandCoverTile tiles = entry.getValue();
+				if (tiles == null) {
+					continue;
+				}
+				xmlWriter.writeStartElement(lcType.getName());
+				
+				xmlWriter.writeStartElement("convertible");
+				for (int a = 0; a <= LandCoverTile.getMaxAgeBin(); a++) {
+					double area = tiles.getConvertibleArea(a);
+					if (area == 0) 
+						continue;
+					writeElement("X"+Integer.toString(a), area);
+				}
+				xmlWriter.writeEndElement();
+				
+				xmlWriter.writeStartElement("protected");
+				for (int a = 0; a <= LandCoverTile.getMaxAgeBin(); a++) {
+					double area = tiles.getProtectedArea(a);
+					if (area == 0) 
+						continue;
+					writeElement("X"+Integer.toString(a), area);
+				}
+				xmlWriter.writeEndElement();
+				
+				writeElement("unavailable", tiles.getUnavailableArea());
+				
+				xmlWriter.writeEndElement();
+			}
+			xmlWriter.writeEndElement();
+
+
+		} catch (XMLStreamException e) {
+			LogWriter.print(e);
+		} 
+
+	}
+
+	private void writeElement(String element, double value) {
+		try {
+			xmlWriter.writeStartElement(element);
+			xmlWriter.writeCharacters(Double.toString(value));
+			xmlWriter.writeEndElement();
+		} catch (XMLStreamException e) {
+			LogWriter.print(e);
+		}
+	}
+	
+	private void writeElement(String element, int value) {
+		try {
+			xmlWriter.writeStartElement(element);
+			xmlWriter.writeCharacters(Integer.toString(value));
+			xmlWriter.writeEndElement();
+		} catch (XMLStreamException e) {
+			LogWriter.print(e);
+		}
+	}
+	
+	public void deserializeLandUse(RasterSet<LandUseItem> luRaster) {
+		DocumentBuilderFactory fact = DocumentBuilderFactory.newInstance();
+		DocumentBuilder builder;
+
+		Document doc;
+		try {
+			builder = fact.newDocumentBuilder();
+			doc = builder.parse(new File(ModelConfig.SERIALIZED_LAND_COVER_FILE));
+			doc.getDocumentElement().normalize();
+			
+			Element root = doc.getDocumentElement();
+			NodeList nodeLParent = doc.getElementsByTagName("LandUseItems");
+			NodeList nodeL = nodeLParent.item(0).getChildNodes();
+			
+			for (int i = 0; i < nodeL.getLength(); i++) {
+				Node node = nodeL.item(i);
+				Map<LandCoverType, LandCoverTile> landCoverAreas = new HashMap<LandCoverType, LandCoverTile>();
+				for (LandCoverType lcType: LandCoverType.values()) {
+					landCoverAreas.put(lcType, new LandCoverTile());
+				}
+				if (node.getNodeType() == Node.ELEMENT_NODE) {
+					Element itemElement = (Element) node;
+					int col = Integer.parseInt(itemElement.getElementsByTagName("col").item(0).getTextContent()); // TODO store as attribute
+					int row = Integer.parseInt(itemElement.getElementsByTagName("row").item(0).getTextContent());
+					
+					NodeList lcNodeListParent = itemElement.getElementsByTagName("landCover");
+					NodeList lcNodeList = lcNodeListParent.item(0).getChildNodes();
+					Element lcNodeElement = (Element) lcNodeList;
+					
+					
+					for (int j = 0; j < lcNodeList.getLength(); j++) {
+						Node lcNode = lcNodeList.item(j);
+						
+						if (node.getNodeType() == Node.ELEMENT_NODE) {
+							LandCoverType lcType = LandCoverType.getForName(lcNode.getNodeName());
+							Element lcElement = (Element) lcNode;
+							NodeList convElements = lcElement.getElementsByTagName("convertible").item(0).getChildNodes();
+							NodeList protElements = lcElement.getElementsByTagName("protected").item(0).getChildNodes();
+							double unavArea = Double.parseDouble(lcElement.getElementsByTagName("unavailable").item(0).getTextContent());
+							landCoverAreas.get(lcType).addUnavailableArea(unavArea);
+							
+							for (int k = 0; k < convElements.getLength(); k++) {
+								Node areaNode = convElements.item(k);
+								int age = Integer.parseInt(areaNode.getNodeName().replace("X", ""));
+								//double area = Double.parseDouble(areaNode.getNodeValue());
+								double area = Double.parseDouble(areaNode.getChildNodes().item(0).getNodeValue());
+								landCoverAreas.get(lcType).setConvertibleArea(age, area);
+							}
+							
+							for (int k = 0; k < protElements.getLength(); k++) {
+								Node areaNode = protElements.item(k);
+								int age = Integer.parseInt(areaNode.getNodeName().replace("X", ""));
+								//double area = Double.parseDouble(areaNode.getNodeValue());
+								double area = Double.parseDouble(areaNode.getChildNodes().item(0).getNodeValue());
+								landCoverAreas.get(lcType).setProtectedArea(age, area);
+							}
+						}
+					}	
+					luRaster.get(new RasterKey(col, row)).overwriteLandCoverAreas(landCoverAreas);
+				}
+				
+			}
+			
+		} catch (SAXException | IOException | ParserConfigurationException e) {
+			// TODO Auto-generated catch block
+			LogWriter.print(e);;
+		}
+
+	}
+}