From 363fc645ad95ae242bb6a77f9e6120baa481887a Mon Sep 17 00:00:00 2001
From: s1924442 <b.arendarczyk@sms.ed.ac.uk>
Date: Mon, 11 Oct 2021 20:55:24 +0100
Subject: [PATCH] Land cover areas serialized to XML for improved performance.
---
debug_config.properties | 11 +-
src/ac/ed/lurg/ModelConfig.java | 1 +
src/ac/ed/lurg/ModelMain.java | 20 +-
src/ac/ed/lurg/carbon/CarbonFluxReader.java | 4 +-
src/ac/ed/lurg/forestry/WoodYieldReader.java | 6 +-
src/ac/ed/lurg/landuse/Intensity.java | 4 +
src/ac/ed/lurg/landuse/LandCoverTile.java | 8 +
src/ac/ed/lurg/landuse/LandUseItem.java | 15 +-
src/ac/ed/lurg/landuse/LandUseSerializer.java | 270 ++++++++++++++++++
9 files changed, 326 insertions(+), 13 deletions(-)
create mode 100644 src/ac/ed/lurg/landuse/LandUseSerializer.java
diff --git a/debug_config.properties b/debug_config.properties
index 8c6c1246..9c61e2bd 100644
--- a/debug_config.properties
+++ b/debug_config.properties
@@ -1,6 +1,7 @@
BASE_DIR=C:/Users/Bart/git/plumv2
YIELD_DIR=C:/Users/Bart/Documents/PhD/LURG/rcp60
+YIELD_DIR_TOP=rcp60
OUTPUT_DIR = C:/Users/Bart/git/plumv2/output
@@ -9,18 +10,18 @@ WOOD_AND_CARBON_DIR = C:/Users/Bart/Documents/PhD/Carbon and wood yields/forestr
BASE_YEAR=2010
START_TIMESTEP=0
TIMESTEP_SIZE=1
-END_TIMESTEP=0
+END_TIMESTEP=10
-IS_CALIBRATION_RUN=true
+IS_CALIBRATION_RUN=false
END_FIRST_STAGE_CALIBRATION=0
GENERATE_NEW_YIELD_CLUSTERS=false
YIELD_FILENAME=yield.out
-DEBUG_LIMIT_COUNTRIES=false
-DEBUG_COUNTRY_NAME=United States of America
-GAMS_COUNTRY_TO_SAVE=United States of America
+DEBUG_LIMIT_COUNTRIES=true
+DEBUG_COUNTRY_NAME=United Kingdom
+GAMS_COUNTRY_TO_SAVE=United Kingdom
INIT_WOOD_PRICE=0.4
INIT_WOOD_STOCK=1000
diff --git a/src/ac/ed/lurg/ModelConfig.java b/src/ac/ed/lurg/ModelConfig.java
index da4518f5..17c3ee4d 100755
--- a/src/ac/ed/lurg/ModelConfig.java
+++ b/src/ac/ed/lurg/ModelConfig.java
@@ -276,6 +276,7 @@ public class ModelConfig {
public static final String CALIB_DIR = IS_CALIBRATION_RUN ? OUTPUT_DIR : getProperty("CALIB_DIR", OUTPUT_DIR);
public static final int END_FIRST_STAGE_CALIBRATION = getIntProperty("END_FIRST_STAGE_CALIBRATION", 10);
public static final String SERIALIZED_LAND_USE_FILE = CALIB_DIR + File.separator + "landUseRaster.ser";
+ public static final String SERIALIZED_LAND_COVER_FILE = CALIB_DIR + File.separator + "landCoverData.xml";
public static final String SERIALIZED_CROP_USAGE_FILE = CALIB_DIR + File.separator + "countryCropUsages.ser";
public static final String SERIALIZED_INTERNATIONAL_MARKET_FILE = CALIB_DIR + File.separator + "internationalMarket.ser";
public static final boolean MARKET_ADJ_PRICE = getBooleanProperty("MARKET_ADJ_PRICE", true);
diff --git a/src/ac/ed/lurg/ModelMain.java b/src/ac/ed/lurg/ModelMain.java
index 12c0b33a..6eca89f0 100644
--- a/src/ac/ed/lurg/ModelMain.java
+++ b/src/ac/ed/lurg/ModelMain.java
@@ -8,6 +8,7 @@ import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.Collection;
+import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import ac.ed.lurg.carbon.CarbonFluxRasterSet;
@@ -41,7 +42,9 @@ import ac.ed.lurg.landuse.IrrigiationCostReader;
import ac.ed.lurg.landuse.LccKey;
import ac.ed.lurg.landuse.LandCoverItem;
import ac.ed.lurg.landuse.LandCoverReader;
+import ac.ed.lurg.landuse.LandCoverTile;
import ac.ed.lurg.landuse.LandUseItem;
+import ac.ed.lurg.landuse.LandUseSerializer;
import ac.ed.lurg.landuse.LandTileReader;
import ac.ed.lurg.landuse.MaxCropAreaReader;
import ac.ed.lurg.landuse.ProtectedAreasReader;
@@ -477,7 +480,7 @@ public class ModelMain {
}
outputWaterAvailablity(timestep, currentIrrigationData); // uses the year directory structure created above
}
-
+
if (ModelConfig.IS_CALIBRATION_RUN) {
if (timestep.isFinalTimestep()) {
serializeLandUse(landUseRaster);
@@ -614,17 +617,28 @@ public class ModelMain {
}
private void serializeLandUse(RasterSet<LandUseItem> landUseRaster) {
+ RasterSet<LandUseItem> rasterToSerialise = new RasterSet<LandUseItem>(desiredProjection);
+ for (Map.Entry<RasterKey, LandUseItem> entry : landUseRaster.entrySet()) {
+ LandUseItem newLuItem = new LandUseItem(entry.getValue());
+ newLuItem.overwriteLandCoverAreas(new HashMap<LandCoverType, LandCoverTile>()); // Clear data
+ rasterToSerialise.put(entry.getKey(), newLuItem);
+ }
try {
LogWriter.println("Starting serializing LandUse to " + ModelConfig.SERIALIZED_LAND_USE_FILE);
FileOutputStream fileOut = new FileOutputStream(ModelConfig.SERIALIZED_LAND_USE_FILE);
ObjectOutputStream out = new ObjectOutputStream(fileOut);
- out.writeObject(landUseRaster);
+ out.writeObject(rasterToSerialise);
out.close();
fileOut.close();
LogWriter.println("Serialized data is saved");
} catch (IOException i) {
i.printStackTrace();
}
+
+ LogWriter.println("Starting serializing LandCover to " + ModelConfig.SERIALIZED_LAND_COVER_FILE);
+ LandUseSerializer luSer = new LandUseSerializer();
+ luSer.serialiseLandUse(globalLandUseRaster);
+ LogWriter.println("LandCover data is saved");
}
@SuppressWarnings("unchecked")
@@ -637,6 +651,8 @@ public class ModelMain {
in.close();
fileIn.close();
LogWriter.println("Deserialized " + ModelConfig.SERIALIZED_LAND_USE_FILE);
+ LandUseSerializer luSer = new LandUseSerializer();
+ luSer.deserializeLandUse(initLU);
return initLU;
} catch (IOException i) {
LogWriter.printlnError("Problem deserializing " + ModelConfig.SERIALIZED_LAND_USE_FILE);
diff --git a/src/ac/ed/lurg/carbon/CarbonFluxReader.java b/src/ac/ed/lurg/carbon/CarbonFluxReader.java
index 79f40500..3c4d8c7e 100644
--- a/src/ac/ed/lurg/carbon/CarbonFluxReader.java
+++ b/src/ac/ed/lurg/carbon/CarbonFluxReader.java
@@ -68,7 +68,7 @@ public class CarbonFluxReader {
protected void setData(RasterKey key, CarbonFluxItem item, Map<String, Double> rowValues) {
Double[] fluxes = getArrayFromRowValues(rowValues);
- item.calcConversionCarbonFlux(lccKey, fluxes, landUseRaster.get(key).getLandUseTiles(), timestep);
+ item.calcConversionCarbonFlux(lccKey, fluxes, landUseRaster.get(key).getLandCoverTiles(), timestep);
}
};
@@ -81,7 +81,7 @@ public class CarbonFluxReader {
protected void setData(RasterKey key, CarbonFluxItem item, Map<String, Double> rowValues) {
Double[] fluxes = getArrayFromRowValues(rowValues);
- item.calcEcosystemCarbonFlux(lcType, fluxes, landUseRaster.get(key).getLandUseTiles(), timestep);
+ item.calcEcosystemCarbonFlux(lcType, fluxes, landUseRaster.get(key).getLandCoverTiles(), timestep);
}
};
diff --git a/src/ac/ed/lurg/forestry/WoodYieldReader.java b/src/ac/ed/lurg/forestry/WoodYieldReader.java
index ab2c998c..467b23f6 100644
--- a/src/ac/ed/lurg/forestry/WoodYieldReader.java
+++ b/src/ac/ed/lurg/forestry/WoodYieldReader.java
@@ -50,8 +50,8 @@ public class WoodYieldReader {
}
}
- item.calcYieldData(yieldMap, landUseRaster.get(key).getLandUseTiles(), timestep);
- item.calcRotationData(yieldMap, landUseRaster.get(key).getLandUseTiles(), timestep, woodPrice);
+ item.calcYieldData(yieldMap, landUseRaster.get(key).getLandCoverTiles(), timestep);
+ item.calcRotationData(yieldMap, landUseRaster.get(key).getLandCoverTiles(), timestep, woodPrice);
}
};
@@ -78,7 +78,7 @@ public class WoodYieldReader {
}
}
- item.calcYieldData(yieldMap, landUseRaster.get(key).getLandUseTiles(), timestep);
+ item.calcYieldData(yieldMap, landUseRaster.get(key).getLandCoverTiles(), timestep);
}
};
diff --git a/src/ac/ed/lurg/landuse/Intensity.java b/src/ac/ed/lurg/landuse/Intensity.java
index f694ef0c..fecb2b2c 100644
--- a/src/ac/ed/lurg/landuse/Intensity.java
+++ b/src/ac/ed/lurg/landuse/Intensity.java
@@ -58,6 +58,10 @@ public class Intensity implements Serializable {
return maxIrrigRate * irrigationIntensity;
}
+ public double getMaxIrrigRate() {
+ return maxIrrigRate;
+ }
+
public Intensity(Intensity from, Intensity to, double factor) {
this(
Interpolator.interpolate(from.fertiliserIntensity, to.fertiliserIntensity, factor),
diff --git a/src/ac/ed/lurg/landuse/LandCoverTile.java b/src/ac/ed/lurg/landuse/LandCoverTile.java
index 659f11b7..73c2b0f5 100644
--- a/src/ac/ed/lurg/landuse/LandCoverTile.java
+++ b/src/ac/ed/lurg/landuse/LandCoverTile.java
@@ -57,6 +57,10 @@ public class LandCoverTile implements Serializable, InterpolatingRasterItem<Land
}
}
+ public double getProtectedArea(int age) {
+ return protectedAreas[age];
+ }
+
public void setProtectedArea(int age, double area) {
protectedAreas[age] = area;
}
@@ -89,6 +93,10 @@ public class LandCoverTile implements Serializable, InterpolatingRasterItem<Land
}
}
+ public double getUnavailableArea() {
+ return unavailableArea;
+ }
+
public void addUnavailableArea(double d) {
unavailableArea += d;
}
diff --git a/src/ac/ed/lurg/landuse/LandUseItem.java b/src/ac/ed/lurg/landuse/LandUseItem.java
index 003ca8ef..58c0f3f2 100644
--- a/src/ac/ed/lurg/landuse/LandUseItem.java
+++ b/src/ac/ed/lurg/landuse/LandUseItem.java
@@ -46,6 +46,11 @@ public class LandUseItem implements InterpolatingRasterItem<LandUseItem>, Serial
protectedFraction = (luItemToCopy.protectedFraction);
}
+ public void overwriteLandCoverAreas(Map<LandCoverType, LandCoverTile> landCoverAreas) {
+ this.landCoverAreas = new HashMap<LandCoverType, LandCoverTile>();
+ this.landCoverAreas.putAll(landCoverAreas);
+ }
+
public void addLandCoverTiles(LandCoverItem landCover) {
double protectableFraction = 0;
for (LandCoverType lcType : LandCoverType.getProtectibleTypes()) {
@@ -184,7 +189,7 @@ public class LandUseItem implements InterpolatingRasterItem<LandUseItem>, Serial
}
}
- public Map<LandCoverType, LandCoverTile> getLandUseTiles() {
+ public Map<LandCoverType, LandCoverTile> getLandCoverTiles() {
return landCoverAreas;
}
@@ -311,6 +316,10 @@ public class LandUseItem implements InterpolatingRasterItem<LandUseItem>, Serial
protectedFraction = 1.0;
}
+ public Map<CropType, Intensity> getIntensityMap() {
+ return intensityMap;
+ }
+
public Intensity getIntensity(CropType crop) {
return intensityMap.get(crop);
}
@@ -430,6 +439,10 @@ public class LandUseItem implements InterpolatingRasterItem<LandUseItem>, Serial
}
return totalFract;
}
+
+ public Map<CropType, Double> getCropFractionMap() {
+ return cropFractions;
+ }
public void setCropFraction(CropType c, double areaFract) {
cropFractions.put(c, areaFract);
diff --git a/src/ac/ed/lurg/landuse/LandUseSerializer.java b/src/ac/ed/lurg/landuse/LandUseSerializer.java
new file mode 100644
index 00000000..3b9cdb4e
--- /dev/null
+++ b/src/ac/ed/lurg/landuse/LandUseSerializer.java
@@ -0,0 +1,270 @@
+package ac.ed.lurg.landuse;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.UnsupportedEncodingException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.xml.stream.FactoryConfigurationError;
+import javax.xml.stream.XMLOutputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamWriter;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+
+import ac.ed.lurg.ModelConfig;
+import ac.ed.lurg.types.CropType;
+import ac.ed.lurg.types.LandCoverType;
+import ac.ed.lurg.utils.LogWriter;
+import ac.sac.raster.RasterKey;
+import ac.sac.raster.RasterSet;
+
+public class LandUseSerializer {
+
+ XMLStreamWriter xmlWriter;
+
+ public void LandUseSerialiser() {}
+
+ public void serialiseLandUse(RasterSet<LandUseItem> luRaster) {
+
+ try {
+
+ OutputStream outStream = new FileOutputStream(new File(ModelConfig.SERIALIZED_LAND_COVER_FILE));
+
+ xmlWriter = XMLOutputFactory.newInstance().createXMLStreamWriter(
+ new OutputStreamWriter(outStream, "utf-8"));
+
+ xmlWriter.writeStartDocument("UTF-8", "1.0");
+ xmlWriter.writeStartElement("LandUseItems");
+
+ int counter = 1;
+ for (Map.Entry<RasterKey, LandUseItem> entry : luRaster.entrySet()) {
+ RasterKey key = entry.getKey();
+ LandUseItem luItem = entry.getValue();
+
+ xmlWriter.writeStartElement("LandUseItem");
+ xmlWriter.writeAttribute("id", Integer.toString(counter));
+ writeElement("col", key.getCol());
+ writeElement("row", key.getRow());
+
+ //writeIntensityData(luItem.getIntensityMap());
+ //writeCropData(luItem.getCropFractionMap());
+ writeLandCoverData(luItem.getLandCoverTiles());
+ //writeElement("protectedFraction", luItem.getProtectedFraction());
+
+ xmlWriter.writeEndElement(); // LandUseItem
+ xmlWriter.flush();
+ counter++;
+ //if (counter > 1) break;
+ }
+
+ xmlWriter.writeEndElement(); // LandUseItems
+ xmlWriter.writeEndDocument();
+
+ xmlWriter.flush();
+ xmlWriter.close();
+
+ } catch (FileNotFoundException e) {
+ LogWriter.print(e);
+ } catch (XMLStreamException e) {
+ LogWriter.print(e);
+ } catch (UnsupportedEncodingException e) {
+ LogWriter.print(e);
+ } catch (FactoryConfigurationError e) {
+ LogWriter.print(e.getException());
+ }
+ }
+
+ private void writeIntensityData(Map<CropType, Intensity> intensityMap) {
+ try {
+ xmlWriter.writeStartElement("intensity");
+ for (Map.Entry<CropType, Intensity> entry : intensityMap.entrySet()) {
+ CropType crop = entry.getKey();
+ Intensity inten = entry.getValue();
+ if (inten == null) {
+ continue;
+ }
+ xmlWriter.writeStartElement(crop.getGamsName());
+ writeElement("fertiliserIntensity", inten.getFertiliserIntensity());
+ writeElement("irrigationIntensity", inten.getIrrigationIntensity());
+ writeElement("otherIntensity", inten.getOtherIntensity());
+ writeElement("yield", inten.getYield());
+ writeElement("unitEnergy", inten.getUnitEnergy());
+ writeElement("maxIrrigRate", inten.getMaxIrrigRate());
+ xmlWriter.writeEndElement();
+ }
+ xmlWriter.writeEndElement();
+
+
+ } catch (XMLStreamException e) {
+ LogWriter.print(e);
+ }
+
+ }
+
+ private void writeCropData(Map<CropType, Double> cropFractMap) {
+ try {
+ xmlWriter.writeStartElement("cropFractions");
+ for (Map.Entry<CropType, Double> entry : cropFractMap.entrySet()) {
+ CropType crop = entry.getKey();
+ Double fract = entry.getValue();
+ if (fract == null) {
+ continue;
+ }
+ writeElement(crop.getGamsName(), fract.doubleValue());
+
+ }
+ xmlWriter.writeEndElement();
+
+ } catch (XMLStreamException e) {
+ LogWriter.print(e);
+ }
+
+ }
+
+ private void writeLandCoverData(Map<LandCoverType, LandCoverTile> lcData) {
+ try {
+ xmlWriter.writeStartElement("landCover");
+ for (Map.Entry<LandCoverType, LandCoverTile> entry : lcData.entrySet()) {
+ LandCoverType lcType = entry.getKey();
+ LandCoverTile tiles = entry.getValue();
+ if (tiles == null) {
+ continue;
+ }
+ xmlWriter.writeStartElement(lcType.getName());
+
+ xmlWriter.writeStartElement("convertible");
+ for (int a = 0; a <= LandCoverTile.getMaxAgeBin(); a++) {
+ double area = tiles.getConvertibleArea(a);
+ if (area == 0)
+ continue;
+ writeElement("X"+Integer.toString(a), area);
+ }
+ xmlWriter.writeEndElement();
+
+ xmlWriter.writeStartElement("protected");
+ for (int a = 0; a <= LandCoverTile.getMaxAgeBin(); a++) {
+ double area = tiles.getProtectedArea(a);
+ if (area == 0)
+ continue;
+ writeElement("X"+Integer.toString(a), area);
+ }
+ xmlWriter.writeEndElement();
+
+ writeElement("unavailable", tiles.getUnavailableArea());
+
+ xmlWriter.writeEndElement();
+ }
+ xmlWriter.writeEndElement();
+
+
+ } catch (XMLStreamException e) {
+ LogWriter.print(e);
+ }
+
+ }
+
+ private void writeElement(String element, double value) {
+ try {
+ xmlWriter.writeStartElement(element);
+ xmlWriter.writeCharacters(Double.toString(value));
+ xmlWriter.writeEndElement();
+ } catch (XMLStreamException e) {
+ LogWriter.print(e);
+ }
+ }
+
+ private void writeElement(String element, int value) {
+ try {
+ xmlWriter.writeStartElement(element);
+ xmlWriter.writeCharacters(Integer.toString(value));
+ xmlWriter.writeEndElement();
+ } catch (XMLStreamException e) {
+ LogWriter.print(e);
+ }
+ }
+
+ public void deserializeLandUse(RasterSet<LandUseItem> luRaster) {
+ DocumentBuilderFactory fact = DocumentBuilderFactory.newInstance();
+ DocumentBuilder builder;
+
+ Document doc;
+ try {
+ builder = fact.newDocumentBuilder();
+ doc = builder.parse(new File(ModelConfig.SERIALIZED_LAND_COVER_FILE));
+ doc.getDocumentElement().normalize();
+
+ Element root = doc.getDocumentElement();
+ NodeList nodeLParent = doc.getElementsByTagName("LandUseItems");
+ NodeList nodeL = nodeLParent.item(0).getChildNodes();
+
+ for (int i = 0; i < nodeL.getLength(); i++) {
+ Node node = nodeL.item(i);
+ Map<LandCoverType, LandCoverTile> landCoverAreas = new HashMap<LandCoverType, LandCoverTile>();
+ for (LandCoverType lcType: LandCoverType.values()) {
+ landCoverAreas.put(lcType, new LandCoverTile());
+ }
+ if (node.getNodeType() == Node.ELEMENT_NODE) {
+ Element itemElement = (Element) node;
+ int col = Integer.parseInt(itemElement.getElementsByTagName("col").item(0).getTextContent()); // TODO store as attribute
+ int row = Integer.parseInt(itemElement.getElementsByTagName("row").item(0).getTextContent());
+
+ NodeList lcNodeListParent = itemElement.getElementsByTagName("landCover");
+ NodeList lcNodeList = lcNodeListParent.item(0).getChildNodes();
+ Element lcNodeElement = (Element) lcNodeList;
+
+
+ for (int j = 0; j < lcNodeList.getLength(); j++) {
+ Node lcNode = lcNodeList.item(j);
+
+ if (node.getNodeType() == Node.ELEMENT_NODE) {
+ LandCoverType lcType = LandCoverType.getForName(lcNode.getNodeName());
+ Element lcElement = (Element) lcNode;
+ NodeList convElements = lcElement.getElementsByTagName("convertible").item(0).getChildNodes();
+ NodeList protElements = lcElement.getElementsByTagName("protected").item(0).getChildNodes();
+ double unavArea = Double.parseDouble(lcElement.getElementsByTagName("unavailable").item(0).getTextContent());
+ landCoverAreas.get(lcType).addUnavailableArea(unavArea);
+
+ for (int k = 0; k < convElements.getLength(); k++) {
+ Node areaNode = convElements.item(k);
+ int age = Integer.parseInt(areaNode.getNodeName().replace("X", ""));
+ //double area = Double.parseDouble(areaNode.getNodeValue());
+ double area = Double.parseDouble(areaNode.getChildNodes().item(0).getNodeValue());
+ landCoverAreas.get(lcType).setConvertibleArea(age, area);
+ }
+
+ for (int k = 0; k < protElements.getLength(); k++) {
+ Node areaNode = protElements.item(k);
+ int age = Integer.parseInt(areaNode.getNodeName().replace("X", ""));
+ //double area = Double.parseDouble(areaNode.getNodeValue());
+ double area = Double.parseDouble(areaNode.getChildNodes().item(0).getNodeValue());
+ landCoverAreas.get(lcType).setProtectedArea(age, area);
+ }
+ }
+ }
+ luRaster.get(new RasterKey(col, row)).overwriteLandCoverAreas(landCoverAreas);
+ }
+
+ }
+
+ } catch (SAXException | IOException | ParserConfigurationException e) {
+ // TODO Auto-generated catch block
+ LogWriter.print(e);;
+ }
+
+ }
+}
--
GitLab