From 3877670dc5837311fbba9d21bf51efb8744116ff Mon Sep 17 00:00:00 2001 From: afabiani Date: Thu, 14 May 2015 14:54:28 +0200 Subject: [PATCH] Pull Request for Issue #148 --- pom.xml | 39 ++- .../rest/GeoServerRESTPublisher.java | 50 +++ .../geoserver/rest/HTTPUtils.java | 91 ++++++ .../manager/GeoServerRESTImporterManager.java | 296 ++++++++++++++++++ .../publisher/GeoserverRESTGeoTiffTest.java | 3 +- .../publisher/GeoserverRESTImporterTest.java | 96 ++++++ 6 files changed, 564 insertions(+), 11 deletions(-) create mode 100644 src/main/java/it/geosolutions/geoserver/rest/manager/GeoServerRESTImporterManager.java create mode 100644 src/test/java/it/geosolutions/geoserver/rest/publisher/GeoserverRESTImporterTest.java diff --git a/pom.xml b/pom.xml index ef8c343..213cd00 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,8 @@ ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. --> - + 4.0.0 it.geosolutions @@ -203,6 +204,7 @@ 1.5.11 + 2.8-SNAPSHOT @@ -248,9 +250,7 @@ + --> @@ -268,11 +268,32 @@ 2.5.6.SEC02 test - - commons-logging - commons-logging - - + + commons-logging + commons-logging + + + + + + org.geoserver.importer + gs-importer-core + ${gs.version} + + + org.geoserver + gs-restconfig + ${gs.version} + + + org.restlet + org.restlet.ext.fileupload + 1.0.8 + + + commons-fileupload + commons-fileupload + 1.2.1 diff --git a/src/main/java/it/geosolutions/geoserver/rest/GeoServerRESTPublisher.java b/src/main/java/it/geosolutions/geoserver/rest/GeoServerRESTPublisher.java index 3804042..a851035 100644 --- a/src/main/java/it/geosolutions/geoserver/rest/GeoServerRESTPublisher.java +++ b/src/main/java/it/geosolutions/geoserver/rest/GeoServerRESTPublisher.java @@ -39,6 +39,7 @@ import it.geosolutions.geoserver.rest.encoder.GSResourceEncoder.ProjectionPolicy import it.geosolutions.geoserver.rest.encoder.GSWorkspaceEncoder; import it.geosolutions.geoserver.rest.encoder.coverage.GSCoverageEncoder; import it.geosolutions.geoserver.rest.encoder.feature.GSFeatureTypeEncoder; +import it.geosolutions.geoserver.rest.manager.GeoServerRESTImporterManager; import it.geosolutions.geoserver.rest.manager.GeoServerRESTStructuredGridCoverageReaderManager; import it.geosolutions.geoserver.rest.manager.GeoServerRESTStructuredGridCoverageReaderManager.ConfigureCoveragesOption; import it.geosolutions.geoserver.rest.manager.GeoServerRESTStyleManager; @@ -53,6 +54,8 @@ import java.net.URL; import java.net.URLEncoder; import java.util.zip.ZipFile; +import net.sf.json.JSONObject; + import org.apache.commons.httpclient.NameValuePair; import org.apache.commons.io.FilenameUtils; import org.slf4j.Logger; @@ -90,6 +93,7 @@ public class GeoServerRESTPublisher { private final GeoServerRESTStyleManager styleManager; + private final GeoServerRESTImporterManager importerManager; /** * Creates a GeoServerRESTPublisher to connect against a GeoServer instance with the given URL and user credentials. * @@ -109,6 +113,7 @@ public class GeoServerRESTPublisher { LOGGER.error("Bad URL: Calls to GeoServer are going to fail" , ex); } styleManager = new GeoServerRESTStyleManager(url, username, password); + importerManager = new GeoServerRESTImporterManager(url, username, password); } // ========================================================================== @@ -2929,4 +2934,49 @@ public class GeoServerRESTPublisher { } + /** + * Refers to {@link it.geosolutions.geoserver.rest.manager.GeoServerRESTImporterManager#postNewImport() postNewImport} method + * + * @throws Exception + */ + public int postNewImport() throws Exception { + return importerManager.postNewImport(); + } + + /** + * Refers to {@link it.geosolutions.geoserver.rest.manager.GeoServerRESTImporterManager#postNewTaskAsMultiPartForm(int, String) postNewTaskAsMultiPartForm} method + * + * @throws Exception + */ + public int postNewTaskAsMultiPartForm(int i, String data) throws Exception { + return importerManager.postNewTaskAsMultiPartForm(i, data); + } + + /** + * Refers to {@link it.geosolutions.geoserver.rest.manager.GeoServerRESTImporterManager#getTask(int, int) getTask} method + * + * @throws Exception + */ + public JSONObject getTask(int i, int t) throws Exception { + return importerManager.getTask(i, t); + } + + /** + * Refers to {@link it.geosolutions.geoserver.rest.manager.GeoServerRESTImporterManager#putTask(int, int, String) putTask} method + * + * @throws Exception + */ + public void putTask(int i, int t, String json) throws Exception { + importerManager.putTask(i, t, json); + } + + /** + * Refers to {@link it.geosolutions.geoserver.rest.manager.GeoServerRESTImporterManager#postImport(int) postImport} method + * + * @throws Exception + */ + public void postImport(int i) throws Exception { + importerManager.postImport(i); + } + } diff --git a/src/main/java/it/geosolutions/geoserver/rest/HTTPUtils.java b/src/main/java/it/geosolutions/geoserver/rest/HTTPUtils.java index f3cc6e6..ac2d8ef 100644 --- a/src/main/java/it/geosolutions/geoserver/rest/HTTPUtils.java +++ b/src/main/java/it/geosolutions/geoserver/rest/HTTPUtils.java @@ -25,6 +25,7 @@ package it.geosolutions.geoserver.rest; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -33,6 +34,11 @@ import java.net.ConnectException; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; +import java.util.ArrayList; +import java.util.List; + +import net.sf.json.JSON; +import net.sf.json.JSONSerializer; import org.apache.commons.httpclient.Credentials; import org.apache.commons.httpclient.HttpClient; @@ -48,6 +54,9 @@ import org.apache.commons.httpclient.methods.PostMethod; import org.apache.commons.httpclient.methods.PutMethod; import org.apache.commons.httpclient.methods.RequestEntity; import org.apache.commons.httpclient.methods.StringRequestEntity; +import org.apache.commons.httpclient.methods.multipart.FilePart; +import org.apache.commons.httpclient.methods.multipart.MultipartRequestEntity; +import org.apache.commons.httpclient.methods.multipart.Part; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -117,6 +126,22 @@ public class HTTPUtils { return null; } + /** + * Executes a request using the GET method and parses the result as a json object. + * + * @param path The path to request. + * + * @return The result parsed as json. + */ + public static JSON getAsJSON(String url, String username, String pw) throws Exception { + String response = get(url, username, pw); + return json(response); + } + + public static JSON json(String content) { + return JSONSerializer.toJSON(content); + } + /** * PUTs a File to the given URL.
* Basic auth is used if both username and pw are not null. @@ -175,6 +200,23 @@ public class HTTPUtils { return put(url, content, "text/xml", username, pw); } + /** + * PUTs a String representing an JSON Object to the given URL.
+ * Basic auth is used if both username and pw are not null. + * + * @param url The URL where to connect to. + * @param content The JSON Object to be sent as a String. + * @param username Basic auth credential. No basic auth if null. + * @param pw Basic auth credential. No basic auth if null. + * @return The HTTP response as a String if the HTTP response code was 200 + * (OK). + * @throws MalformedURLException + * @return the HTTP response or null on errors. + */ + public static String putJson(String url, String content, String username, String pw) { + return put(url, content, "application/json", username, pw); + } + /** * Performs a PUT to the given URL.
* Basic auth is used if both username and pw are not null. @@ -233,6 +275,38 @@ public class HTTPUtils { } } + /** + * POSTs a list of files as attachments to the given URL.
+ * Basic auth is used if both username and pw are not null. + * + * @param url The URL where to connect to. + * @param dir The folder containing the attachments. + * @param username Basic auth credential. No basic auth if null. + * @param pw Basic auth credential. No basic auth if null. + * @return The HTTP response as a String if the HTTP response code was 200 + * (OK). + * @throws MalformedURLException + * @return the HTTP response or null on errors. + */ + public static String postMultipartForm(String url, File dir, String username, String pw) { + try { + List parts = new ArrayList(); + for (File f : dir.listFiles()) { + parts.add(new FilePart(f.getName(), f)); + } + MultipartRequestEntity multipart = new MultipartRequestEntity( + parts.toArray(new Part[parts.size()]), new PostMethod().getParams()); + + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + multipart.writeRequest(bout); + + return post(url, multipart, username, pw); + } catch (Exception ex) { + LOGGER.error("Cannot POST " + url, ex); + return null; + } + } + /** * POSTs a String representing an XML document to the given URL.
* Basic auth is used if both username and pw are not null. @@ -250,6 +324,23 @@ public class HTTPUtils { return post(url, content, "text/xml", username, pw); } + /** + * POSTs a String representing an JSON Object to the given URL.
+ * Basic auth is used if both username and pw are not null. + * + * @param url The URL where to connect to. + * @param content The JSON content to be sent as a String. + * @param username Basic auth credential. No basic auth if null. + * @param pw Basic auth credential. No basic auth if null. + * @return The HTTP response as a String if the HTTP response code was 200 + * (OK). + * @throws MalformedURLException + * @return the HTTP response or null on errors. + */ + public static String postJson(String url, String content, String username, String pw) { + return post(url, content, "application/json", username, pw); + } + /** * Performs a POST to the given URL.
* Basic auth is used if both username and pw are not null. diff --git a/src/main/java/it/geosolutions/geoserver/rest/manager/GeoServerRESTImporterManager.java b/src/main/java/it/geosolutions/geoserver/rest/manager/GeoServerRESTImporterManager.java new file mode 100644 index 0000000..05e7fdb --- /dev/null +++ b/src/main/java/it/geosolutions/geoserver/rest/manager/GeoServerRESTImporterManager.java @@ -0,0 +1,296 @@ +/* + * GeoTools - The Open Source Java GIS Toolkit + * http://geotools.org + * + * (C) 2015, Open Source Geospatial Foundation (OSGeo) + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ +package it.geosolutions.geoserver.rest.manager; + +import it.geosolutions.geoserver.rest.HTTPUtils; + +import java.io.File; +import java.io.IOException; +import java.net.URL; + +import net.sf.json.JSON; +import net.sf.json.JSONObject; + +import org.geoserver.importer.VFSWorker; +import org.restlet.data.MediaType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Alessio Fabiani + * + */ +public class GeoServerRESTImporterManager extends GeoServerRESTAbstractManager { + + private final static Logger LOGGER = LoggerFactory.getLogger(GeoServerRESTImporterManager.class); + + /** + * Default constructor. + * + * @param restURL GeoServer REST API endpoint + * @param username GeoServer REST API authorized username + * @param password GeoServer REST API password for the former username + */ + public GeoServerRESTImporterManager(URL restURL, String username, String password) + throws IllegalArgumentException { + super(restURL, username, password); + } + + /** + * Retrieves the Import JSON Object given its identifier + * + * @param imp int: Import context number ID + */ + public JSONObject getImport(int imp) throws Exception { + JSON json = HTTPUtils.getAsJSON(String.format(buildUrl()+"/%d", imp), gsuser , gspass); + return ((JSONObject)json).getJSONObject("import"); + } + + /** + * Retrieves the Import Task JSON Object given its identifier and task number + * + * @param imp int: Import context number ID + * @param task int: Task number + */ + public JSONObject getTask(int imp, int task) throws Exception { + JSON json = HTTPUtils.getAsJSON(String.format(buildUrl()+"/%d/tasks/%d?expand=all", imp, task), gsuser , gspass); + return ((JSONObject)json).getJSONObject("task"); + } + + /** + * Example usage: + * 
+     *  // Creates a new Importer Context and gets back the ID
+     *  int i = postNewImport();
+     *  
+     *  // Attaches to the new Importer Context a Task pointing to a shapefile's zip archive
+     *  int t = postNewTaskAsMultiPartForm(i, "/path_to/shape/archsites_no_crs.zip");
+     *
+     *  // Check that the Task was actually created and that the CRS has not recognized in this case
+     *  JSONObject task = getTask(i, t);
+     *  assertEquals("NO_CRS", task.getString("state"));
+     *  
+     *  // Prepare the JSON String instructing the Task about the SRS to use
+     *  String json = 
+     *  "{" +
+     *    "\"task\": {" +
+     *      "\"layer\": {" +
+     *              "\"srs\": \"EPSG:4326\"" + 
+     *       "}" +
+     *     "}" + 
+     *  "}";
+     *  
+     *  // Performing the Task update
+     *  putTask(i, t, json);
+     *
+     *  // Double check that the Task is in the READY state
+     *  task = getTask(i, t);
+     *  assertEquals("READY", task.getString("state"));
+     *  assertEquals("gs_archsites", task.getJSONObject("layer").getJSONObject("style").getString("name"));
+     *  
+     *  // Prepare the JSON String instructing the Task avout the SLD to use for the new Layer
+     *  json = 
+     *  "{" +
+     *    "\"task\": {" +
+     *      "\"layer\": {" +
+     *        "\"style\": {" +
+     *              "\"name\": \"point\"" + 
+     *           "}" +
+     *         "}" +
+     *     "}" + 
+     *  "}";
+     *  
+     *  // Performing the Task update
+     *  putTask(i, t,json);
+     *
+     *  // Double check that the Task is in the READY state and that the Style has been correctly updated
+     *  task = getTask(i, t);
+     *  assertEquals("READY", task.getString("state"));
+     *  assertEquals("point", task.getJSONObject("layer").getJSONObject("style").getString("name"));
+     *  
+     *  // Finally starts the Import ...
+     *  postImport(i);
+     *
+ * + * @param imp int: Import context number ID + * @param task int: Task number + * @param json String: JSON containing the Task properties to be updated + * @throws Exception + */ + public void putTask(int imp, int task, final String json) throws Exception { + HTTPUtils.putJson(String.format(buildUrl()+"/%d/tasks/%d", imp, task), json, gsuser, gspass); + } + + /** + * Just update the Layers properties associated to a Task (t) in the Importer Context (i). + * + * e.g.: + * 
+     * putTaskLayer(i, t, "{\"title\":\"Archsites\", \"abstract\":\"Archeological Sites\"}");
+     *
+ * + * @param imp int: Import context number ID + * @param task int: Task number + * @param json String: JSON containing the Layer properties to be updated + * @throws Exception + */ + public void putTaskLayer(int imp, int task, final String json) throws Exception { + HTTPUtils.putJson(String.format(buildUrl()+"/%d/tasks/%d/layer", imp, task), json, gsuser, gspass); + } + + /** + * Creates an empty Importer Context. + * + * @return The new Importer Context ID + * @throws Exception + */ + public int postNewImport() throws Exception { + return postNewImport(null); + } + + /** + * e.g.: + * 
+     * String body = 
+     *         "{" + 
+     *              "\"import\": { " + 
+     *                  "\"data\": {" +
+     *                     "\"type\": \"mosaic\", " + 
+     *                     "\"time\": {" +
+     *                        " \"mode\": \"auto\"" + 
+     *                     "}" + 
+     *                   "}" +
+     *              "}" + 
+     *         "}";
+     *
+ * + * @param body JSON String representing the Importer Context definition + * @return The new Importer Context ID + * @throws Exception + */ + public int postNewImport(String body) throws Exception { + String resp = body == null ? HTTPUtils.postJson(buildUrl(), "", gsuser, gspass) + : HTTPUtils.postJson(buildUrl(), body, gsuser, gspass); + + JSONObject json = (JSONObject) HTTPUtils.json(resp); + JSONObject imprt = json.getJSONObject("import"); + return imprt.getInt("id"); + } + + /** + * Actually starts the READY State Import. + * + * @param imp int: Import context number ID + * @throws Exception + */ + public void postImport(int imp) throws Exception { + HTTPUtils.postJson(buildUrl()+"/" + imp, "", gsuser, gspass); + } + + /** + * + * @param imp int: Import context number ID + * @param data + * @return + * @throws Exception + */ + public int postNewTaskAsMultiPartForm(int imp, String data) throws Exception { + String resp = HTTPUtils.postMultipartForm(buildUrl()+"/" + imp + "/tasks", unpack(data), gsuser, gspass); + + JSONObject json = (JSONObject) HTTPUtils.json(resp); + + JSONObject task = json.getJSONObject("task"); + return task.getInt("id"); + } + + /** + * Allows to attach a new zip file to an existing Importer Context. + * + * @param imp int: Import context number ID + * @param path + * @return + * @throws Exception + */ + public int putNewTask(int imp, String path) throws Exception { + File zip = new File(path); + + String resp = HTTPUtils.put(buildUrl()+"/" + imp + "/tasks/" + zip.getName(), zip, MediaType.APPLICATION_ZIP.toString(), gsuser, gspass); + + JSONObject json = (JSONObject) HTTPUtils.json(resp); + + JSONObject task = json.getJSONObject("task"); + return task.getInt("id"); + } + + //========================================================================= + // Util methods + //========================================================================= + + /** + * Creates the base REST URL for the imports + */ + protected String buildUrl() { + StringBuilder sUrl = new StringBuilder(gsBaseUrl.toString()).append("/rest/imports"); + + return sUrl.toString(); + } + + /** + * Creates a temporary file + * + * @return Path to the temporary file + * @throws Exception + */ + public static File tmpDir() throws Exception { + File dir = File.createTempFile("importer", "data", new File("target")); + dir.delete(); + dir.mkdirs(); + return dir; + } + + /** + * Expands a zip archive into the temporary folder. + * + * @param path The absolute path to the source zip file + * @return Path to the temporary folder containing the expanded files + * @throws Exception + */ + public static File unpack(String path) throws Exception { + return unpack(path, tmpDir()); + } + + /** + * Expands a zip archive into the target folder. + * + * @param path The absolute path to the source zip file + * @param dir Full path of the target folder where to expand the archive + * @return Path to the temporary folder containing the expanded files + * @throws Exception + */ + public static File unpack(String path, File dir) throws Exception { + + File file = new File(path); + + new VFSWorker().extractTo(file, dir); + if (!file.delete()) { + // fail early as tests will expect it's deleted + throw new IOException("deletion failed during extraction"); + } + + return dir; + } +} diff --git a/src/test/java/it/geosolutions/geoserver/rest/publisher/GeoserverRESTGeoTiffTest.java b/src/test/java/it/geosolutions/geoserver/rest/publisher/GeoserverRESTGeoTiffTest.java index 7056b2f..594a619 100644 --- a/src/test/java/it/geosolutions/geoserver/rest/publisher/GeoserverRESTGeoTiffTest.java +++ b/src/test/java/it/geosolutions/geoserver/rest/publisher/GeoserverRESTGeoTiffTest.java @@ -88,7 +88,7 @@ public class GeoserverRESTGeoTiffTest extends GeoserverRESTTest { assertFalse("Bad unpublish()", publisher.unpublishCoverage(DEFAULT_WS, storeName, layerName)); assertFalse(existsLayer(layerName)); } - + @Test public void testGeotiff() throws FileNotFoundException, IOException { if (!enabled()) return; @@ -125,7 +125,6 @@ public class GeoserverRESTGeoTiffTest extends GeoserverRESTTest { assertFalse(reader.existsCoveragestore(DEFAULT_WS, storeName)); } - @Test public void testReloadCoverageStore() throws FileNotFoundException, IOException { if (!enabled()) return; diff --git a/src/test/java/it/geosolutions/geoserver/rest/publisher/GeoserverRESTImporterTest.java b/src/test/java/it/geosolutions/geoserver/rest/publisher/GeoserverRESTImporterTest.java new file mode 100644 index 0000000..26739bd --- /dev/null +++ b/src/test/java/it/geosolutions/geoserver/rest/publisher/GeoserverRESTImporterTest.java @@ -0,0 +1,96 @@ +/* + * GeoTools - The Open Source Java GIS Toolkit + * http://geotools.org + * + * (C) 2015, Open Source Geospatial Foundation (OSGeo) + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ +package it.geosolutions.geoserver.rest.publisher; + +import it.geosolutions.geoserver.rest.GeoserverRESTTest; +import net.sf.json.JSONObject; + +import org.junit.Test; + +import static org.junit.Assert.*; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.io.ClassPathResource; + +/** + * Testcase for publishing layers on geoserver. + * We need a running GeoServer to properly run the tests. + * If such geoserver instance cannot be contacted, tests will be skipped. + * + * @author Alessio Fabiani + * + */ +public class GeoserverRESTImporterTest extends GeoserverRESTTest { + + private final static Logger LOGGER = LoggerFactory.getLogger(GeoserverRESTImporterTest.class); + + @Test + public void testShapeFileImport() throws Exception { + // Creates a new Importer Context and gets back the ID + int i = publisher.postNewImport(); + + // Attaches to the new Importer Context a Task pointing to a shapefile's zip archive + int t = publisher.postNewTaskAsMultiPartForm(i, new ClassPathResource("testdata/test_noepsg.zip").getPath()); + + // Check that the Task was actually created and that the CRS has not recognized in this case + JSONObject task = publisher.getTask(i, t); + assertEquals("NO_CRS", task.getString("state")); + + // Prepare the JSON String instructing the Task about the SRS to use + String json = + "{" + + "\"task\": {" + + "\"layer\": {" + + "\"srs\": \"EPSG:4326\"" + + "}" + + "}" + + "}"; + + // Performing the Task update + publisher.putTask(i, t, json); + + // Double check that the Task is in the READY state + task = publisher.getTask(i, t); + assertEquals("READY", task.getString("state")); + assertEquals("gs_archsites", task.getJSONObject("layer").getJSONObject("style").getString("name")); + + // Prepare the JSON String instructing the Task avout the SLD to use for the new Layer + json = + "{" + + "\"task\": {" + + "\"layer\": {" + + "\"style\": {" + + "\"name\": \"point\"" + + "}" + + "}" + + "}" + + "}"; + + // Performing the Task update + publisher.putTask(i, t,json); + + // Double check that the Task is in the READY state and that the Style has been correctly updated + task = publisher.getTask(i, t); + assertEquals("READY", task.getString("state")); + assertEquals("point", task.getJSONObject("layer").getJSONObject("style").getString("name")); + + // Finally starts the Import ... + publisher.postImport(i); + } + +}