Merge branch 'geoserver-importer' of https://github.com/afabiani/geoserver-manager into afabiani-geoserver-importer

This commit is contained in:
Lorenzo Natali 2015-05-29 11:06:36 +02:00
commit 9f9a022e79
7 changed files with 570 additions and 13 deletions

39
pom.xml
View File

@ -15,7 +15,8 @@
ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN * THE SOFTWARE. -->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>it.geosolutions</groupId>
@ -203,6 +204,7 @@
<properties>
<slf4j.version>1.5.11</slf4j.version>
<gs.version>2.8-SNAPSHOT</gs.version>
</properties>
<dependencies>
@ -248,9 +250,7 @@
<!-- <dependency> <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId>
<version>1.1.1</version> <scope>provided</scope> </dependency> <dependency>
<groupId>log4j</groupId> <artifactId>log4j</artifactId> <version>1.2.16</version>
</dependency>
</dependency>-->
</dependency> </dependency> -->
<!--================================================================= -->
<!-- TEST -->
<!--================================================================= -->
@ -268,11 +268,32 @@
<version>2.5.6.SEC02</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.geoserver.importer</groupId>
<artifactId>gs-importer-core</artifactId>
<version>${gs.version}</version>
</dependency>
<dependency>
<groupId>org.geoserver</groupId>
<artifactId>gs-restconfig</artifactId>
<version>${gs.version}</version>
</dependency>
<dependency>
<groupId>org.restlet</groupId>
<artifactId>org.restlet.ext.fileupload</artifactId>
<version>1.0.8</version>
</dependency>
<dependency>
<groupId>commons-fileupload</groupId>
<artifactId>commons-fileupload</artifactId>
<version>1.2.1</version>
</dependency>
<dependency>

View File

@ -39,6 +39,7 @@ import it.geosolutions.geoserver.rest.encoder.GSResourceEncoder.ProjectionPolicy
import it.geosolutions.geoserver.rest.encoder.GSWorkspaceEncoder;
import it.geosolutions.geoserver.rest.encoder.coverage.GSCoverageEncoder;
import it.geosolutions.geoserver.rest.encoder.feature.GSFeatureTypeEncoder;
import it.geosolutions.geoserver.rest.manager.GeoServerRESTImporterManager;
import it.geosolutions.geoserver.rest.manager.GeoServerRESTStructuredGridCoverageReaderManager;
import it.geosolutions.geoserver.rest.manager.GeoServerRESTStructuredGridCoverageReaderManager.ConfigureCoveragesOption;
import it.geosolutions.geoserver.rest.manager.GeoServerRESTStyleManager;
@ -53,6 +54,8 @@ import java.net.URL;
import java.net.URLEncoder;
import java.util.zip.ZipFile;
import net.sf.json.JSONObject;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.io.FilenameUtils;
import org.slf4j.Logger;
@ -90,6 +93,7 @@ public class GeoServerRESTPublisher {
private final GeoServerRESTStyleManager styleManager;
private final GeoServerRESTImporterManager importerManager;
/**
* Creates a <TT>GeoServerRESTPublisher</TT> to connect against a GeoServer instance with the given URL and user credentials.
*
@ -109,6 +113,7 @@ public class GeoServerRESTPublisher {
LOGGER.error("Bad URL: Calls to GeoServer are going to fail" , ex);
}
styleManager = new GeoServerRESTStyleManager(url, username, password);
importerManager = new GeoServerRESTImporterManager(url, username, password);
}
// ==========================================================================
@ -2929,4 +2934,58 @@ public class GeoServerRESTPublisher {
}
/**
* Refers to {@link it.geosolutions.geoserver.rest.manager.GeoServerRESTImporterManager#postNewImport() postNewImport} method
*
* @throws Exception
*/
public int postNewImport() throws Exception {
return importerManager.postNewImport();
}
/**
* Refers to {@link it.geosolutions.geoserver.rest.manager.GeoServerRESTImporterManager#postNewTaskAsMultiPartForm(int, String) postNewTaskAsMultiPartForm} method
*
* @throws Exception
*/
public int postNewTaskAsMultiPartForm(int i, String data) throws Exception {
return importerManager.postNewTaskAsMultiPartForm(i, data);
}
/**
* Refers to {@link it.geosolutions.geoserver.rest.manager.GeoServerRESTImporterManager#getTask(int, int) getTask} method
*
* @throws Exception
*/
public JSONObject getTask(int i, int t) throws Exception {
return importerManager.getTask(i, t);
}
/**
* Refers to {@link it.geosolutions.geoserver.rest.manager.GeoServerRESTImporterManager#putTask(int, int, String) putTask} method
*
* @throws Exception
*/
public void putTask(int i, int t, String json) throws Exception {
importerManager.putTask(i, t, json);
}
/**
* Refers to {@link it.geosolutions.geoserver.rest.manager.GeoServerRESTImporterManager#putTaskLayer(int, int, String) putTaskLayer} method
*
* @throws Exception
*/
public void putTaskLayer(int i, int t, String json) throws Exception {
importerManager.putTaskLayer(i, t, json);
}
/**
* Refers to {@link it.geosolutions.geoserver.rest.manager.GeoServerRESTImporterManager#postImport(int) postImport} method
*
* @throws Exception
*/
public void postImport(int i) throws Exception {
importerManager.postImport(i);
}
}

View File

@ -25,6 +25,7 @@
package it.geosolutions.geoserver.rest;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@ -33,6 +34,11 @@ import java.net.ConnectException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import net.sf.json.JSON;
import net.sf.json.JSONSerializer;
import org.apache.commons.httpclient.Credentials;
import org.apache.commons.httpclient.HttpClient;
@ -48,6 +54,9 @@ import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.PutMethod;
import org.apache.commons.httpclient.methods.RequestEntity;
import org.apache.commons.httpclient.methods.StringRequestEntity;
import org.apache.commons.httpclient.methods.multipart.FilePart;
import org.apache.commons.httpclient.methods.multipart.MultipartRequestEntity;
import org.apache.commons.httpclient.methods.multipart.Part;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -117,6 +126,22 @@ public class HTTPUtils {
return null;
}
/**
* Executes a request using the GET method and parses the result as a json object.
*
* @param path The path to request.
*
* @return The result parsed as json.
*/
public static JSON getAsJSON(String url, String username, String pw) throws Exception {
String response = get(url, username, pw);
return json(response);
}
public static JSON json(String content) {
return JSONSerializer.toJSON(content);
}
/**
* PUTs a File to the given URL. <BR>
* Basic auth is used if both username and pw are not null.
@ -175,6 +200,23 @@ public class HTTPUtils {
return put(url, content, "text/xml", username, pw);
}
/**
* PUTs a String representing an JSON Object to the given URL. <BR>
* Basic auth is used if both username and pw are not null.
*
* @param url The URL where to connect to.
* @param content The JSON Object to be sent as a String.
* @param username Basic auth credential. No basic auth if null.
* @param pw Basic auth credential. No basic auth if null.
* @return The HTTP response as a String if the HTTP response code was 200
* (OK).
* @throws MalformedURLException
* @return the HTTP response or <TT>null</TT> on errors.
*/
public static String putJson(String url, String content, String username, String pw) {
return put(url, content, "application/json", username, pw);
}
/**
* Performs a PUT to the given URL. <BR>
* Basic auth is used if both username and pw are not null.
@ -233,6 +275,38 @@ public class HTTPUtils {
}
}
/**
* POSTs a list of files as attachments to the given URL. <BR>
* Basic auth is used if both username and pw are not null.
*
* @param url The URL where to connect to.
* @param dir The folder containing the attachments.
* @param username Basic auth credential. No basic auth if null.
* @param pw Basic auth credential. No basic auth if null.
* @return The HTTP response as a String if the HTTP response code was 200
* (OK).
* @throws MalformedURLException
* @return the HTTP response or <TT>null</TT> on errors.
*/
public static String postMultipartForm(String url, File dir, String username, String pw) {
try {
List<Part> parts = new ArrayList<Part>();
for (File f : dir.listFiles()) {
parts.add(new FilePart(f.getName(), f));
}
MultipartRequestEntity multipart = new MultipartRequestEntity(
parts.toArray(new Part[parts.size()]), new PostMethod().getParams());
ByteArrayOutputStream bout = new ByteArrayOutputStream();
multipart.writeRequest(bout);
return post(url, multipart, username, pw);
} catch (Exception ex) {
LOGGER.error("Cannot POST " + url, ex);
return null;
}
}
/**
* POSTs a String representing an XML document to the given URL. <BR>
* Basic auth is used if both username and pw are not null.
@ -250,6 +324,23 @@ public class HTTPUtils {
return post(url, content, "text/xml", username, pw);
}
/**
* POSTs a String representing an JSON Object to the given URL. <BR>
* Basic auth is used if both username and pw are not null.
*
* @param url The URL where to connect to.
* @param content The JSON content to be sent as a String.
* @param username Basic auth credential. No basic auth if null.
* @param pw Basic auth credential. No basic auth if null.
* @return The HTTP response as a String if the HTTP response code was 200
* (OK).
* @throws MalformedURLException
* @return the HTTP response or <TT>null</TT> on errors.
*/
public static String postJson(String url, String content, String username, String pw) {
return post(url, content, "application/json", username, pw);
}
/**
* Performs a POST to the given URL. <BR>
* Basic auth is used if both username and pw are not null.
@ -293,6 +384,7 @@ public class HTTPUtils {
httpMethod.setRequestEntity(requestEntity);
int status = client.executeMethod(httpMethod);
InputStream responseBody;
switch (status) {
case HttpURLConnection.HTTP_OK:
case HttpURLConnection.HTTP_CREATED:
@ -303,9 +395,10 @@ public class HTTPUtils {
LOGGER.info("HTTP " + httpMethod.getStatusText() + ": " + response);
return response;
default:
responseBody = httpMethod.getResponseBodyAsStream();
LOGGER.warn("Bad response: code[" + status + "]" + " msg[" + httpMethod.getStatusText() + "]"
+ " url[" + url + "]" + " method[" + httpMethod.getClass().getSimpleName()
+ "]: " + IOUtils.toString(httpMethod.getResponseBodyAsStream()));
+ "]: " + (responseBody != null ? IOUtils.toString(responseBody) : ""));
return null;
}
} catch (ConnectException e) {

View File

@ -0,0 +1,297 @@
/*
* GeoTools - The Open Source Java GIS Toolkit
* http://geotools.org
*
* (C) 2015, Open Source Geospatial Foundation (OSGeo)
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation;
* version 2.1 of the License.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*/
package it.geosolutions.geoserver.rest.manager;
import it.geosolutions.geoserver.rest.HTTPUtils;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import net.sf.json.JSON;
import net.sf.json.JSONObject;
import org.geoserver.importer.VFSWorker;
import org.restlet.data.MediaType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author Alessio Fabiani
*
*/
public class GeoServerRESTImporterManager extends GeoServerRESTAbstractManager {
private final static Logger LOGGER = LoggerFactory.getLogger(GeoServerRESTImporterManager.class);
/**
* Default constructor.
*
* @param restURL GeoServer REST API endpoint
* @param username GeoServer REST API authorized username
* @param password GeoServer REST API password for the former username
*/
public GeoServerRESTImporterManager(URL restURL, String username, String password)
throws IllegalArgumentException {
super(restURL, username, password);
}
/**
* Retrieves the Import JSON Object given its identifier
*
* @param imp int: Import context number ID
*/
public JSONObject getImport(int imp) throws Exception {
JSON json = HTTPUtils.getAsJSON(String.format(buildUrl()+"/%d", imp), gsuser , gspass);
return ((JSONObject)json).getJSONObject("import");
}
/**
* Retrieves the Import Task JSON Object given its identifier and task number
*
* @param imp int: Import context number ID
* @param task int: Task number
*/
public JSONObject getTask(int imp, int task) throws Exception {
JSON json = HTTPUtils.getAsJSON(String.format(buildUrl()+"/%d/tasks/%d?expand=all", imp, task), gsuser , gspass);
return ((JSONObject)json).getJSONObject("task");
}
/**
* Example usage:
* <pre>
* // Creates a new Importer Context and gets back the ID
* int i = postNewImport();
*
* // Attaches to the new Importer Context a Task pointing to a shapefile's zip archive
* int t = postNewTaskAsMultiPartForm(i, "/path_to/shape/archsites_no_crs.zip");
*
* // Check that the Task was actually created and that the CRS has not recognized in this case
* JSONObject task = getTask(i, t);
* assertEquals("NO_CRS", task.getString("state"));
*
* // Prepare the JSON String instructing the Task about the SRS to use
* String json =
* "{" +
* "\"task\": {" +
* "\"layer\": {" +
* "\"srs\": \"EPSG:4326\"" +
* "}" +
* "}" +
* "}";
*
* // Performing the Task update
* putTask(i, t, json);
*
* // Double check that the Task is in the READY state
* task = getTask(i, t);
* assertEquals("READY", task.getString("state"));
* assertEquals("gs_archsites", task.getJSONObject("layer").getJSONObject("style").getString("name"));
*
* // Prepare the JSON String instructing the Task avout the SLD to use for the new Layer
* json =
* "{" +
* "\"task\": {" +
* "\"layer\": {" +
* "\"style\": {" +
* "\"name\": \"point\"" +
* "}" +
* "}" +
* "}" +
* "}";
*
* // Performing the Task update
* putTask(i, t,json);
*
* // Double check that the Task is in the READY state and that the Style has been correctly updated
* task = getTask(i, t);
* assertEquals("READY", task.getString("state"));
* assertEquals("point", task.getJSONObject("layer").getJSONObject("style").getString("name"));
*
* // Finally starts the Import ...
* postImport(i);
* </pre>
*
* @param imp int: Import context number ID
* @param task int: Task number
* @param json String: JSON containing the Task properties to be updated
* @throws Exception
*/
public void putTask(int imp, int task, final String json) throws Exception {
//HTTPUtils.putJson(String.format(buildUrl()+"/%d/tasks/%d", imp, task), json, gsuser, gspass);
HTTPUtils.put(String.format(buildUrl()+"/%d/tasks/%d", imp, task), json, "text/plain", gsuser, gspass);
}
/**
* Just update the Layers properties associated to a Task (t) in the Importer Context (i).
*
* e.g.:
* <pre>
* putTaskLayer(i, t, "{\"title\":\"Archsites\", \"abstract\":\"Archeological Sites\"}");
* </pre>
*
* @param imp int: Import context number ID
* @param task int: Task number
* @param json String: JSON containing the Layer properties to be updated
* @throws Exception
*/
public void putTaskLayer(int imp, int task, final String json) throws Exception {
HTTPUtils.putJson(String.format(buildUrl()+"/%d/tasks/%d/layer", imp, task), json, gsuser, gspass);
}
/**
* Creates an empty Importer Context.
*
* @return The new Importer Context ID
* @throws Exception
*/
public int postNewImport() throws Exception {
return postNewImport(null);
}
/**
* e.g.:
* <pre>
* String body =
* "{" +
* "\"import\": { " +
* "\"data\": {" +
* "\"type\": \"mosaic\", " +
* "\"time\": {" +
* " \"mode\": \"auto\"" +
* "}" +
* "}" +
* "}" +
* "}";
* </pre>
*
* @param body JSON String representing the Importer Context definition
* @return The new Importer Context ID
* @throws Exception
*/
public int postNewImport(String body) throws Exception {
String resp = body == null ? HTTPUtils.post(buildUrl(), "", "text/plain", gsuser, gspass)
: HTTPUtils.postJson(buildUrl(), body, gsuser, gspass);
JSONObject json = (JSONObject) HTTPUtils.json(resp);
JSONObject imprt = json.getJSONObject("import");
return imprt.getInt("id");
}
/**
* Actually starts the READY State Import.
*
* @param imp int: Import context number ID
* @throws Exception
*/
public void postImport(int imp) throws Exception {
HTTPUtils.post(buildUrl()+"/" + imp, "", "text/plain", gsuser, gspass);
}
/**
*
* @param imp int: Import context number ID
* @param data
* @return
* @throws Exception
*/
public int postNewTaskAsMultiPartForm(int imp, String data) throws Exception {
String resp = HTTPUtils.postMultipartForm(buildUrl()+"/" + imp + "/tasks", unpack(data), gsuser, gspass);
JSONObject json = (JSONObject) HTTPUtils.json(resp);
JSONObject task = json.getJSONObject("task");
return task.getInt("id");
}
/**
* Allows to attach a new zip file to an existing Importer Context.
*
* @param imp int: Import context number ID
* @param path
* @return
* @throws Exception
*/
public int putNewTask(int imp, String path) throws Exception {
File zip = new File(path);
String resp = HTTPUtils.put(buildUrl()+"/" + imp + "/tasks/" + zip.getName(), zip, MediaType.APPLICATION_ZIP.toString(), gsuser, gspass);
JSONObject json = (JSONObject) HTTPUtils.json(resp);
JSONObject task = json.getJSONObject("task");
return task.getInt("id");
}
//=========================================================================
// Util methods
//=========================================================================
/**
* Creates the base REST URL for the imports
*/
protected String buildUrl() {
StringBuilder sUrl = new StringBuilder(gsBaseUrl.toString()).append("/rest/imports");
return sUrl.toString();
}
/**
* Creates a temporary file
*
* @return Path to the temporary file
* @throws Exception
*/
public static File tmpDir() throws Exception {
File dir = File.createTempFile("importer", "data", new File("target"));
dir.delete();
dir.mkdirs();
return dir;
}
/**
* Expands a zip archive into the temporary folder.
*
* @param path The absolute path to the source zip file
* @return Path to the temporary folder containing the expanded files
* @throws Exception
*/
public static File unpack(String path) throws Exception {
return unpack(path, tmpDir());
}
/**
* Expands a zip archive into the target folder.
*
* @param path The absolute path to the source zip file
* @param dir Full path of the target folder where to expand the archive
* @return Path to the temporary folder containing the expanded files
* @throws Exception
*/
public static File unpack(String path, File dir) throws Exception {
File file = new File(path);
new VFSWorker().extractTo(file, dir);
if (!file.delete()) {
// fail early as tests will expect it's deleted
throw new IOException("deletion failed during extraction");
}
return dir;
}
}

View File

@ -98,7 +98,7 @@ public abstract class GeoserverRESTTest {
RESTURL = getenv("gsmgr_resturl", "http://localhost:8080/geoserver");
RESTUSER = getenv("gsmgr_restuser", "admin");
RESTPW = getenv("gsmgr_restpw", "geoserver");
GS_VERSION = getenv("gsmgr_version", "2.4");
GS_VERSION = getenv("gsmgr_version", "28");
// These tests will destroy data, so let's make sure we do want to run them
enabled = getenv("gsmgr_resttest", "false").equalsIgnoreCase("true");

View File

@ -125,7 +125,6 @@ public class GeoserverRESTGeoTiffTest extends GeoserverRESTTest {
assertFalse(reader.existsCoveragestore(DEFAULT_WS, storeName));
}
@Test
public void testReloadCoverageStore() throws FileNotFoundException, IOException {
if (!enabled()) return;

View File

@ -0,0 +1,88 @@
/*
* GeoTools - The Open Source Java GIS Toolkit
* http://geotools.org
*
* (C) 2015, Open Source Geospatial Foundation (OSGeo)
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation;
* version 2.1 of the License.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*/
package it.geosolutions.geoserver.rest.publisher;
import static org.junit.Assert.assertEquals;
import it.geosolutions.geoserver.rest.GeoserverRESTTest;
import net.sf.json.JSONObject;
import org.junit.After;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.io.ClassPathResource;
/**
* Testcase for publishing layers on geoserver.
* We need a running GeoServer to properly run the tests.
* If such geoserver instance cannot be contacted, tests will be skipped.
*
* @author Alessio Fabiani
*
*/
public class GeoserverRESTImporterTest extends GeoserverRESTTest {
private final static Logger LOGGER = LoggerFactory.getLogger(GeoserverRESTImporterTest.class);
@After
public void cleanUp() {
}
@Test
public void testShapeFileImport() throws Exception {
if (!enabled())
return;
// Creates a new Importer Context and gets back the ID
int i = publisher.postNewImport();
// Attaches to the new Importer Context a Task pointing to a shapefile's zip archive
String data = new ClassPathResource("testdata/test_noepsg.zip").getFile().getAbsolutePath();
int t = publisher.postNewTaskAsMultiPartForm(i, data);
// Check that the Task was actually created and that the CRS has not recognized in this case
JSONObject task = publisher.getTask(i, t);
//assertEquals("NO_CRS", task.getString("state"));
assertEquals("READY", task.getString("state"));
// Prepare the JSON String instructing the Task about the SRS to use
String json = "{\"layer\":{\"srs\":\"EPSG:26713\"}}";
// Performing the Task update
publisher.putTaskLayer(i, t, json);
// Double check that the Task is in the READY state
task = publisher.getTask(i, t);
assertEquals("READY", task.getString("state"));
assertEquals("nurc_10m_populated_places", task.getJSONObject("layer").getJSONObject("style").getString("name"));
// Prepare the JSON String instructing the Task avout the SLD to use for the new Layer
json = "{\"layer\":{\"style\":{\"name\": \"point\"}}}";
// Performing the Task update
publisher.putTaskLayer(i, t,json);
// Double check that the Task is in the READY state and that the Style has been correctly updated
task = publisher.getTask(i, t);
assertEquals("READY", task.getString("state"));
assertEquals("point", task.getJSONObject("layer").getJSONObject("style").getString("name"));
// Finally starts the Import ...
publisher.postImport(i);
}
}