Skip to content

Commit

Permalink
* SNAC Join model: Verify Related CPF/Resource IDs exist in the targe…
Browse files Browse the repository at this point in the history 
…t SNAC

  environment before joining.  This prevents crashes and locked CPF in SNAC,
  particularly when related CPF IDs do not exist.

* CPFs: upon successful upload, populate the "*SNAC*: Link" column with the
  ARK for production SNAC, otherwise a direct link to the constellation.
  This is because ARKs are only valid in production SNAC, and it's useful
  to be able to link to created constellations when testing in dev SNAC.

* improvements to API client use, response parsing, and lookup caching

* additional logging
  • Loading branch information
@jlj5aj
jlj5aj committed Jul 21, 2023
1 parent 081e5e6 commit b9ce5ed
Show file tree
Hide file tree
Showing 14 changed files with 441 additions and 129 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1 +1,3 @@
target/
module/
*.swp
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ More complete instructions for installing OpenRefine Extensions can be found in

To install this extension,
1. Download the extension from the [latest release](https://github.com/snac-cooperative/snac-openrefine-extension/releases/latest).
2. Extract the zip contents into OpenRefine's extensions directory. You may have to create the `extensions` directory if it does not already exist in your workspace (see below). If you already have the snac extension, you whould delete the original before unzipping.
2. Extract the zip contents into OpenRefine's extensions directory. You may have to create the `extensions` directory if it does not already exist in your workspace (see below). If you already have the snac extension, you should delete the original before unzipping.
3. Start (or restart) OpenRefine.

### Finding Workspace directory
Expand Down
63 changes: 55 additions & 8 deletions src/org/snaccooperative/commands/SNACAPIClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,34 +4,81 @@
import org.apache.http.*;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.snaccooperative.connection.SNACConnector;

public class SNACAPIClient {

static final Logger logger = LoggerFactory.getLogger("SNACAPIClient");

protected DefaultHttpClient _client;
protected CloseableHttpClient _client;
protected HttpPost _post;
protected String _webURL;
protected String _apiURL;
protected String _apiKey;
protected Boolean _isProd;

public SNACAPIClient(String apiURL) {
this._client = new DefaultHttpClient();
this._post = new HttpPost(apiURL);
public SNACAPIClient(String snacEnv) {
SNACConnector keyManager = SNACConnector.getInstance();

this._webURL = "";
this._apiURL = "";
this._apiKey = keyManager.getKey();
this._isProd = false;

switch (snacEnv.toLowerCase()) {
case "prod":
this._webURL = "https://snaccooperative.org/";
this._apiURL = "https://api.snaccooperative.org/";
this._isProd = true;
break;

case "dev":
this._webURL = "https://snac-dev.iath.virginia.edu/";
this._apiURL = "https://snac-dev.iath.virginia.edu/api/";
this._isProd = false;
break;
}

this._client = HttpClientBuilder.create().build();
this._post = new HttpPost(this._apiURL);

logger.debug("web url: [" + this._webURL + "]");
logger.debug("api url: [" + this._apiURL + "]");
logger.debug("api key: [" + this._apiKey + "]");
}

public String webURL() {
return this._webURL;
}

public String apiURL() {
return this._apiURL;
}

public String apiKey() {
return this._apiKey;
}

public Boolean isProd() {
return this._isProd;
}

public SNACAPIResponse post(String apiJSON) {
try {
logger.debug("API POST data: [" + apiJSON + "]");
StringEntity apiCasted = new StringEntity(apiJSON, "UTF-8");
_post.setEntity(apiCasted);
HttpResponse response = _client.execute(_post);
String result = EntityUtils.toString(response.getEntity());

return new SNACAPIResponse(result);
return new SNACAPIResponse(this, result);
} catch (IOException e) {
logger.error(e.toString());
return new SNACAPIResponse(e.toString());
return new SNACAPIResponse(this, e.toString());
}
}
}
34 changes: 28 additions & 6 deletions src/org/snaccooperative/commands/SNACAPIResponse.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package org.snaccooperative.commands;

import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
Expand All @@ -21,8 +22,15 @@ public class SNACAPIResponse {
protected int _id;
protected int _version;

public SNACAPIResponse(String apiResponse) {
public SNACAPIResponse(SNACAPIClient client, String apiResponse) {
this._apiResponse = apiResponse;
this._result = "";
this._message = "";
this._uri = "";
this._resource = null;
this._constellation = null;
this._id = 0;
this._version = 0;

// attempt to parse json; if it does not parse, it's either
// a badly-formed response or (most likely) an exception
Expand All @@ -45,6 +53,7 @@ public SNACAPIResponse(String apiResponse) {
Object resource = jsonResponse.get("resource");
Object constellation = jsonResponse.get("constellation");
Object results = jsonResponse.get("results");
Object relatedConstellations = jsonResponse.get("related_constellations");

// populate result/message
if (result instanceof String) {
Expand Down Expand Up @@ -72,8 +81,11 @@ public SNACAPIResponse(String apiResponse) {
this._result = "error";
this._message = errorFull;
} else {
if (results instanceof JSONObject) {
// things like vocabulary lookups may just have a "results" section
if ((results instanceof JSONObject) || (relatedConstellations instanceof JSONArray)) {
// "results": things like vocabulary lookups or elasticsearch queries may just have this
// section
// "related_constellations": a "read_resource" for a non-existent resource may just have
// this section
this._result = "success";
} else {
this._result = "unknown";
Expand All @@ -89,16 +101,22 @@ public SNACAPIResponse(String apiResponse) {
this._resource = res;
this._id = res.getID();
this._version = res.getVersion();
this._uri = "https://snaccooperative.org/vocab_administrator/resources/" + res.getID();
this._uri = client.webURL() + "vocab_administrator/resources/" + res.getID();
} else if (constellation instanceof JSONObject) {
Constellation con = Constellation.fromJSON(((JSONObject) constellation).toString());
this._constellation = con;
this._id = con.getID();
this._version = con.getVersion();
this._uri = con.getArk();

// only use ARK for prod CPF, since they are only valid there
if (client.isProd()) {
this._uri = con.getArk();
} else {
this._uri = client.webURL() + "view/" + con.getID();
}
}
} catch (ParseException e) {
// assume apiResponse is an exception string, not a badly-formed API responsee
// assume apiResponse is an exception string, not a badly-formed API response
this._result = "exception";
this._message = apiResponse;
}
Expand Down Expand Up @@ -170,4 +188,8 @@ public Resource getResource() {
public Constellation getConstellation() {
return _constellation;
}

public Boolean isSuccess() {
return _result.toLowerCase().contains("success");
}
}
8 changes: 8 additions & 0 deletions src/org/snaccooperative/commands/SNACExportJSONCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ public class SNACExportJSONCommand extends Command {
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {

logger.info("exporting SNAC JSON...");

try {
Project project = getProject(request);
Engine engine = getEngine(request, project);
Expand All @@ -65,13 +67,15 @@ public void doPost(HttpServletRequest request, HttpServletResponse response)
try {
schema = SNACSchema.reconstruct(schemaJSON);
} catch (IOException e) {
logger.error("SNAC JSON export: could not reconstruct schema: [" + e + "]");
respondError(response, "SNAC schema could not be parsed.");
return;
}
} else {
schema = (SNACSchema) project.overlayModels.get("snacSchema");
}
if (schema == null) {
logger.error("SNAC JSON export: missing schema");
respondError(response, "No SNAC schema provided.");
return;
}
Expand All @@ -86,14 +90,18 @@ public void doPost(HttpServletRequest request, HttpServletResponse response)
try {
jsonItems.add((JSONObject) jsonParser.parse(items.get(i).toJSON()));
} catch (ParseException e) {
logger.warn("SNAC JSON export: skipping item " + (i + 1) + ": [" + e + "]");
continue;
}
}

jsonResp.put(schema.getSchemaType() + "s", jsonItems);

logger.info("SNAC JSON export succeeded");

respondJSON(response, jsonResp);
} catch (Exception e) {
logger.error("SNAC JSON export: exception: [" + e + "]");
respondException(response, e);
}
}
Expand Down
7 changes: 7 additions & 0 deletions src/org/snaccooperative/commands/SNACPreviewSchemaCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ public class SNACPreviewSchemaCommand extends Command {
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {

logger.info("generating SNAC preview...");

try {
Project project = getProject(request);
Engine engine = getEngine(request, project);
Expand All @@ -61,13 +63,15 @@ public void doPost(HttpServletRequest request, HttpServletResponse response)
try {
schema = SNACSchema.reconstruct(schemaJSON);
} catch (IOException e) {
logger.error("SNAC preview generation: could not reconstruct schema: [" + e + "]");
respondError(response, "SNAC schema could not be parsed.");
return;
}
} else {
schema = (SNACSchema) project.overlayModels.get("snacSchema");
}
if (schema == null) {
logger.error("SNAC preview generation: missing schema");
respondError(response, "No SNAC schema provided.");
return;
}
Expand All @@ -84,8 +88,11 @@ public void doPost(HttpServletRequest request, HttpServletResponse response)
previewItems.addPreviewItem(item.getPreviewText());
}

logger.info("SNAC preview generation succeeded");

respondJSON(response, previewItems);
} catch (Exception e) {
logger.error("SNAC preview generation: exception: [" + e + "]");
respondException(response, e);
}
}
Expand Down
10 changes: 10 additions & 0 deletions src/org/snaccooperative/commands/SNACSaveSchemaCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,10 @@ public class SNACSaveSchemaCommand extends Command {
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {

logger.info("saving SNAC schema...");

if (!hasValidCSRFToken(request)) {
logger.error("SNAC schema save: invalid CSRF token");
respondCSRFError(response);
return;
}
Expand All @@ -59,6 +62,7 @@ public void doPost(HttpServletRequest request, HttpServletResponse response)

String schemaJSON = request.getParameter("schema");
if (schemaJSON == null) {
logger.error("SNAC schema save: missing schema");
respondError(response, "No SNAC schema provided.");
return;
}
Expand All @@ -68,15 +72,21 @@ public void doPost(HttpServletRequest request, HttpServletResponse response)
AbstractOperation op = new SNACSaveSchemaOperation(schema);
Process process = op.createProcess(project, new Properties());

logger.info("SNAC schema save initiated");

performProcessAndRespond(request, response, project, process);

logger.info("SNAC schema save completed");

} catch (IOException e) {
// We do not use respondException here because this is an expected
// exception which happens every time a user tries to save an incomplete
// schema - the exception should not be logged.
logger.warn("SNAC schema save: incomplete schema?: [" + e + "]");
respondError(response, "SNAC schema could not be parsed.");
} catch (Exception e) {
// This is an unexpected exception, so we log it.
logger.error("SNAC schema save: exception: [" + e + "]");
respondException(response, e);
}
}
Expand Down
Loading

0 comments on commit b9ce5ed

Please sign in to comment.