Skip to content

Commit

Permalink
Merge pull request #7 from xubo245/optimizeVOCLabelProperties
Browse files Browse the repository at this point in the history 
Add new feature and optimize for dataset sdk
  • Loading branch information
@gclouding
gclouding authored Jul 15, 2019
2 parents b6769b1 + cc2716b commit 40bf541
Show file tree
Hide file tree
Showing 36 changed files with 3,195 additions and 210 deletions.
134 changes: 69 additions & 65 deletions ...sdk-java-modelarts-dataset/src/main/java/com/huaweicloud/modelarts/dataset/FieldName.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,135 +20,139 @@
*/
public class FieldName
{
public static String SOURCE = "source";
public static final String SOURCE = "source";

public static String SOURCE_TYPE = "source-type";
public static final String SOURCE_TYPE = "source-type";

public static String USAGE = "usage";
public static final String USAGE = "usage";

public static String INFERENCE_LOC = "inference-loc";
public static final String INFERENCE_LOC = "inference-loc";

public static String INFERENCE_LOC2 = "inference_loc";
public static final String INFERENCE_LOC2 = "inference_loc";

public static String ID = "id";
public static final String ID = "id";

public static String SOURCE_PROPERTY = "property";
public static final String SOURCE_PROPERTY = "property";

public static String ANNOTATIONS = "annotation";
public static final String ANNOTATIONS = "annotation";

public static String NAME = "name";
public static final String NAME = "name";

public static String ANNOTATION_TYPE = "type";
public static final String ANNOTATION_TYPE = "type";

public static String ANNOTATION_LOC = "annotation-loc";
public static final String ANNOTATION_LOC = "annotation-loc";

public static String ANNOTATION_LOC2 = "annotation_loc";
public static final String ANNOTATION_LOC2 = "annotation_loc";

public static String ANNOTATION_FORMAT = "annotation-format";
public static final String ANNOTATION_FORMAT = "annotation-format";

public static String ANNOTATION_FORMAT2 = "annotation_format";
public static final String ANNOTATION_FORMAT2 = "annotation_format";

public static String ANNOTATION_NAME = "name";
public static final String ANNOTATION_NAME = "name";

public static String ANNOTATION_NAMES = "annotation_names";
public static final String ANNOTATION_NAMES = "annotation_names";

public static String ANNOTATION_PROPERTY = "property";
public static final String ANNOTATION_PROPERTY = "property";

public static String ANNOTATION_HARD = "hard";
public static final String ANNOTATION_HARD = "hard";

public static String ANNOTATION_CONFIDENCE = "confidence";
public static final String ANNOTATION_CONFIDENCE = "confidence";

public static String ANNOTATION_CREATION_TIME = "creation-time";
public static final String ANNOTATION_CREATION_TIME = "creation-time";

public static String ANNOTATION_CREATION_TIME2 = "creation_time";
public static final String ANNOTATION_CREATION_TIME2 = "creation_time";

public static String ANNOTATION_ANNOTATED_BY = "annotated-by";
public static final String ANNOTATION_ANNOTATED_BY = "annotated-by";

public static String ANNOTATION_ANNOTATED_BY2 = "annotated_by";
public static final String ANNOTATION_ANNOTATED_BY2 = "annotated_by";

public static String SIZE = "size";
public static final String SIZE = "size";

public static String IMAGE_CLASSIFICATION = "image_classification";
public static final String IMAGE_CLASSIFICATION = "image_classification";

public static String AUDIO_CLASSIFICATION = "audio_classification";
public static final String AUDIO_CLASSIFICATION = "audio_classification";

public static String TEXT_CLASSIFICATION = "text_classification";
public static final String TEXT_CLASSIFICATION = "text_classification";

public static String OBJECT_DETECTION = "object_detection";
public static final String OBJECT_DETECTION = "object_detection";

// PASCAL VOC FORMAT
public static String FOLDER_NAME = "folder";
public static final String FOLDER_NAME = "folder";

public static String FILE_NAME = "filename";
public static final String FILE_NAME = "filename";

public static String DATABASE = "database";
public static final String DATABASE = "database";

public static String IMAGE = "image";
public static final String IMAGE = "image";

public static String WIDTH = "width";
public static final String WIDTH = "width";

public static String HEIGHT = "height";
public static final String HEIGHT = "height";

public static String DEPTH = "depth";
public static final String DEPTH = "depth";

public static String SEGMENTED = "segmented";
public static final String SEGMENTED = "segmented";

public static String OBJECT = "object";
public static final String OBJECT = "object";

public static String VOC_PROPERTIES = "properties";
public static final String VOC_PROPERTIES = "properties";

public static String POSE = "pose";
public static final String VOC_PROPERTY_KEY = "key";

public static String TRUNCATED = "truncated";
public static final String VOC_PROPERTY_VALUE = "value";

public static String OCCLUDED = "occluded";
public static final String POSE = "pose";

public static String DIFFICULT = "difficult";
public static final String TRUNCATED = "truncated";

public static String CONFIDENCE = "confidence";
public static final String OCCLUDED = "occluded";

public static String XMIN = "xmin";
public static final String DIFFICULT = "difficult";

public static String YMIN = "ymin";
public static final String CONFIDENCE = "confidence";

public static String XMAX = "xmax";
public static final String XMIN = "xmin";

public static String YMAX = "ymax";
public static final String YMIN = "ymin";

public static String X1 = "x1";
public static final String XMAX = "xmax";

public static String Y1 = "y1";
public static final String YMAX = "ymax";

public static String X2 = "x2";
public static final String X1 = "x1";

public static String Y2 = "y2";
public static final String Y1 = "y1";

public static String CX = "cx";
public static final String X2 = "x2";

public static String CY = "cy";
public static final String Y2 = "y2";

public static String R = "r";
public static final String CX = "cx";

public static String X = "x";
public static final String CY = "cy";

public static String Y = "y";
public static final String R = "r";

public static String PART = "part";
public static final String X = "x";

public static String PROPERTY_COLOR = "@modelarts:color";
public static final String Y = "y";

public static String PROPERTY_START_INDEX = "@modelarts:start_index";
public static final String PART = "part";

public static String PROPERTY_END_INDEX = "@modelarts:end_index";
public static final String PROPERTY_COLOR = "@modelarts:color";

public static String PROPERTY_CONTENT = "@modelarts:content";
public static final String PROPERTY_START_INDEX = "@modelarts:start_index";

public static String PROPERTY_SCHEMA = "@modelarts:schema";
public static final String PROPERTY_END_INDEX = "@modelarts:end_index";

public static String PARSE_PASCAL_VOC = "parsePascalVOC";
public static final String PROPERTY_CONTENT = "@modelarts:content";

public static String RELATIVE_PATH = "RelativePath";
public static final String PROPERTY_SCHEMA = "@modelarts:schema";

public static String RELATIVE_PATH_PREFIX = "./";
public static final String PARSE_PASCAL_VOC = "parsePascalVOC";

public static final String RELATIVE_PATH = "RelativePath";

public static final String RELATIVE_PATH_PREFIX = "./";

}
45 changes: 23 additions & 22 deletions ...-sdk-java-modelarts-dataset/src/main/java/com/huaweicloud/modelarts/dataset/Manifest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ else if (path.toLowerCase().startsWith(HDFS_PREFIX))
return false;
}
}

/**
* read manifest data from local and return with dataset format for manifest
*
Expand Down Expand Up @@ -98,9 +98,10 @@ private static Dataset readFromLocal(String path, Map properties)
}
}
dataset.setSize(sum);
bufferedReader.close();
return dataset;
}

private static Map addRelativePath(Map properties, String relativePath)
{
if (null == properties)
Expand All @@ -110,7 +111,7 @@ private static Map addRelativePath(Map properties, String relativePath)
properties.put(RELATIVE_PATH, relativePath.substring(0, relativePath.lastIndexOf('/')));
return properties;
}

/**
* get double from json object by key
*
Expand All @@ -130,7 +131,7 @@ private static Double getDouble(JSONObject jsonObject, String key)
return jsonObject.getDouble(key);
}
}

/**
* get json object by key
* For compatible the property
Expand Down Expand Up @@ -158,7 +159,7 @@ private static JSONObject getJSONObject(JSONObject jsonObject, String key)
}
}
}

/**
* parse annotation by json Array
*
Expand All @@ -180,7 +181,7 @@ private static List<Annotation> parseAnnotations(JSONArray jsonArray, Map proper
{
annotationLoc = properties.get(RELATIVE_PATH) + annotationLoc.substring(1, annotationLoc.length());
}

Annotation annotation = new Annotation(jsonObject.getString(NAME),
jsonObject.getString(ANNOTATION_TYPE),
annotationLoc,
Expand All @@ -203,7 +204,7 @@ private static List<Annotation> parseAnnotations(JSONArray jsonArray, Map proper
annotation.setPascalVoc(new PascalVocIO(annotationLoc, obsClient));
}
}

if (null != properties && 0 != properties.size())
{
if (Boolean.parseBoolean(String.valueOf(properties.get(ANNOTATION_HARD))))
Expand Down Expand Up @@ -241,7 +242,7 @@ else if ("false".equals(String.valueOf(properties.get(ANNOTATION_HARD)).toLowerC
}
return annotationList;
}

public static boolean isFilterAnnotations(Annotation annotation, Map properties, ObsClient obsClient)
{
if (null != properties && 0 != properties.size() &&
Expand Down Expand Up @@ -280,7 +281,7 @@ public static boolean isFilterAnnotations(Annotation annotation, Map properties,
}
return true;
}

private static boolean isHard(Annotation annotation, ObsClient obsClient)
{
if (annotation.isHard())
Expand All @@ -302,7 +303,7 @@ private static boolean isHard(Annotation annotation, ObsClient obsClient)
}
return false;
}

private static boolean isNotHard(Annotation annotation, ObsClient obsClient)
{
if (annotation.isHard())
Expand All @@ -328,7 +329,7 @@ private static boolean isNotHard(Annotation annotation, ObsClient obsClient)
}
return false;
}

private static String getString(JSONObject jObject, String key1, String key2)
{
String value = jObject.getString(key1);
Expand All @@ -338,7 +339,7 @@ private static String getString(JSONObject jObject, String key1, String key2)
}
return value;
}

/**
* parse sample by line string of manifest
*
Expand All @@ -348,7 +349,7 @@ private static String getString(JSONObject jObject, String key1, String key2)
private static Sample parseSample(String line, Map properties, ObsClient obsClient)
{
JSONObject jObject = JSONObject.parseObject(line);

List<Annotation> annotationList =
parseAnnotations(jObject.getJSONArray(FieldName.ANNOTATIONS), properties, obsClient);
if (0 != properties.size() && (Boolean.parseBoolean(String.valueOf(properties.get(ANNOTATION_HARD)))
Expand Down Expand Up @@ -379,7 +380,7 @@ private static Sample parseSample(String line, Map properties, ObsClient obsClie
);
}
}

/**
* parse manifest by manifest path
* default properties value is false.
Expand All @@ -393,7 +394,7 @@ public static Dataset parseManifest(String path)
{
return parseManifest(path, new HashMap());
}

/**
* parse manifest by manifest path
*
Expand Down Expand Up @@ -422,9 +423,9 @@ public static Dataset parseManifest(String path, Map properties)
return readFromLocal(path, properties);
}
}

private static final Logger LOGGER = Logger.getLogger(Manifest.class.getName());

/**
* parse manifest from S3, with obsClient.
*
Expand Down Expand Up @@ -469,7 +470,7 @@ private static Dataset readFromOBS(String path, ObsClient obsClient, Map propert
dataset.setSize(sum);
return dataset;
}

/**
* parse manifest from S3, with access_key, secret_key and end_point.
* It will parse manifest from local if the path is local, even though configure access_key, secret_key and end_point.
Expand All @@ -486,7 +487,7 @@ public static Dataset parseManifest(String path, String access_key, String secre
{
return parseManifest(path, access_key, secret_key, end_point, new HashMap());
}

/**
* parse manifest from S3, with access_key, secret_key and end_point.
* It will parse manifest from local if the path is local, even though configure access_key, secret_key and end_point.
Expand Down Expand Up @@ -523,7 +524,7 @@ public static Dataset parseManifest(String path, String access_key, String secre
return readFromOBS(path, obsClient, properties);
}
}

/**
* parse manifest from S3, with obsClient.
* It will parse manifest from local if the path is local, even though configure access_key, secret_key and end_point.
Expand All @@ -538,7 +539,7 @@ public static Dataset parseManifest(String path, ObsClient obsClient)
{
return parseManifest(path, obsClient, new HashMap());
}

/**
* parse manifest from S3, with obsClient.
* It will parse manifest from local if the path is local, even though configure access_key, secret_key and end_point.
Expand Down Expand Up @@ -571,5 +572,5 @@ public static Dataset parseManifest(String path, ObsClient obsClient, Map proper
return readFromOBS(path, obsClient, properties);
}
}

}
Loading

0 comments on commit 40bf541

Please sign in to comment.