Skip to content

Commit

Permalink
refactor:
Browse files Browse the repository at this point in the history
  • Loading branch information
zengde committed Feb 23, 2016
1 parent 10663aa commit 1a0feb7
Show file tree
Hide file tree
Showing 4 changed files with 59 additions and 74 deletions.
27 changes: 24 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,15 @@ Image Plugin for Elasticsearch

[![Build Status](https://travis-ci.org/kzwang/elasticsearch-image.png?branch=master)](https://travis-ci.org/kzwang/elasticsearch-image)

The Image Plugin is an Content Based Image Retrieval Plugin for Elasticsearch using [LIRE (Lucene Image Retrieval)](https://code.google.com/p/lire/). It allows users to index images and search for similar images.
The Image Plugin is an Content Based Image Retrieval Plugin for Elasticsearch using [LIRE (Lucene Image Retrieval)](https://github.com/dermotte/LIRE/). It allows users to index images and search for similar images.

It adds an `image` field type and an `image` query

In order to install the plugin, simply run: `bin\plugin install zengde/elasticsearch-image`.

| Image Plugin | elasticsearch | Release date |
|---------------------------|-------------------|:------------:|
| 2.1.1 | 2.1.1 | |
| 2.1.1 | 2.1.1 | 2016-02-23 |
| 1.3.0-SNAPSHOT (master) | 1.1.0 | |
| 1.2.0 | 1.0.1 | 2014-03-20 |
| 1.1.0 | 1.0.1 | 2014-03-13 |
Expand Down Expand Up @@ -152,8 +152,29 @@ See [Large image data sets with LIRE ?some new numbers](http://www.semanticmetad
| index.image.use_thread_pool | use multiple thread when multiple features are required | True |
| index.image.ignore_metadata_error| ignore errors happened during extract metadata from image | True |

## TODO:
- only test with search
```sh
curl -XPOST 'localhost:9200/test/test/_search' -d '{
"query": {
"image": {
"my_img": {
"feature": "CEDD",
"image": "... base64 encoded image to search ...",
"hash": "BIT_SAMPLING",
"boost": 2.1
}
}
}
}'
```
should works with limit and some others


## ChangeLog
#### 2.1.1 (2016-01-06)
#### 2.1.1 (2016-02-23)
- support es 2.1.1


#### 1.2.0 (2014-03-20)

Expand Down
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<name>elasticsearch-image</name>
<modelVersion>4.0.0</modelVersion>
<groupId>com.github.angleto</groupId>
<groupId>com.github.zengde</groupId>
<artifactId>elasticsearch-image</artifactId>
<version>2.1.1</version>
<packaging>jar</packaging>
Expand Down
90 changes: 21 additions & 69 deletions src/main/java/org/elasticsearch/index/mapper/image/ImageMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchGenerationException;
import org.elasticsearch.ElasticsearchImageProcessException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import com.google.common.collect.Lists;
import com.google.common.collect.MapMaker;
Expand All @@ -28,22 +29,17 @@
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.threadpool.ThreadPool;

import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executor;

import static org.elasticsearch.index.mapper.MapperBuilders.binaryField;
import static org.elasticsearch.index.mapper.MapperBuilders.stringField;



public class ImageMapper extends FieldMapper {

private static ESLogger logger = ESLoggerFactory.getLogger(ImageMapper.class.getName());
Expand All @@ -70,10 +66,9 @@ public class ImageMapper extends FieldMapper {
}

public static class Defaults {

public static final ImageFieldType FIELD_TYPE = new ImageFieldType();
static {
FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
FIELD_TYPE.setTokenized(false);
FIELD_TYPE.freeze();
}
Expand Down Expand Up @@ -103,16 +98,13 @@ public String value(Object value) {

public static class Builder extends FieldMapper.Builder<Builder, ImageMapper> {

private ThreadPool threadPool;

private Map<FeatureEnum, Map<String, Object>> features = Maps.newHashMap();

private Map<String, FieldMapper.Builder> metadataBuilders = Maps.newHashMap();

public Builder(String name, ThreadPool threadPool) {
super(name,new ImageFieldType());
this.threadPool = threadPool;
this.builder = this;
public Builder(String name) {
super(name, Defaults.FIELD_TYPE);
builder = this;
}

public Builder addFeature(FeatureEnum featureEnum, Map<String, Object> featureMap) {
Expand Down Expand Up @@ -141,7 +133,6 @@ public ImageMapper build(BuilderContext context) {
// add feature mapper
featureMappers.put(featureName, binaryField(featureName).store(true).includeInAll(false).index(false).build(context));


// add hash mapper if hash is required
if (featureMap.containsKey(HASH)){
List<String> hashes = (List<String>) featureMap.get(HASH);
Expand All @@ -162,38 +153,30 @@ public ImageMapper build(BuilderContext context) {
context.path().remove(); // remove METADATA
context.path().remove(); // remove name

MappedFieldType defaultFieldType = Defaults.FIELD_TYPE.clone();
return new ImageMapper(name, threadPool, context.indexSettings(), features, featureMappers, hashMappers, metadataMappers,
setupFieldType(context);
return new ImageMapper(name,context.indexSettings(), features, featureMappers, hashMappers, metadataMappers,
fieldType,defaultFieldType,multiFieldsBuilder.build(this, context), copyTo);
}
}

public static class TypeParser implements Mapper.TypeParser {
private ThreadPool threadPool;

public TypeParser() {

}

public TypeParser(ThreadPool threadPool) {
this.threadPool = threadPool;
}

@Override
@SuppressWarnings("unchecked")
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
ImageMapper.Builder builder = new ImageMapper.Builder(name, threadPool);
ImageMapper.Builder builder = new ImageMapper.Builder(name);
Map<String, Object> features = Maps.newHashMap();
Map<String, Object> metadatas = Maps.newHashMap();

for (Map.Entry<String, Object> entry : node.entrySet()) {
String fieldName = entry.getKey();
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
Map.Entry<String, Object> entry = iterator.next();
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();

if (FEATURE.equals(fieldName)) {
features = (Map<String, Object>) fieldNode;
features = (Map<String, Object>) fieldNode;
iterator.remove();
} else if (METADATA.equals(fieldName)) {
metadatas = (Map<String, Object>) fieldNode;
iterator.remove();
}
}

Expand All @@ -205,7 +188,6 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
for (Map.Entry<String, Object> entry : features.entrySet()) {
String feature = entry.getKey();
Map<String, Object> featureMap = (Map<String, Object>) entry.getValue();

// process hash for each feature
if (featureMap.containsKey(HASH)) {
Object hashVal = featureMap.get(HASH);
Expand All @@ -232,7 +214,10 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
String metadataName = entry.getKey();
Map<String, Object> metadataMap = (Map<String, Object>) entry.getValue();
String fieldType = (String) metadataMap.get("type");
builder.addMetadata(metadataName, (FieldMapper.Builder) parserContext.typeParser(fieldType).parse(metadataName, metadataMap, parserContext));
builder.addMetadata(metadataName, (FieldMapper.Builder) parserContext
.typeParser(fieldType)
.parse(metadataName, metadataMap, parserContext)
);
}

return builder;
Expand All @@ -241,8 +226,6 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext

private final String name;

private final ThreadPool threadPool;

private final Settings settings;

private volatile ImmutableOpenMap<FeatureEnum, Map<String, Object>> features = ImmutableOpenMap.of();
Expand All @@ -253,13 +236,11 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext

private volatile ImmutableOpenMap<String, FieldMapper> metadataMappers = ImmutableOpenMap.of();


public ImageMapper(String name, ThreadPool threadPool, Settings indexSettings, Map<FeatureEnum, Map<String, Object>> features, Map<String, FieldMapper> featureMappers,
public ImageMapper(String name,Settings indexSettings, Map<FeatureEnum, Map<String, Object>> features, Map<String, FieldMapper> featureMappers,
Map<String, FieldMapper> hashMappers, Map<String, FieldMapper> metadataMappers,
MappedFieldType type, MappedFieldType defaultFieldType,MultiFields multiFields, CopyTo copyTo) {
super(name, type, defaultFieldType, indexSettings, multiFields, copyTo);
this.name = name;
this.threadPool = threadPool;
this.settings = indexSettings;
if (features != null) {
this.features = ImmutableOpenMap.builder(this.features).putAll(features).build();
Expand Down Expand Up @@ -304,40 +285,12 @@ public Mapper parse(ParseContext context) throws IOException {
}
final BufferedImage finalImg = img;



final Map<FeatureEnum, LireFeature> featureExtractMap = new MapMaker().makeMap();

// have multiple features, use ThreadPool to process each feature
if (useThreadPool && features.size() > 1) {
final CountDownLatch latch = new CountDownLatch(features.size());
Executor executor = threadPool.generic();

for (ObjectObjectCursor<FeatureEnum, Map<String, Object>> cursor : features) {
final FeatureEnum featureEnum = cursor.key;
executor.execute(new Runnable() {
@Override
public void run() {
try {
LireFeature lireFeature = featureEnum.getFeatureClass().newInstance();
lireFeature.extract(finalImg);
featureExtractMap.put(featureEnum, lireFeature);
} catch (Throwable e){
logger.error("Failed to extract feature from image", e);
} finally {
latch.countDown();
}
}
});
}
try {
latch.await();
} catch (InterruptedException e) {
logger.debug("Interrupted extract feature from image", e);
Thread.currentThread().interrupt();
}
}

}

for (ObjectObjectCursor<FeatureEnum, Map<String, Object>> cursor : features) {
FeatureEnum featureEnum = cursor.key;
Expand Down Expand Up @@ -368,7 +321,6 @@ public void run() {
} else if (hashEnum.equals(HashEnum.LSH)) {
hashVals = LocalitySensitiveHashing.generateHashes(lireFeature.getDoubleHistogram());
}

String mapperName = featureEnum.name() + "." + HASH + "." + h;
FieldMapper hashMapper = hashMappers.get(mapperName) ;
hashMapper.parse(context.createExternalValueContext(SerializationUtils.arrayToString(hashVals)));
Expand Down
14 changes: 13 additions & 1 deletion src/test/java/org/elasticsearch/plugin/image/test/ImageIntegrationTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@
import org.apache.sanselan.ImageFormat;
import org.apache.sanselan.ImageWriteException;
import org.apache.sanselan.Sanselan;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchResponse;
import com.google.common.collect.Maps;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.get.GetField;
import org.elasticsearch.index.mapper.image.FeatureEnum;
import org.elasticsearch.index.mapper.image.HashEnum;
import org.elasticsearch.index.query.BoolQueryBuilder;
Expand All @@ -16,6 +18,7 @@
import org.elasticsearch.plugin.image.ImagePlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHitField;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.test.ESIntegTestCase;
import org.junit.Before;
Expand All @@ -25,6 +28,8 @@
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Collection;
import java.util.Iterator;
import java.util.Map;

import static org.elasticsearch.client.Requests.putMappingRequest;
import static org.elasticsearch.common.io.Streams.copyToString;
Expand Down Expand Up @@ -93,13 +98,20 @@ public void test_index_search_image() throws Exception {

// test search with hash
ImageQueryBuilder imageQueryBuilder = new ImageQueryBuilder("img").feature(FeatureEnum.CEDD.name()).image(imgToSearch).hash(HashEnum.BIT_SAMPLING.name());
SearchResponse searchResponse = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE_NAME).setQuery(imageQueryBuilder).addFields("img.metadata.exif_ifd0.x_resolution", "name").setSize(totalImages).get();
SearchResponse searchResponse = client().prepareSearch(INDEX_NAME)
.setTypes(DOC_TYPE_NAME)
.setQuery(imageQueryBuilder)
.addFields("img.metadata.exif_ifd0.x_resolution", "name")
.setSize(totalImages)
.get();
assertNoFailures(searchResponse);
SearchHits hits = searchResponse.getHits();
assertThat("Should match at least one image", hits.getTotalHits(), greaterThanOrEqualTo(1l)); // if using hash, total result maybe different than number of images
SearchHit hit = hits.getHits()[0];
assertThat("First should be exact match and has score 1", hit.getScore(), equalTo(2.0f));
assertImageScore(hits, nameToSearch, 2.0f);


assertThat("Should have metadata", hit.getFields().get("img.metadata.exif_ifd0.x_resolution").getValues(), hasSize(1));

// test search without hash and with boost
Expand Down

0 comments on commit 1a0feb7

Please sign in to comment.