diff --git a/docs/source/admin/fs/rest.rst b/docs/source/admin/fs/rest.rst index 02ebeea8c..7b965b3c5 100644 --- a/docs/source/admin/fs/rest.rst +++ b/docs/source/admin/fs/rest.rst @@ -143,7 +143,7 @@ You will get back your document as it has been stored by elasticsearch: } } -If you started FSCrawler in debug mode with ``--debug`` or if you pass +If you started FSCrawler in debug mode or if you pass ``debug=true`` query parameter, then the response will be much more complete: @@ -279,6 +279,15 @@ The field ``external`` doesn't necessarily be a flat structure. This is a more a } } +You can use this technique to add for example the filesize of the file your are uploading:: + +.. code:: sh + + echo "This is my text" > test.txt + curl -F "file=@test.txt" \ + -F "tags={\"file\":{\"filesize\":$(ls -l test.txt | awk '{print $5}')}}" \ + "http://127.0.0.1:8080/fscrawler/_document" + .. attention:: Only standard :ref:`FSCrawler fields ` can be set outside ``external`` field name. Remove a document diff --git a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerRestIT.java b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerRestIT.java index 0cee30850..76bda1e3f 100644 --- a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerRestIT.java +++ b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerRestIT.java @@ -149,7 +149,7 @@ public void testUploadDocumentWithIdUsingPut() throws Exception { // We wait until we have our document ESSearchResponse response = countTestHelper(new ESSearchRequest().withIndex(getCrawlerName()), 1L, null); assertThat(response.getHits().get(0).getId(), is("1234")); - assertThat(JsonPath.read(response.getHits().get(0).getSource(), "$.file.filesize"), notNullValue()); + assertThat(JsonPath.read(response.getHits().get(0).getSource(), "$.file.filesize"), greaterThan(0)); } @Test @@ -224,7 +224,14 @@ public void testAllDocumentsWithRestExternalIndex() throws Exception { .timeValueMinutes(2)); for (ESSearchHit hit : response.getHits()) { assertThat(JsonPath.read(hit.getSource(), "$.file.extension"), notNullValue()); - assertThat(JsonPath.read(hit.getSource(), "$.file.filesize"), notNullValue()); + int filesize = JsonPath.read(hit.getSource(), "$.file.filesize"); + if (filesize <= 0) { + // On some machines (ie Github Actions), the size is not provided + logger.warn("File [{}] has a size of [{}]", + JsonPath.read(hit.getSource(), "$.file.filename"), filesize); + } else { + assertThat(JsonPath.read(hit.getSource(), "$.file.filesize"), greaterThan(0)); + } } } @@ -249,7 +256,7 @@ public void testDocumentWithExternalTags() throws Exception { checkDocument("add_external.txt", hit -> { assertThat(JsonPath.read(hit.getSource(), "$.content"), containsString("This file content will be extracted")); assertThat(JsonPath.read(hit.getSource(), "$.file.extension"), notNullValue()); - assertThat(JsonPath.read(hit.getSource(), "$.file.filesize"), notNullValue()); + assertThat(JsonPath.read(hit.getSource(), "$.file.filesize"), greaterThan(0)); expectThrows(PathNotFoundException.class, () -> JsonPath.read(hit.getSource(), "$.meta")); assertThat(JsonPath.read(hit.getSource(), "$.external.tenantId"), is(23)); assertThat(JsonPath.read(hit.getSource(), "$.external.company"), is("shoe company")); @@ -265,7 +272,7 @@ public void testDocumentWithExternalTags() throws Exception { checkDocument("replace_content_and_external.txt", hit -> { assertThat(JsonPath.read(hit.getSource(), "$.content"), is("OVERWRITTEN CONTENT")); assertThat(JsonPath.read(hit.getSource(), "$.file.extension"), notNullValue()); - assertThat(JsonPath.read(hit.getSource(), "$.file.filesize"), notNullValue()); + assertThat(JsonPath.read(hit.getSource(), "$.file.filesize"), greaterThan(0)); expectThrows(PathNotFoundException.class, () -> JsonPath.read(hit.getSource(), "$.meta")); assertThat(JsonPath.read(hit.getSource(), "$.external.tenantId"), is(23)); assertThat(JsonPath.read(hit.getSource(), "$.external.company"), is("shoe company")); @@ -281,7 +288,7 @@ public void testDocumentWithExternalTags() throws Exception { checkDocument("replace_content_only.txt", hit -> { assertThat(JsonPath.read(hit.getSource(), "$.content"), is("OVERWRITTEN CONTENT")); assertThat(JsonPath.read(hit.getSource(), "$.file.extension"), notNullValue()); - assertThat(JsonPath.read(hit.getSource(), "$.file.filesize"), notNullValue()); + assertThat(JsonPath.read(hit.getSource(), "$.file.filesize"), greaterThan(0)); expectThrows(PathNotFoundException.class, () -> JsonPath.read(hit.getSource(), "$.meta")); expectThrows(PathNotFoundException.class, () -> JsonPath.read(hit.getSource(), "$.external")); });