From 72a20493fee70ae52c9628ad438ab4e76b8b0145 Mon Sep 17 00:00:00 2001 From: Marc Worrell Date: Tue, 21 Jun 2022 12:30:21 +0200 Subject: [PATCH] Add mod_elasticsearch2 support for Elasticsearch v7+ (#702) * Add mod_elasticsearch2 support for Elasticsearch v7+ * Should be ginger_adlib_elasticsearch2_mapping --- .../mod_ginger_adlib_elasticsearch2/LICENSE | 201 ++++++++++++++++++ .../mod_ginger_adlib_elasticsearch2/README.md | 9 + .../mod_ginger_adlib_elasticsearch2.erl | 70 ++++++ ...ger_adlib_elasticsearch2_index_mapping.erl | 121 +++++++++++ .../ginger_adlib_elasticsearch2_mapper.erl | 56 +++++ .../ginger_adlib_elasticsearch2_mapping.erl | 182 ++++++++++++++++ .../templates/_admin_configure_module.tpl | 11 + .../mod_ginger_collection.erl | 17 +- .../models/m_collection_object.erl | 59 +++-- .../models/m_ginger_collection.erl | 49 +++++ .../support/collection_search.erl | 1 - ...admin_edit_content.beeldenzoeker_query.tpl | 16 +- .../templates/collection/prevnext.tpl | 2 +- .../collection/search-query-wrapper.tpl | 2 +- 14 files changed, 760 insertions(+), 36 deletions(-) create mode 100644 modules/mod_ginger_adlib_elasticsearch2/LICENSE create mode 100644 modules/mod_ginger_adlib_elasticsearch2/README.md create mode 100644 modules/mod_ginger_adlib_elasticsearch2/mod_ginger_adlib_elasticsearch2.erl create mode 100644 modules/mod_ginger_adlib_elasticsearch2/support/ginger_adlib_elasticsearch2_index_mapping.erl create mode 100644 modules/mod_ginger_adlib_elasticsearch2/support/ginger_adlib_elasticsearch2_mapper.erl create mode 100644 modules/mod_ginger_adlib_elasticsearch2/support/ginger_adlib_elasticsearch2_mapping.erl create mode 100644 modules/mod_ginger_adlib_elasticsearch2/templates/_admin_configure_module.tpl create mode 100644 modules/mod_ginger_collection/models/m_ginger_collection.erl diff --git a/modules/mod_ginger_adlib_elasticsearch2/LICENSE b/modules/mod_ginger_adlib_elasticsearch2/LICENSE new file mode 100644 index 000000000..8dada3eda --- /dev/null +++ b/modules/mod_ginger_adlib_elasticsearch2/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/modules/mod_ginger_adlib_elasticsearch2/README.md b/modules/mod_ginger_adlib_elasticsearch2/README.md new file mode 100644 index 000000000..2b4f65d1e --- /dev/null +++ b/modules/mod_ginger_adlib_elasticsearch2/README.md @@ -0,0 +1,9 @@ +mod_ginger_adlib_elasticsearch +============================== + +This module makes Adlib data searchable by indexing it in Elasticsearch. This +module is part of [Ginger](http://github.com/driebit/ginger). + +Features: + +* Custom Elasticsearch mappings. diff --git a/modules/mod_ginger_adlib_elasticsearch2/mod_ginger_adlib_elasticsearch2.erl b/modules/mod_ginger_adlib_elasticsearch2/mod_ginger_adlib_elasticsearch2.erl new file mode 100644 index 000000000..092904d07 --- /dev/null +++ b/modules/mod_ginger_adlib_elasticsearch2/mod_ginger_adlib_elasticsearch2.erl @@ -0,0 +1,70 @@ +%% @doc Zotonic/Adlib integration +-module(mod_ginger_adlib_elasticsearch2). +-author("Driebit "). + +-mod_title("Adlib in ElasticSearch"). +-mod_prio(500). +-mod_description("Makes Adlib data searchable by indexing it in ElasticSearch (v7+)"). +-mod_depends([mod_elasticsearch2, mod_ginger_adlib]). +-mod_schema(1). + +-export([ + init/1, + index/1, + types/1, + manage_schema/2, + observe_adlib_update/2 +]). + +-include_lib("mod_ginger_adlib/include/ginger_adlib.hrl"). +-include_lib("zotonic.hrl"). + +init(Context) -> + default_config(index, index(Context), Context), + prepare_elasticsearch_index(Context). + +manage_schema(_, Context) -> + prepare_elasticsearch_index(Context). + +%% @doc Create index and mappings for each type in the index. +-spec prepare_elasticsearch_index(z:context()) -> ok. +prepare_elasticsearch_index(Context) -> + {Version, Mapping} = ginger_adlib_elasticsearch2_index_mapping:default_mapping(Context), + {ok, _} = elasticsearch2_index:upgrade(index(Context), Mapping, Version, Context), + ok. + +observe_adlib_update(#adlib_update{date = _Date, database = Database, record = #{<<"@attributes">> := #{<<"priref">> := Priref}} = Record}, Context) -> + lager:debug("Indexing Adlib record ~s from database ~s", [Priref, Database]), + MappedRecord = ginger_adlib_elasticsearch2_mapper:map(Record, ginger_adlib_elasticsearch2_mapping), + MappedRecord1 = MappedRecord#{ + <<"es_type">> => Database + }, + case elasticsearch2:put_doc(index(Context), Priref, MappedRecord1, Context) of + {ok, _} -> + ok; + {error, Message} -> + lager:error("Record with priref ~s from database ~p could not be saved to Elasticsearch: ~p", [Priref, Database, Message]) + end. + +%% @doc Get Elasticsearch index name for Adlib resources +-spec index(z:context()) -> binary(). +index(Context) -> + Default = <<(elasticsearch2:index(Context))/binary, "_adlib">>, + case m_config:get_value(mod_ginger_adlib_elasticsearch2, index, Context) of + undefined -> Default; + <<>> -> Default; + Value -> z_convert:to_binary(Value) + end. + +%% @doc Get Elasticsearch types used for Adlib resources +-spec types(z:context()) -> [binary()]. +types(Context) -> + mod_ginger_adlib:enabled_databases(Context). + +default_config(Key, Value, Context) -> + case m_config:get_value(?MODULE, Key, Context) of + undefined -> + m_config:set_value(?MODULE, Key, Value, Context); + _ -> + ok + end. diff --git a/modules/mod_ginger_adlib_elasticsearch2/support/ginger_adlib_elasticsearch2_index_mapping.erl b/modules/mod_ginger_adlib_elasticsearch2/support/ginger_adlib_elasticsearch2_index_mapping.erl new file mode 100644 index 000000000..cdc06d8a7 --- /dev/null +++ b/modules/mod_ginger_adlib_elasticsearch2/support/ginger_adlib_elasticsearch2_index_mapping.erl @@ -0,0 +1,121 @@ +-module(ginger_adlib_elasticsearch2_index_mapping). + +-export([ + default_mapping/1 +]). + +%% @doc Default mappings are based on linked data predicates. +default_mapping(Context) -> + Mapping = #{ + <<"properties">> => #{ + <<"es_type">> => #{ + <<"type">> => <<"keyword">> + }, + <<"acquisition.date">> => #{ + <<"type">> => <<"date">> + }, + <<"dcterms:created">> => #{ + <<"type">> => <<"date">> + }, + <<"dbo:creationYear">> => #{ + <<"type">> => <<"date">> + }, + <<"dbo:productionStartYear">> => #{ + <<"type">> => <<"date">> + }, + <<"dbo:productionEndDate">> => #{ + <<"type">> => <<"date">> + }, + <<"dbo:productionEndYear">> => #{ + <<"type">> => <<"date">> + }, + <<"dcterms:creator">> => #{ + <<"type">> => <<"nested">>, + <<"properties">> => #{ + <<"dbpedia-owl:birthDate">> => #{ + <<"type">> => <<"date">> + }, + <<"dbpedia-owl:deathDate">> => #{ + <<"type">> => <<"date">> + }, + <<"rdfs:label">> => #{ + <<"type">> => <<"text">>, + <<"fields">> => #{ + <<"keyword">> => #{ + <<"type">> => <<"keyword">>, + <<"ignore_above">> => 256 + } + } + } + } + }, + <<"rdf:type.rdfs:label">> => #{ + <<"type">> => <<"text">>, + <<"fields">> => #{ + <<"keyword">> => #{ + <<"type">> => <<"keyword">>, + <<"ignore_above">> => 256 + } + } + }, + <<"dcterms:subject.rdfs:label">> => #{ + <<"type">> => <<"text">>, + <<"fields">> => #{ + <<"keyword">> => #{ + <<"type">> => <<"keyword">>, + <<"ignore_above">> => 256 + } + } + }, + <<"schema:about.rdfs:label">> => #{ + <<"type">> => <<"text">>, + <<"fields">> => #{ + <<"keyword">> => #{ + <<"type">> => <<"keyword">>, + <<"ignore_above">> => 256 + }, + <<"keyword_global">> => #{ + <<"type">> => <<"keyword">>, + <<"ignore_above">> => 256 + } + } + } + }, + <<"dynamic_templates">> => [ + #{<<"date">> => #{ + <<"match">> => <<"dcterms:date">>, + <<"mapping">> => #{ + <<"type">> => <<"date">> + } + }}, + #{<<"strings">> => #{ + <<"match_mapping_type">> => <<"string">>, + <<"match">> => <<"title*">>, + <<"mapping">> => #{ + <<"type">> => <<"text">> + } + }}, + #{<<"strings">> => #{ + <<"match_mapping_type">> => <<"string">>, + <<"match">> => <<"dc:title*">>, + <<"mapping">> => #{ + <<"type">> => <<"text">> + } + }}, + #{<<"strings">> => #{ + <<"match_mapping_type">> => <<"string">>, + <<"match">> => <<"dcterms:title*">>, + <<"mapping">> => #{ + <<"type">> => <<"text">> + } + }}, + #{<<"strings">> => #{ + <<"match_mapping_type">> => <<"string">>, + <<"match">> => <<"schema:title*">>, + <<"mapping">> => #{ + <<"type">> => <<"text">> + } + }} + ] ++ elasticsearch2_mapping:dynamic_language_mapping(Context) + }, + {elasticsearch2_mapping:hash(Mapping), Mapping}. diff --git a/modules/mod_ginger_adlib_elasticsearch2/support/ginger_adlib_elasticsearch2_mapper.erl b/modules/mod_ginger_adlib_elasticsearch2/support/ginger_adlib_elasticsearch2_mapper.erl new file mode 100644 index 000000000..333badd81 --- /dev/null +++ b/modules/mod_ginger_adlib_elasticsearch2/support/ginger_adlib_elasticsearch2_mapper.erl @@ -0,0 +1,56 @@ +%% @doc Maps an Adlib record map to an Elasticsearch document. +-module(ginger_adlib_elasticsearch2_mapper). + +-export([ + map/2 +]). + +-include_lib("zotonic.hrl"). + +%% @doc Maps an Adlib record to an Elasticsearch document, taking care of nested +%% fields. As Mapping, pass a module that implements behaviour +%% ginger_adlib_elasticsearch_mapping. +-spec map(map(), module()) -> map(). +map(Map, Mapping) -> + maps:fold( + fun(Key, Value, Acc) -> + map_value(Key, Value, Acc, Mapping) + end, + #{}, + Map + ). + +map_value(Key, Value, Acc, Mapping) when is_map(Value) -> + Mapped = map(Value, Mapping), + map_property(Key, Mapped, Acc, Mapping); +map_value(Key, Value, Acc, Mapping) when is_list(Value) -> + %% Map any value that is itself a map. + Mapped = lists:filtermap( + fun(V) when is_map(V) -> + {true, map(V, Mapping)}; + (<<>>) -> + false; + ([]) -> + false; + (V) -> + {true, V} + end, + Value + ), + map_property(Key, Mapped, Acc, Mapping); +map_value(Key, Value, Acc, Mapping) -> + map_property(Key, Value, Acc, Mapping). + +map_property(_, [<<"0000">>], Acc, _Mapping) -> + %% Incorrect date + Acc; +map_property(_, [], Acc, _Mapping) -> + Acc; +map_property(_, [<<>>], Acc, _Mapping) -> + Acc; +map_property(_, <<>>, Acc, _Mapping) -> + Acc; +map_property(Key, [SingleValue], Acc, Mapping) when is_binary(SingleValue) -> + map_property(Key, SingleValue, Acc, Mapping); +map_property(Key, Value, Acc, Mapping) -> + Mapping:map_property(Key, Value, Acc). diff --git a/modules/mod_ginger_adlib_elasticsearch2/support/ginger_adlib_elasticsearch2_mapping.erl b/modules/mod_ginger_adlib_elasticsearch2/support/ginger_adlib_elasticsearch2_mapping.erl new file mode 100644 index 000000000..176397fa5 --- /dev/null +++ b/modules/mod_ginger_adlib_elasticsearch2/support/ginger_adlib_elasticsearch2_mapping.erl @@ -0,0 +1,182 @@ +%% @doc Default +-module(ginger_adlib_elasticsearch2_mapping). + +-export([ + map/1, + map_property/3, + year/1, + to_list/1, + to_labelled_list/1, + extract_values/1, + parse_name/1 +]). + +-include_lib("zotonic.hrl"). + +-callback map_property(Property :: binary(), Value :: binary(), Acc :: map()) -> NewAcc :: map(). + +map(Record) -> + Record. + +map_property(Key, [#{<<"value">> := _} | _] = Values, Acc) -> + E = extract_values(Values), + %% Flatten Adlib's @lang/value xmltype=grouped structure. + map_property(Key, E, Acc); +map_property(Key, #{<<"value">> := Value}, Acc) -> + %% Flatten Adlib's @lang/value xmltype=grouped structure. + map_property(Key, Value, Acc); +map_property(Key, Value, Acc) when Key =:= <<"object_number">>; Key =:= <<"object.object_number">> -> + Acc#{ + <<"dcterms:identifier">> => Value + }; +map_property(<<"creator">> = Key, Value, Acc) -> + Acc#{Key => [ parse_name(Creator) || Creator <- to_list(Value) ]}; +map_property(<<"creator.role">>, Value, Acc) -> + Acc#{<<"role">> => Value}; +map_property(<<"production.date.start">> = Key, Value, Acc) when value =/= <<"?">> -> + Acc2 = Acc#{ + Key => Value, + <<"dcterms:date">> => Value + }, + case year(Value) of + undefined -> + Acc2; + Year -> + Acc2#{<<"dbo:productionStartYear">> => Year} + end; +map_property(<<"production.date.end">> = Key, Value, Acc) when value =/= <<"?">> -> + Acc2 = Acc#{ + Key => Value, + <<"dcterms:date">> => Value + }, + case year(Value) of + undefined -> + Acc2; + Year -> + Acc2#{<<"dbo:productionEndYear">> => Year} + end; +map_property(<<"reproduction.creator">>, [Value], Acc) -> + map_property(<<"reproduction.creator">>, Value, Acc); +map_property(<<"reproduction.creator">>, Value, Acc) when Value =/= <<>> -> + Acc#{<<"dcterms:creator">> => parse_name(Value)}; +map_property(<<"reproduction.reference">>, Value, Acc) -> + Acc#{<<"reference">> => Value}; +map_property(Key, Values, Acc) when Key =:= <<"Dimension">>; Key =:= <<"dimension">> -> + lists:foldl( + fun map_dimension/2, + Acc, + to_list(Values) + ); +%% @doc See e.g. http://wikidata.dbpedia.org/page/Q1248830 +map_property(<<"material">>, Values, Acc) -> + %% List of materials as single value + %% @doc See e.g. http://wikidata.dbpedia.org/page/Q1248830 + Acc#{<<"dbpedia-owl:constructionMaterial">> => to_list(Values)}; +map_property(<<"notes">>, Values, Acc) -> + Acc#{<<"dbpedia-owl:notes">> => to_list(Values)}; +map_property(<<"rights">> = Key, Value, Acc) -> + Acc2 = Acc#{Key => Value}, + case m_creative_commons:url_for(Value) of + undefined -> + Acc2; + Url -> + Acc2#{<<"dcterms:license">> => Url} + end; +map_property(<<"dimension.unit">>, Value, Acc) -> + Acc#{<<"schema:unitCode">> => map_dimension_unit(Value), + <<"schema:unitText">> => Value + }; +map_property(<<"dimension.value">>, Value, Acc) -> + Acc#{<<"schema:value">> => Value}; +map_property(<<"@attributes">> = Key, #{<<"modification">> := Date} = Value, Acc) -> + Acc#{ + Key => Value, + <<"dcterms:modified">> => Date, + <<"modified">> => Date %% For sorting together with Zotonic resources + }; +map_property(Key, [Value], Acc) -> + map_property(Key, Value, Acc); +map_property(Key, Value, Acc) -> + Acc#{Key => Value}. + +map_dimension(#{<<"schema:value">> := _, <<"dimension.type">> := Type} = Dimension, Acc) -> + Dimension2 = maps:remove(<<"dimension.type">>, Dimension), + case map_dimension_type(Type) of + undefined -> + Acc; + MappedType -> + Acc#{MappedType => Dimension2#{ + <<"rdf:type">> => <<"schema:QuantitativeValue">> + }} + end; +map_dimension(FreeTextDimension, Acc) when is_binary(FreeTextDimension) -> + Acc#{<<"dcterms:format">> => FreeTextDimension}; +map_dimension(_OtherDimension, Acc) -> + Acc. + + +%% @doc Map dimension unit to UN/CEFACT Common Codes for Units of Measurement +map_dimension_unit(<<"cm">>) -> + <<"CMT">>; +map_dimension_unit(Unit) -> + Unit. + +map_dimension_type(<<"breedte">>) -> + <<"schema:width">>; +map_dimension_type(<<"diepte">>) -> + <<"schema:depth">>; +map_dimension_type(<<"hoogte">>) -> + <<"schema:height">>; +map_dimension_type(<<"lengte">>) -> + <<"dbpedia-owl:length">>; +map_dimension_type(<<"diameter">>) -> + <<"dbpedia-owl:diameter">>; +map_dimension_type(_Type) -> + undefined. + +parse_name(#{<<"name">> := Name} = Person) -> + maps:merge(parse_name(Name), maps:remove(<<"name">>, Person)); +parse_name(Name) -> + case binary:split(Name, <<", ">>) of + [Last, First] -> + #{ + <<"rdfs:label">> => <>, + <<"foaf:firstName">> => First, + <<"foaf:familyName">> => Last + }; + _ -> + #{ + <<"rdfs:label">> => Name + } + end. + +to_list(Values) when is_list(Values) -> + Values; +to_list(Value) -> + %% Wrap single value in a list for consistency + [Value]. + +extract_values(Values) when is_list(Values) -> + lists:filtermap(fun extract_value/1, Values); +extract_values(Values) -> + %% Wrap single value in a list for consistency + extract_values([Values]). + +extract_value(#{<<"value">> := Value}) -> + {true, Value}; +extract_value(Value) when map_size(Value) =:= 0 -> + false; +extract_value(Value) -> + {true, Value}. + +to_labelled_list(Values) -> + ListValues = to_list(Values), + [#{<<"rdfs:label">> => Value} || Value <- ListValues]. + +%% @doc Extract YYYY value (removing '?' etc.) +-spec year(binary()) -> binary(). +year(Value) -> + case re:run(Value, "^(\\d+).*$", [{capture, all_but_first, binary}]) of + {match, [Year]} -> Year; + _ -> undefined + end. diff --git a/modules/mod_ginger_adlib_elasticsearch2/templates/_admin_configure_module.tpl b/modules/mod_ginger_adlib_elasticsearch2/templates/_admin_configure_module.tpl new file mode 100644 index 000000000..c104470d4 --- /dev/null +++ b/modules/mod_ginger_adlib_elasticsearch2/templates/_admin_configure_module.tpl @@ -0,0 +1,11 @@ +
+
+ + + {% wire id="index" type="blur" action={config_toggle module="mod_ginger_adlib_elasticsearch2" key="index" on="keyup"} %} +
+
+ +
+ {% button class="btn btn-default" text=_"Close" action={dialog_close} tag="a" %} +
diff --git a/modules/mod_ginger_collection/mod_ginger_collection.erl b/modules/mod_ginger_collection/mod_ginger_collection.erl index 3f533669a..b042bde16 100644 --- a/modules/mod_ginger_collection/mod_ginger_collection.erl +++ b/modules/mod_ginger_collection/mod_ginger_collection.erl @@ -4,14 +4,13 @@ -mod_title("Linked data collections"). -mod_description("Linked data and media collection view and search interface powered by Elasticsearch"). -mod_prio(200). --mod_depends([mod_ginger_base, mod_elasticsearch, mod_ginger_rdf]). +-mod_depends([mod_ginger_base, elasticsearch, mod_ginger_rdf]). -mod_schema(6). -include_lib("zotonic.hrl"). -export([ init/1, - index/1, manage_schema/2, observe_search_query/2, observe_acl_is_allowed/2, @@ -19,21 +18,11 @@ ]). init(Context) -> - default_config(index, index(Context), Context). - -%% @doc Get Elasticsearch index name used for collections. --spec index(z:context()) -> binary(). -index(Context) -> - Default = <<(elasticsearch:index(Context))/binary, "_collection">>, - case m_config:get_value(?MODULE, index, Context) of - undefined -> Default; - <<>> -> Default; - Value -> z_convert:to_binary(Value) - end. + default_config(index, m_ginger_collection:collection_index(Context), Context). manage_schema(_, _Context) -> datamodel(). - + datamodel() -> #datamodel{ categories = [ diff --git a/modules/mod_ginger_collection/models/m_collection_object.erl b/modules/mod_ginger_collection/models/m_collection_object.erl index 0d055ff1b..1786213bb 100644 --- a/modules/mod_ginger_collection/models/m_collection_object.erl +++ b/modules/mod_ginger_collection/models/m_collection_object.erl @@ -8,6 +8,10 @@ m_find_value/3, m_to_list/2, m_value/2, + + get/2, + store/3, + get/3, store/4 ]). @@ -32,24 +36,49 @@ m_find_value(ObjectId, #m{value = Type}, Context) -> m_to_list(_, _Context) -> []. -m_value(#m{value = Object}, _Context) -> +m_value(#m{ value = ObjectId }, Context) when is_binary(ObjectId) -> + % ES 7+ object lookup + get(ObjectId, Context); +m_value(#m{ value = Object }, _Context) -> Object. +%% @doc For ES 7.x+ +get(Id, Context) -> + Index = m_ginger_collection:collection_index(Context), + elasticsearch2:get_doc(Index, Id, Context). + +store(Id, Document, Context) -> + Index = m_ginger_collection:collection_index(Context), + elasticsearch2:put_doc(Index, Id, Document, Context). + + +%% @doc For ES 5.x with Types get(Type, Id, Context) -> - case erlastic_search:get_doc( - mod_ginger_collection:index(Context), - z_convert:to_binary(Type), - z_convert:to_binary(Id) - ) of - {ok, Object} when is_list(Object) -> - %% BC with jsx 2.0 - maps:from_list(Object); - {ok, Object} when is_map(Object) -> - Object; - {error, _} -> - undefined + Index = m_ginger_collection:collection_index(Context), + case m_ginger_collection:is_elastic2(Context) of + true -> + elasticsearch2:get_doc(Index, Id, Context); + false -> + case erlastic_search:get_doc( + Index, + z_convert:to_binary(Type), + z_convert:to_binary(Id) + ) of + {ok, Object} when is_map(Object) -> + Object; + {error, _} -> + undefined + end end. store(Type, Id, Document, Context) -> - Index = mod_ginger_collection:index(Context), - elasticsearch:put_doc(Index, Type, Id, Document, Context). + Index = m_ginger_collection:collection_index(Context), + case m_ginger_collection:is_elastic2(Context) of + true -> + Document1 = Document#{ + <<"es_type">> => Type + }, + elasticsearch2:put_doc(Index, Id, Document1, Context); + false -> + elasticsearch:put_doc(Index, Type, Id, Document, Context) + end. diff --git a/modules/mod_ginger_collection/models/m_ginger_collection.erl b/modules/mod_ginger_collection/models/m_ginger_collection.erl new file mode 100644 index 000000000..280ee3736 --- /dev/null +++ b/modules/mod_ginger_collection/models/m_ginger_collection.erl @@ -0,0 +1,49 @@ +-module(m_ginger_collection). + +-export([ + m_find_value/3, + + default_index/1, + collection_index/1, + is_elastic2/1 +]). + +-include("zotonic.hrl"). + +m_find_value(default_index, #m{}, Context) -> + default_index(Context); +m_find_value(collection_index, #m{}, Context) -> + collection_index(Context); +m_find_value(query_preview_delegate, #m{}, Context) -> + case is_elastic2(Context) of + true -> elasticsearch2_admin; + false -> controller_admin_elasticsearch_edit + end. + +%% @doc Check if we use mod_elasticsearch2 or mod_elasticsearch +is_elastic2(Context) -> + case m_config:get_value(mod_elasticsearch2, index, Context) of + undefined -> + false; + _Index -> + true + end. + +default_index(Context) -> + case is_elastic2(Context) of + true -> + elasticsearch2:index(Context); + false -> + elasticsearch:index(Context) + end. + +%% @doc Get Elasticsearch index name used for collections. +-spec collection_index(z:context()) -> binary(). +collection_index(Context) -> + case z_convert:to_binary(m_config:get_value(mod_ginger_collection, index, Context)) of + <<>> -> + Default = default_index(Context), + <>; + Index -> + Index + end. diff --git a/modules/mod_ginger_collection/support/collection_search.erl b/modules/mod_ginger_collection/support/collection_search.erl index 4d451761b..d93dc36de 100644 --- a/modules/mod_ginger_collection/support/collection_search.erl +++ b/modules/mod_ginger_collection/support/collection_search.erl @@ -24,7 +24,6 @@ search(#search_query{search = {_, Args}} = Query, Context) -> [], z_context:get_q_all_noz(Context) ++ Args ), - %% Forward to Elasticsearch. ElasticQuery = Query#search_query{search = {elastic, Args3}}, result(z_notifier:first(ElasticQuery, Context), Context). diff --git a/modules/mod_ginger_collection/templates/_admin_edit_content.beeldenzoeker_query.tpl b/modules/mod_ginger_collection/templates/_admin_edit_content.beeldenzoeker_query.tpl index 7c80ae2da..cc70bc43f 100644 --- a/modules/mod_ginger_collection/templates/_admin_edit_content.beeldenzoeker_query.tpl +++ b/modules/mod_ginger_collection/templates/_admin_edit_content.beeldenzoeker_query.tpl @@ -10,7 +10,6 @@ {% block widget_show_minimized %}false{% endblock %} {% block widget_content %} -{% with m.rsc[id] as r %}

{_ Here you can edit your elasticsearch query. _} @@ -20,9 +19,19 @@

{% with "[]" as placeholder %} - + {% endwith %} - {% wire id=#elastic_query type="change" postback={elastic_query_preview query_type="beeldenzoeker" rsc_id=id div_id=#elastic_query_preview target_id=#elastic_query index=m.config.mod_ginger_collection.index.value} delegate="controller_admin_elasticsearch_edit" %} + {% wire id=#elastic_query + type="change" + postback={elastic_query_preview + query_type="beeldenzoeker" + rsc_id=id + div_id=#elastic_query_preview + target_id=#elastic_query + index=m.ginger_collection.collection_index + } + delegate=m.ginger_collection.query_preview_delegate + %}
@@ -37,5 +46,4 @@ {% catinclude "_admin_query_preview.tpl" id result=m.search[{beeldenzoeker query_id=id id=id index=m.config.mod_ginger_collection.index.value pagelen=20}] %}
-{% endwith %} {% endblock %} diff --git a/modules/mod_ginger_collection/templates/collection/prevnext.tpl b/modules/mod_ginger_collection/templates/collection/prevnext.tpl index a4fced503..8297e151d 100644 --- a/modules/mod_ginger_collection/templates/collection/prevnext.tpl +++ b/modules/mod_ginger_collection/templates/collection/prevnext.tpl @@ -1,5 +1,5 @@ {% if q.query_id and q.current %} - {% with index|default:(m.config.mod_ginger_collection.index.value ++ "," ++ m.config.mod_elasticsearch.index.value) as index %} + {% with index|default:(m.ginger_collection.collection_index ++ "," ++ m.ginger_collection.default_index) as index %} {% with search|collection_query_pager:index:q.query_id:q.current as prevnext %} {% with prevnext|first as prev %} {% with prevnext|last as next %} diff --git a/modules/mod_ginger_collection/templates/collection/search-query-wrapper.tpl b/modules/mod_ginger_collection/templates/collection/search-query-wrapper.tpl index 46ccc0b6e..5c4e1b070 100644 --- a/modules/mod_ginger_collection/templates/collection/search-query-wrapper.tpl +++ b/modules/mod_ginger_collection/templates/collection/search-query-wrapper.tpl @@ -1,5 +1,5 @@ {% with - index|default:(m.config.mod_ginger_collection.index.value ++ "," ++ m.config.mod_elasticsearch.index.value), + index|default:(m.ginger_collection.collection_index ++ "," ++ m.ginger_collection.default_index), results_template|default:"list/list.tpl", cat|default:['beeldenzoeker_query'] as