Merge pull request #85 from qld-gov-au/merge-upstream

Merge upstream changes into our fork

README.rst

@@ -191,6 +191,11 @@ Configuration:
 
 See the extension's `config_declaration.yaml <ckanext/xloader/config_declaration.yaml>`_ file.
 
+This plugin also supports the `ckan.download_proxy` setting, to use a proxy server when downloading files.
+This setting is shared with other plugins that download resource files, such as ckanext-archiver. Eg:
+
+    ckan.download_proxy = http://my-proxy:1234/
+
 You may also wish to configure the database to use your preferred date input style on COPY.
 For example, to make [PostgreSQL](https://www.postgresql.org/docs/current/runtime-config-client.html#RUNTIME-CONFIG-CLIENT-FORMAT)
 expect European (day-first) dates, you could add to ``postgresql.conf``:

ckanext/xloader/helpers.py

@@ -31,10 +31,8 @@ def xloader_status_description(status):
 def is_resource_supported_by_xloader(res_dict, check_access=True):
     is_supported_format = XLoaderFormats.is_it_an_xloader_format(res_dict.get('format'))
     is_datastore_active = res_dict.get('datastore_active', False)
-    if check_access:
-        user_has_access = toolkit.h.check_access('package_update', {'id': res_dict.get('package_id')})
-    else:
-        user_has_access = True
+    user_has_access = not check_access or toolkit.h.check_access('package_update',
+                                                                 {'id':res_dict.get('package_id')})
     url_type = res_dict.get('url_type')
     if url_type:
         try:

ckanext/xloader/templates/package/resource_read.html

@@ -6,3 +6,10 @@
     <li>{% link_for _('DataStore'), named_route='xloader.resource_data', id=pkg.name, resource_id=res.id, class_='btn btn-light', icon='cloud-upload' %}</li>
   {% endif %}
 {% endblock %}
+
+{% block resource_actions_inner %}
+  {% if h.check_ckan_version(max_version='2.10') and h.is_resource_supported_by_xloader(res) %}
+    <li>{% link_for _('DataStore'), named_route='xloader.resource_data', id=pkg.name, resource_id=res.id, class_='btn btn-light', icon='cloud-upload' %}</li>
+  {% endif %}
+  {{ super() }}
+{% endblock %}

ckanext/xloader/templates/package/snippets/resource_item.html

@@ -6,3 +6,10 @@
     <li>{% link_for _('DataStore'), named_route='xloader.resource_data', id=pkg.name, resource_id=res.id, class_='dropdown-item', icon='cloud-upload' %}</li>
   {% endif %}
 {% endblock %}
+
+{% block resource_item_explore_links %}
+  {% if h.check_ckan_version(max_version='2.10') and h.is_resource_supported_by_xloader(res) %}
+    <li>{% link_for _('DataStore'), named_route='xloader.resource_data', id=pkg.name, resource_id=res.id, class_='dropdown-item', icon='cloud-upload' %}</li>
+  {% endif %}
+  {{ super() }}
+{% endblock %}

ckanext/xloader/templates/xloader/resource_data.html

@@ -6,31 +6,34 @@
 
   {% set show_table = true %}
 
+  {% block upload_ds_button %}
+    {% set action = h.url_for('xloader.resource_data', id=pkg.name, resource_id=res.id) %}
+    <form method="post" action="{{ action }}" class="datapusher-form mb-3 mr-3 d-inline-block">
+      {{ h.csrf_input() if 'csrf_input' in h }}
+      <button class="btn btn-primary" name="save" type="submit">
+        <i class="fa fa-cloud-upload"></i> {{ _('Upload to DataStore') }}
+      </button>
+    </form>
+  {% endblock %}
+
+  <div class="clear-fix mx-1 d-inline-block"></div>
+
   {% block delete_ds_button %}
     {% if res.datastore_active %}
       {% set delete_action = h.url_for('xloader.delete_datastore_table', id=pkg.id, resource_id=res.id) %}
-      <form method="post" action="{{ delete_action }}" class="mb-3 d-inline-block pull-right">
+      <form method="post" action="{{ delete_action }}" class="mb-3 d-inline-block">
         {{ h.csrf_input() if 'csrf_input' in h }}
         <a href="{{ delete_action }}"
-          class="btn btn-danger pull-left"
+          class="btn btn-danger"
           type="submit"
           data-module="confirm-action"
           data-module-with-data=true
           data-module-content="{{ _('Are you sure you want to delete the DataStore and Data Dictionary?') }}"
-          >{% block delete_datastore_button_text %}<i class="fa fa-remove"></i> {{ _('Delete from DataStore') }}{% endblock %}</a>
+          >{% block delete_datastore_button_text %}<i class="fa fa-trash-alt"></i> {{ _('Delete from DataStore') }}{% endblock %}</a>
       </form>
     {% endif %}
   {% endblock %}
 
-  {% block upload_ds_button %}
-    <form method="post" action="{{ h.url_for('xloader.resource_data', id=pkg.name, resource_id=res.id) }}" class="datapusher-form mb-3 d-inline-block">
-      {{ h.csrf_input() if 'csrf_input' in h }}
-      <button class="btn btn-primary" name="save" type="submit">
-        <i class="fa fa-cloud-upload"></i> {{ _('Upload to DataStore') }}
-      </button>
-    </form>
-  {% endblock %}
-
   {% if status.error and status.error.message %}
     {% set show_table = false %}
     <div class="alert alert-error">

ckanext/xloader/tests/samples/mixed_numeric_string_sample.csv

@@ -1,3 +1,3 @@
-Funding agency,Program title,Maximum (indicative) grant amount
-DTIS,Accessible Tourism Infrastructure Grants,Five hundred thousand dollars
-DTIS,Boosting Accessible Tourism Experiences Grants,5000
+Funding agency,Program title,Maximum (indicative) grant amount
+DTIS,Accessible Tourism Infrastructure Grants,Five hundred thousand dollars
+DTIS,Boosting Accessible Tourism Experiences Grants,5000

ckanext/xloader/tests/samples/sample_with_blanks.csv

@@ -1,4 +1,4 @@
-Funding agency,Program title,Opening date,Service ID
-DTIS,Visitor First Experiences Fund,23/03/2023,63039
-DTIS,First Nations Sport and Recreation Program Round 2,22/03/2023,63040
-,,,63041
+Funding agency,Program title,Opening date,Service ID
+DTIS,Visitor First Experiences Fund,23/03/2023,63039
+DTIS,First Nations Sport and Recreation Program Round 2,22/03/2023,63040
+,,,63041

ckanext/xloader/utils.py

@@ -37,6 +37,7 @@ def is_it_an_xloader_format(cls, format_):
         if cls.formats is None:
             cls._formats = config.get("ckanext.xloader.formats")
             if cls._formats is not None:
+                # use config value. preserves empty list as well.
                 cls._formats = cls._formats.lower().split()
             else:
                 cls._formats = DEFAULT_FORMATS