Merge pull request #4 from ekorkut/ekorkut1_virtual

Do not urlquote values coming from default of virtual columns

pycsvw/generator_utils.py

@@ -114,6 +114,7 @@ def parse_datetime(input_str):
 
 def process_dates_times(value, base):
     """Entry function to process any date, time or dateTime."""
+    out = ""
     if base == "date":
         out = parse_date(value)
     elif base == "time":

pycsvw/nt_serializer.py

@@ -249,8 +249,8 @@ def write_row(output, row_num, row, table_info):
                 obj_val = apply_all_subs(value_url, row_num, row, column_info)
                 write_objs_as_uri(output, subject, predicate, obj_val)
         elif column_spec["default"]:
-            # Apply any substitution first
-            obj_val = apply_all_subs(column_spec["default"], row_num, row, column_info)
+            # Apply any substitution first, but without quoting
+            obj_val = apply_all_subs(column_spec["default"], row_num, row, column_info, False)
             write_objs_as_literal(output, subject, predicate, obj_val, column_spec)
 
 

pycsvw/rdf_utils.py

@@ -48,7 +48,7 @@ def get_column_map(table_schema):
     return column_map
 
 
-def apply_sub(url, row, column_name_to_sub, column_info):
+def apply_sub(url, row, column_name_to_sub, column_info, quote_sub=True):
     """ Apply a given substitution and raise if it is a null value. """
     try:
         column_ind, column_spec = column_info['column_map'][column_name_to_sub]
@@ -59,10 +59,13 @@ def apply_sub(url, row, column_name_to_sub, column_info):
         raise NullValueException("'{}' is one of the null values specified")
 
     rep_before = "{" + column_name_to_sub + "}"
-    return url.replace(rep_before, quote(rep_after.encode('utf-8'), safe=':/'))
+    if quote_sub:
+        return url.replace(rep_before, quote(rep_after.encode('utf-8'), safe=':/'))
+    else:
+        return url.replace(rep_before, rep_after)
 
 
-def apply_all_subs(url, row_num, row, column_info):
+def apply_all_subs(url, row_num, row, column_info, quote_sub=True):
     """ Apply all substitutions (in format of {columnName}) and resolve the url """
 
     # Early return just with resolving if nothing to substitute
@@ -74,7 +77,7 @@ def apply_all_subs(url, row_num, row, column_info):
     subs = SUB_PATTERN.findall(out)
     for sub in subs:
         try:
-            out = apply_sub(out, row, sub, column_info)
+            out = apply_sub(out, row, sub, column_info, quote_sub)
         except NullValueException:
             raise
         except Exception as e:

setup.py

@@ -13,7 +13,7 @@
 
 setup(
     name='pycsvw',
-    version="1.0.0",
+    version="1.0.1",
     description='Generate JSON and RDF from csv files with metadata',
     url='https://github.com/bloomberg/pycsvw',
     author='Dev Ramudit, Erman Korkut',

tests/rdf/test_virtual_columns.py

@@ -86,6 +86,18 @@ def test_default_with_datatype():
         assert active_val.datatype == XSD.boolean
         assert active_val.value
 
+        string_vals = list(g.triples((ns['sub-{}'.format(x)], ns['stringprop1'], None)))
+        assert len(string_vals) == 1
+        string_val = string_vals[0][2]
+        assert isinstance(string_val, Literal)
+        assert string_val.value == "some string"
+
+        string_vals = list(g.triples((ns['sub-{}'.format(x)], ns['stringprop2'], None)))
+        assert len(string_vals) == 1
+        string_val = string_vals[0][2]
+        assert isinstance(string_val, Literal)
+        assert "%20" not in string_val.value
+
 
 def test_negative_no_default_or_value():
     with pytest.raises(NoDefaultOrValueUrlError):

tests/virtual1.csv

@@ -1,3 +1,3 @@
-expense,amount
-taxi,25
-dinner,50
+expense,amount,description
+taxi,25,taxi from airport to hotel
+dinner,50,dinner on Tuesday

tests/virtual1.default.datatype.csv-metadata.json

@@ -8,16 +8,40 @@
   "url": "http://example.org/simple.csv",
   "tableSchema": {
     "aboutUrl": "ns:sub-{_row}",
-    "columns": [{
+    "columns": [
+      {
       "titles": "t1"
-    },{
+      },
+      {
       "titles": "t2"
-    }, {
+      },
+      {
+        "name": "description",
+        "titles": "description",
+        "suppressOutput": true
+      },
+      {
       "name": "v1",
       "virtual": true,
       "propertyUrl": "ns:active",
       "default": "true",
       "datatype": "boolean"
-    }]
+      },
+      {
+      "name": "v2",
+      "virtual": true,
+      "propertyUrl": "ns:stringprop1",
+      "default": "some string",
+      "datatype": "string"
+      },
+      {
+      "name": "v3",
+      "virtual": true,
+      "propertyUrl": "ns:stringprop2",
+      "default": "{description}",
+      "datatype": "string"
+      }
+
+    ]
   }
 }