Merge pull request #82 from bcgsc/fix-wrappers

Fix wrappers

.gitignore

@@ -1,3 +1,4 @@
 /install/
 /__build-compile/
 /Doxyfile.bak
+__pycache__/

meson.build

@@ -1,5 +1,5 @@
 project('btllib', 'cpp',
-        version : '1.5.0',
+        version : '1.5.1',
         license : 'GPL3',
         default_options : [ 'cpp_std=c++11', 'warning_level=3', 'werror=true' ],
         meson_version : '>= 0.60.0')

scripts/test-wrappers

@@ -9,5 +9,6 @@ set -e
 
 cd "${MESON_SOURCE_ROOT}"
 
-cd install/lib/btllib/python
-python3 -c 'import btllib; nt = btllib.NtHash("ACTG", 2, 3); nt.roll(); b = btllib.BloomFilter(1024, 2); b.insert(nt.hashes())'
+export PYTHONPATH="$(pwd)/install/lib/btllib/python"
+cd tests/python
+python -m unittest

scripts/wrap

@@ -1,33 +1,20 @@
-#!/bin/bash
+#!/usr/bin/env python3
 
-if [ -z "${MESON_SOURCE_ROOT}" ]; then
-  echo "[ERROR] This script can only be ran with meson!"
-  exit 1
-fi
+import os
+import re
+import functools
+import subprocess
 
-set -e
 
-# Remove old wrapper files
-rm -f ${MESON_SOURCE_ROOT}/wrappers/python/btllib.py
+class PythonWrapper:
 
-# Generate python swig files
-cd ${MESON_SOURCE_ROOT}
+    INTERFACE_TEMPLATE = """%module btllib
 
-include_files=$(scripts/get-files include)
-
-echo "%module btllib
-
-%{
+%{{
 #define SWIG_FILE_WITH_INIT
-" > wrappers/python/btllib.i
 
-for file in ${include_files}; do
-  relative=$(scripts/get-include-relative ${file})
-  path="${relative}/$(basename ${file})"
-  echo "#include \"$path\"" >> wrappers/python/btllib.i
-done
-
-echo "%}
+{cpp_includes}
+%}}
 
 %include <stdint.i>
 %include <typemaps.i>
@@ -42,23 +29,101 @@ echo "%}
 %include <std_vector.i>
 %include <stl.i>
 
-%include \"../extra_common.i\"
-%include \"extra.i\"
-" >> wrappers/python/btllib.i
+%include "../extra_common.i"
+%include "extra.i"
+
+{swig_includes}
+
+%include "../extra_templates.i"
+"""
+
+    def __init__(self, out_path: str, include_path: str):
+        self._out_path = out_path
+        self._include_path = include_path
+
+    def _remove_old_files(self):
+        if os.path.exists(os.path.join(self._out_path, 'btllib.py')):
+            os.remove(os.path.join(self._out_path, 'btllib.py'))
+
+    def _load_include_file_names(self):
+        include_files = os.listdir(os.path.join(self._include_path, 'btllib'))
+        code_filter = functools.partial(re.match, r'.*\.(h|hpp|cpp|cxx)$')
+        code_files = filter(code_filter, include_files)
+        path_fixer = functools.partial(os.path.join, 'btllib')
+        include_files = map(path_fixer, code_files)
+        return list(include_files)
+
+    def _update_interface_file(self):
+        with open(os.path.join(self._out_path, 'btllib.i')) as f:
+            current_lines = list(map(str.strip, f.readlines()))
+        # Add include statements only for newly added files.
+        # This will prevent changing the ordering of the includes.
+        include_files = []
+        for line in current_lines:
+            include_match = re.match(r'#include "(.*)"', line)
+            if include_match:
+                include_files.append(include_match.group(1))
+        for file in self._load_include_file_names():
+            if file not in include_files:
+                include_files.append(file)
+        cpp = os.linesep.join(f'#include "{f}"' for f in include_files)
+        swig = os.linesep.join(f'%include "{f}"' for f in include_files)
+        interface = PythonWrapper.INTERFACE_TEMPLATE.format(cpp_includes=cpp,
+                                                            swig_includes=swig)
+        with open(os.path.join(self._out_path, 'btllib.i'), 'w') as f:
+            f.write(interface)
+
+    def _call_swig(self):
+        swig_cmd = [
+            'swig', '-python', '-fastproxy', '-fastdispatch', '-builtin',
+            '-c++', f'-I{self._include_path}', 'btllib.i'
+        ]
+        return subprocess.run(swig_cmd,
+                              capture_output=True,
+                              text=True,
+                              cwd=self._out_path)
+
+    def _fix_unsigned_long_long(self):
+        # The following is necessary because SWIG produces inconsistent code that cannot be compiled on all platforms.
+        # On some platforms, uint64_t is unsigned long int and unsigned long long int on others.
+        cxx_path = os.path.join(self._out_path, 'btllib_wrap.cxx')
+        with open(cxx_path) as f:
+            cxx_contents = f.read()
+        cxx_contents = cxx_contents.replace('unsigned long long', 'uint64_t')
+        with open(cxx_path, 'w') as f:
+            f.write(cxx_contents)
+
+    def generate(self):
+        self._remove_old_files()
+        self._update_interface_file()
+        swig_result = self._call_swig()
+        self._fix_unsigned_long_long()
+        return swig_result
+
+
+def check_meson_env():
+    if 'MESON_SOURCE_ROOT' not in os.environ:
+        print("[ERROR] This script can only be ran with meson!")
+        exit(1)
+
+
+def check_swig_result(swig_result, wrapper_language: str):
+    lang = wrapper_language.capitalize()
+    if swig_result.returncode == 0:
+        print(f"{lang} wrappers generated successfully")
+    elif swig_result.returncode == 1:
+        print(f"Error when calling SWIG for {lang}, stderr:")
+        print(swig_result.stderr)
 
-for file in ${include_files}; do
-  relative=$(scripts/get-include-relative ${file})
-  path="${relative}/$(basename ${file})"
-  echo "%include \"$path\"" >> wrappers/python/btllib.i
-done
 
-echo "%include \"../extra_templates.i\"" >> wrappers/python/btllib.i
+def main():
+    check_meson_env()
+    project_root = os.environ['MESON_SOURCE_ROOT']
+    include_path = os.path.join(project_root, 'include')
+    python_dir = os.path.join(project_root, 'wrappers', 'python')
+    swig_result_py = PythonWrapper(python_dir, include_path).generate()
+    check_swig_result(swig_result_py, 'python')
 
-ln -sf $PWD/include wrappers/python/
-cd wrappers/python
-swig -python -py3 -fastproxy -fastdispatch -builtin -c++ -Iinclude btllib.i
-rm -f include
 
-# The following line is necessary because SWIG produces inconsistent code that cannot be compiled on all platforms. On some platforms, uint64_t is unsigned long int and unsigned long long int on others.
-sed -i'.tmp' 's~unsigned long long~uint64_t~g' btllib_wrap.cxx
-rm -f btllib_wrap.cxx.tmp
+if __name__ == '__main__':
+    main()

tests/python/single_seq.fa

@@ -0,0 +1,2 @@
+>1
+CGCGTGAAAGCAAAACAAGA

tests/python/test_bloom_filter.py

@@ -0,0 +1,10 @@
+import btllib
+import unittest
+
+class BloomFilterTests(unittest.TestCase):
+
+    def test_seq_insertion(self):
+        seq, k = 'AGTCATCGACTGATGC', 5
+        bf = btllib.KmerBloomFilter(1024, 3, k)
+        bf.insert(seq)
+        self.assertEqual(bf.contains('TCATC'), 1)

tests/python/test_seq_reader.py

@@ -0,0 +1,16 @@
+import os
+import btllib
+import unittest
+
+
+class SeqReaderTests(unittest.TestCase):
+
+    def setUp(self):
+        self.base_dir = os.path.dirname(__file__)
+
+    def test_seq_reader_single_seq(self):
+        seq = 'CGCGTGAAAGCAAAACAAGA'
+        path = os.path.join(self.base_dir, 'single_seq.fa')
+        with btllib.SeqReader(path, btllib.SeqReaderFlag.SHORT_MODE) as reader:
+            read = [record.seq for record in reader]
+        self.assertListEqual([seq], read)

wrappers/python/btllib.i

@@ -3,30 +3,30 @@
 %{
 #define SWIG_FILE_WITH_INIT
 
+#include "btllib/indexlr.hpp"
 #include "btllib/graph.hpp"
-#include "btllib/seq_writer.hpp"
-#include "btllib/seq_reader_sam_module.hpp"
-#include "btllib/nthash_lowlevel.hpp"
-#include "btllib/counting_bloom_filter.hpp"
-#include "btllib/mi_bloom_filter.hpp"
-#include "btllib/data_stream.hpp"
+#include "btllib/order_queue.hpp"
 #include "btllib/bloom_filter.hpp"
-#include "btllib/seq_reader.hpp"
-#include "btllib/cstring.hpp"
 #include "btllib/seq_reader_gfa2_module.hpp"
-#include "btllib/indexlr.hpp"
-#include "btllib/random_seq_generator.hpp"
+#include "btllib/seq_reader_fasta_module.hpp"
+#include "btllib/counting_bloom_filter.hpp"
+#include "btllib/seq_reader.hpp"
+#include "btllib/seq_reader_sam_module.hpp"
+#include "btllib/seq_reader_multiline_fasta_module.hpp"
 #include "btllib/util.hpp"
-#include "btllib/order_queue.hpp"
+#include "btllib/seq_writer.hpp"
 #include "btllib/seq.hpp"
-#include "btllib/seq_reader_fastq_module.hpp"
-#include "btllib/nthash.hpp"
-#include "btllib/status.hpp"
 #include "btllib/nthash_consts.hpp"
+#include "btllib/nthash_lowlevel.hpp"
+#include "btllib/data_stream.hpp"
 #include "btllib/process_pipeline.hpp"
-#include "btllib/seq_reader_fasta_module.hpp"
-#include "btllib/seq_reader_multiline_fasta_module.hpp"
 #include "btllib/seq_reader_multiline_fastq_module.hpp"
+#include "btllib/status.hpp"
+#include "btllib/seq_reader_fastq_module.hpp"
+#include "btllib/mi_bloom_filter.hpp"
+#include "btllib/cstring.hpp"
+#include "btllib/nthash.hpp"
+#include "btllib/random_seq_generator.hpp"
 %}
 
 %include <stdint.i>
@@ -45,28 +45,29 @@
 %include "../extra_common.i"
 %include "extra.i"
 
+%include "btllib/indexlr.hpp"
 %include "btllib/graph.hpp"
-%include "btllib/seq_writer.hpp"
-%include "btllib/seq_reader_sam_module.hpp"
-%include "btllib/nthash_lowlevel.hpp"
-%include "btllib/counting_bloom_filter.hpp"
-%include "btllib/mi_bloom_filter.hpp"
-%include "btllib/data_stream.hpp"
+%include "btllib/order_queue.hpp"
 %include "btllib/bloom_filter.hpp"
-%include "btllib/seq_reader.hpp"
-%include "btllib/cstring.hpp"
 %include "btllib/seq_reader_gfa2_module.hpp"
-%include "btllib/indexlr.hpp"
-%include "btllib/random_seq_generator.hpp"
+%include "btllib/seq_reader_fasta_module.hpp"
+%include "btllib/counting_bloom_filter.hpp"
+%include "btllib/seq_reader.hpp"
+%include "btllib/seq_reader_sam_module.hpp"
+%include "btllib/seq_reader_multiline_fasta_module.hpp"
 %include "btllib/util.hpp"
-%include "btllib/order_queue.hpp"
+%include "btllib/seq_writer.hpp"
 %include "btllib/seq.hpp"
-%include "btllib/seq_reader_fastq_module.hpp"
-%include "btllib/nthash.hpp"
-%include "btllib/status.hpp"
 %include "btllib/nthash_consts.hpp"
+%include "btllib/nthash_lowlevel.hpp"
+%include "btllib/data_stream.hpp"
 %include "btllib/process_pipeline.hpp"
-%include "btllib/seq_reader_fasta_module.hpp"
-%include "btllib/seq_reader_multiline_fasta_module.hpp"
 %include "btllib/seq_reader_multiline_fastq_module.hpp"
+%include "btllib/status.hpp"
+%include "btllib/seq_reader_fastq_module.hpp"
+%include "btllib/mi_bloom_filter.hpp"
+%include "btllib/cstring.hpp"
+%include "btllib/nthash.hpp"
+%include "btllib/random_seq_generator.hpp"
+
 %include "../extra_templates.i"