Bfsvtab Extension (#3)

* Add new extension for breadth-first search queries

- https://github.com/abetlen/sqlite3-bfsvtab-ext

+ tests

* Simplify Makefile

- extract downloading for vendored libs to a bash script

* Rename gen -> genMake

* Rename runner -> testrunner for more clarity.

* Makefile: add cleanup for SQLITE downloading



.github/workflows/build.yml

@@ -40,7 +40,7 @@ jobs:
           sudo apt install nodejs yarn
           sudo npm install --global yarn
           make compile-linux
-          pushd runner && yarn install && popd
+          pushd testrunner && yarn install && popd
           bin/test.sh CI
 
       - name: Build for Windows

CHANGELOG.md

@@ -1,3 +1,10 @@
+### 0.8.9-3360
+
+- Files: https://github.com/mindreframer/sqlean/releases/tag/0.8.9-3360
+
+- EXT: bfsvtab extension (breadth-first search VTab)
+  - https://github.com/abetlen/sqlite3-bfsvtab-ext
+
 ### 0.8.8-3360
 
 - Files: https://github.com/mindreframer/sqlean/releases/tag/0.8.8-3360

Makefile

@@ -1,6 +1,6 @@
-## **** GENERATED CODE! see gen/generateMakefile.js for more details!
+## **** GENERATED CODE! see genMake/index.js for more details!
 
-.PHONY: prepare-dist download-sqlite download-native compile-linux compile-windows compile-macos test
+.PHONY: prepare-dist download-sqlite compile-linux compile-windows compile-macos
 
 SQLITE_RELEASE_YEAR ?= "2021"
 SQLITE_VERSION ?= "3360000"
@@ -14,22 +14,11 @@ download-sqlite:
 	curl -L http://sqlite.org/$(SQLITE_RELEASE_YEAR)/sqlite-amalgamation-$(SQLITE_VERSION).zip --output src.zip
 	unzip src.zip
 	mv sqlite-amalgamation-$(SQLITE_VERSION)/* src
-
-download-native:
-	curl -L https://github.com/sqlite/sqlite/raw/master/ext/misc/json1.c --output src/sqlite3-json1.c
-	curl -L https://github.com/sqlite/sqlite/raw/master/ext/misc/series.c --output src/sqlite3-series.c
-	curl -L https://github.com/sqlite/sqlite/raw/master/ext/misc/spellfix.c --output src/sqlite3-spellfix.c
-	curl -L https://github.com/sqlite/sqlite/raw/master/ext/misc/memstat.c --output src/sqlite3-memstat.c
-	curl -L https://github.com/shawnw/useful_sqlite_extensions/raw/master/src/math_funcs.c --output src/sqlite3-shawnw_math.c
-	patch -p0 < diffs/sqlite3-shawnw_math.diff
-	curl -L https://github.com/shawnw/useful_sqlite_extensions/raw/master/src/bloom_filter.c --output src/sqlite3-bloom_filter.c
-	patch -p0 < diffs/sqlite3-bloom_filter.diff
-
-test:
-	bin/test.sh
-
+	rm -rf sqlite-amalgamation-$(SQLITE_VERSION)/
+	rm src.zip
 
 compile-linux:
+	gcc -fPIC -shared src/sqlite3-bfsvtab.c -o dist/bfsvtab.so -lm
 	gcc -fPIC -shared src/sqlite3-bloom_filter.c -o dist/bloom_filter.so -lm
 	gcc -fPIC -shared src/sqlite3-crypto.c src/crypto/*.c -o dist/crypto.so -lm
 	gcc -fPIC -shared src/sqlite3-ipaddr.c -o dist/ipaddr.so -lm
@@ -46,6 +35,7 @@ compile-linux:
 	gcc -fPIC -shared src/sqlite3-vsv.c -o dist/vsv.so -lm
 
 compile-macos:
+	gcc -fPIC -dynamiclib -I src src/sqlite3-bfsvtab.c -o dist/bfsvtab.dylib -lm
 	gcc -fPIC -dynamiclib -I src src/sqlite3-bloom_filter.c -o dist/bloom_filter.dylib -lm
 	gcc -fPIC -dynamiclib -I src src/sqlite3-crypto.c src/crypto/*.c -o dist/crypto.dylib -lm
 	gcc -fPIC -dynamiclib -I src src/sqlite3-ipaddr.c -o dist/ipaddr.dylib -lm
@@ -62,6 +52,7 @@ compile-macos:
 	gcc -fPIC -dynamiclib -I src src/sqlite3-vsv.c -o dist/vsv.dylib -lm
 
 compile-windows:
+	gcc -shared -I. src/sqlite3-bfsvtab.c -o dist/bfsvtab.dll -lm
 	gcc -shared -I. src/sqlite3-bloom_filter.c -o dist/bloom_filter.dll -lm
 	gcc -shared -I. src/sqlite3-crypto.c src/crypto/*.c -o dist/crypto.dll -lm
 	gcc -shared -I. src/sqlite3-json1.c -o dist/json1.dll -lm
@@ -77,6 +68,7 @@ compile-windows:
 	gcc -shared -I. src/sqlite3-vsv.c -o dist/vsv.dll -lm
 
 compile-windows-32:
+	gcc -shared -I. src/sqlite3-bfsvtab.c -o dist/bfsvtab-win32.dll -lm
 	gcc -shared -I. src/sqlite3-bloom_filter.c -o dist/bloom_filter-win32.dll -lm
 	gcc -shared -I. src/sqlite3-crypto.c src/crypto/*.c -o dist/crypto-win32.dll -lm
 	gcc -shared -I. src/sqlite3-json1.c -o dist/json1-win32.dll -lm

bin/test.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
 
 ## call with `bin/test.sh CI` for CI testing!
-node runner/index.js $@
+node testrunner/index.js $@
 exit $?

bin/updateMake.sh

@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-node gen/generateMakefile.js
+node genMake/index.js

bin/vendorLibs.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+### official SQLite extensions
+
+curl -L https://github.com/sqlite/sqlite/raw/master/ext/misc/json1.c --output src/sqlite3-json1.c
+curl -L https://github.com/sqlite/sqlite/raw/master/ext/misc/series.c --output src/sqlite3-series.c
+curl -L https://github.com/sqlite/sqlite/raw/master/ext/misc/spellfix.c --output src/sqlite3-spellfix.c
+curl -L https://github.com/sqlite/sqlite/raw/master/ext/misc/memstat.c --output src/sqlite3-memstat.c
+
+### community maintained extensions
+
+curl -L https://github.com/shawnw/useful_sqlite_extensions/raw/master/src/math_funcs.c --output src/sqlite3-shawnw_math.c
+patch -p0 < diffs/sqlite3-shawnw_math.diff
+
+curl -L https://github.com/shawnw/useful_sqlite_extensions/raw/master/src/bloom_filter.c --output src/sqlite3-bloom_filter.c
+patch -p0 < diffs/sqlite3-bloom_filter.diff
+
+curl -L https://github.com/abetlen/sqlite3-bfsvtab-ext/raw/main/bfsvtab.c --output src/sqlite3-bfsvtab.c

diffs/Readme.md

@@ -1,3 +1,5 @@
+### Custom patches to fix some minor issues on our side
+
 $ cp src/sqlite3-shawnw_math.c src/sqlite3-shawnw_math-fixed.c
 
 ## fix src/sqlite3-shawnw_math-fixed.c...

gen/generateMakefile.js → genMake/index.js

@@ -2,6 +2,7 @@ const fs = require("fs");
 const { GenBase } = require("./genbase");
 
 const activeExts = [
+  ["src/sqlite3-bfsvtab.c", "bfsvtab"],
   ["src/sqlite3-bloom_filter.c", "bloom_filter"],
   ["src/sqlite3-crypto.c src/crypto/*.c", "crypto"],
   ["src/sqlite3-ipaddr.c", "ipaddr"],
@@ -22,13 +23,12 @@ const activeExtsWindows = activeExts.filter((elem) => elem[1] != "ipaddr");
 /**
  * Keeps Makefile consistent
  *
- * to update run: `node gen/generateMakefile.js`
+ * to update run: `node genMake/index.js`
  */
 
 class Generator extends GenBase {
   run() {
-    const makeheader = fs.readFileSync("./gen/makeheader.txt", "utf-8");
-    this.push("");
+    const makeheader = fs.readFileSync("./genMake/makeheader.txt", "utf-8");
     this.push("");
     this.genLinux();
     this.push("");

genMake/makeheader.txt

@@ -0,0 +1,18 @@
+## **** GENERATED CODE! see genMake/index.js for more details!
+
+.PHONY: prepare-dist download-sqlite compile-linux compile-windows compile-macos
+
+SQLITE_RELEASE_YEAR ?= "2021"
+SQLITE_VERSION ?= "3360000"
+
+
+prepare-dist:
+	mkdir -p dist
+	rm -f dist/*
+
+download-sqlite:
+	curl -L http://sqlite.org/$(SQLITE_RELEASE_YEAR)/sqlite-amalgamation-$(SQLITE_VERSION).zip --output src.zip
+	unzip src.zip
+	mv sqlite-amalgamation-$(SQLITE_VERSION)/* src
+	rm -rf sqlite-amalgamation-$(SQLITE_VERSION)/
+	rm src.zip