使用 Python 的 SQLite JSON1 和 FTS5 扩展

json1

json1

fts5

fts5

使用入门

fossil

tcl

awk

~/bin/jqlite

export JQLITE="$HOME/bin/jqlite"
mkdir -p $JQLITE
cd $JQLITE

fossil

fossil clone http://www.sqlite.org/cgi/src sqlite.fossil
fossil open sqlite.fossil

curl 'https://www.sqlite.org/src/tarball/sqlite.tar.gz?ci=trunk' | tar xz
mv sqlite/* .

$JQLITE

利用 json1 和 fts5 编译 SQLite

json1

fts5

configure -> make -> make install

export CFLAGS="-DSQLITE_ENABLE_COLUMN_METADATA=1 \
-DSQLITE_ENABLE_DBSTAT_VTAB=1 \
-DSQLITE_ENABLE_FTS3=1 \
-DSQLITE_ENABLE_FTS3_PARENTHESIS=1 \
-DSQLITE_ENABLE_FTS5=1 \
-DSQLITE_ENABLE_JSON1=1 \
-DSQLITE_ENABLE_RTREE=1 \
-DSQLITE_ENABLE_UNLOCK_NOTIFY \
-DSQLITE_ENABLE_UPDATE_DELETE_LIMIT \
-DSQLITE_SECURE_DELETE \
-DSQLITE_SOUNDEX \
-DSQLITE_TEMP_STORE=3 \
-fPIC"
LIBS="-lm" ./configure --prefix=$JQLITE --enable-static --enable-shared
make
make install

lib/libsqlite3.a

创建 pysqlite

pysqlite

sqlite3

setup.cfg

include

lib

git clone https://github.com/ghaering/pysqlite cd pysqlite/
cp ../sqlite3.c .
echo -e "library_dirs=$JQLITE/lib" >> setup.cfg
echo -e "include_dirs=$JQLITE/include" >> setup.cfg
LIBS="-lm" python setup.py build_static

build/lib.linux-xfoobar/

>>> from pysqlite2 import dbapi2 as sqlite
>>> conn = sqlite.connect(':memory:')
>>> conn.execute('CREATE VIRTUAL TABLE testing USING fts5(data);')
<pysqlite2.dbapi2.Cursor object at 0x7ff7d0a2dc60>
>>> conn.execute('SELECT json(?)', (1337,)).fetchone()
(u'1337',)

python setup.py

pysqlite2

build/lib.linux.../

$PYTHONPATH

virtualenv

pysqlite

setup.py

创建 apsw

pysqlite

cd $JQLITE
git clone https://github.com/rogerbinns/apsw cd apsw
cp ../sqlite3{ext.h,.h,.c} .
echo -e "library_dirs=$SQLITE_SRC/lib" >> setup.cfg
echo -e "include_dirs=$SQLITE_SRC/include" >> setup.cfg
LIBS="-lm" python setup.py build

apsw

build/libXXX

>>> import apsw
>>> conn = apsw.Connection(':memory:')
>>> cursor = conn.cursor()
>>> cursor.execute('CREATE VIRTUAL TABLE testing USING fts5(data);')
<apsw.Cursor at 0x7fcf6b17fa80>
>>> cursor.execute('SELECT json(?)', (1337,)).fetchone()
(u'1337',)

Python setup.py

apsw

apsw.so

build/lib.linux.../

apsw

apsw

setup.py

使用 JSON 扩展

json1

json_tree

json_each

[{
"title": "My List of Python and SQLite Resources",
"url": "http://charlesleifer.com/blog/my-list-of-python-and-sqlite-resources/",
"metadata": {"tags": ["python", "sqlite"]}
},
{
"title": "Using SQLite4's LSM Storage Engine as a Stand-alone NoSQL Database with Python"
"url": "http://charlesleifer.com/blog/using-sqlite4-s-lsm-storage-engine-as-a-stand-alone-nosql-database-with-python/",
"metadata": {"tags": ["nosql", "python", "sqlite", "cython"]}
},
...]

填充数据库

>>> import json, urllib2
>>> fh = urllib2.urlopen('http://media.charlesleifer.com/downloads/misc/blogs.json')
>>> data = json.loads(fh.read())
>>> data[0]
{u'metadata': {u'tags': [u'python', u'sqlite']},
u'title': u'My List of Python and SQLite Resources',
u'url': u'http://charlesleifer.com/blog/my-list-of-python-and-sqlite-resources/'}

pysqlite

pysqlite2

sqlite3

pysqlite2

>>> from pysqlite2 import dbapi2 as jqlite
>>> from peewee import *
>>> from playhouse.sqlite_ext import *
>>> class JQLiteDatabase(SqliteExtDatabase):
...     def _connect(self, database, **kwargs):
...         conn = jqlite.connect(database, **kwargs)
...         conn.isolation_level = None
...         self._add_conn_hooks(conn)
...         return conn
...
>>> db = JQLiteDatabase(':memory:')

TEXT

TextField

>>> class Entry(Model):
...     data = TextField()
...     class Meta:
...         database = db
...
>>> Entry.create_table()
>>> with db.atomic():
...     for entry_json in data:
...         Entry.create(data=json.dumps(entry_json))
...

JSON 的功能

json_extract()

Entry

URL

>>> title = fn.json_extract(Entry.data, '$.title')
>>> query = (Entry
...          .select(title.alias('title'))
...          .order_by(title)
...          .limit(5))
...
>>> [row for row in query.dicts()]
[{'title': u'A Tour of Tagging Schemas: Many-to-many, Bitmaps and More'},
{'title': u'Alternative Redis-Like Databases with Python'},
{'title': u'Building the SQLite FTS5 Search Extension'},
{'title': u'Connor Thomas Leifer'},
{'title': u'Extending SQLite with Python'}]

SELECT json_extract("t1"."data", '$.title') AS title
FROM "entry" AS t1
ORDER BY json_extract("t1"."data", '$.title')
LIMIT 5

json_each()

>>> from peewee import Entity
>>> tags_src = fn.json_each(Entry.data, '$.metadata.tags').alias('tags')
>>> tags_ref = Entity('tags')

>>> query = (Entry
...          .select(title.alias('title'))
...          .from_(Entry, tags_src)
...          .where(tags_ref.value == 'sqlite')
...          .order_by(title))
...
>>> [row for row, in query.tuples()]
[u'Building the SQLite FTS5 Search Extension',
u'Extending SQLite with Python',
u'Meet Scout, a Search Server Powered by SQLite',
u'My List of Python and SQLite Resources',
u'Querying Tree Structures in SQLite using Python and the Transitive Closure Extension',
u"Using SQLite4's LSM Storage Engine as a Stand-alone NoSQL Database with Python",
u'Web-based SQLite Database Browser, powered by Flask and Peewee']

SELECT json_extract("t1"."data", '$.title') AS title
FROM
"entry" AS t1,
json_each("t1"."data", '$.metadata.tags') AS tags
WHERE ("tags"."value" = 'sqlite')
ORDER BY json_extract("t1"."data", '$.title')

json_each()

tags_src

tags_ref

>>> query = (Entry
...          .select(
...              title.alias('title'),
...              fn.group_concat(tags_ref.value, ', ').alias('tags'))
...          .from_(Entry, tags_src)
...          .group_by(title)
...          .limit(5))
...
>>> [row for row in query.tuples()]
[(u'A Tour of Tagging Schemas: Many-to-many, Bitmaps and More',
u'peewee, sql, python'),
(u'Alternative Redis-Like Databases with Python',
u'python, walrus, redis, nosql'),
(u'Building the SQLite FTS5 Search Extension',
u'sqlite, search, python, peewee'),
(u'Connor Thomas Leifer', u'thoughts'),
(u'Extending SQLite with Python', u'peewee, python, sqlite')]

SELECT
json_extract("t1"."data", '$.title') AS title,
group_concat("tags"."value", ', ') AS tags
FROM
"entry" AS t1,
json_each("t1"."data", '$.metadata.tags') AS tags
GROUP BY json_extract("t1"."data", '$.title')
LIMIT 5

json_tree()

json_each()

json_tree()

json_each()

json_tree()

>>> tree = fn.json_tree(Entry.data, '$').alias('tree')
>>> parent = fn.json_tree(Entry.data, '$').alias('parent')

>>> tree_ref = Entity('tree')
>>> parent_ref = Entity('parent')

>>> query = (Entry
...          .select(title.alias('title'))
...          .from_(Entry, tree, parent)
...          .where(
...              (tree_ref.parent == parent_ref.id) &
...              (parent_ref.key == 'tags') &
...              (tree_ref.value == 'sqlite'))
...          .order_by(title))
...
>>> [title for title, in query.tuples()]
[u'Building the SQLite FTS5 Search Extension',
u'Extending SQLite with Python',
u'Meet Scout, a Search Server Powered by SQLite',
u'My List of Python and SQLite Resources',
u'Querying Tree Structures in SQLite using Python and the Transitive Closure Extension',
u"Using SQLite4's LSM Storage Engine as a Stand-alone NoSQL Database with Python",
u'Web-based SQLite Database Browser, powered by Flask and Peewee']

SELECT json_extract("t1"."data", '$.title') AS title
FROM
"entry" AS t1,
json_tree("t1"."data", '$') AS tree,
json_tree("t1"."data", '$') AS parent
WHERE (
("tree"."parent" = "parent"."id") AND
("parent"."key" = 'tags') AND
("tree"."value" = 'sqlite'))
ORDER BY json_extract("t1"."data", '$.title')

FTS5 与 Python

Entry

FTS5Model

fts5

class EntryIndex(FTS5Model):
title = SearchField()
url = SearchField(unindexed=True)
class Meta:
database = db
options = {'tokenize': 'porter', 'prefix': '2,3'}

EntryIndex.create_table()

fts5

CREATE VIRTUAL TABLE "entryindex" USING fts5 (
"title" ,
"url"  UNINDEXED,
prefix=2,3,
tokenize=porter)

title = fn.json_extract(Entry.data, '$.title').alias('title')
url = fn.json_extract(Entry.data, '$.url').alias('url')
query = Entry.select(title, url).dicts()
with db.atomic():
for entry in query:
EntryIndex.create(**entry)

>>> query = EntryIndex.search('sqlite').limit(3)
>>> for result in query:
...     print result.title

Extending SQLite with Python
Building the SQLite FTS5 Search Extension
My List of Python and SQLite Resources

SELECT "t1"."title", "t1"."url"
FROM "entryindex" AS t1
WHERE ("entryindex" MATCH 'sqlite')
ORDER BY rank

>>> query = EntryIndex.search('sqlite AND python', with_score=True)
>>> for result in query:
...     print round(result.score, 3), result.title

-1.259 Extending SQLite with Python
-1.059 My List of Python and SQLite Resources
-0.838 Querying Tree Structures in SQLite using Python and the Transitive Closure Extension

SELECT "t1"."title", "t1"."url", rank AS score
FROM "entryindex" AS t1
WHERE ("entryindex" MATCH 'sqlite AND python')
ORDER BY rank

fts5

感谢阅读

json1.c