update readme

.gitignore

@@ -0,0 +1,88 @@
+.*.swp
+*.o
+*.log
+dump.rdb
+redis-benchmark
+redis-check-aof
+redis-check-rdb
+redis-check-dump
+redis-cli
+redis-sentinel
+redis-server
+doc-tools
+release
+misc/*
+src/release.h
+appendonly.aof
+SHORT_TERM_TODO
+release.h
+src/transfer.sh
+src/configs
+redis.ds
+src/redis.conf
+src/nodes.conf
+deps/lua/src/lua
+deps/lua/src/luac
+deps/lua/src/liblua.a
+.make-*
+.prerequisites
+*.dSYM
+Makefile.dep
+
+
+### C++ template
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+### JetBrains template
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+.idea/*
+cmake-build-debug
+
+## File-based project format:
+*.iws
+
+## Plugin-specific files:
+
+# IntelliJ
+/out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties

Dockerfile

@@ -0,0 +1,14 @@
+FROM ubuntu:19.04
+
+RUN apt-get update \
+ && apt-get install -y ca-certificates \
+                       curl \
+                       git \
+                       make \
+                       libtinfo5
+
+RUN curl -L https://github.com/CraneStation/wasi-sdk/releases/download/wasi-sdk-6/wasi-sdk-6.0-linux.tar.gz | tar xz --strip-components=1 -C /
+
+VOLUME /code
+WORKDIR /code
+CMD make

Makefile

@@ -0,0 +1,24 @@
+TARGET = sqlite3
+CC = /opt/wasi-sdk/bin/clang
+SYSROOT = /opt/wasi-sdk/share/wasi-sysroot
+TARGET_TRIPLE = wasm32-unknown-wasi
+CFLAGS = -nostartfiles -fvisibility=hidden
+LDFLAGS = -Wl,--no-entry,--demangle,--allow-undefined
+EXPORT_FUNCS = --export=allocate,--export=deallocate,--export=invoke,--export=load,--export=store,--export=sqlite_allocate,--export=sqlite_deallocate,--export=sqlite_invoke,--export=sqlite_load,--export=sqlite_store
+SQLITE_SRC = src/alter.c src/analyze.c src/attach.c src/auth.c src/backup.c src/bitvec.c src/btmutex.c src/btree.c src/build.c src/callback.c src/complete.c src/ctime.c src/date.c src/dbpage.c src/dbstat.c src/delete.c src/expr.c src/fault.c src/fkey.c src/fts3.c src/fts3_write.c src/fts3_aux.c src/fts3_expr.c src/fts3_hash.c src/fts3_icu.c src/fts3_porter.c src/fts3_snippet.c src/fts3_tokenize_vtab.c src/fts3_tokenizer.c src/fts3_tokenizer1.c src/fts3_unicode.c src/fts3_unicode2.c src/fts5.c src/func.c src/global.c src/hash.c src/insert.c src/json1.c src/legacy.c src/loadext.c src/main.c src/malloc.c src/memdb.c src/mem0.c src/mem1.c src/mem2.c src/memjournal.c src/notify.c src/opcodes.c src/os.c src/pager.c src/parse.c src/pcache.c src/pcache1.c src/pragma.c src/prepare.c src/printf.c src/random.c src/resolve.c src/rowset.c src/rtree.c src/select.c src/sqlite3session.c src/status.c src/stmt.c src/table.c src/tokenize.c src/treeview.c src/trigger.c src/update.c src/upsert.c src/userauth.c src/utf.c src/util.c src/vacuum.c src/vdbe.c src/vdbeapi.c src/vdbeaux.c src/vdbeblob.c src/vdbemem.c src/vdbesort.c src/vdbetrace.c src/vtab.c src/wal.c src/walker.c src/where.c src/wherecode.c src/whereexpr.c src/window.c
+WRAPPER_SRC = src/wrapper.c
+SQLITE_FLAGS = -DSQLITE_CORE -D_HAVE_SQLITE_CONFIG_H -DSQLITE_ENABLE_JSON1 -DBUILD_sqlite -DNDEBUG -DSQLITE_THREADSAFE=0 -DHAVE_READLINE=0 -DHAVE_EDITLINE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_DBSTAT_VTAB -DSQLITE_ENABLE_OFFSET_SQL_FUNC -DSQLITE_ENABLE_DESERIALIZE -DSQLITE_INTROSPECTION_PRAGMAS -DSQLITE_OMIT_POPEN
+SDK = sdk/logger.c
+
+.PHONY: default all clean
+
+default: $(TARGET)
+all: default
+
+$(TARGET): $(SDK) $(SQLITE_SRC) $(WRAPPER_SRC)
+	$(CC) -O3 --sysroot=$(SYSROOT) --target=$(TARGET_TRIPLE) $(SQLITE_FLAGS) $(CFLAGS) $(LDFLAGS) -Wl,$(EXPORT_FUNCS) $^ -o $@.wasm
+
+.PRECIOUS: $(TARGET)
+
+clean:
+	-rm -f $(TARGET).wasm

Readme.md

@@ -0,0 +1,85 @@
+# SQLite
+
+Sqlite fork ported to WebAssembly and adapted for the Fluence network. Could be launched and played on the Fluence [dashboard](http://dash.fluence.network/deploy/sqlite).
+
+# How to build
+
+This app could be built either with docker 
+
+```bash
+docker-compose up
+```
+
+or by Makefile with [wasi-sdk](https://github.com/CraneStation/wasi-sdk) installed
+```bash
+make
+```
+
+# How to use
+
+At now, this fork exports five API functions:
+```cpp
+
+/**
+ * Executes given SQL request and returns result in as a pointer to the following structure: | result size (4 bytes, le)| result (size bytes) |.
+ *
+ * @param sql a pointer to the supplied sql request
+ * @param length a size of the supplied sql request
+ * @return a pointer to the struct contains result_size and result
+ */
+char *invoke(const char *sql, size_t length);
+
+/**
+ * Allocates a memory region of a given size. Could be used by Wasm execution environments for byte array passing.
+ *
+ * @param size a size of allocated memory region
+ * @return a pointer to the allocated memory region
+ */
+void *allocate(size_t size);
+
+/**
+ * Frees a memory region. Could be used by Wasm execution environments for freeing previous memory allocated by `allocate` function.
+ *
+ * @param ptr a pointer to the previously allocated memory region
+ * @param size a size of the previously allocated memory region
+ */
+void deallocate(void *ptr, size_t size);
+
+/**
+ * Stores one byte by a given address in the module memory.
+ *
+ * @param ptr a address where byte should be stored
+ * @param value a byte to be stored
+ */
+void store(void *ptr, char value);
+
+/**
+ * Returns one byte by a given address in the module memory.
+ *
+ * @param ptr a address at which the needed byte is located
+ * @return the byte at the given address
+ */
+char load(void *ptr);
+```
+
+Given char string `sql` as the request, the general scheme to use it is following:
+ 1. `void *ptr = allocate(strlen(sql))` that returns a pointer to the memory region enough for the string
+ 2. `void *res = invoke(ptr, strlen(sql))` to execute the request
+ 3. read a result from the `res` by reading 4 bytes as little-endian `result_size` and the read `result_size` bytes as the final result.
+ 4. `deallocate(res, strlen(sql))` to clean memory.
+ 
+Depends on your Wasm execution environment, `load`/`store` could be used for reading and writing a module memory.
+
+## More insights
+
+At Fluence, we use WebAssembly (Wasm) to run applications in a trustless network of independent nodes. This trustless environment mandates that every piece of the code executed by any network node must be verified by another node to check whether the execution was performed correctly. In order to be verifiable, every computation must be made deterministic, which means that to run WebAssembly code in a trustless network, we need to make its execution deterministic.
+
+There are three primary sources of nondeterminism in WebAssembly: external functions invocation, NaN payloads, and VM resource exhaustion. The first one is the most problematic; currently, we deal with it by simply prohibiting any interactions with the host environment from Wasm programs. This means that submitted WebAssembly code should not contain any external imports.
+
+The main purpose of this fork is to compile to Wasm without any imports. SQlite is famous for it embeddability and allows to configure itself by variety of macros. E.g. multithreading that produces a lot of imports of host functions could be completely disabled by setting SQLITE_THREADSAFE to 0. Also this fork has been patched to use only memory database without any interaction with a hard disk. So, now it is Wasm in-memory database.
+
+More information about ways of porting C/C++ code to Wasm could be found in our [article](https://medium.com/fluence-network/porting-redis-to-webassembly-with-clang-wasi-af99b264ca8) about porting of Redis.
+
+## Future plans
+
+We are working on supporting of subset of WASI syscalls to allow interaction with hard disk in a deterministic way.

docker-compose.yml

@@ -0,0 +1,7 @@
+version: '3'
+services:
+  sqlite3:
+    build:
+      context: .
+    volumes:
+     - .:/code

sdk/allocator.c

@@ -0,0 +1,13 @@
+#include "allocator.h"
+#include <stdlib.h>
+
+#define UNUSED(x) (void)(x)
+
+void *allocate(size_t size) {
+    return malloc(size);
+}
+
+void deallocate(void *ptr, size_t size) {
+    UNUSED(size);
+    free(ptr);
+}

sdk/allocator.h

@@ -0,0 +1,27 @@
+#ifndef C_SDK_ALLOCATOR_H
+#define C_SDK_ALLOCATOR_H
+
+#include <stddef.h> // for size_t
+
+/**
+ * Allocates a memory region of given size.
+ *
+ * Used by Wasm VM for byte array passing. Should be exported from module.
+ *
+ * @param size a size of needed memory region.
+ * @return a pointer to allocated memory region.
+ */
+void *allocate(size_t size);
+
+/**
+ * Frees a memory region.
+ *
+ * Used by Wasm VM for freeing previous memory allocated by `allocate` function.
+ * Should be exported from module.
+ *
+ * @param ptr the pointer to the previously allocated memory region.
+ * @param size the size of the previously allocated memory region.
+ */
+void deallocate(void *ptr, size_t size);
+
+#endif //C_SDK_ALLOCATOR_H

sdk/logger.c

@@ -0,0 +1,15 @@
+#include "logger.h"
+
+#define __LOGGER_IMPORT(name) \
+    __attribute__((__import_module__("logger"), __import_name__(#name)))
+
+void __write(char ch) __LOGGER_IMPORT(write);
+void __flush() __LOGGER_IMPORT(flush);
+
+void wasm_log(const char *str, int len) {
+    for(int byteId = 0; byteId < len; ++byteId) {
+        __write(str[byteId]);
+    }
+
+    __flush();
+}

sdk/logger.h

@@ -0,0 +1,11 @@
+#ifndef C_SDK_LOGGER_H
+#define C_SDK_LOGGER_H
+
+/**
+ * Writes provided string to Wasm VM logger.
+ *
+ * @param log_message a message that should be logged.
+ */
+void wasm_log(const char *str, int len);
+
+#endif //C_SDK_LOGGER_H

src/config.h

@@ -0,0 +1,132 @@
+/* config.h.  Generated from config.h.in by configure.  */
+/* config.h.in.  Generated from configure.ac by autoheader.  */
+
+/* Define to 1 if you have the <dlfcn.h> header file. */
+#define HAVE_DLFCN_H 1
+
+/* Define to 1 if you have the `fdatasync' function. */
+#define HAVE_FDATASYNC 1
+
+/* Define to 1 if you have the `gmtime_r' function. */
+#define HAVE_GMTIME_R 1
+
+/* Define to 1 if the system has the type `int16_t'. */
+#define HAVE_INT16_T 1
+
+/* Define to 1 if the system has the type `int32_t'. */
+#define HAVE_INT32_T 1
+
+/* Define to 1 if the system has the type `int64_t'. */
+#define HAVE_INT64_T 1
+
+/* Define to 1 if the system has the type `int8_t'. */
+#define HAVE_INT8_T 1
+
+/* Define to 1 if the system has the type `intptr_t'. */
+#define HAVE_INTPTR_T 1
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#define HAVE_INTTYPES_H 1
+
+/* Define to 1 if you have the `isnan' function. */
+#define HAVE_ISNAN 1
+
+/* Define to 1 if you have the `localtime_r' function. */
+#define HAVE_LOCALTIME_R 1
+
+/* Define to 1 if you have the `localtime_s' function. */
+/* #undef HAVE_LOCALTIME_S */
+
+/* Define to 1 if you have the <malloc.h> header file. */
+#define HAVE_MALLOC_H 1
+
+/* Define to 1 if you have the `malloc_usable_size' function. */
+#define HAVE_MALLOC_USABLE_SIZE 1
+
+/* Define to 1 if you have the <memory.h> header file. */
+#define HAVE_MEMORY_H 1
+
+/* Define to 1 if you have the pread() function. */
+#define HAVE_PREAD 1
+
+/* Define to 1 if you have the pread64() function. */
+#define HAVE_PREAD64 1
+
+/* Define to 1 if you have the pwrite() function. */
+#define HAVE_PWRITE 1
+
+/* Define to 1 if you have the pwrite64() function. */
+#define HAVE_PWRITE64 1
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#define HAVE_STDINT_H 1
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#define HAVE_STDLIB_H 1
+
+/* Define to 1 if you have the strchrnul() function */
+#define HAVE_STRCHRNUL 1
+
+/* Define to 1 if you have the <strings.h> header file. */
+#define HAVE_STRINGS_H 1
+
+/* Define to 1 if you have the <string.h> header file. */
+#define HAVE_STRING_H 1
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#define HAVE_SYS_STAT_H 1
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#define HAVE_SYS_TYPES_H 1
+
+/* Define to 1 if the system has the type `uint16_t'. */
+#define HAVE_UINT16_T 1
+
+/* Define to 1 if the system has the type `uint32_t'. */
+#define HAVE_UINT32_T 1
+
+/* Define to 1 if the system has the type `uint64_t'. */
+#define HAVE_UINT64_T 1
+
+/* Define to 1 if the system has the type `uint8_t'. */
+#define HAVE_UINT8_T 1
+
+/* Define to 1 if the system has the type `uintptr_t'. */
+#define HAVE_UINTPTR_T 1
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#define HAVE_UNISTD_H 1
+
+/* Define to 1 if you have the `usleep' function. */
+#define HAVE_USLEEP 1
+
+/* Define to 1 if you have the utime() library function. */
+#define HAVE_UTIME 1
+
+/* Define to the sub-directory in which libtool stores uninstalled libraries.
+   */
+#define LT_OBJDIR ".libs/"
+
+/* Define to the address where bug reports for this package should be sent. */
+#define PACKAGE_BUGREPORT ""
+
+/* Define to the full name of this package. */
+#define PACKAGE_NAME "sqlite"
+
+/* Define to the full name and version of this package. */
+#define PACKAGE_STRING "sqlite 3.30.0"
+
+/* Define to the one symbol short name of this package. */
+#define PACKAGE_TARNAME "sqlite"
+
+/* Define to the version of this package. */
+#define PACKAGE_VERSION "3.30.0"
+
+/* Define to 1 if you have the ANSI C header files. */
+#define STDC_HEADERS 1
+
+/* Number of bits in a file offset, on hosts where this is settable. */
+/* #undef _FILE_OFFSET_BITS */
+
+/* Define for large files, on AIX-style hosts. */
+/* #undef _LARGE_FILES */

src/main.c

@@ -237,7 +237,9 @@ int sqlite3_initialize(void){
     }
     if( rc==SQLITE_OK ){
       sqlite3GlobalConfig.isPCacheInit = 1;
+#if __sqlite_unmodified_upstream
       rc = sqlite3OsInit();
+#endif
     }
 #ifdef SQLITE_ENABLE_DESERIALIZE
     if( rc==SQLITE_OK ){
@@ -321,7 +323,9 @@ int sqlite3_shutdown(void){
     void SQLITE_EXTRA_SHUTDOWN(void);
     SQLITE_EXTRA_SHUTDOWN();
 #endif
+#if __sqlite_unmodified_upstream
     sqlite3_os_end();
+#endif
     sqlite3_reset_auto_extension();
     sqlite3GlobalConfig.isInit = 0;
   }

src/memdb.c

@@ -17,6 +17,10 @@
 ** sqlite3_deserialize().
 */
 #include "sqliteInt.h"
+#if __sqlite_unmodified_upstream
+#else
+#include <stdlib.h>
+#endif
 #ifdef SQLITE_ENABLE_DESERIALIZE
 
 /*
@@ -422,7 +426,15 @@ static void memdbDlClose(sqlite3_vfs *pVfs, void *pHandle){
 ** random data.
 */
 static int memdbRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
+#if __sqlite_unmodified_upstream
   return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut);
+#else
+  zBufOut = malloc(nByte);
+  for(int i = 0; i < nByte; ++i) {
+    zBufOut[i] = rand() % 256;
+  }
+  return SQLITE_OK;
+#endif
 }
 
 /*
@@ -610,14 +622,23 @@ end_deserialize:
 ** Register the new VFS.
 */
 int sqlite3MemdbInit(void){
+#if __sqlite_unmodified_upstream
   sqlite3_vfs *pLower = sqlite3_vfs_find(0);
   int sz = pLower->szOsFile;
   memdb_vfs.pAppData = pLower;
+#else
+  memdb_vfs.pAppData = (void *)0xFFFFFFFF;
+  int sz = sizeof(MemFile);
+#endif
   /* In all known configurations of SQLite, the size of a default
   ** sqlite3_file is greater than the size of a memdb sqlite3_file.
   ** Should that ever change, remove the following NEVER() */
   if( NEVER(sz<sizeof(MemFile)) ) sz = sizeof(MemFile);
   memdb_vfs.szOsFile = sz;
+#if __sqlite_unmodified_upstream // Make memdb the default database
   return sqlite3_vfs_register(&memdb_vfs, 0);
+#else
+  return sqlite3_vfs_register(&memdb_vfs, 1);
+#endif
 }
 #endif /* SQLITE_ENABLE_DESERIALIZE */

src/wrapper.c

@@ -0,0 +1,174 @@
+#include <stdlib.h>
+#include "../sdk/logger.h"
+#include "sqliteInt.h"
+
+sqlite3 *state;
+
+int init() {
+  const int rc = sqlite3_initialize();
+  if(rc != 0) {
+    return rc;
+  }
+
+  return sqlite3_open(":memory:", &state);
+}
+
+int g_isInited = 0;
+
+void store(char *ptr, unsigned char byte) {
+  *ptr = byte;
+}
+
+void sqlite_store(char *ptr, unsigned char byte) {
+  store(ptr, byte);
+}
+
+unsigned char load(const unsigned char *ptr) {
+  return *ptr;
+}
+
+unsigned char sqlite_load(const unsigned char *ptr) {
+  return load(ptr);
+}
+
+void* allocate(size_t size) {
+  return malloc(size + 1);
+}
+
+void* sqlite_allocate(size_t size) {
+  return allocate(size);
+}
+
+void deallocate(void *ptr, int size) {
+  free(ptr);
+}
+
+void sqlite_deallocate(void *ptr, int size) {
+  deallocate(ptr, size);
+}
+
+char *write_response(char *response, int response_size) {
+  char *result_response = allocate(response_size + 4);
+
+  for(int i = 0; i < 4; ++i) {
+    result_response[i] = (response_size >> 8*i) & 0xFF;
+  }
+
+  memcpy(result_response + 4, response, response_size);
+  return result_response;
+}
+
+typedef struct ShellText ShellText;
+struct ShellText {
+    char *z;
+    int n;
+    int nAlloc;
+};
+
+static void initText(ShellText *p){
+  memset(p, 0, sizeof(*p));
+}
+
+static void freeText(ShellText *p){
+  free(p->z);
+  initText(p);
+}
+
+static int strlen30(const char *z){
+  const char *z2 = z;
+  while( *z2 ){ z2++; }
+  return 0x3fffffff & (int)(z2 - z);
+}
+
+static void appendText(ShellText *p, char const *zAppend, char quote){
+  int len;
+  int i;
+  int nAppend = strlen30(zAppend);
+
+  len = nAppend+p->n+1;
+  if( quote ){
+    len += 2;
+    for(i=0; i<nAppend; i++){
+      if( zAppend[i]==quote ) len++;
+    }
+  }
+
+  if( p->n+len>=p->nAlloc ){
+    p->nAlloc = p->nAlloc*2 + len + 20;
+    p->z = realloc(p->z, p->nAlloc);
+    // TODO: more solid work with OOM
+    if( p->z==0 ) __builtin_unreachable();
+  }
+
+  if( quote ){
+    char *zCsr = p->z+p->n;
+    *zCsr++ = quote;
+    for(i=0; i<nAppend; i++){
+      *zCsr++ = zAppend[i];
+      if( zAppend[i]==quote ) *zCsr++ = quote;
+    }
+    *zCsr++ = quote;
+    p->n = (int)(zCsr - p->z);
+    *zCsr = '\0';
+  }else{
+    memcpy(p->z+p->n, zAppend, nAppend);
+    p->n += nAppend;
+    p->z[p->n] = '\0';
+  }
+}
+
+static int captureOutputCallback(void *pArg, int nArg, char **azArg, char **az){
+  ShellText *p = (ShellText*)pArg;
+  int i;
+  UNUSED_PARAMETER(az);
+  if( azArg==0 ) return 0;
+  if( p->n ) appendText(p, "|", 0);
+  for(i=0; i<nArg; i++){
+    if( i ) appendText(p, ",", 0);
+    if( azArg[i] ) appendText(p, azArg[i], 0);
+  }
+  return 0;
+}
+
+const char *invoke(char *request, int request_size) {
+  if(g_isInited == 0) {
+    // TODO: check the return code
+    init();
+    const char successInitMessage[] = "Sqlite has been initialized";
+    wasm_log(successInitMessage, sizeof(successInitMessage));
+    g_isInited = 1;
+  }
+
+  request[request_size] = 0;
+
+  wasm_log(request, request_size);
+
+  ShellText str;
+  initText(&str);
+  char *errorMessage = 0;
+
+  int rc = sqlite3_exec(state, request, captureOutputCallback, &str, &errorMessage);
+
+  char *response = 0;
+  if(rc || errorMessage) {
+    response = write_response(errorMessage, strlen(errorMessage));
+  }
+  else {
+    if(str.n != 0) {
+      response = write_response(str.z, str.n);
+    } else {
+      // if a request was successfull, sqlite doesn't return anything as the result string
+      const char success_result[] = "OK";
+      response = write_response((char *)success_result, sizeof(success_result));
+    }
+  }
+
+  deallocate(request, request_size + 1);
+  freeText(&str);
+
+  return response;
+}
+
+const char *sqlite_invoke(char *request, int request_size) {
+  return invoke(request, request_size);
+}