v4.5.2-beta.3

This reverts commit 9c21452d0b.

.github/workflows/build-app-beta.yaml

@@ -19,13 +19,13 @@ jobs:
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Use Node.js 10.x
+      - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
-          node-version: 10.x
+          node-version: 14.x
       - name: yarn install
         run: |
           yarn install
@@ -47,8 +47,8 @@ jobs:
           yarn run build:app
         env:
           GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
-          WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
-          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
+          # WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
+          # WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
 
       - name: Publish Mac
         if: matrix.os == 'macOS-10.15'
@@ -78,8 +78,9 @@ jobs:
           cp app/dist/*x86*.AppImage artifacts/dbgate-beta.AppImage || true
           cp app/dist/*arm64*.AppImage artifacts/dbgate-beta-arm64.AppImage || true
           cp app/dist/*armv7l*.AppImage artifacts/dbgate-beta-armv7l.AppImage || true
-          cp app/dist/*.exe artifacts/dbgate-beta.exe || true
-          cp app/dist/*win*.zip artifacts/dbgate-windows-beta.zip || true
+          cp app/dist/*win*.exe artifacts/dbgate-beta.exe || true
+          cp app/dist/*win_x64.zip artifacts/dbgate-windows-beta.zip || true
+          cp app/dist/*win_arm64.zip artifacts/dbgate-windows-beta-arm64.zip || true
           cp app/dist/*-mac.dmg artifacts/dbgate-beta.dmg || true
           cp app/dist/*-mac_arm64.dmg artifacts/dbgate-beta-arm64.dmg || true
 

.github/workflows/build-app.yaml

@@ -23,13 +23,13 @@ jobs:
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Use Node.js 10.x
+      - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
-          node-version: 10.x
+          node-version: 14.x
       - name: yarn install
         run: |
           yarn install
@@ -79,7 +79,8 @@ jobs:
           cp app/dist/*arm64*.AppImage artifacts/dbgate-latest-arm64.AppImage || true
           cp app/dist/*armv7l*.AppImage artifacts/dbgate-latest-armv7l.AppImage || true
           cp app/dist/*.exe artifacts/dbgate-latest.exe || true
-          cp app/dist/*win*.zip artifacts/dbgate-windows-latest.zip || true
+          cp app/dist/*win_x64.zip artifacts/dbgate-windows-latest.zip || true
+          cp app/dist/*win_arm64.zip artifacts/dbgate-windows-latest-arm64.zip || true
           cp app/dist/*-mac.dmg artifacts/dbgate-latest.dmg || true
           cp app/dist/*-mac_arm64.dmg artifacts/dbgate-latest-arm64.dmg || true
 

.github/workflows/build-docker-beta.yaml

@@ -21,13 +21,13 @@ jobs:
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Use Node.js 10.x
+      - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
-          node-version: 10.x
+          node-version: 14.x
       - name: yarn install
         run: |
           yarn install

.github/workflows/build-docker.yaml

@@ -27,13 +27,13 @@ jobs:
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Use Node.js 10.x
+      - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
-          node-version: 10.x
+          node-version: 14.x
       - name: yarn install
         run: |
           yarn install

.github/workflows/build-npm.yaml

@@ -6,7 +6,7 @@ on:
   push:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+'
-      # - 'v[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+'
 
 # on:
 #   push:
@@ -27,13 +27,13 @@ jobs:
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Use Node.js 10.x
+      - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
-          node-version: 10.x
+          node-version: 14.x
 
       - name: Configure NPM token
         env:

.github/workflows/run-tests.yaml

@@ -3,18 +3,19 @@ on:
   push:
     branches:
       - master
+      - develop
 
 jobs:
   test-runner:
     runs-on: ubuntu-latest
-    container: node:10.18-jessie
+    container: node:14.18
 
     steps:
       - name: Context
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
       - name: yarn install

CHANGELOG.md

@@ -1,6 +1,140 @@
 # ChangeLog
+### 4.5.1
+- FIXED: MongoId detection
+- FIXED: #203 disabled spellchecker
+- FIXED: Prevented display filters in form view twice
+- FIXED: Query designer fixes
 
-### 4.2.4 - to be released
+### 4.5.0
+- ADDED: #220 functions, materialized views and stored procedures in code completion
+- ADDED: Query result in statusbar
+- ADDED: Highlight and execute current query
+- CHANGED: Code completion offers objects only from current query
+- CHANGED: Big optimalizations of electron app - removed embedded web server, removed remote module, updated electron to version 13
+- CHANGED: Removed dependency to electron-store module
+- FIXED: #201 fixed database URL definition, when running from Docvker container
+- FIXED: #192 Docker container stops in 1 second, ability to stop container with Ctrl+C
+- CHANGED: Web app - websocket replaced with SSE technology
+- CHANGED: Changed tab order, tabs are ordered by creation time
+- ADDED: Reorder tabs with drag & drop
+- CHANGED: Collapse left column in datagrid - removed from settings, remember last used state
+- ADDED: Ability to select multiple columns in column manager in datagrid + copy column names
+- ADDED: Show used filters in left datagrid column
+- FIXED: Fixed delete dependency cycle detection (delete didn't work for some tables)
+
+### 4.4.4
+- FIXED: Database colors
+- CHANGED: Precise work with MongoDB ObjectId
+- FIXED: Run macro works on MongoDB collection data editor
+- ADDED: Type conversion macros
+- CHANGED: Improved UX of import into current database or current archive
+- ADDED: Posibility to create string MongoDB IDs when importing into MongoDB collections
+- CHANGED: Better crash recovery
+- FIXED: Context menu of data editor when using views - some commands didn't work for views
+- ADDED: Widget lists (on left side) now supports add operation, where it has sense
+- CHANGED: Improved UX of saved data sheets
+- ADDED: deploy - preloadedRows: impelemnted onsertOnly columns
+- ADDED: Show change log after app upgrade
+
+### 4.4.3
+- ADDED: Connection and database colors
+- ADDED: Ability to pin connection or table
+- ADDED: MongoDb: create, drop collection from menu
+- ADDED: Copy as MongoDB insert
+- ADDED: MongoDB support for multiple statements in script (dbgate-query-splitter)
+- ADDED: View JSON in tab
+- ADDED: Open DB model as JSON
+- ADDED: Open JSON array as data sheet
+- ADDED: Open JSON from data grid
+- FIXED: Mongo update command when using string IDs resembling Mongo IDs
+- CHANGED: Imrpoved add JSON document, change JSON document commands
+- ADDED: Possibility to add column to JSON grid view
+- FIXED: Hiding columns #1
+- REMOVED: Copy JSON document menu command (please use Copy advanced instead)
+- CHANGED: Save widget visibility and size
+
+### 4.4.2
+- ADDED: Open SQL script from SQL confirm
+- CHANGED: Better looking statusbar
+- ADDED: Create table from database popup menu
+- FIXED: Some fixes for DB compare+deploy (eg. #196)
+- ADDED: Archives + DB models from external directories
+- ADDED: DB deploy supports preloaded data
+- ADDED: Support for Command key on Mac (#199)
+
+### 4.4.1
+- FIXED: #188 Fixed problem with datetime values in PostgreSQL and mysql
+- ADDED: #194 Close tabs by DB
+- FIXED: Improved form view width calculations
+- CHANGED: Form view - highlight matched columns instead of filtering
+- ADDED: Lookup distinct values
+- ADDED: Copy advanced command, Copy as CSV, JSON, YAML, SQL
+- CHANGED: Hide column manager by default
+- ADDED: Change database status command
+- CHANGED: Table structure and view structure tabs have different icons
+- ADDED: #186 - zoom setting
+- ADDED: Row count information moved into status bar, when only one grid on tab is used (typical case)
+
+### 4.4.0
+- ADDED: Database structure compare, export report to HTML
+- ADDED: Experimental: Deploy DB structure changes between databases
+- ADDED: Lookup dialog, available in table view on columns with foreign key
+- ADDED: Customize foreign key lookups
+- ADDED: Chart improvements, export charts as HTML page
+- ADDED: Experimental: work with DB model, deploy model, compare model with real DB
+- ADDED: #193 new SQLite db command
+- CHANGED: #190 code completion improvements
+- ADDED: #189 Copy JSON document - context menu command in data grid for MongoDB
+- ADDED: #191 Connection to POstgreSQL can be defined also with connection string
+- ADDED: #187 dbgate-query-splitter: Transform stream support
+- CHANGED: Upgraded to node 12 in docker app
+- FIXED: Upgraded to node 12 in docker app
+- FIXED: Fixed import into SQLite and PostgreSQL databases, added integration test for this
+
+### 4.3.4
+- FIXED: Delete row with binary ID in MySQL (#182)
+- ADDED: Using 'ODBC Driver 17 for SQL Server' or 'SQL Server Native Client 11.0', when connecting to MS SQL using windows auth #183
+
+### 4.3.3
+- ADDED: Generate SQL from data (#176 - Copy row as INSERT/UPDATE statement)
+- ADDED: Datagrid keyboard column operations (Ctrl+F - find column, Ctrl+H - hide column) #180
+- FIXED: Make window remember that it was maximized
+- FIXED: Fixed lost focus after copy to clipboard and after inserting SQL join
+
+### 4.3.2
+- FIXED: Sorted database list in PostgreSQL (#178)
+- FIXED: Loading stricture of PostgreSQL database, when it contains indexes on expressions (#175)
+- ADDED: Hotkey Shift+Alt+F for formatting SQL code
+
+### 4.3.1
+- FIXED: #173 Using key phrase for SSH key file connection
+- ADDED: #172 Abiloity to quick search within database names
+- ADDED: Database search added to command palette (Ctrl+P)
+- FIXED: #171 fixed PostgreSQL analyser for older versions than 9.3 (matviews don't exist)
+- ADDED: DELETE cascade option - ability to delete all referenced rows, when deleting rows
+
+### 4.3.0
+- ADDED: Table structure editor
+- ADDED: Index support
+- ADDED: Unique constraint support
+- ADDED: Context menu for drop/rename table/columns and for drop view/procedure/function
+- ADDED: Added support for Windows arm64 platform
+- FIXED: Search by _id in MongoDB
+
+### 4.2.6
+- FIXED: Fixed MongoDB import
+- ADDED: Configurable thousands separator #136
+- ADDED: Using case insensitive text search in postgres
+
+### 4.2.5
+- FIXED: Fixed crash when using large model on some installations
+- FIXED: Postgre SQL CREATE function 
+- FIXED: Analysing of MySQL when modifyDate is not known
+
+### 4.2.4
+- ADDED: Query history
+- ADDED: One-click exports in desktop app
+- ADDED: JSON array export
 - FIXED: Procedures in PostgreSQL #122
 - ADDED: Support of materialized views for PostgreSQL #123
 - ADDED: Integration tests

README.md

@@ -4,15 +4,21 @@
 [![dbgate](https://snapcraft.io/dbgate/trending.svg?name=0)](https://snapcraft.io/dbgate)
 [![styled with prettier](https://img.shields.io/badge/styled_with-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
 
-# DbGate - database administration tool
+<img src="https://raw.githubusercontent.com/dbgate/dbgate/master/app/icon.png" width="64" align="right"/>
 
-DbGate modern, fast and easy to use database manager
+# DbGate - (no)SQL database client
+
+DbGate is cross-platform database manager. 
+It's designed to be simple to use and effective, when working with more databases simultaneously.
+But there are also many advanced features like schema compare, visual query designer, chart visualisation or batch export and import.
+
+DbGate is licensed under MIT license and is completely free.
 
 * Try it online - [demo.dbgate.org](https://demo.dbgate.org) - online demo application
-* Download application for Windows, Linux or Mac from [dbgate.org](https://dbgate.org/download/)
+* **Download** application for Windows, Linux or Mac from [dbgate.org](https://dbgate.org/download/)
 * Run web version as [NPM package](https://www.npmjs.com/package/dbgate) or as [docker image](https://hub.docker.com/r/dbgate/dbgate)
 
-Supported databases:
+## Supported databases
 * MySQL
 * PostgreSQL
 * SQL Server
@@ -22,12 +28,30 @@ Supported databases:
 * CockroachDB
 * MariaDB
 
-![Screenshot](https://raw.githubusercontent.com/dbgate/dbgate/master/screenshot.png)
+<!-- Learn more about DbGate features at the [DbGate website](https://dbgate.org/), or try our online [demo application](https://demo.dbgate.org) -->
+
+
+<a href="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot1.png">
+    <img src="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot1.png" width="400"/>
+</a>
+<a href="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot2.png">
+    <img src="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot2.png" width="400"/>
+</a>
+<a href="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot4.png">
+    <img src="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot4.png" width="400"/>
+</a>
+<a href="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot3.png">
+    <img src="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot3.png" width="400"/>
+</a>
+
+<!-- ![Screenshot](https://raw.githubusercontent.com/dbgate/dbgate/master/screenshot.png) -->
 
 ## Features
 * Table data editing, with SQL change script preview
+* Edit table schema, indexes, primary and foreign keys
+* Compare and synchronize database structure
 * Light and dark theme
-* Master/detail views
+* Master/detail views, foreign key lookups
 * Query designer
 * Form view for comfortable work with tables with many columns
 * JSON view on MongoDB collections
@@ -42,20 +66,25 @@ Supported databases:
 * Import, export from/to CSV, Excel, JSON
 * Free table editor - quick table data editing (cleanup data after import/before export, prototype tables etc.)
 * Archives - backup your data in JSON files on local filesystem (or on DbGate server, when using web application)
-* Charts
+* Charts, export chart to HTML page
 * For detailed info, how to run DbGate in docker container, visit [docker hub](https://hub.docker.com/r/dbgate/dbgate)
 * Extensible plugin architecture
 
 ## How to contribute
 Any contributions are welcome. If you want to contribute without coding, consider following:
-* Create issue, if you find problem in app, or you have idea to new feature. If issue already exists, you could leave comment on it, to prioritise most wanted issues.
+
 * Tell your friends about DbGate or share on social networks - when more people will use DbGate, it will grow to be better
+* Write review on [Slant.co](https://www.slant.co/improve/options/41086/~dbgate-review) or [G2](https://www.g2.com/products/dbgate/reviews) 
+* Create issue, if you find problem in app, or you have idea to new feature. If issue already exists, you could leave comment on it, to prioritise most wanted issues.
+* Become a backer on [Open collective](https://opencollective.com/dbgate)
+
+Thank you!
 
 ## Why is DbGate different
 There are many database managers now, so why DbGate? 
 * Works everywhere - Windows, Linux, Mac, Web browser (+mobile web is planned), without compromises in features
 * Based on standalone NPM packages, scripts can be run without DbGate (example - [CSV export](https://www.npmjs.com/package/dbgate-plugin-csv) )
-* Many data browsing functions based using foreign keys - master/detail, expand columns, expandable form view (on screenshot above)
+* Many data browsing functions based using foreign keys - master/detail, expand columns, expandable form view
 
 ## Design goals
 * Application simplicity - DbGate takes the best and only the best from old [DbGate](http://www.jenasoft.com/dbgate), [DatAdmin](http://www.jenasoft.com/datadmin) and [DbMouse](http://www.jenasoft.com/dbmouse) .
@@ -64,9 +93,9 @@ There are many database managers now, so why DbGate?
     * Backend - NodeJs, ExpressJs, socket.io, database connection drivers
     * JavaScript + TypeScript
     * App - electron
-* Platform independed - will run as web application in single docker container on server, or as application using Electron platform on Linux, Windows and Mac
+* Platform independent - runs as web application in single docker container on server, or as application using Electron platform on Linux, Windows and Mac
 
-## Plugins
+<!-- ## Plugins
 Plugins are standard NPM packages published on [npmjs.com](https://www.npmjs.com).  
 See all [existing DbGate plugins](https://www.npmjs.com/search?q=keywords:dbgateplugin).  
 Visit [dbgate generator homepage](https://github.com/dbgate/generator-dbgate) to see, how to create your own plugin.  
@@ -75,31 +104,46 @@ Currently following extensions can be implemented using plugins:
 - File format parsers/writers
 - Database engine connectors
 
-Basic set of plugins is part of DbGate git repository and is installed with app. Additional plugins pust be downloaded from NPM (this task is handled by DbGate)
+Basic set of plugins is part of DbGate git repository and is installed with app. Additional plugins pust be downloaded from NPM (this task is handled by DbGate) -->
 
 ## How to run development environment
 
+Simple variant - runs WEB application:
 ```sh
 yarn
 yarn start
 ```
 
-If you want to make modifications in libraries or plugins, run library compiler in watch mode in the second terminal:
+If you want more control, run WEB application:
 ```sh
-yarn lib
+yarn # install NPM packages
 ```
 
+And than run following 3 commands concurrently in 3 terminals:
+```
+yarn start:api # run API on port 3000
+yarn start:web # run web on port 5000
+yarn lib # watch typescript libraries and plugins modifications
+```
+This runs API on port 3000 and web application on port 5000  
 Open http://localhost:5000 in your browser
 
-You could run electron app (requires running localhost:5000):
+If you want to run electron app:
 ```sh
+yarn # install NPM packages
 cd app
-yarn
-yarn start
+yarn # install NPM packages for electron
+```
+
+And than run following 3 commands concurrently in 3 terminals:
+```
+yarn start:web # run web on port 5000 (only static JS and HTML files)
+yarn lib # watch typescript libraries and plugins modifications
+yarn start:app # run electron app
 ```
 
 ## How to run built electron app locally
-This mode is very similar to production run of electron app. Electron app forks process with API on dynamically allocated port, works with compiled javascript files and uses compiled version of plugins (doesn't use localhost:5000)
+This mode is very similar to production run of electron app. Electron doesn't use localhost:5000.
 
 ```sh
 cd app

app/package.json

@@ -5,10 +5,8 @@
   "author": "Jan Prochazka <jenasoft.database@gmail.com>",
   "description": "Opensource database administration tool",
   "dependencies": {
-    "better-sqlite3-with-prebuilds": "^7.1.8",
-    "electron-log": "^4.3.1",
-    "electron-store": "^5.1.1",
-    "electron-updater": "^4.3.5",
+    "electron-log": "^4.4.1",
+    "electron-updater": "^4.6.1",
     "patch-package": "^6.4.7"
   },
   "repository": {
@@ -45,7 +43,7 @@
           ]
         }
       ],
-      "icon": "icon.png",
+      "icon": "icons/",
       "category": "Development",
       "synopsis": "Database manager for SQL Server, MySQL, PostgreSQL, MongoDB and SQLite",
       "publish": [
@@ -68,7 +66,13 @@
     "win": {
       "target": [
         "nsis",
-        "zip"
+        {
+          "target": "zip",
+          "arch": [
+            "x64",
+            "arm64"
+          ]
+        }
       ],
       "icon": "icon.ico",
       "publish": [
@@ -84,23 +88,25 @@
   },
   "homepage": "./",
   "scripts": {
-    "start": "cross-env ELECTRON_START_URL=http://localhost:5000 electron .",
+    "start": "cross-env ELECTRON_START_URL=http://localhost:5000 DEVMODE=1 electron .",
     "start:local": "cross-env electron .",
     "dist": "electron-builder",
     "build": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn dist",
     "build:mac": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && node setMacPlatform x64 && yarn dist && node setMacPlatform arm64 && yarn dist",
     "build:local": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn predist",
-    "postinstall": "electron-builder install-app-deps && patch-package",
+    "postinstall": "yarn rebuild && patch-package",
+    "rebuild": "electron-builder install-app-deps",
     "predist": "copyfiles ../packages/api/dist/* packages && copyfiles \"../packages/web/public/*\" packages && copyfiles \"../packages/web/public/**/*\" packages && copyfiles --up 3 \"../plugins/dist/**/*\" packages/plugins"
   },
   "main": "src/electron.js",
   "devDependencies": {
     "copyfiles": "^2.2.0",
     "cross-env": "^6.0.3",
-    "electron": "11.2.3",
-    "electron-builder": "22.10.5"
+    "electron": "13.6.3",
+    "electron-builder": "22.14.5"
   },
   "optionalDependencies": {
-    "msnodesqlv8": "^2.0.10"
+    "better-sqlite3": "7.4.5",
+    "msnodesqlv8": "^2.4.4"
   }
-}
+}

app/src/electron.js

@@ -1,9 +1,8 @@
 const electron = require('electron');
 const os = require('os');
+const fs = require('fs');
 const { Menu, ipcMain } = require('electron');
-const { fork } = require('child_process');
 const { autoUpdater } = require('electron-updater');
-const Store = require('electron-store');
 const log = require('electron-log');
 
 // Module to control application life.
@@ -14,7 +13,17 @@ const BrowserWindow = electron.BrowserWindow;
 const path = require('path');
 const url = require('url');
 
-const store = new Store();
+// require('@electron/remote/main').initialize();
+
+const configRootPath = path.join(app.getPath('userData'), 'config-root.json');
+let initialConfig = {};
+
+try {
+  initialConfig = JSON.parse(fs.readFileSync(configRootPath, { encoding: 'utf-8' }));
+} catch (err) {
+  console.log('Error loading config-root:', err.message);
+  initialConfig = {};
+}
 
 // Keep a global reference of the window object, if you don't, the window will
 // be closed automatically when the JavaScript object is garbage collected.
@@ -36,7 +45,7 @@ function commandItem(id) {
     accelerator: command ? command.keyText : undefined,
     enabled: command ? command.enabled : false,
     click() {
-      mainWindow.webContents.executeJavaScript(`dbgate_runCommand('${id}')`);
+      mainWindow.webContents.send('run-command', id);
     },
   };
 }
@@ -47,11 +56,19 @@ function buildMenu() {
       label: 'File',
       submenu: [
         commandItem('new.connection'),
+        commandItem('new.sqliteDatabase'),
+        commandItem('new.modelCompare'),
+        commandItem('new.freetable'),
+        { type: 'separator' },
         commandItem('file.open'),
+        commandItem('file.openArchive'),
+        { type: 'separator' },
         commandItem('group.save'),
         commandItem('group.saveAs'),
+        commandItem('database.search'),
         { type: 'separator' },
-        { role: 'close' },
+        commandItem('tabs.closeTab'),
+        commandItem('file.exit'),
       ],
     },
     {
@@ -91,27 +108,28 @@ function buildMenu() {
         {
           label: 'dbgate.org',
           click() {
-            require('electron').shell.openExternal('https://dbgate.org');
+            electron.shell.openExternal('https://dbgate.org');
           },
         },
         {
           label: 'DbGate on GitHub',
           click() {
-            require('electron').shell.openExternal('https://github.com/dbgate/dbgate');
+            electron.shell.openExternal('https://github.com/dbgate/dbgate');
           },
         },
         {
           label: 'DbGate on docker hub',
           click() {
-            require('electron').shell.openExternal('https://hub.docker.com/r/dbgate/dbgate');
+            electron.shell.openExternal('https://hub.docker.com/r/dbgate/dbgate');
           },
         },
         {
           label: 'Report problem or feature request',
           click() {
-            require('electron').shell.openExternal('https://github.com/dbgate/dbgate/issues/new');
+            electron.shell.openExternal('https://github.com/dbgate/dbgate/issues/new');
           },
         },
+        commandItem('tabs.changelog'),
         commandItem('about.show'),
       ],
     },
@@ -137,10 +155,27 @@ ipcMain.on('update-commands', async (event, arg) => {
     menu.enabled = command.enabled;
   }
 });
+ipcMain.on('close-window', async (event, arg) => {
+  mainWindow.close();
+});
+
+ipcMain.handle('showOpenDialog', async (event, options) => {
+  const res = electron.dialog.showOpenDialogSync(mainWindow, options);
+  return res;
+});
+ipcMain.handle('showSaveDialog', async (event, options) => {
+  const res = electron.dialog.showSaveDialogSync(mainWindow, options);
+  return res;
+});
+ipcMain.handle('showItemInFolder', async (event, path) => {
+  electron.shell.showItemInFolder(path);
+});
+ipcMain.handle('openExternal', async (event, url) => {
+  electron.shell.openExternal(url);
+});
 
 function createWindow() {
-  const bounds = store.get('winBounds');
-
+  const bounds = initialConfig['winBounds'];
   mainWindow = new BrowserWindow({
     width: 1200,
     height: 800,
@@ -149,10 +184,15 @@ function createWindow() {
     icon: os.platform() == 'win32' ? 'icon.ico' : path.resolve(__dirname, '../icon.png'),
     webPreferences: {
       nodeIntegration: true,
-      enableRemoteModule: true,
+      contextIsolation: false,
+      spellcheck: false,
     },
   });
 
+  if (initialConfig['winIsMaximized']) {
+    mainWindow.maximize();
+  }
+
   mainMenu = buildMenu();
   mainWindow.setMenu(mainMenu);
 
@@ -164,11 +204,19 @@ function createWindow() {
         protocol: 'file:',
         slashes: true,
       });
-    mainWindow.webContents.on('did-finish-load', function () {
-      // hideSplash();
-    });
     mainWindow.on('close', () => {
-      store.set('winBounds', mainWindow.getBounds());
+      try {
+        fs.writeFileSync(
+          configRootPath,
+          JSON.stringify({
+            winBounds: mainWindow.getBounds(),
+            winIsMaximized: mainWindow.isMaximized(),
+          }),
+          'utf-8'
+        );
+      } catch (err) {
+        console.log('Error saving config-root:', err.message);
+      }
     });
     mainWindow.loadURL(startUrl);
     if (os.platform() == 'linux') {
@@ -176,31 +224,27 @@ function createWindow() {
     }
   }
 
-  if (process.env.ELECTRON_START_URL) {
-    loadMainWindow();
-  } else {
-    const apiProcess = fork(path.join(__dirname, '../packages/api/dist/bundle.js'), [
-      '--dynport',
-      '--is-electron-bundle',
-      '--native-modules',
-      path.join(__dirname, 'nativeModules'),
-      // '../../../src/nativeModules'
-    ]);
-    apiProcess.on('message', msg => {
-      if (msg.msgtype == 'listening') {
-        const { port, authorization } = msg;
-        global['port'] = port;
-        global['authorization'] = authorization;
-        loadMainWindow();
-      }
-    });
-  }
+  const apiPackage = path.join(
+    __dirname,
+    process.env.DEVMODE ? '../../packages/api/src/index' : '../packages/api/dist/bundle.js'
+  );
 
-  // and load the index.html of the app.
-  // mainWindow.loadURL('http://localhost:3000');
+  global.API_PACKAGE = apiPackage;
+  global.NATIVE_MODULES = path.join(__dirname, 'nativeModules');
 
-  // Open the DevTools.
-  // mainWindow.webContents.openDevTools();
+  // console.log('global.API_PACKAGE', global.API_PACKAGE);
+  const api = require(apiPackage);
+  // console.log(
+  //   'REQUIRED',
+  //   path.resolve(
+  //     path.join(__dirname, process.env.DEVMODE ? '../../packages/api/src/index' : '../packages/api/dist/bundle.js')
+  //   )
+  // );
+  const main = api.getMainModule();
+  main.initializeElectronSender(mainWindow.webContents);
+  main.useAllControllers(null, electron);
+
+  loadMainWindow();
 
   // Emitted when the window is closed.
   mainWindow.on('closed', function () {
@@ -212,7 +256,9 @@ function createWindow() {
 }
 
 function onAppReady() {
-  autoUpdater.checkForUpdatesAndNotify();
+  if (!process.env.DEVMODE) {
+    autoUpdater.checkForUpdatesAndNotify();
+  }
   createWindow();
 }
 

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:12-alpine
+FROM node:14-alpine
 
 WORKDIR /home/dbgate-docker
 

fillNativeModules.js

@@ -5,7 +5,7 @@ let fillContent = '';
 if (process.platform == 'win32') {
   fillContent += `content.msnodesqlv8 = () => require('msnodesqlv8');`;
 }
-fillContent += `content['better-sqlite3-with-prebuilds'] = () => require('better-sqlite3-with-prebuilds');`;
+fillContent += `content['better-sqlite3'] = () => require('better-sqlite3');`;
 
 const getContent = (empty) => `
 // this file is generated automatically by script fillNativeModules.js, do not edit it manually

integration-tests/__tests__/alter-database.spec.js

@@ -0,0 +1,68 @@
+const stableStringify = require('json-stable-stringify');
+const _ = require('lodash');
+const fp = require('lodash/fp');
+const uuidv1 = require('uuid/v1');
+const { testWrapper } = require('../tools');
+const engines = require('../engines');
+const { getAlterDatabaseScript, extendDatabaseInfo, generateDbPairingId } = require('dbgate-tools');
+
+function flatSource() {
+  return _.flatten(
+    engines.map(engine => (engine.objects || []).map(object => [engine.label, object.type, object, engine]))
+  );
+}
+
+async function testDatabaseDiff(conn, driver, mangle, createObject = null) {
+  await driver.query(conn, `create table t1 (id int not null primary key)`);
+
+  await driver.query(
+    conn,
+    `create table t2 (
+    id int not null primary key, 
+    t1_id int null references t1(id)
+  )`
+  );
+
+  if (createObject) await driver.query(conn, createObject);
+
+  const structure1 = generateDbPairingId(extendDatabaseInfo(await driver.analyseFull(conn)));
+  let structure2 = _.cloneDeep(structure1);
+  mangle(structure2);
+  structure2 = extendDatabaseInfo(structure2);
+
+  const { sql } = getAlterDatabaseScript(structure1, structure2, {}, structure1, structure2, driver);
+  console.log('RUNNING ALTER SQL', driver.engine, ':', sql);
+
+  await driver.script(conn, sql);
+
+  const structure2Real = extendDatabaseInfo(await driver.analyseFull(conn));
+
+  expect(structure2Real.tables.length).toEqual(structure2.tables.length);
+  return structure2Real;
+}
+
+describe('Alter database', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Drop referenced table - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDiff(conn, driver, db => {
+        _.remove(db.tables, x => x.pureName == 't1');
+      });
+    })
+  );
+
+  test.each(flatSource())(
+    'Drop object - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      const db = await testDatabaseDiff(
+        conn,
+        driver,
+        db => {
+          _.remove(db[type], x => x.pureName == 'obj1');
+        },
+        object.create1
+      );
+      expect(db[type].length).toEqual(0);
+    })
+  );
+});

integration-tests/__tests__/alter-table.spec.js

@@ -0,0 +1,124 @@
+const stableStringify = require('json-stable-stringify');
+const _ = require('lodash');
+const fp = require('lodash/fp');
+const uuidv1 = require('uuid/v1');
+const { testWrapper } = require('../tools');
+const engines = require('../engines');
+const { getAlterTableScript, extendDatabaseInfo, generateDbPairingId } = require('dbgate-tools');
+
+function pickImportantTableInfo(table) {
+  return {
+    pureName: table.pureName,
+    columns: table.columns
+      .filter(x => x.columnName != 'rowid')
+      .map(fp.pick(['columnName', 'notNull', 'autoIncrement'])),
+  };
+}
+
+function checkTableStructure(t1, t2) {
+  // expect(t1.pureName).toEqual(t2.pureName)
+  expect(pickImportantTableInfo(t1)).toEqual(pickImportantTableInfo(t2));
+}
+
+async function testTableDiff(conn, driver, mangle) {
+  await driver.query(conn, `create table t0 (id int not null primary key)`);
+
+  await driver.query(
+    conn,
+    `create table t1 (
+    col_pk int not null primary key, 
+    col_std int null, 
+    col_def int null default 12,
+    col_fk int null references t0(id),
+    col_idx int null,
+    col_uq int null unique,
+    col_ref int null unique
+  )`
+  );
+
+  await driver.query(conn, `create index idx1 on t1(col_idx)`);
+
+  await driver.query(conn, `create table t2 (id int not null primary key, fkval int null references t1(col_ref))`);
+
+  const tget = x => x.tables.find(y => y.pureName == 't1');
+  const structure1 = generateDbPairingId(extendDatabaseInfo(await driver.analyseFull(conn)));
+  let structure2 = _.cloneDeep(structure1);
+  mangle(tget(structure2));
+  structure2 = extendDatabaseInfo(structure2);
+
+  const { sql } = getAlterTableScript(tget(structure1), tget(structure2), {}, structure1, structure2, driver);
+  console.log('RUNNING ALTER SQL', driver.engine, ':', sql);
+
+  await driver.script(conn, sql);
+
+  const structure2Real = extendDatabaseInfo(await driver.analyseFull(conn));
+
+  checkTableStructure(tget(structure2Real), tget(structure2));
+  // expect(stableStringify(structure2)).toEqual(stableStringify(structure2Real));
+}
+
+const TESTED_COLUMNS = ['col_pk', 'col_std', 'col_def', 'col_fk', 'col_ref', 'col_idx', 'col_uq'];
+// const TESTED_COLUMNS = ['col_pk'];
+// const TESTED_COLUMNS = ['col_idx'];
+// const TESTED_COLUMNS = ['col_def'];
+// const TESTED_COLUMNS = ['col_std'];
+// const TESTED_COLUMNS = ['col_ref'];
+
+function engines_columns_source() {
+  return _.flatten(engines.map(engine => TESTED_COLUMNS.map(column => [engine.label, column, engine])));
+}
+
+describe('Alter table', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Add column - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableDiff(conn, driver, tbl => {
+        tbl.columns.push({
+          columnName: 'added',
+          dataType: 'int',
+          pairingId: uuidv1(),
+          notNull: false,
+          autoIncrement: false,
+        });
+      });
+    })
+  );
+
+  test.each(engines_columns_source())(
+    'Drop column - %s - %s',
+    testWrapper(async (conn, driver, column, engine) => {
+      await testTableDiff(conn, driver, tbl => (tbl.columns = tbl.columns.filter(x => x.columnName != column)));
+    })
+  );
+
+  test.each(engines_columns_source())(
+    'Change nullability - %s - %s',
+    testWrapper(async (conn, driver, column, engine) => {
+      await testTableDiff(
+        conn,
+        driver,
+        tbl => (tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, notNull: true } : x)))
+      );
+    })
+  );
+
+  test.each(engines_columns_source())(
+    'Rename column - %s - %s',
+    testWrapper(async (conn, driver, column, engine) => {
+      await testTableDiff(
+        conn,
+        driver,
+        tbl => (tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, columnName: 'col_renamed' } : x)))
+      );
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Drop index - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableDiff(conn, driver, tbl => {
+        tbl.indexes = [];
+      });
+    })
+  );
+});

integration-tests/__tests__/db-import.spec.js

@@ -0,0 +1,75 @@
+const engines = require('../engines');
+const stream = require('stream');
+const { testWrapper } = require('../tools');
+const tableWriter = require('dbgate-api/src/shell/tableWriter');
+const copyStream = require('dbgate-api/src/shell/copyStream');
+const fakeObjectReader = require('dbgate-api/src/shell/fakeObjectReader');
+
+function createImportStream() {
+  const pass = new stream.PassThrough({
+    objectMode: true,
+  });
+  pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }], __isStreamHeader: true });
+  pass.write({ id: 1, country: 'Czechia' });
+  pass.write({ id: 2, country: 'Austria' });
+  pass.write({ country: 'Germany', id: 3 });
+  pass.write({ country: 'Romania', id: 4 });
+  pass.write({ country: 'Great Britain', id: 5 });
+  pass.write({ country: 'Bosna, Hecegovina', id: 6 });
+  pass.end();
+
+  return pass;
+}
+
+describe('DB Import', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Import one table - %s',
+    testWrapper(async (conn, driver, engine) => {
+      // const reader = await fakeObjectReader({ delay: 10 });
+      // const reader = await fakeObjectReader();
+      const reader = createImportStream();
+      const writer = await tableWriter({
+        systemConnection: conn,
+        driver,
+        pureName: 't1',
+        createIfNotExists: true,
+      });
+      await copyStream(reader, writer);
+
+      const res = await driver.query(conn, `select count(*) as cnt from t1`);
+      expect(res.rows[0].cnt.toString()).toEqual('6');
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Import two tables - %s',
+    testWrapper(async (conn, driver, engine) => {
+      // const reader = await fakeObjectReader({ delay: 10 });
+      // const reader = await fakeObjectReader();
+      const reader1 = createImportStream();
+      const writer1 = await tableWriter({
+        systemConnection: conn,
+        driver,
+        pureName: 't1',
+        createIfNotExists: true,
+      });
+      await copyStream(reader1, writer1);
+
+      const reader2 = createImportStream();
+      const writer2 = await tableWriter({
+        systemConnection: conn,
+        driver,
+        pureName: 't2',
+        createIfNotExists: true,
+      });
+      await copyStream(reader2, writer2);
+
+      const res1 = await driver.query(conn, `select count(*) as cnt from t1`);
+      expect(res1.rows[0].cnt.toString()).toEqual('6');
+
+      const res2 = await driver.query(conn, `select count(*) as cnt from t2`);
+      expect(res2.rows[0].cnt.toString()).toEqual('6');
+    })
+  );
+
+});

integration-tests/__tests__/deploy-database.spec.js

@@ -0,0 +1,300 @@
+/// TODO
+
+const { testWrapper } = require('../tools');
+const _ = require('lodash');
+const engines = require('../engines');
+const deployDb = require('dbgate-api/src/shell/deployDb');
+const { databaseInfoFromYamlModel } = require('dbgate-tools');
+const generateDeploySql = require('dbgate-api/src/shell/generateDeploySql');
+
+function checkStructure(structure, model) {
+  const expected = databaseInfoFromYamlModel(model);
+  expect(structure.tables.length).toEqual(expected.tables.length);
+
+  for (const [realTable, expectedTable] of _.zip(
+    _.sortBy(structure.tables, 'pureName'),
+    _.sortBy(expected.tables, 'pureName')
+  )) {
+    expect(realTable.columns.length).toBeGreaterThanOrEqual(expectedTable.columns.length);
+  }
+}
+
+async function testDatabaseDeploy(conn, driver, dbModelsYaml, testEmptyLastScript) {
+  let index = 0;
+  for (const loadedDbModel of dbModelsYaml) {
+    const { sql, isEmpty } = await generateDeploySql({
+      systemConnection: conn,
+      driver,
+      loadedDbModel,
+    });
+    console.debug('Generated deploy script:', sql);
+    expect(sql.toUpperCase().includes('DROP ')).toBeFalsy();
+
+    console.log('dbModelsYaml.length', dbModelsYaml.length, index);
+    if (testEmptyLastScript && index == dbModelsYaml.length - 1) {
+      expect(isEmpty).toBeTruthy();
+    }
+
+    await deployDb({
+      systemConnection: conn,
+      driver,
+      loadedDbModel,
+    });
+
+    index++;
+  }
+
+  const structure = await driver.analyseFull(conn);
+  checkStructure(structure, dbModelsYaml[dbModelsYaml.length - 1]);
+}
+
+describe('Deploy database', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Deploy database simple - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(conn, driver, [
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [{ name: 'id', type: 'int' }],
+              primaryKey: ['id'],
+            },
+          },
+        ],
+      ]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Deploy database simple twice - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(
+        conn,
+        driver,
+        [
+          [
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [{ name: 'id', type: 'int' }],
+                primaryKey: ['id'],
+              },
+            },
+          ],
+          [
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [{ name: 'id', type: 'int' }],
+                primaryKey: ['id'],
+              },
+            },
+          ],
+        ],
+        true
+      );
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Add column - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(conn, driver, [
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [{ name: 'id', type: 'int' }],
+              primaryKey: ['id'],
+            },
+          },
+        ],
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [
+                { name: 'id', type: 'int' },
+                { name: 'val', type: 'int' },
+              ],
+              primaryKey: ['id'],
+            },
+          },
+        ],
+      ]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Dont drop column - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(
+        conn,
+        driver,
+        [
+          [
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [
+                  { name: 'id', type: 'int' },
+                  { name: 'val', type: 'int' },
+                ],
+                primaryKey: ['id'],
+              },
+            },
+          ],
+          [
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [{ name: 'id', type: 'int' }],
+                primaryKey: ['id'],
+              },
+            },
+          ],
+        ],
+        true
+      );
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Foreign keys - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(
+        conn,
+        driver,
+        [
+          [
+            {
+              name: 't2.table.yaml',
+              json: {
+                name: 't2',
+                columns: [
+                  { name: 't2id', type: 'int' },
+                  { name: 't1id', type: 'int', references: 't1' },
+                ],
+                primaryKey: ['t2id'],
+              },
+            },
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [{ name: 't1id', type: 'int' }],
+                primaryKey: ['t1id'],
+              },
+            },
+          ],
+          [
+            {
+              name: 't2.table.yaml',
+              json: {
+                name: 't2',
+                columns: [
+                  { name: 't2id', type: 'int' },
+                  { name: 't1id', type: 'int', references: 't1' },
+                ],
+                primaryKey: ['t2id'],
+              },
+            },
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [{ name: 't1id', type: 'int' }],
+                primaryKey: ['t1id'],
+              },
+            },
+          ],
+        ],
+        true
+      );
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Deploy preloaded data - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(conn, driver, [
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [
+                { name: 'id', type: 'int' },
+                { name: 'value', type: 'int' },
+              ],
+              primaryKey: ['id'],
+              data: [
+                { id: 1, value: 1 },
+                { id: 2, value: 2 },
+                { id: 3, value: 3 },
+              ],
+            },
+          },
+        ],
+      ]);
+
+      const res = await driver.query(conn, `select count(*) as cnt from t1`);
+      expect(res.rows[0].cnt.toString()).toEqual('3');
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Deploy preloaded data - update - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(conn, driver, [
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [
+                { name: 'id', type: 'int' },
+                { name: 'val', type: 'int' },
+              ],
+              primaryKey: ['id'],
+              data: [
+                { id: 1, val: 1 },
+                { id: 2, val: 2 },
+                { id: 3, val: 3 },
+              ],
+            },
+          },
+        ],
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [
+                { name: 'id', type: 'int' },
+                { name: 'val', type: 'int' },
+              ],
+              primaryKey: ['id'],
+              data: [
+                { id: 1, val: 1 },
+                { id: 2, val: 5 },
+                { id: 3, val: 3 },
+              ],
+            },
+          },
+        ],
+      ]);
+
+      const res = await driver.query(conn, `select val from t1 where id = 2`);
+      expect(res.rows[0].val.toString()).toEqual('5');
+    })
+  );
+});

integration-tests/__tests__/table-analyse.spec.js

@@ -2,7 +2,10 @@ const engines = require('../engines');
 const { testWrapper } = require('../tools');
 
 const t1Sql = 'CREATE TABLE t1 (id int not null primary key, val1 varchar(50) null)';
-const t2Sql = 'CREATE TABLE t2 (id int not null primary key, val2 varchar(50) null)';
+const ix1Sql = 'CREATE index ix1 ON t1(val1, id)';
+const t2Sql = 'CREATE TABLE t2 (id int not null primary key, val2 varchar(50) null unique)';
+const t3Sql = 'CREATE TABLE t3 (id int not null primary key, valfk int, foreign key (valfk) references t2(id))';
+// const fkSql = 'ALTER TABLE t3 ADD FOREIGN KEY (valfk) REFERENCES t2(id)'
 
 const txMatch = (tname, vcolname, nextcol) =>
   expect.objectContaining({
@@ -98,7 +101,7 @@ describe('Table analyse', () => {
       const structure1 = await driver.analyseFull(conn);
 
       if (engine.dbSnapshotBySeconds) await new Promise(resolve => setTimeout(resolve, 1100));
-      
+
       await driver.query(conn, 'ALTER TABLE t2 ADD nextcol varchar(50)');
       const structure2 = await driver.analyseIncremental(conn, structure1);
 
@@ -109,4 +112,53 @@ describe('Table analyse', () => {
       expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2NextColMatch);
     })
   );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Index - full analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+      await driver.query(conn, ix1Sql);
+      const structure = await driver.analyseFull(conn);
+
+      const t1 = structure.tables.find(x => x.pureName == 't1');
+      expect(t1.indexes.length).toEqual(1);
+      expect(t1.indexes[0].columns.length).toEqual(2);
+      expect(t1.indexes[0].columns[0]).toEqual(expect.objectContaining({ columnName: 'val1' }));
+      expect(t1.indexes[0].columns[1]).toEqual(expect.objectContaining({ columnName: 'id' }));
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Unique - full analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t2Sql);
+      const structure = await driver.analyseFull(conn);
+
+      const t2 = structure.tables.find(x => x.pureName == 't2');
+      // const indexesAndUniques = [...t2.uniques, ...t2.indexes];
+      expect(t2.uniques.length).toEqual(1);
+      expect(t2.uniques[0].columns.length).toEqual(1);
+      expect(t2.uniques[0].columns[0]).toEqual(expect.objectContaining({ columnName: 'val2' }));
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Foreign key - full analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t2Sql);
+      await driver.query(conn, t3Sql);
+      // await driver.query(conn, fkSql);
+
+      const structure = await driver.analyseFull(conn);
+
+      const t3 = structure.tables.find(x => x.pureName == 't3');
+      console.log('T3', t3.foreignKeys[0].columns);
+      expect(t3.foreignKeys.length).toEqual(1);
+      expect(t3.foreignKeys[0].columns.length).toEqual(1);
+      expect(t3.foreignKeys[0]).toEqual(expect.objectContaining({ refTableName: 't2' }));
+      expect(t3.foreignKeys[0].columns[0]).toEqual(
+        expect.objectContaining({ columnName: 'valfk', refColumnName: 'id' })
+      );
+    })
+  );
 });

integration-tests/__tests__/table-create.spec.js

@@ -0,0 +1,153 @@
+const _ = require('lodash');
+const fp = require('lodash/fp');
+const engines = require('../engines');
+const { testWrapper } = require('../tools');
+const { extendDatabaseInfo } = require('dbgate-tools');
+
+function createExpector(value) {
+  return _.cloneDeepWith(value, x => {
+    if (_.isPlainObject(x)) {
+      return expect.objectContaining(_.mapValues(x, y => createExpector(y)));
+    }
+  });
+}
+
+function omitTableSpecificInfo(table) {
+  return {
+    ...table,
+    columns: table.columns.map(fp.omit(['dataType'])),
+  };
+}
+
+function checkTableStructure2(t1, t2) {
+  // expect(t1.pureName).toEqual(t2.pureName)
+  expect(t2).toEqual(createExpector(omitTableSpecificInfo(t1)));
+}
+
+async function testTableCreate(conn, driver, table) {
+  await driver.query(conn, `create table t0 (id int not null primary key)`);
+
+  const dmp = driver.createDumper();
+  const table1 = {
+    ...table,
+    pureName: 'tested',
+  };
+  dmp.createTable(table1);
+
+  console.log('RUNNING CREATE SQL', driver.engine, ':', dmp.s);
+  await driver.script(conn, dmp.s);
+
+  const db = extendDatabaseInfo(await driver.analyseFull(conn));
+  const table2 = db.tables.find(x => x.pureName == 'tested');
+
+  checkTableStructure2(table1, table2);
+}
+
+describe('Table create', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Simple table - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableCreate(conn, driver, {
+        columns: [
+          {
+            columnName: 'col1',
+            dataType: 'int',
+            notNull: true,
+          },
+        ],
+        primaryKey: {
+          columns: [{ columnName: 'col1' }],
+        },
+      });
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table with index - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableCreate(conn, driver, {
+        columns: [
+          {
+            columnName: 'col1',
+            dataType: 'int',
+            notNull: true,
+          },
+          {
+            columnName: 'col2',
+            dataType: 'int',
+            notNull: true,
+          },
+        ],
+        primaryKey: {
+          columns: [{ columnName: 'col1' }],
+        },
+        indexes: [
+          {
+            constraintName: 'ix1',
+            pureName: 'tested',
+            columns: [{ columnName: 'col2' }],
+          },
+        ],
+      });
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table with foreign key - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableCreate(conn, driver, {
+        columns: [
+          {
+            columnName: 'col1',
+            dataType: 'int',
+            notNull: true,
+          },
+          {
+            columnName: 'col2',
+            dataType: 'int',
+            notNull: true,
+          },
+        ],
+        primaryKey: {
+          columns: [{ columnName: 'col1' }],
+        },
+        foreignKeys: [
+          {
+            pureName: 'tested',
+            refTableName: 't0',
+            columns: [{ columnName: 'col2', refColumnName: 'id' }],
+          },
+        ],
+      });
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table with unique - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableCreate(conn, driver, {
+        columns: [
+          {
+            columnName: 'col1',
+            dataType: 'int',
+            notNull: true,
+          },
+          {
+            columnName: 'col2',
+            dataType: 'int',
+            notNull: true,
+          },
+        ],
+        primaryKey: {
+          columns: [{ columnName: 'col1' }],
+        },
+        uniques: [
+          {
+            pureName: 'tested',
+            columns: [{ columnName: 'col2' }],
+          },
+        ],
+      });
+    })
+  );
+});

integration-tests/engines.js

@@ -54,6 +54,15 @@ const engines = [
         drop1: 'DROP PROCEDURE obj1',
         drop2: 'DROP PROCEDURE obj2',
       },
+      {
+        type: 'functions',
+        create1:
+          'CREATE FUNCTION obj1() returns int LANGUAGE plpgsql AS $$ declare  res integer; begin select count(*) into res from t1; return res; end; $$',
+        create2:
+          'CREATE FUNCTION obj2() returns int LANGUAGE plpgsql AS $$ declare res integer; begin select count(*) into res from t2; return res; end; $$',
+        drop1: 'DROP FUNCTION obj1',
+        drop2: 'DROP FUNCTION obj2',
+      },
     ],
   },
   {
@@ -105,4 +114,15 @@ const engines = [
   },
 ];
 
-module.exports = process.env.CITEST ? engines.filter(x => !x.skipOnCI) : engines;
+const filterLocal = [
+  // filter local testing
+  '-MySQL',
+  '-PostgreSQL',
+  '-SQL Server',
+  'SQLite',
+  '-CockroachDB',
+];
+
+module.exports = process.env.CITEST
+  ? engines.filter(x => !x.skipOnCI)
+  : engines.filter(x => filterLocal.find(y => x.label == y));

integration-tests/package.json

@@ -17,6 +17,9 @@
     
     "run:local": "docker-compose down && docker-compose up -d && yarn wait:local && yarn test:local"
   },
+  "jest": {
+    "testTimeout": 5000
+  },
   "devDependencies": {
     "cross-env": "^7.0.3",
     "jest": "^27.0.1"

package.json

@@ -1,6 +1,6 @@
 {
   "private": true,
-  "version": "4.2.4-beta.2",
+  "version": "4.5.2-beta.3",
   "name": "dbgate-all",
   "workspaces": [
     "packages/*",
@@ -9,6 +9,7 @@
   ],
   "scripts": {
     "start:api": "yarn workspace dbgate-api start",
+    "start:app": "cd app && yarn start",
     "start:api:portal": "yarn workspace dbgate-api start:portal",
     "start:api:covid": "yarn workspace dbgate-api start:covid",
     "start:web": "yarn workspace dbgate-web dev",

packages/api/.env-portal

@@ -1,6 +1,6 @@
 DEVMODE=1
 
-CONNECTIONS=mysql,postgres
+CONNECTIONS=mysql,postgres,mongo,mongo2
 
 LABEL_mysql=MySql localhost
 SERVER_mysql=localhost
@@ -15,3 +15,13 @@ USER_postgres=postgres
 PASSWORD_postgres=test
 PORT_postgres=5433
 ENGINE_postgres=postgres@dbgate-plugin-postgres
+
+LABEL_mongo=Mongo URL
+URL_mongo=mongodb://localhost:27017
+ENGINE_mongo=mongo@dbgate-plugin-mongo
+
+LABEL_mongo2=Mongo Server
+SERVER_mongo2=localhost
+ENGINE_mongo2=mongo@dbgate-plugin-mongo
+
+# docker run -p 3000:3000 -e CONNECTIONS=mongo -e URL_mongo=mongodb://localhost:27017 -e ENGINE_mongo=mongo@dbgate-plugin-mongo -e LABEL_mongo=mongo dbgate/dbgate:beta

packages/api/package.json

@@ -19,15 +19,17 @@
   "dependencies": {
     "async-lock": "^1.2.4",
     "axios": "^0.21.1",
-    "better-sqlite3-with-prebuilds": "^7.1.8",
     "body-parser": "^1.19.0",
     "bufferutil": "^4.0.1",
     "byline": "^5.0.0",
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
+    "dbgate-query-splitter": "^4.1.1",
     "dbgate-sqltree": "^4.1.1",
     "dbgate-tools": "^4.1.1",
+    "diff": "^5.0.0",
+    "diff2html": "^3.4.13",
     "eslint": "^6.8.0",
     "express": "^4.17.1",
     "express-basic-auth": "^1.2.0",
@@ -36,6 +38,8 @@
     "fs-reverse": "^0.0.3",
     "get-port": "^5.1.1",
     "http": "^0.0.0",
+    "is-electron": "^2.2.1",
+    "js-yaml": "^4.1.0",
     "json-stable-stringify": "^1.0.1",
     "line-reader": "^0.4.0",
     "lodash": "^4.17.21",
@@ -45,7 +49,6 @@
     "node-ssh-forward": "^0.7.2",
     "portfinder": "^1.0.28",
     "simple-encryptor": "^4.0.0",
-    "socket.io": "^2.3.0",
     "tar": "^6.0.5",
     "uuid": "^3.4.0"
   },
@@ -65,11 +68,12 @@
     "env-cmd": "^10.1.0",
     "node-loader": "^1.0.2",
     "nodemon": "^2.0.2",
-    "typescript": "^3.7.4",
+    "typescript": "^4.4.3",
     "webpack": "^4.42.0",
     "webpack-cli": "^3.3.11"
   },
   "optionalDependencies": {
-    "msnodesqlv8": "^2.0.10"
+    "better-sqlite3": "7.4.5",
+    "msnodesqlv8": "^2.4.4"
   }
 }

packages/api/src/controllers/archive.js

@@ -3,13 +3,14 @@ const stream = require('stream');
 const readline = require('readline');
 const path = require('path');
 const { formatWithOptions } = require('util');
-const { archivedir } = require('../utility/directories');
+const { archivedir, clearArchiveLinksCache, resolveArchiveFolder } = require('../utility/directories');
 const socket = require('../utility/socket');
 const JsonLinesDatastore = require('../utility/JsonLinesDatastore');
 const { saveFreeTableData } = require('../utility/freeTableStorage');
+const loadFilesRecursive = require('../utility/loadFilesRecursive');
 
 module.exports = {
-  folders_meta: 'get',
+  folders_meta: true,
   async folders() {
     const folders = await fs.readdir(archivedir());
     return [
@@ -26,59 +27,103 @@ module.exports = {
     ];
   },
 
-  createFolder_meta: 'post',
+  createFolder_meta: true,
   async createFolder({ folder }) {
     await fs.mkdir(path.join(archivedir(), folder));
     socket.emitChanged('archive-folders-changed');
     return true;
   },
 
-  files_meta: 'get',
-  async files({ folder }) {
-    const dir = path.join(archivedir(), folder);
-    if (!(await fs.exists(dir))) return [];
-    const files = await fs.readdir(dir);
-    return files
-      .filter(name => name.endsWith('.jsonl'))
-      .map(name => ({
-        name: name.slice(0, -'.jsonl'.length),
-        type: 'jsonl',
-      }));
+  createLink_meta: true,
+  async createLink({ linkedFolder }) {
+    const folder = await this.getNewArchiveFolder({ database: path.parse(linkedFolder).name + '.link' });
+    fs.writeFile(path.join(archivedir(), folder), linkedFolder);
+    clearArchiveLinksCache();
+    socket.emitChanged('archive-folders-changed');
+    return folder;
   },
 
-  refreshFiles_meta: 'post',
+  files_meta: true,
+  async files({ folder }) {
+    const dir = resolveArchiveFolder(folder);
+    if (!(await fs.exists(dir))) return [];
+    const files = await loadFilesRecursive(dir); // fs.readdir(dir);
+
+    function fileType(ext, type) {
+      return files
+        .filter(name => name.endsWith(ext))
+        .map(name => ({
+          name: name.slice(0, -ext.length),
+          label: path.parse(name.slice(0, -ext.length)).base,
+          type,
+        }));
+    }
+
+    return [
+      ...fileType('.jsonl', 'jsonl'),
+      ...fileType('.table.yaml', 'table.yaml'),
+      ...fileType('.view.sql', 'view.sql'),
+      ...fileType('.proc.sql', 'proc.sql'),
+      ...fileType('.func.sql', 'func.sql'),
+      ...fileType('.trigger.sql', 'trigger.sql'),
+      ...fileType('.matview.sql', 'matview.sql'),
+    ];
+  },
+
+  refreshFiles_meta: true,
   async refreshFiles({ folder }) {
     socket.emitChanged(`archive-files-changed-${folder}`);
   },
 
-  refreshFolders_meta: 'post',
+  refreshFolders_meta: true,
   async refreshFolders() {
     socket.emitChanged(`archive-folders-changed`);
   },
 
-  deleteFile_meta: 'post',
-  async deleteFile({ folder, file }) {
-    await fs.unlink(path.join(archivedir(), folder, `${file}.jsonl`));
+  deleteFile_meta: true,
+  async deleteFile({ folder, file, fileType }) {
+    await fs.unlink(path.join(resolveArchiveFolder(folder), `${file}.${fileType}`));
     socket.emitChanged(`archive-files-changed-${folder}`);
   },
 
-  deleteFolder_meta: 'post',
-  async deleteFolder({ folder }) {
-    if (!folder) throw new Error('Missing folder parameter');
-    await fs.rmdir(path.join(archivedir(), folder), { recursive: true });
+  renameFile_meta: true,
+  async renameFile({ folder, file, newFile, fileType }) {
+    await fs.rename(
+      path.join(resolveArchiveFolder(folder), `${file}.${fileType}`),
+      path.join(resolveArchiveFolder(folder), `${newFile}.${fileType}`)
+    );
+    socket.emitChanged(`archive-files-changed-${folder}`);
+  },
+
+  renameFolder_meta: true,
+  async renameFolder({ folder, newFolder }) {
+    const uniqueName = await this.getNewArchiveFolder({ database: newFolder });
+    await fs.rename(path.join(archivedir(), folder), path.join(archivedir(), uniqueName));
     socket.emitChanged(`archive-folders-changed`);
   },
 
-  saveFreeTable_meta: 'post',
+  deleteFolder_meta: true,
+  async deleteFolder({ folder }) {
+    if (!folder) throw new Error('Missing folder parameter');
+    if (folder.endsWith('.link')) {
+      await fs.unlink(path.join(archivedir(), folder));
+    } else {
+      await fs.rmdir(path.join(archivedir(), folder), { recursive: true });
+    }
+    socket.emitChanged(`archive-folders-changed`);
+  },
+
+  saveFreeTable_meta: true,
   async saveFreeTable({ folder, file, data }) {
-    saveFreeTableData(path.join(archivedir(), folder, `${file}.jsonl`), data);
+    await saveFreeTableData(path.join(resolveArchiveFolder(folder), `${file}.jsonl`), data);
+    socket.emitChanged(`archive-files-changed-${folder}`);
     return true;
   },
 
-  loadFreeTable_meta: 'post',
+  loadFreeTable_meta: true,
   async loadFreeTable({ folder, file }) {
     return new Promise((resolve, reject) => {
-      const fileStream = fs.createReadStream(path.join(archivedir(), folder, `${file}.jsonl`));
+      const fileStream = fs.createReadStream(path.join(resolveArchiveFolder(folder), `${file}.jsonl`));
       const liner = readline.createInterface({
         input: fileStream,
       });
@@ -95,4 +140,16 @@ module.exports = {
       });
     });
   },
+
+  async getNewArchiveFolder({ database }) {
+    const isLink = database.endsWith(database);
+    const name = isLink ? database.slice(0, -5) : database;
+    const suffix = isLink ? '.link' : '';
+    if (!(await fs.exists(path.join(archivedir(), database)))) return database;
+    let index = 2;
+    while (await fs.exists(path.join(archivedir(), `${name}${index}${suffix}`))) {
+      index += 1;
+    }
+    return `${name}${index}${suffix}`;
+  },
 };

packages/api/src/controllers/config.js

@@ -1,5 +1,6 @@
 const fs = require('fs-extra');
 const path = require('path');
+const axios = require('axios');
 const { datadir } = require('../utility/directories');
 const hasPermission = require('../utility/hasPermission');
 const socket = require('../utility/socket');
@@ -20,7 +21,7 @@ module.exports = {
     }
   },
 
-  get_meta: 'get',
+  get_meta: true,
   async get() {
     const permissions = process.env.PERMISSIONS ? process.env.PERMISSIONS.split(',') : null;
 
@@ -32,17 +33,17 @@ module.exports = {
     };
   },
 
-  platformInfo_meta: 'get',
+  platformInfo_meta: true,
   async platformInfo() {
     return platformInfo;
   },
 
-  getSettings_meta: 'get',
+  getSettings_meta: true,
   async getSettings() {
     return this.settingsValue;
   },
 
-  updateSettings_meta: 'post',
+  updateSettings_meta: true,
   async updateSettings(values) {
     if (!hasPermission(`settings/change`)) return false;
     try {
@@ -58,4 +59,10 @@ module.exports = {
       return false;
     }
   },
+
+  changelog_meta: true,
+  async changelog() {
+    const resp = await axios.default.get('https://raw.githubusercontent.com/dbgate/dbgate/master/CHANGELOG.md');
+    return resp.data;
+  },
 };

packages/api/src/controllers/connections.js

@@ -2,11 +2,15 @@ const path = require('path');
 const { fork } = require('child_process');
 const _ = require('lodash');
 const nedb = require('nedb-promises');
+const fs = require('fs-extra');
 
-const { datadir } = require('../utility/directories');
+const { datadir, filesdir } = require('../utility/directories');
 const socket = require('../utility/socket');
 const { encryptConnection } = require('../utility/crypting');
 const { handleProcessCommunication } = require('../utility/processComm');
+const { pickSafeConnectionInfo } = require('../utility/crypting');
+
+const processArgs = require('../utility/processArgs');
 
 function getNamedArgs() {
   const res = {};
@@ -36,7 +40,7 @@ function getDatabaseFileLabel(databaseFile) {
 
 function getPortalCollections() {
   if (process.env.CONNECTIONS) {
-    return _.compact(process.env.CONNECTIONS.split(',')).map(id => ({
+    const connections = _.compact(process.env.CONNECTIONS.split(',')).map(id => ({
       _id: id,
       engine: process.env[`ENGINE_${id}`],
       server: process.env[`SERVER_${id}`],
@@ -44,11 +48,22 @@ function getPortalCollections() {
       password: process.env[`PASSWORD_${id}`],
       port: process.env[`PORT_${id}`],
       databaseUrl: process.env[`URL_${id}`],
+      useDatabaseUrl: !!process.env[`URL_${id}`],
       databaseFile: process.env[`FILE_${id}`],
       defaultDatabase: process.env[`DATABASE_${id}`],
       singleDatabase: !!process.env[`DATABASE_${id}`],
       displayName: process.env[`LABEL_${id}`],
     }));
+    console.log('Using connections from ENV variables:');
+    console.log(JSON.stringify(connections.map(pickSafeConnectionInfo), undefined, 2));
+    const noengine = connections.filter(x => !x.engine);
+    if (noengine.length > 0) {
+      console.log(
+        'Warning: Invalid CONNECTIONS configutation, missing ENGINE for connection ID:',
+        noengine.map(x => x._id)
+      );
+    }
+    return connections;
   }
 
   const args = getNamedArgs();
@@ -125,29 +140,34 @@ module.exports = {
     }
   },
 
-  list_meta: 'get',
+  list_meta: true,
   async list() {
     return portalConnections || this.datastore.find();
   },
 
-  test_meta: {
-    method: 'post',
-    raw: true,
-  },
-  test(req, res) {
-    const subprocess = fork(process.argv[1], ['--start-process', 'connectProcess', ...process.argv.slice(3)]);
-    subprocess.on('message', resp => {
-      if (handleProcessCommunication(resp, subprocess)) return;
-      // @ts-ignore
-      const { msgtype } = resp;
-      if (msgtype == 'connected' || msgtype == 'error') {
-        res.json(resp);
-      }
+  test_meta: true,
+  test(connection) {
+    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+      '--is-forked-api',
+      '--start-process',
+      'connectProcess',
+      ...processArgs.getPassArgs(),
+      // ...process.argv.slice(3),
+    ]);
+    subprocess.send(connection);
+    return new Promise(resolve => {
+      subprocess.on('message', resp => {
+        if (handleProcessCommunication(resp, subprocess)) return;
+        // @ts-ignore
+        const { msgtype } = resp;
+        if (msgtype == 'connected' || msgtype == 'error') {
+          resolve(resp);
+        }
+      });
     });
-    subprocess.send(req.body);
   },
 
-  save_meta: 'post',
+  save_meta: true,
   async save(connection) {
     if (portalConnections) return;
     let res;
@@ -161,7 +181,30 @@ module.exports = {
     return res;
   },
 
-  delete_meta: 'post',
+  update_meta: true,
+  async update({ _id, values }) {
+    if (portalConnections) return;
+    const res = await this.datastore.update({ _id }, { $set: values });
+    socket.emitChanged('connection-list-changed');
+    return res;
+  },
+
+  updateDatabase_meta: true,
+  async updateDatabase({ conid, database, values }) {
+    if (portalConnections) return;
+    const conn = await this.datastore.find({ _id: conid });
+    let databases = conn[0].databases || [];
+    if (databases.find(x => x.name == database)) {
+      databases = databases.map(x => (x.name == database ? { ...x, ...values } : x));
+    } else {
+      databases = [...databases, { name: database, ...values }];
+    }
+    const res = await this.datastore.update({ _id: conid }, { $set: { databases } });
+    socket.emitChanged('connection-list-changed');
+    return res;
+  },
+
+  delete_meta: true,
   async delete(connection) {
     if (portalConnections) return;
     const res = await this.datastore.remove(_.pick(connection, '_id'));
@@ -169,10 +212,26 @@ module.exports = {
     return res;
   },
 
-  get_meta: 'get',
+  get_meta: true,
   async get({ conid }) {
-    if (portalConnections) return portalConnections.find(x => x._id == conid);
+    if (portalConnections) return portalConnections.find(x => x._id == conid) || null;
     const res = await this.datastore.find({ _id: conid });
-    return res[0];
+    return res[0] || null;
+  },
+
+  newSqliteDatabase_meta: true,
+  async newSqliteDatabase({ file }) {
+    const sqliteDir = path.join(filesdir(), 'sqlite');
+    if (!(await fs.exists(sqliteDir))) {
+      await fs.mkdir(sqliteDir);
+    }
+    const databaseFile = path.join(sqliteDir, `${file}.sqlite`);
+    const res = await this.save({
+      engine: 'sqlite@dbgate-plugin-sqlite',
+      databaseFile,
+      singleDatabase: true,
+      defaultDatabase: `${file}.sqlite`,
+    });
+    return res;
   },
 };

packages/api/src/controllers/databaseConnections.js

@@ -1,10 +1,31 @@
 const uuidv1 = require('uuid/v1');
 const connections = require('./connections');
+const archive = require('./archive');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
-const { DatabaseAnalyser } = require('dbgate-tools');
+const {
+  DatabaseAnalyser,
+  computeDbDiffRows,
+  getCreateObjectScript,
+  getAlterDatabaseScript,
+  generateDbPairingId,
+  matchPairedObjects,
+  extendDatabaseInfo,
+  modelCompareDbDiffOptions,
+} = require('dbgate-tools');
+const { html, parse } = require('diff2html');
 const { handleProcessCommunication } = require('../utility/processComm');
 const config = require('./config');
+const fs = require('fs-extra');
+const exportDbModel = require('../utility/exportDbModel');
+const { archivedir, resolveArchiveFolder, uploadsdir } = require('../utility/directories');
+const path = require('path');
+const importDbModel = require('../utility/importDbModel');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const generateDeploySql = require('../shell/generateDeploySql');
+const { createTwoFilesPatch } = require('diff');
+const diff2htmlPage = require('../utility/diff2htmlPage');
+const processArgs = require('../utility/processArgs');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedDatabaseConnection[]} */
@@ -54,10 +75,12 @@ module.exports = {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) return existing;
     const connection = await connections.get({ conid });
-    const subprocess = fork(process.argv[1], [
+    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+      '--is-forked-api',
       '--start-process',
       'databaseConnectionProcess',
-      ...process.argv.slice(3),
+      ...processArgs.getPassArgs(),
+      // ...process.argv.slice(3),
     ]);
     const lastClosed = this.closed[`${conid}/${database}`];
     const newOpened = {
@@ -101,7 +124,7 @@ module.exports = {
     return promise;
   },
 
-  queryData_meta: 'post',
+  queryData_meta: true,
   async queryData({ conid, database, sql }) {
     console.log(`Processing query, conid=${conid}, database=${database}, sql=${sql}`);
     const opened = await this.ensureOpened(conid, database);
@@ -112,7 +135,7 @@ module.exports = {
     return res;
   },
 
-  runScript_meta: 'post',
+  runScript_meta: true,
   async runScript({ conid, database, sql }) {
     console.log(`Processing script, conid=${conid}, database=${database}, sql=${sql}`);
     const opened = await this.ensureOpened(conid, database);
@@ -120,21 +143,21 @@ module.exports = {
     return res;
   },
 
-  collectionData_meta: 'post',
+  collectionData_meta: true,
   async collectionData({ conid, database, options }) {
     const opened = await this.ensureOpened(conid, database);
     const res = await this.sendRequest(opened, { msgtype: 'collectionData', options });
     return res.result;
   },
 
-  updateCollection_meta: 'post',
+  updateCollection_meta: true,
   async updateCollection({ conid, database, changeSet }) {
     const opened = await this.ensureOpened(conid, database);
     const res = await this.sendRequest(opened, { msgtype: 'updateCollection', changeSet });
     return res.result;
   },
 
-  status_meta: 'get',
+  status_meta: true,
   async status({ conid, database }) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) {
@@ -156,7 +179,7 @@ module.exports = {
     };
   },
 
-  ping_meta: 'post',
+  ping_meta: true,
   async ping({ conid, database }) {
     let existing = this.opened.find(x => x.conid == conid && x.database == database);
 
@@ -172,7 +195,7 @@ module.exports = {
     };
   },
 
-  refresh_meta: 'post',
+  refresh_meta: true,
   async refresh({ conid, database, keepOpen }) {
     if (!keepOpen) this.close(conid, database);
 
@@ -180,7 +203,7 @@ module.exports = {
     return { status: 'ok' };
   },
 
-  syncModel_meta: 'post',
+  syncModel_meta: true,
   async syncModel({ conid, database }) {
     const conn = await this.ensureOpened(conid, database);
     conn.subprocess.send({ msgtype: 'syncModel' });
@@ -204,14 +227,19 @@ module.exports = {
     }
   },
 
-  disconnect_meta: 'post',
+  disconnect_meta: true,
   async disconnect({ conid, database }) {
     await this.close(conid, database, true);
     return { status: 'ok' };
   },
 
-  structure_meta: 'get',
+  structure_meta: true,
   async structure({ conid, database }) {
+    if (conid == '__model') {
+      const model = await importDbModel(database);
+      return model;
+    }
+
     const opened = await this.ensureOpened(conid, database);
     return opened.structure;
     // const existing = this.opened.find((x) => x.conid == conid && x.database == database);
@@ -222,13 +250,14 @@ module.exports = {
     // };
   },
 
-  serverVersion_meta: 'get',
+  serverVersion_meta: true,
   async serverVersion({ conid, database }) {
+    if (!conid) return null;
     const opened = await this.ensureOpened(conid, database);
-    return opened.serverVersion;
+    return opened.serverVersion || null;
   },
 
-  sqlPreview_meta: 'post',
+  sqlPreview_meta: true,
   async sqlPreview({ conid, database, objects, options }) {
     // wait for structure
     await this.structure({ conid, database });
@@ -238,11 +267,99 @@ module.exports = {
     return res;
   },
 
-  // runCommand_meta: 'post',
+  exportModel_meta: true,
+  async exportModel({ conid, database }) {
+    const archiveFolder = await archive.getNewArchiveFolder({ database });
+    await fs.mkdir(path.join(archivedir(), archiveFolder));
+    const model = await this.structure({ conid, database });
+    await exportDbModel(model, path.join(archivedir(), archiveFolder));
+    socket.emitChanged(`archive-folders-changed`);
+    return { archiveFolder };
+  },
+
+  generateDeploySql_meta: true,
+  async generateDeploySql({ conid, database, archiveFolder }) {
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, {
+      msgtype: 'generateDeploySql',
+      modelFolder: resolveArchiveFolder(archiveFolder),
+    });
+    return res;
+
+    // const connection = await connections.get({ conid });
+    // return generateDeploySql({
+    //   connection,
+    //   analysedStructure: await this.structure({ conid, database }),
+    //   modelFolder: resolveArchiveFolder(archiveFolder),
+    // });
+
+    // const deployedModel = generateDbPairingId(await importDbModel(path.join(archivedir(), archiveFolder)));
+    // const currentModel = generateDbPairingId(await this.structure({ conid, database }));
+    // const currentModelPaired = matchPairedObjects(deployedModel, currentModel);
+    // const connection = await connections.get({ conid });
+    // const driver = requireEngineDriver(connection);
+    // const { sql } = getAlterDatabaseScript(currentModelPaired, deployedModel, {}, deployedModel, driver);
+    // return {
+    //   deployedModel,
+    //   currentModel,
+    //   currentModelPaired,
+    //   sql,
+    // };
+    // return sql;
+  },
+  // runCommand_meta: true,
   // async runCommand({ conid, database, sql }) {
   //   console.log(`Running SQL command , conid=${conid}, database=${database}, sql=${sql}`);
   //   const opened = await this.ensureOpened(conid, database);
   //   const res = await this.sendRequest(opened, { msgtype: 'queryData', sql });
   //   return res;
   // },
+
+  async getUnifiedDiff({ sourceConid, sourceDatabase, targetConid, targetDatabase }) {
+    const dbDiffOptions = sourceConid == '__model' ? modelCompareDbDiffOptions : {};
+
+    const sourceDb = generateDbPairingId(
+      extendDatabaseInfo(await this.structure({ conid: sourceConid, database: sourceDatabase }))
+    );
+    const targetDb = generateDbPairingId(
+      extendDatabaseInfo(await this.structure({ conid: targetConid, database: targetDatabase }))
+    );
+    // const sourceConnection = await connections.get({conid:sourceConid})
+    const connection = await connections.get({ conid: targetConid });
+    const driver = requireEngineDriver(connection);
+    const targetDbPaired = matchPairedObjects(sourceDb, targetDb, dbDiffOptions);
+    const diffRows = computeDbDiffRows(sourceDb, targetDbPaired, dbDiffOptions, driver);
+
+    // console.log('sourceDb', sourceDb);
+    // console.log('targetDb', targetDb);
+    // console.log('sourceConid, sourceDatabase', sourceConid, sourceDatabase);
+
+    let res = '';
+    for (const row of diffRows) {
+      // console.log('PAIR', row.source && row.source.pureName, row.target && row.target.pureName);
+      const unifiedDiff = createTwoFilesPatch(
+        (row.target && row.target.pureName) || '',
+        (row.source && row.source.pureName) || '',
+        getCreateObjectScript(row.target, driver),
+        getCreateObjectScript(row.source, driver),
+        '',
+        ''
+      );
+      res += unifiedDiff;
+    }
+    return res;
+  },
+
+  generateDbDiffReport_meta: true,
+  async generateDbDiffReport({ filePath, sourceConid, sourceDatabase, targetConid, targetDatabase }) {
+    const unifiedDiff = await this.getUnifiedDiff({ sourceConid, sourceDatabase, targetConid, targetDatabase });
+
+    const diffJson = parse(unifiedDiff);
+    // $: diffHtml = html(diffJson, { outputFormat: 'side-by-side', drawFileList: false });
+    const diffHtml = html(diffJson, { outputFormat: 'side-by-side' });
+
+    await fs.writeFile(filePath, diff2htmlPage(diffHtml));
+
+    return true;
+  },
 };

packages/api/src/controllers/files.js

@@ -1,9 +1,12 @@
+const uuidv1 = require('uuid/v1');
 const fs = require('fs-extra');
 const path = require('path');
-const { filesdir } = require('../utility/directories');
+const { filesdir, archivedir, resolveArchiveFolder, uploadsdir } = require('../utility/directories');
+const getChartExport = require('../utility/getChartExport');
 const hasPermission = require('../utility/hasPermission');
 const socket = require('../utility/socket');
 const scheduler = require('./scheduler');
+const getDiagramExport = require('../utility/getDiagramExport');
 
 function serialize(format, data) {
   if (format == 'text') return data;
@@ -18,7 +21,7 @@ function deserialize(format, text) {
 }
 
 module.exports = {
-  list_meta: 'get',
+  list_meta: true,
   async list({ folder }) {
     if (!hasPermission(`files/${folder}/read`)) return [];
     const dir = path.join(filesdir(), folder);
@@ -27,7 +30,7 @@ module.exports = {
     return files;
   },
 
-  listAll_meta: 'get',
+  listAll_meta: true,
   async listAll() {
     const folders = await fs.readdir(filesdir());
     const res = [];
@@ -40,7 +43,7 @@ module.exports = {
     return res;
   },
 
-  delete_meta: 'post',
+  delete_meta: true,
   async delete({ folder, file }) {
     if (!hasPermission(`files/${folder}/write`)) return;
     await fs.unlink(path.join(filesdir(), folder, file));
@@ -48,7 +51,7 @@ module.exports = {
     socket.emitChanged(`all-files-changed`);
   },
 
-  rename_meta: 'post',
+  rename_meta: true,
   async rename({ folder, file, newFile }) {
     if (!hasPermission(`files/${folder}/write`)) return;
     await fs.rename(path.join(filesdir(), folder, file), path.join(filesdir(), folder, newFile));
@@ -56,34 +59,57 @@ module.exports = {
     socket.emitChanged(`all-files-changed`);
   },
 
-  load_meta: 'post',
-  async load({ folder, file, format }) {
-    if (!hasPermission(`files/${folder}/read`)) return null;
-    const text = await fs.readFile(path.join(filesdir(), folder, file), { encoding: 'utf-8' });
-    return deserialize(format, text);
-  },
-
-  save_meta: 'post',
-  async save({ folder, file, data, format }) {
+  copy_meta: true,
+  async copy({ folder, file, newFile }) {
     if (!hasPermission(`files/${folder}/write`)) return;
-    const dir = path.join(filesdir(), folder);
-    if (!(await fs.exists(dir))) {
-      await fs.mkdir(dir);
-    }
-    await fs.writeFile(path.join(dir, file), serialize(format, data));
+    await fs.copyFile(path.join(filesdir(), folder, file), path.join(filesdir(), folder, newFile));
     socket.emitChanged(`files-changed-${folder}`);
     socket.emitChanged(`all-files-changed`);
-    if (folder == 'shell') {
-      scheduler.reload();
+  },
+
+  load_meta: true,
+  async load({ folder, file, format }) {
+    if (folder.startsWith('archive:')) {
+      const text = await fs.readFile(path.join(resolveArchiveFolder(folder.substring('archive:'.length)), file), {
+        encoding: 'utf-8',
+      });
+      return deserialize(format, text);
+    } else {
+      if (!hasPermission(`files/${folder}/read`)) return null;
+      const text = await fs.readFile(path.join(filesdir(), folder, file), { encoding: 'utf-8' });
+      return deserialize(format, text);
     }
   },
 
-  saveAs_meta: 'post',
+  save_meta: true,
+  async save({ folder, file, data, format }) {
+    if (folder.startsWith('archive:')) {
+      const dir = resolveArchiveFolder(folder.substring('archive:'.length));
+      await fs.writeFile(path.join(dir, file), serialize(format, data));
+      socket.emitChanged(`archive-files-changed-${folder.substring('archive:'.length)}`);
+      return true;
+    } else {
+      if (!hasPermission(`files/${folder}/write`)) return false;
+      const dir = path.join(filesdir(), folder);
+      if (!(await fs.exists(dir))) {
+        await fs.mkdir(dir);
+      }
+      await fs.writeFile(path.join(dir, file), serialize(format, data));
+      socket.emitChanged(`files-changed-${folder}`);
+      socket.emitChanged(`all-files-changed`);
+      if (folder == 'shell') {
+        scheduler.reload();
+      }
+      return true;
+    }
+  },
+
+  saveAs_meta: true,
   async saveAs({ filePath, data, format }) {
     await fs.writeFile(filePath, serialize(format, data));
   },
 
-  favorites_meta: 'get',
+  favorites_meta: true,
   async favorites() {
     if (!hasPermission(`files/favorites/read`)) return [];
     const dir = path.join(filesdir(), 'favorites');
@@ -101,4 +127,36 @@ module.exports = {
     }
     return res;
   },
+
+  generateUploadsFile_meta: true,
+  async generateUploadsFile() {
+    const fileName = `${uuidv1()}.html`;
+    return {
+      fileName,
+      filePath: path.join(uploadsdir(), fileName),
+    };
+  },
+
+  exportChart_meta: true,
+  async exportChart({ filePath, title, config, image }) {
+    const fileName = path.parse(filePath).base;
+    const imageFile = fileName.replace('.html', '-preview.png');
+    const html = getChartExport(title, config, imageFile);
+    await fs.writeFile(filePath, html);
+    if (image) {
+      const index = image.indexOf('base64,');
+      if (index > 0) {
+        const data = image.substr(index + 'base64,'.length);
+        const buf = Buffer.from(data, 'base64');
+        await fs.writeFile(filePath.replace('.html', '-preview.png'), buf);
+      }
+    }
+    return true;
+  },
+
+  exportDiagram_meta: true,
+  async exportDiagram({ filePath, html, css }) {
+    await fs.writeFile(filePath, getDiagramExport(html, css));
+    return true;
+  },
 };

packages/api/src/controllers/jsldata.js

@@ -104,7 +104,7 @@ module.exports = {
     return datastore;
   },
 
-  getInfo_meta: 'get',
+  getInfo_meta: true,
   async getInfo({ jslid }) {
     const file = getJslFileName(jslid);
     const firstLine = await readFirstLine(file);
@@ -112,13 +112,13 @@ module.exports = {
     return null;
   },
 
-  getRows_meta: 'post',
+  getRows_meta: true,
   async getRows({ jslid, offset, limit, filters }) {
     const datastore = await this.ensureDatastore(jslid);
     return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters);
   },
 
-  getStats_meta: 'get',
+  getStats_meta: true,
   getStats({ jslid }) {
     const file = `${getJslFileName(jslid)}.stats`;
     if (fs.existsSync(file)) {
@@ -146,7 +146,7 @@ module.exports = {
     // }
   },
 
-  saveFreeTable_meta: 'post',
+  saveFreeTable_meta: true,
   async saveFreeTable({ jslid, data }) {
     saveFreeTableData(getJslFileName(jslid), data);
     return true;

packages/api/src/controllers/metadata.js

@@ -7,14 +7,14 @@ function pickObjectNames(array) {
 }
 
 module.exports = {
-  // tableData_meta: 'get',
+  // tableData_meta: true,
   // async tableData({ conid, database, schemaName, pureName }) {
   //   const opened = await databaseConnections.ensureOpened(conid, database);
   //   const res = await databaseConnections.sendRequest(opened, { msgtype: 'tableData', schemaName, pureName });
   //   return res;
   // },
 
-  listObjects_meta: 'get',
+  listObjects_meta: true,
   async listObjects({ conid, database }) {
     const opened = await databaseConnections.ensureOpened(conid, database);
     const types = ['tables', 'collections', 'views', 'procedures', 'functions', 'triggers'];
@@ -27,7 +27,7 @@ module.exports = {
     );
   },
 
-  tableInfo_meta: 'get',
+  tableInfo_meta: true,
   async tableInfo({ conid, database, schemaName, pureName }) {
     const opened = await databaseConnections.ensureOpened(conid, database);
     const table = opened.structure.tables.find(x => x.pureName == pureName && x.schemaName == schemaName);
@@ -38,7 +38,7 @@ module.exports = {
     };
   },
 
-  sqlObjectInfo_meta: 'get',
+  sqlObjectInfo_meta: true,
   async sqlObjectInfo({ objectTypeField, conid, database, schemaName, pureName }) {
     const opened = await databaseConnections.ensureOpened(conid, database);
     const res = opened.structure[objectTypeField].find(x => x.pureName == pureName && x.schemaName == schemaName);

packages/api/src/controllers/plugins.js

@@ -12,7 +12,7 @@ const _ = require('lodash');
 const packagedPluginsContent = require('../packagedPluginsContent');
 
 module.exports = {
-  script_meta: 'get',
+  script_meta: true,
   async script({ packageName }) {
     const packagedContent = packagedPluginsContent();
 
@@ -30,7 +30,7 @@ module.exports = {
     return data;
   },
 
-  search_meta: 'get',
+  search_meta: true,
   async search({ filter }) {
     // DOCS: https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#get-v1search
     const resp = await axios.default.get(
@@ -40,7 +40,7 @@ module.exports = {
     return (objects || []).map(x => x.package);
   },
 
-  info_meta: 'get',
+  info_meta: true,
   async info({ packageName }) {
     try {
       const infoResp = await axios.default.get(`https://registry.npmjs.org/${packageName}`);
@@ -63,7 +63,7 @@ module.exports = {
     }
   },
 
-  installed_meta: 'get',
+  installed_meta: true,
   async installed() {
     const packagedContent = packagedPluginsContent();
 
@@ -107,7 +107,7 @@ module.exports = {
   //   await fs.writeFile(path.join(datadir(), 'removed-plugins'), this.removedPlugins.join('\n'));
   // },
 
-  install_meta: 'post',
+  install_meta: true,
   async install({ packageName }) {
     if (!hasPermission(`plugins/install`)) return;
     const dir = path.join(pluginsdir(), packageName);
@@ -120,7 +120,7 @@ module.exports = {
     // await this.saveRemovePlugins();
   },
 
-  uninstall_meta: 'post',
+  uninstall_meta: true,
   async uninstall({ packageName }) {
     if (!hasPermission(`plugins/install`)) return;
     const dir = path.join(pluginsdir(), packageName);
@@ -130,7 +130,7 @@ module.exports = {
     await this.saveRemovePlugins();
   },
 
-  upgrade_meta: 'post',
+  upgrade_meta: true,
   async upgrade({ packageName }) {
     if (!hasPermission(`plugins/install`)) return;
     const dir = path.join(pluginsdir(), packageName);
@@ -143,13 +143,13 @@ module.exports = {
     socket.emitChanged(`installed-plugins-changed`);
   },
 
-  command_meta: 'post',
+  command_meta: true,
   async command({ packageName, command, args }) {
     const content = requirePlugin(packageName);
     return content.commands[command](args);
   },
 
-  authTypes_meta: 'get',
+  authTypes_meta: true,
   async authTypes({ engine }) {
     const packageName = extractPackageName(engine);
     const content = requirePlugin(packageName);

packages/api/src/controllers/queryHistory.js

@@ -34,7 +34,7 @@ function readCore(reader, skip, limit, filter) {
 }
 
 module.exports = {
-  read_meta: 'get',
+  read_meta: true,
   async read({ skip, limit, filter }) {
     const fileName = path.join(datadir(), 'query-history.jsonl');
     // @ts-ignore
@@ -44,7 +44,7 @@ module.exports = {
     return res;
   },
 
-  write_meta: 'post',
+  write_meta: true,
   async write({ data }) {
     const fileName = path.join(datadir(), 'query-history.jsonl');
     await fs.appendFile(fileName, JSON.stringify(data) + '\n');

packages/api/src/controllers/runners.js

@@ -8,6 +8,7 @@ const { fork } = require('child_process');
 const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories');
 const { extractShellApiPlugins, extractShellApiFunctionName } = require('dbgate-tools');
 const { handleProcessCommunication } = require('../utility/processComm');
+const processArgs = require('../utility/processArgs');
 
 function extractPlugins(script) {
   const requireRegex = /\s*\/\/\s*@require\s+([^\s]+)\s*\n/g;
@@ -15,17 +16,20 @@ function extractPlugins(script) {
   return matches.map(x => x[1]);
 }
 
-const requirePluginsTemplate = plugins =>
+const requirePluginsTemplate = (plugins, isExport) =>
   plugins
     .map(
-      packageName => `const ${_.camelCase(packageName)} = require(process.env.PLUGIN_${_.camelCase(packageName)});\n`
+      packageName =>
+        `const ${_.camelCase(packageName)} = require(${
+          isExport ? `'${packageName}'` : `process.env.PLUGIN_${_.camelCase(packageName)}`
+        });\n`
     )
     .join('') + `dbgateApi.registerPlugins(${plugins.map(x => _.camelCase(x)).join(',')});\n`;
 
-const scriptTemplate = script => `
-const dbgateApi = require(process.env.DBGATE_API);
+const scriptTemplate = (script, isExport) => `
+const dbgateApi = require(${isExport ? `'dbgate-api'` : 'process.env.DBGATE_API'});
 dbgateApi.initializeApiEnvironment();
-${requirePluginsTemplate(extractPlugins(script))}
+${requirePluginsTemplate(extractPlugins(script), isExport)}
 require=null;
 async function run() {
 ${script}
@@ -95,15 +99,22 @@ module.exports = {
     const pluginNames = _.union(fs.readdirSync(pluginsdir()), packagedPluginList);
     console.log(`RUNNING SCRIPT ${scriptFile}`);
     // const subprocess = fork(scriptFile, ['--checkParent', '--max-old-space-size=8192'], {
-    const subprocess = fork(scriptFile, ['--checkParent', ...process.argv.slice(3)], {
-      cwd: directory,
-      stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
-      env: {
-        ...process.env,
-        DBGATE_API: global['dbgateApiModulePath'] || process.argv[1],
-        ..._.fromPairs(pluginNames.map(name => [`PLUGIN_${_.camelCase(name)}`, getPluginBackendPath(name)])),
-      },
-    });
+    const subprocess = fork(
+      scriptFile,
+      [
+        '--checkParent', // ...process.argv.slice(3)
+        ...processArgs.getPassArgs(),
+      ],
+      {
+        cwd: directory,
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+        env: {
+          ...process.env,
+          DBGATE_API: global['API_PACKAGE'] || global['dbgateApiModulePath'] || process.argv[1],
+          ..._.fromPairs(pluginNames.map(name => [`PLUGIN_${_.camelCase(name)}`, getPluginBackendPath(name)])),
+        },
+      }
+    );
     const pipeDispatcher = severity => data =>
       this.dispatchMessage(runid, { severity, message: data.toString().trim() });
 
@@ -133,16 +144,21 @@ module.exports = {
       if (handleProcessCommunication(message, subprocess)) return;
       this[`handle_${msgtype}`](runid, message);
     });
-    return newOpened;
+    return _.pick(newOpened, ['runid']);
   },
 
-  start_meta: 'post',
+  start_meta: true,
   async start({ script }) {
     const runid = uuidv1();
-    return this.startCore(runid, scriptTemplate(script));
+    return this.startCore(runid, scriptTemplate(script, false));
   },
 
-  cancel_meta: 'post',
+  getNodeScript_meta: true,
+  async getNodeScript({ script }) {
+    return scriptTemplate(script, true);
+  },
+
+  cancel_meta: true,
   async cancel({ runid }) {
     const runner = this.opened.find(x => x.runid == runid);
     if (!runner) {
@@ -152,7 +168,7 @@ module.exports = {
     return { state: 'ok' };
   },
 
-  files_meta: 'get',
+  files_meta: true,
   async files({ runid }) {
     const directory = path.join(rundir(), runid);
     const files = await fs.readdir(directory);
@@ -168,7 +184,7 @@ module.exports = {
     return res;
   },
 
-  loadReader_meta: 'post',
+  loadReader_meta: true,
   async loadReader({ functionName, props }) {
     const promise = new Promise((resolve, reject) => {
       const runid = uuidv1();

packages/api/src/controllers/serverConnections.js

@@ -6,6 +6,7 @@ const AsyncLock = require('async-lock');
 const { handleProcessCommunication } = require('../utility/processComm');
 const lock = new AsyncLock();
 const config = require('./config');
+const processArgs = require('../utility/processArgs');
 
 module.exports = {
   opened: [],
@@ -37,10 +38,12 @@ module.exports = {
       const existing = this.opened.find(x => x.conid == conid);
       if (existing) return existing;
       const connection = await connections.get({ conid });
-      const subprocess = fork(process.argv[1], [
+      const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+        '--is-forked-api',
         '--start-process',
         'serverConnectionProcess',
-        ...process.argv.slice(3),
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
       ]);
       const newOpened = {
         conid,
@@ -86,25 +89,25 @@ module.exports = {
     }
   },
 
-  disconnect_meta: 'post',
+  disconnect_meta: true,
   async disconnect({ conid }) {
     await this.close(conid, true);
     return { status: 'ok' };
   },
 
-  listDatabases_meta: 'get',
+  listDatabases_meta: true,
   async listDatabases({ conid }) {
     const opened = await this.ensureOpened(conid);
     return opened.databases;
   },
 
-  version_meta: 'get',
+  version_meta: true,
   async version({ conid }) {
     const opened = await this.ensureOpened(conid);
     return opened.version;
   },
 
-  serverStatus_meta: 'get',
+  serverStatus_meta: true,
   async serverStatus() {
     return {
       ...this.closed,
@@ -112,7 +115,7 @@ module.exports = {
     };
   },
 
-  ping_meta: 'post',
+  ping_meta: true,
   async ping({ connections }) {
     await Promise.all(
       _.uniq(connections).map(async conid => {
@@ -128,7 +131,7 @@ module.exports = {
     return { status: 'ok' };
   },
 
-  refresh_meta: 'post',
+  refresh_meta: true,
   async refresh({ conid, keepOpen }) {
     if (!keepOpen) this.close(conid);
 
@@ -136,7 +139,7 @@ module.exports = {
     return { status: 'ok' };
   },
 
-  createDatabase_meta: 'post',
+  createDatabase_meta: true,
   async createDatabase({ conid, name }) {
     const opened = await this.ensureOpened(conid);
     opened.subprocess.send({ msgtype: 'createDatabase', name });

packages/api/src/controllers/sessions.js

@@ -5,6 +5,7 @@ const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const jsldata = require('./jsldata');
 const { handleProcessCommunication } = require('../utility/processComm');
+const processArgs = require('../utility/processArgs');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedSession[]} */
@@ -61,11 +62,17 @@ module.exports = {
 
   handle_ping() {},
 
-  create_meta: 'post',
+  create_meta: true,
   async create({ conid, database }) {
     const sesid = uuidv1();
     const connection = await connections.get({ conid });
-    const subprocess = fork(process.argv[1], ['--start-process', 'sessionProcess', ...process.argv.slice(3)]);
+    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+      '--is-forked-api',
+      '--start-process',
+      'sessionProcess',
+      ...processArgs.getPassArgs(),
+      // ...process.argv.slice(3),
+    ]);
     const newOpened = {
       conid,
       database,
@@ -81,10 +88,10 @@ module.exports = {
       this[`handle_${msgtype}`](sesid, message);
     });
     subprocess.send({ msgtype: 'connect', ...connection, database });
-    return newOpened;
+    return _.pick(newOpened, ['conid', 'database', 'sesid']);
   },
 
-  executeQuery_meta: 'post',
+  executeQuery_meta: true,
   async executeQuery({ sesid, sql }) {
     const session = this.opened.find(x => x.sesid == sesid);
     if (!session) {
@@ -98,7 +105,7 @@ module.exports = {
     return { state: 'ok' };
   },
 
-  // cancel_meta: 'post',
+  // cancel_meta: true,
   // async cancel({ sesid }) {
   //   const session = this.opened.find((x) => x.sesid == sesid);
   //   if (!session) {
@@ -108,7 +115,7 @@ module.exports = {
   //   return { state: 'ok' };
   // },
 
-  kill_meta: 'post',
+  kill_meta: true,
   async kill({ sesid }) {
     const session = this.opened.find(x => x.sesid == sesid);
     if (!session) {
@@ -119,7 +126,7 @@ module.exports = {
     return { state: 'ok' };
   },
 
-  // runCommand_meta: 'post',
+  // runCommand_meta: true,
   // async runCommand({ conid, database, sql }) {
   //   console.log(`Running SQL command , conid=${conid}, database=${database}, sql=${sql}`);
   //   const opened = await this.ensureOpened(conid, database);

packages/api/src/controllers/uploads.js

@@ -25,4 +25,12 @@ module.exports = {
       });
     });
   },
+
+  get_meta: {
+    method: 'get',
+    raw: true,
+  },
+  get(req, res) {
+    res.sendFile(path.join(uploadsdir(), req.query.file));
+  },
 };

packages/api/src/index.js

@@ -8,7 +8,7 @@ if (processArgs.startProcess) {
   const proc = require('./proc');
   const module = proc[processArgs.startProcess];
   module.start();
-} else if (!module['parent'] && !processArgs.checkParent) {
+} else if (!processArgs.checkParent && !global['API_PACKAGE'] && !global['dbgateApiModulePath']) {
   const main = require('./main');
 
   main.start();

packages/api/src/main.js

@@ -4,12 +4,8 @@ const bodyParser = require('body-parser');
 const fileUpload = require('express-fileupload');
 const http = require('http');
 const cors = require('cors');
-const io = require('socket.io');
-const fs = require('fs');
 const getPort = require('get-port');
-const childProcessChecker = require('./utility/childProcessChecker');
 const path = require('path');
-const crypto = require('crypto');
 
 const useController = require('./utility/useController');
 const socket = require('./utility/socket');
@@ -31,10 +27,7 @@ const queryHistory = require('./controllers/queryHistory');
 
 const { rundir } = require('./utility/directories');
 const platformInfo = require('./utility/platformInfo');
-const processArgs = require('./utility/processArgs');
-const timingSafeCheckToken = require('./utility/timingSafeCheckToken');
 
-let authorization = null;
 let checkLocalhostOrigin = null;
 
 function start() {
@@ -43,7 +36,6 @@ function start() {
   const app = express();
 
   const server = http.createServer(app);
-  socket.set(io(server));
 
   if (process.env.LOGIN && process.env.PASSWORD) {
     app.use(
@@ -58,9 +50,6 @@ function start() {
   }
 
   app.use(function (req, res, next) {
-    if (authorization && !timingSafeCheckToken(req.headers.authorization, authorization)) {
-      return res.status(403).json({ error: 'Not authorized!' });
-    }
     if (checkLocalhostOrigin) {
       if (
         req.headers.origin &&
@@ -81,6 +70,20 @@ function start() {
   });
 
   app.use(cors());
+
+  app.get('/stream', async function (req, res) {
+    res.set({
+      'Cache-Control': 'no-cache',
+      'Content-Type': 'text/event-stream',
+      Connection: 'keep-alive',
+    });
+    res.flushHeaders();
+
+    // Tell the client to retry every 10 seconds if connectivity is lost
+    res.write('retry: 10000\n\n');
+    socket.setSseResponse(res);
+  });
+
   app.use(bodyParser.json({ limit: '50mb' }));
 
   app.use(
@@ -90,20 +93,7 @@ function start() {
     })
   );
 
-  useController(app, '/connections', connections);
-  useController(app, '/server-connections', serverConnections);
-  useController(app, '/database-connections', databaseConnections);
-  useController(app, '/metadata', metadata);
-  useController(app, '/sessions', sessions);
-  useController(app, '/runners', runners);
-  useController(app, '/jsldata', jsldata);
-  useController(app, '/config', config);
-  useController(app, '/archive', archive);
-  useController(app, '/uploads', uploads);
-  useController(app, '/plugins', plugins);
-  useController(app, '/files', files);
-  useController(app, '/scheduler', scheduler);
-  useController(app, '/query-history', queryHistory);
+  useAllControllers(app, null);
 
   // if (process.env.PAGES_DIRECTORY) {
   //   app.use('/pages', express.static(process.env.PAGES_DIRECTORY));
@@ -122,19 +112,7 @@ function start() {
     }
   }
 
-  if (processArgs.dynport) {
-    childProcessChecker();
-
-    authorization = crypto.randomBytes(32).toString('hex');
-
-    getPort().then(port => {
-      checkLocalhostOrigin = `localhost:${port}`;
-      server.listen(port, () => {
-        console.log(`DbGate API listening on port ${port}`);
-        process.send({ msgtype: 'listening', port, authorization });
-      });
-    });
-  } else if (platformInfo.isNpmDist) {
+  if (platformInfo.isNpmDist) {
     app.use(express.static(path.join(__dirname, '../../dbgate-web/public')));
     getPort({ port: 5000 }).then(port => {
       server.listen(port, () => {
@@ -142,8 +120,47 @@ function start() {
       });
     });
   } else {
-    server.listen(3000);
+    const port = process.env.PORT || 3000;
+    console.log('DbGate API listening on port', port);
+    server.listen(port);
   }
+
+  function shutdown() {
+    console.log('\nShutting down DbGate API server');
+    server.close(() => {
+      console.log('Server shut down, terminating');
+      process.exit(0);
+    });
+    setTimeout(() => {
+      console.log('Server close timeout, terminating');
+      process.exit(0);
+    }, 1000);
+  }
+
+  process.on('SIGINT', shutdown);
+  process.on('SIGTERM', shutdown);
+  process.on('SIGBREAK', shutdown);
 }
 
-module.exports = { start };
+function useAllControllers(app, electron) {
+  useController(app, electron, '/connections', connections);
+  useController(app, electron, '/server-connections', serverConnections);
+  useController(app, electron, '/database-connections', databaseConnections);
+  useController(app, electron, '/metadata', metadata);
+  useController(app, electron, '/sessions', sessions);
+  useController(app, electron, '/runners', runners);
+  useController(app, electron, '/jsldata', jsldata);
+  useController(app, electron, '/config', config);
+  useController(app, electron, '/archive', archive);
+  useController(app, electron, '/uploads', uploads);
+  useController(app, electron, '/plugins', plugins);
+  useController(app, electron, '/files', files);
+  useController(app, electron, '/scheduler', scheduler);
+  useController(app, electron, '/query-history', queryHistory);
+}
+
+function initializeElectronSender(electronSender) {
+  socket.setElectronSender(electronSender);
+}
+
+module.exports = { start, useAllControllers, initializeElectronSender };

packages/api/src/nativeModules.js

@@ -1,5 +1,13 @@
 const argIndex = process.argv.indexOf('--native-modules');
-const redirectFile = argIndex > 0 ? process.argv[argIndex + 1] : null;
+const redirectFile = global['NATIVE_MODULES'] || (argIndex > 0 ? process.argv[argIndex + 1] : null);
 
-// @ts-ignore
-module.exports = redirectFile ? __non_webpack_require__(redirectFile) : require('./nativeModulesContent');
+function requireDynamic(file) {
+  try {
+    // @ts-ignore
+    return __non_webpack_require__(redirectFile);
+  } catch (err) {
+    return require(redirectFile);
+  }
+}
+
+module.exports = redirectFile ? requireDynamic(redirectFile) : require('./nativeModulesContent');

packages/api/src/proc/connectProcess.js

@@ -2,17 +2,9 @@ const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
+const { pickSafeConnectionInfo } = require('../utility/crypting');
 const _ = require('lodash');
 
-function pickSafeConnectionInfo(connection) {
-  return _.mapValues(connection, (v, k) => {
-    if (k == 'engine' || k == 'port' || k == 'authType' || k == 'sshMode' || k == 'passwordMode') return v;
-    if (v === null || v === true || v === false) return v;
-    if (v) return '***';
-    return undefined;
-  });
-}
-
 const formatErrorDetail = (e, connection) => `${e.stack}
 
 Error JSON: ${JSON.stringify(e, undefined, 2)}

packages/api/src/proc/databaseConnectionProcess.js

@@ -6,14 +6,17 @@ const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { SqlGenerator } = require('dbgate-tools');
+const generateDeploySql = require('../shell/generateDeploySql');
 
 let systemConnection;
 let storedConnection;
 let afterConnectCallbacks = [];
+let afterAnalyseCallbacks = [];
 let analysedStructure = null;
 let lastPing = null;
 let lastStatus = null;
 let analysedTime = 0;
+let serverVersion;
 
 async function checkedAsyncCall(promise) {
   try {
@@ -36,19 +39,23 @@ async function handleFullRefresh() {
   loadingModel = true;
   const driver = requireEngineDriver(storedConnection);
   setStatusName('loadStructure');
-  analysedStructure = await checkedAsyncCall(driver.analyseFull(systemConnection));
+  analysedStructure = await checkedAsyncCall(driver.analyseFull(systemConnection, serverVersion));
   analysedTime = new Date().getTime();
   process.send({ msgtype: 'structure', structure: analysedStructure });
   process.send({ msgtype: 'structureTime', analysedTime });
   setStatusName('ok');
+
   loadingModel = false;
+  resolveAnalysedPromises();
 }
 
 async function handleIncrementalRefresh(forceSend) {
   loadingModel = true;
   const driver = requireEngineDriver(storedConnection);
   setStatusName('checkStructure');
-  const newStructure = await checkedAsyncCall(driver.analyseIncremental(systemConnection, analysedStructure));
+  const newStructure = await checkedAsyncCall(
+    driver.analyseIncremental(systemConnection, analysedStructure, serverVersion)
+  );
   analysedTime = new Date().getTime();
   if (newStructure != null) {
     analysedStructure = newStructure;
@@ -61,6 +68,7 @@ async function handleIncrementalRefresh(forceSend) {
   process.send({ msgtype: 'structureTime', analysedTime });
   setStatusName('ok');
   loadingModel = false;
+  resolveAnalysedPromises();
 }
 
 function handleSyncModel() {
@@ -84,6 +92,7 @@ async function readVersion() {
   const driver = requireEngineDriver(storedConnection);
   const version = await driver.getVersion(systemConnection);
   process.send({ msgtype: 'version', version });
+  serverVersion = version;
 }
 
 async function handleConnect({ connection, structure, globalSettings }) {
@@ -93,7 +102,7 @@ async function handleConnect({ connection, structure, globalSettings }) {
   if (!structure) setStatusName('pending');
   const driver = requireEngineDriver(storedConnection);
   systemConnection = await checkedAsyncCall(connectUtility(driver, storedConnection));
-  readVersion();
+  await checkedAsyncCall(readVersion());
   if (structure) {
     analysedStructure = structure;
     handleIncrementalRefresh(true);
@@ -121,6 +130,20 @@ function waitConnected() {
   });
 }
 
+function waitStructure() {
+  if (analysedStructure) return Promise.resolve();
+  return new Promise((resolve, reject) => {
+    afterAnalyseCallbacks.push([resolve, reject]);
+  });
+}
+
+function resolveAnalysedPromises() {
+  for (const [resolve] of afterAnalyseCallbacks) {
+    resolve();
+  }
+  afterAnalyseCallbacks = [];
+}
+
 async function handleRunScript({ msgid, sql }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
@@ -166,7 +189,7 @@ async function handleUpdateCollection({ msgid, changeSet }) {
 }
 
 async function handleSqlPreview({ msgid, objects, options }) {
-  await waitConnected();
+  await waitStructure();
   const driver = requireEngineDriver(storedConnection);
 
   try {
@@ -186,6 +209,22 @@ async function handleSqlPreview({ msgid, objects, options }) {
   }
 }
 
+async function handleGenerateDeploySql({ msgid, modelFolder }) {
+  await waitStructure();
+
+  try {
+    const res = await generateDeploySql({
+      systemConnection,
+      connection: storedConnection,
+      analysedStructure,
+      modelFolder,
+    });
+    process.send({ ...res, msgtype: 'response', msgid });
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, isError: true, errorMessage: err.message });
+  }
+}
+
 // async function handleRunCommand({ msgid, sql }) {
 //   await waitConnected();
 //   const driver = engines(storedConnection);
@@ -206,6 +245,7 @@ const messageHandlers = {
   sqlPreview: handleSqlPreview,
   ping: handlePing,
   syncModel: handleSyncModel,
+  generateDeploySql: handleGenerateDeploySql,
   // runCommand: handleRunCommand,
 };
 
@@ -230,6 +270,7 @@ function start() {
     try {
       await handleMessage(message);
     } catch (e) {
+      console.error('Error in DB connection', e);
       process.send({ msgtype: 'error', error: e.message });
     }
   });

packages/api/src/shell/archiveReader.js

@@ -1,9 +1,9 @@
 const path = require('path');
-const { archivedir } = require('../utility/directories');
+const { archivedir, resolveArchiveFolder } = require('../utility/directories');
 const jsonLinesReader = require('./jsonLinesReader');
 
 function archiveReader({ folderName, fileName, ...other }) {
-  const jsonlFile = path.join(archivedir(), folderName, `${fileName}.jsonl`);
+  const jsonlFile = path.join(resolveArchiveFolder(folderName), `${fileName}.jsonl`);
   const res = jsonLinesReader({ fileName: jsonlFile, ...other });
   return res;
 }

packages/api/src/shell/archiveWriter.js

@@ -1,11 +1,11 @@
 const path = require('path');
 const fs = require('fs');
-const { archivedir } = require('../utility/directories');
+const { archivedir, resolveArchiveFolder } = require('../utility/directories');
 // const socket = require('../utility/socket');
 const jsonLinesWriter = require('./jsonLinesWriter');
 
 function archiveWriter({ folderName, fileName }) {
-  const dir = path.join(archivedir(), folderName);
+  const dir = resolveArchiveFolder(folderName);
   if (!fs.existsSync(dir)) {
     console.log(`Creating directory ${dir}`);
     fs.mkdirSync(dir);

packages/api/src/shell/deployDb.js

@@ -0,0 +1,17 @@
+const generateDeploySql = require('./generateDeploySql');
+const executeQuery = require('./executeQuery');
+
+async function deployDb({ connection, systemConnection, driver, analysedStructure, modelFolder, loadedDbModel }) {
+  const { sql } = await generateDeploySql({
+    connection,
+    systemConnection,
+    driver,
+    analysedStructure,
+    modelFolder,
+    loadedDbModel,
+  });
+  // console.log('RUNNING DEPLOY SCRIPT:', sql);
+  await executeQuery({ connection, systemConnection, driver, sql });
+}
+
+module.exports = deployDb;

packages/api/src/shell/executeQuery.js

@@ -1,20 +1,14 @@
-const { splitQuery } = require('dbgate-query-splitter');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 
-async function executeQuery({ connection, sql }) {
+async function executeQuery({ connection = undefined, systemConnection = undefined, driver = undefined, sql }) {
   console.log(`Execute query ${sql}`);
 
-  const driver = requireEngineDriver(connection);
-  const pool = await connectUtility(driver, connection);
+  if (!driver) driver = requireEngineDriver(connection);
+  const pool = systemConnection || (await connectUtility(driver, connection));
   console.log(`Connected.`);
 
-  for (const sqlItem of splitQuery(sql, driver.getQuerySplitterOptions('script'))) {
-    console.log('Executing query', sqlItem);
-    await driver.query(pool, sqlItem, { discardResult: true });
-  }
-
-  console.log(`Query finished`);
+  await driver.script(pool, sql);
 }
 
 module.exports = executeQuery;

packages/api/src/shell/fakeObjectReader.js

@@ -5,7 +5,7 @@ async function fakeObjectReader({ delay = 0 } = {}) {
     objectMode: true,
   });
   function doWrite() {
-    pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }] });
+    pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }], __isStreamHeader: true });
     pass.write({ id: 1, country: 'Czechia' });
     pass.write({ id: 2, country: 'Austria' });
     pass.write({ country: 'Germany', id: 3 });

packages/api/src/shell/generateDeploySql.js

@@ -0,0 +1,61 @@
+const {
+  getAlterDatabaseScript,
+  generateDbPairingId,
+  matchPairedObjects,
+  databaseInfoFromYamlModel,
+  extendDatabaseInfo,
+  modelCompareDbDiffOptions,
+  enrichWithPreloadedRows,
+} = require('dbgate-tools');
+const importDbModel = require('../utility/importDbModel');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+
+async function generateDeploySql({
+  connection,
+  systemConnection = undefined,
+  driver = undefined,
+  analysedStructure = undefined,
+  modelFolder = undefined,
+  loadedDbModel = undefined,
+}) {
+  if (!driver) driver = requireEngineDriver(connection);
+
+  const pool = systemConnection || (await connectUtility(driver, connection));
+  if (!analysedStructure) {
+    analysedStructure = await driver.analyseFull(pool);
+  }
+
+  const deployedModel = generateDbPairingId(
+    extendDatabaseInfo(loadedDbModel ? databaseInfoFromYamlModel(loadedDbModel) : await importDbModel(modelFolder))
+  );
+  const currentModel = generateDbPairingId(extendDatabaseInfo(analysedStructure));
+  const opts = {
+    ...modelCompareDbDiffOptions,
+
+    noDropTable: true,
+    noDropColumn: true,
+    noDropConstraint: true,
+    noDropSqlObject: true,
+    noRenameTable: true,
+    noRenameColumn: true,
+  };
+  const currentModelPaired = matchPairedObjects(deployedModel, currentModel, opts);
+  const currentModelPairedPreloaded = await enrichWithPreloadedRows(deployedModel, currentModelPaired, pool, driver);
+
+  // console.log('currentModelPairedPreloaded', currentModelPairedPreloaded.tables[0]);
+  // console.log('deployedModel', deployedModel.tables[0]);
+  // console.log('currentModel', currentModel.tables[0]);
+  // console.log('currentModelPaired', currentModelPaired.tables[0]);
+  const res = getAlterDatabaseScript(
+    currentModelPairedPreloaded,
+    deployedModel,
+    opts,
+    currentModelPairedPreloaded,
+    deployedModel,
+    driver
+  );
+  return res;
+}
+
+module.exports = generateDeploySql;

packages/api/src/shell/index.js

@@ -8,6 +8,7 @@ const consoleObjectWriter = require('./consoleObjectWriter');
 const jsonLinesWriter = require('./jsonLinesWriter');
 const jsonArrayWriter = require('./jsonArrayWriter');
 const jsonLinesReader = require('./jsonLinesReader');
+const sqlDataWriter = require('./sqlDataWriter');
 const jslDataReader = require('./jslDataReader');
 const archiveWriter = require('./archiveWriter');
 const archiveReader = require('./archiveReader');
@@ -18,6 +19,7 @@ const requirePlugin = require('./requirePlugin');
 const download = require('./download');
 const executeQuery = require('./executeQuery');
 const loadFile = require('./loadFile');
+const deployDb = require('./deployDb');
 const initializeApiEnvironment = require('./initializeApiEnvironment');
 
 const dbgateApi = {
@@ -29,6 +31,7 @@ const dbgateApi = {
   jsonLinesWriter,
   jsonArrayWriter,
   jsonLinesReader,
+  sqlDataWriter,
   fakeObjectReader,
   consoleObjectWriter,
   jslDataReader,
@@ -40,6 +43,7 @@ const dbgateApi = {
   registerPlugins,
   executeQuery,
   loadFile,
+  deployDb,
   initializeApiEnvironment,
 };
 

packages/api/src/shell/sqlDataWriter.js

@@ -0,0 +1,54 @@
+const fs = require('fs');
+const stream = require('stream');
+const path = require('path');
+const { driverBase } = require('dbgate-tools');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+
+class SqlizeStream extends stream.Transform {
+  constructor({ fileName }) {
+    super({ objectMode: true });
+    this.wasHeader = false;
+    this.tableName = path.parse(fileName).name;
+    this.driver = driverBase;
+  }
+  _transform(chunk, encoding, done) {
+    let skip = false;
+    if (!this.wasHeader) {
+      if (
+        chunk.__isStreamHeader ||
+        // TODO remove isArray test
+        Array.isArray(chunk.columns)
+      ) {
+        skip = true;
+        this.tableName = chunk.pureName;
+        if (chunk.engine) {
+          // @ts-ignore
+          this.driver = requireEngineDriver(chunk.engine) || driverBase;
+        }
+      }
+      this.wasHeader = true;
+    }
+    if (!skip) {
+      const dmp = this.driver.createDumper();
+      dmp.put(
+        '^insert ^into %f (%,i) ^values (%,v);\n',
+        { pureName: this.tableName },
+        Object.keys(chunk),
+        Object.values(chunk)
+      );
+      this.push(dmp.s);
+    }
+    done();
+  }
+}
+
+async function sqlDataWriter({ fileName, driver, encoding = 'utf-8' }) {
+  console.log(`Writing file ${fileName}`);
+  const stringify = new SqlizeStream({ fileName });
+  const fileStream = fs.createWriteStream(fileName, encoding);
+  stringify.pipe(fileStream);
+  stringify['finisher'] = fileStream;
+  return stringify;
+}
+
+module.exports = sqlDataWriter;

packages/api/src/shell/tableWriter.js

@@ -1,13 +1,15 @@
 const { fullNameToString } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const { decryptConnection } = require('../utility/crypting');
 const connectUtility = require('../utility/connectUtility');
 
-async function tableWriter({ connection, schemaName, pureName, ...options }) {
+async function tableWriter({ connection, schemaName, pureName, driver, systemConnection, ...options }) {
   console.log(`Writing table ${fullNameToString({ schemaName, pureName })}`);
 
-  const driver = requireEngineDriver(connection);
-  const pool = await connectUtility(driver, connection);
+  if (!driver) {
+    driver = requireEngineDriver(connection);
+  }
+  const pool = systemConnection || (await connectUtility(driver, connection));
+
   console.log(`Connected.`);
   return await driver.writeTable(pool, { schemaName, pureName }, options);
 }

packages/api/src/utility/DatastoreProxy.js

@@ -1,6 +1,7 @@
 const { fork } = require('child_process');
 const uuidv1 = require('uuid/v1');
 const { handleProcessCommunication } = require('./processComm');
+const processArgs = require('../utility/processArgs');
 
 class DatastoreProxy {
   constructor(file) {
@@ -29,7 +30,13 @@ class DatastoreProxy {
 
   async ensureSubprocess() {
     if (!this.subprocess) {
-      this.subprocess = fork(process.argv[1], ['--start-process', 'jslDatastoreProcess', ...process.argv.slice(3)]);
+      this.subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+        '--is-forked-api',
+        '--start-process',
+        'jslDatastoreProcess',
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
+      ]);
 
       this.subprocess.on('message', message => {
         // @ts-ignore

packages/api/src/utility/crypting.js

@@ -2,6 +2,7 @@ const crypto = require('crypto');
 const simpleEncryptor = require('simple-encryptor');
 const fs = require('fs');
 const path = require('path');
+const _ = require('lodash');
 
 const { datadir } = require('./directories');
 
@@ -70,19 +71,29 @@ function decryptPasswordField(connection, field) {
 function encryptConnection(connection) {
   connection = encryptPasswordField(connection, 'password');
   connection = encryptPasswordField(connection, 'sshPassword');
-  connection = encryptPasswordField(connection, 'sshKeyFilePassword');
+  connection = encryptPasswordField(connection, 'sshKeyfilePassword');
   return connection;
 }
 
 function decryptConnection(connection) {
   connection = decryptPasswordField(connection, 'password');
   connection = decryptPasswordField(connection, 'sshPassword');
-  connection = decryptPasswordField(connection, 'sshKeyFilePassword');
+  connection = decryptPasswordField(connection, 'sshKeyfilePassword');
   return connection;
 }
 
+function pickSafeConnectionInfo(connection) {
+  return _.mapValues(connection, (v, k) => {
+    if (k == 'engine' || k == 'port' || k == 'authType' || k == 'sshMode' || k == 'passwordMode') return v;
+    if (v === null || v === true || v === false) return v;
+    if (v) return '***';
+    return undefined;
+  });
+}
+
 module.exports = {
   loadEncryptionKey,
   encryptConnection,
   decryptConnection,
+  pickSafeConnectionInfo,
 };

packages/api/src/utility/diff2htmlPage.js

@@ -0,0 +1,8 @@
+const diff2htmlCss =
+  '.d2h-d-none{display:none}.d2h-wrapper{text-align:left}.d2h-file-header{background-color:#f7f7f7;border-bottom:1px solid #d8d8d8;font-family:Source Sans Pro,Helvetica Neue,Helvetica,Arial,sans-serif;height:35px;padding:5px 10px}.d2h-file-header,.d2h-file-stats{display:-webkit-box;display:-ms-flexbox;display:flex}.d2h-file-stats{font-size:14px;margin-left:auto}.d2h-lines-added{border:1px solid #b4e2b4;border-radius:5px 0 0 5px;color:#399839;padding:2px;text-align:right;vertical-align:middle}.d2h-lines-deleted{border:1px solid #e9aeae;border-radius:0 5px 5px 0;color:#c33;margin-left:1px;padding:2px;text-align:left;vertical-align:middle}.d2h-file-name-wrapper{-webkit-box-align:center;-ms-flex-align:center;align-items:center;display:-webkit-box;display:-ms-flexbox;display:flex;font-size:15px;width:100%}.d2h-file-name{overflow-x:hidden;text-overflow:ellipsis;white-space:nowrap}.d2h-file-wrapper{border:1px solid #ddd;border-radius:3px;margin-bottom:1em}.d2h-file-collapse{-webkit-box-pack:end;-ms-flex-pack:end;-webkit-box-align:center;-ms-flex-align:center;align-items:center;border:1px solid #ddd;border-radius:3px;cursor:pointer;display:none;font-size:12px;justify-content:flex-end;padding:4px 8px}.d2h-file-collapse.d2h-selected{background-color:#c8e1ff}.d2h-file-collapse-input{margin:0 4px 0 0}.d2h-diff-table{border-collapse:collapse;font-family:Menlo,Consolas,monospace;font-size:13px;width:100%}.d2h-files-diff{width:100%}.d2h-file-diff{overflow-y:hidden}.d2h-file-side-diff{display:inline-block;margin-bottom:-8px;margin-right:-4px;overflow-x:scroll;overflow-y:hidden;width:50%}.d2h-code-line{padding:0 8em}.d2h-code-line,.d2h-code-side-line{display:inline-block;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;white-space:nowrap;width:100%}.d2h-code-side-line{padding:0 4.5em}.d2h-code-line-ctn{word-wrap:normal;background:none;display:inline-block;padding:0;-webkit-user-select:text;-moz-user-select:text;-ms-user-select:text;user-select:text;vertical-align:middle;white-space:pre;width:100%}.d2h-code-line del,.d2h-code-side-line del{background-color:#ffb6ba}.d2h-code-line del,.d2h-code-line ins,.d2h-code-side-line del,.d2h-code-side-line ins{border-radius:.2em;display:inline-block;margin-top:-1px;text-decoration:none;vertical-align:middle}.d2h-code-line ins,.d2h-code-side-line ins{background-color:#97f295;text-align:left}.d2h-code-line-prefix{word-wrap:normal;background:none;display:inline;padding:0;white-space:pre}.line-num1{float:left}.line-num1,.line-num2{-webkit-box-sizing:border-box;box-sizing:border-box;overflow:hidden;padding:0 .5em;text-overflow:ellipsis;width:3.5em}.line-num2{float:right}.d2h-code-linenumber{background-color:#fff;border:solid #eee;border-width:0 1px;-webkit-box-sizing:border-box;box-sizing:border-box;color:rgba(0,0,0,.3);cursor:pointer;display:inline-block;position:absolute;text-align:right;width:7.5em}.d2h-code-linenumber:after{content:"\200b"}.d2h-code-side-linenumber{background-color:#fff;border:solid #eee;border-width:0 1px;-webkit-box-sizing:border-box;box-sizing:border-box;color:rgba(0,0,0,.3);cursor:pointer;display:inline-block;overflow:hidden;padding:0 .5em;position:absolute;text-align:right;text-overflow:ellipsis;width:4em}.d2h-code-side-linenumber:after{content:"\200b"}.d2h-code-side-emptyplaceholder,.d2h-emptyplaceholder{background-color:#f1f1f1;border-color:#e1e1e1}.d2h-code-line-prefix,.d2h-code-linenumber,.d2h-code-side-linenumber,.d2h-emptyplaceholder{-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.d2h-code-linenumber,.d2h-code-side-linenumber{direction:rtl}.d2h-del{background-color:#fee8e9;border-color:#e9aeae}.d2h-ins{background-color:#dfd;border-color:#b4e2b4}.d2h-info{background-color:#f8fafd;border-color:#d5e4f2;color:rgba(0,0,0,.3)}.d2h-file-diff .d2h-del.d2h-change{background-color:#fdf2d0}.d2h-file-diff .d2h-ins.d2h-change{background-color:#ded}.d2h-file-list-wrapper{margin-bottom:10px}.d2h-file-list-wrapper a{color:#3572b0;text-decoration:none}.d2h-file-list-wrapper a:visited{color:#3572b0}.d2h-file-list-header{text-align:left}.d2h-file-list-title{font-weight:700}.d2h-file-list-line{display:-webkit-box;display:-ms-flexbox;display:flex;text-align:left}.d2h-file-list{display:block;list-style:none;margin:0;padding:0}.d2h-file-list>li{border-bottom:1px solid #ddd;margin:0;padding:5px 10px}.d2h-file-list>li:last-child{border-bottom:none}.d2h-file-switch{cursor:pointer;display:none;font-size:10px}.d2h-icon{fill:currentColor;margin-right:10px;vertical-align:middle}.d2h-deleted{color:#c33}.d2h-added{color:#399839}.d2h-changed{color:#d0b44c}.d2h-moved{color:#3572b0}.d2h-tag{background-color:#fff;display:-webkit-box;display:-ms-flexbox;display:flex;font-size:10px;margin-left:5px;padding:0 2px}.d2h-deleted-tag{border:1px solid #c33}.d2h-added-tag{border:1px solid #399839}.d2h-changed-tag{border:1px solid #d0b44c}.d2h-moved-tag{border:1px solid #3572b0}';
+
+function diff2htmlPage(content) {
+  return `<html><head><style>${diff2htmlCss}</style><body>${content}</body></html>`;
+}
+
+module.exports = diff2htmlPage;

packages/api/src/utility/directories.js

@@ -41,6 +41,10 @@ const archivedir = dirFunc('archive');
 const filesdir = dirFunc('files');
 
 function packagedPluginsDir() {
+  // console.log('CALL DIR FROM', new Error('xxx').stack);
+  // console.log('__dirname', __dirname);
+  // console.log('platformInfo.isElectronBundle', platformInfo.isElectronBundle);
+  // console.log('platformInfo.isForkedApi', platformInfo.isForkedApi);
   if (platformInfo.isDevMode) {
     return path.resolve(__dirname, '../../../../plugins');
   }
@@ -53,6 +57,12 @@ function packagedPluginsDir() {
   }
   if (platformInfo.isElectronBundle) {
     return path.resolve(__dirname, '../../plugins');
+
+    // if (platformInfo.isForkedApi) {
+    //   return path.resolve(__dirname, '../plugins');
+    // } else {
+    //   return path.resolve(__dirname, '../../plugins');
+    // }
   }
   return null;
 }
@@ -71,6 +81,22 @@ function getPluginBackendPath(packageName) {
   return path.join(pluginsdir(), packageName, 'dist', 'backend.js');
 }
 
+let archiveLinksCache = {};
+
+function resolveArchiveFolder(folder) {
+  if (folder.endsWith('.link')) {
+    if (!archiveLinksCache[folder]) {
+      archiveLinksCache[folder] = fs.readFileSync(path.join(archivedir(), folder), 'utf-8');
+    }
+    return archiveLinksCache[folder];
+  }
+  return path.join(archivedir(), folder);
+}
+
+function clearArchiveLinksCache() {
+  archiveLinksCache = {};
+}
+
 module.exports = {
   datadir,
   jsldir,
@@ -83,4 +109,6 @@ module.exports = {
   packagedPluginsDir,
   packagedPluginList,
   getPluginBackendPath,
+  resolveArchiveFolder,
+  clearArchiveLinksCache,
 };

packages/api/src/utility/exportDbModel.js

@@ -0,0 +1,31 @@
+const fs = require('fs-extra');
+const path = require('path');
+const yaml = require('js-yaml');
+const { tableInfoToYaml } = require('dbgate-tools');
+
+async function exportDbModel(dbModel, outputDir) {
+  const { tables, views, procedures, functions, triggers, matviews } = dbModel;
+
+  if (!fs.existsSync(outputDir)) {
+    await fs.mkdir(outputDir);
+  }
+
+  for (const table of tables || []) {
+    const content = yaml.dump(tableInfoToYaml(table));
+    await fs.writeFile(path.join(outputDir, `${table.pureName}.table.yaml`), content);
+  }
+
+  async function writeList(list, ext) {
+    for (const obj of list || []) {
+      await fs.writeFile(path.join(outputDir, `${obj.pureName}.${ext}.sql`), obj.createSql);
+    }
+  }
+
+  await writeList(views, 'view');
+  await writeList(procedures, 'proc');
+  await writeList(functions, 'func');
+  await writeList(triggers, 'trigger');
+  await writeList(matviews, 'matview');
+}
+
+module.exports = exportDbModel;

packages/api/src/utility/getChartExport.js

@@ -0,0 +1,55 @@
+const getChartExport = (title, config, imageFile) => {
+  return `<html>
+<meta charset='utf-8'>
+
+<head>
+    ${title ? `<title>${title}</title>` : ''}
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/3.6.0/chart.min.js" integrity="sha512-GMGzUEevhWh8Tc/njS0bDpwgxdCJLQBWG3Z2Ct+JGOpVnEmjvNx6ts4v6A2XJf1HOrtOsfhv3hBKpK9kE5z8AQ==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.29.1/moment.min.js" integrity="sha512-qTXRIMyZIFb8iQcfjXWCO8+M5Tbc38Qi5WzdPOYZHIlZpzBHG3L3by84BBBOiRGiEb7KKtAOAs5qYdUiZiQNNQ==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/chartjs-adapter-moment/1.0.0/chartjs-adapter-moment.min.js" integrity="sha512-oh5t+CdSBsaVVAvxcZKy3XJdP7ZbYUBSRCXDTVn0ODewMDDNnELsrG9eDm8rVZAQg7RsDD/8K3MjPAFB13o6eA==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/hammer.js/2.0.8/hammer.min.js" integrity="sha512-UXumZrZNiOwnTcZSHLOfcTs0aos2MzBWHXOHOuB0J/R44QB0dwY5JgfbvljXcklVf65Gc4El6RjZ+lnwd2az2g==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/chartjs-plugin-zoom/1.2.0/chartjs-plugin-zoom.min.js" integrity="sha512-TT0wAMqqtjXVzpc48sI0G84rBP+oTkBZPgeRYIOVRGUdwJsyS3WPipsNh///ay2LJ+onCM23tipnz6EvEy2/UA==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+
+    <style>
+        a { text-decoration: none }
+
+        .footer { 
+            float: right;
+            font-family: Arial;
+            color: #888;
+            margin-top: 10px;
+            margin-right: 10px;
+            font-size: 10pt;
+        }
+    </style>
+
+    <script>
+        const config = ${JSON.stringify(config)};
+
+        function showChart() {
+            document.getElementById('myImage').style.display = "none";
+
+            const myChart = new Chart(
+                document.getElementById('myChart'),
+                config
+            );
+        }
+    </script>
+</head>
+
+<body onload="showChart()">
+    <img src="${imageFile}" id="myImage" />
+
+    <div>
+        <canvas id="myChart"></canvas>
+    </div>
+
+    <div class="footer">
+        Exported from <a href='https://dbgate.org/' target='_blank'>DbGate</a>, powered by <a href='https://www.chartjs.org/' target='_blank'>Chart.js</a>
+    </div>
+</body>
+
+</html>`;
+};
+
+module.exports = getChartExport;

packages/api/src/utility/getDiagramExport.js

@@ -0,0 +1,21 @@
+const getDiagramExport = (html,css) => {
+    return `<html>
+  <meta charset='utf-8'>
+  
+  <head>
+      <style>
+        ${css}
+      </style>
+
+      <link rel="stylesheet" href='https://cdn.jsdelivr.net/npm/@mdi/font@6.5.95/css/materialdesignicons.css' />
+  </head>
+  
+  <body class='theme-light'>
+      ${html}
+  </body>
+  
+  </html>`;
+  };
+  
+  module.exports = getDiagramExport;
+  

packages/api/src/utility/getJslFileName.js

@@ -1,10 +1,10 @@
 const path = require('path');
-const { jsldir, archivedir } = require('./directories');
+const { jsldir, archivedir, resolveArchiveFolder } = require('./directories');
 
 function getJslFileName(jslid) {
   const archiveMatch = jslid.match(/^archive:\/\/([^/]+)\/(.*)$/);
   if (archiveMatch) {
-    return path.join(archivedir(), archiveMatch[1], `${archiveMatch[2]}.jsonl`);
+    return path.join(resolveArchiveFolder(archiveMatch[1]), `${archiveMatch[2]}.jsonl`);
   }
   return path.join(jsldir(), `${jslid}.jsonl`);
 }

packages/api/src/utility/importDbModel.js

@@ -0,0 +1,29 @@
+const fs = require('fs-extra');
+const path = require('path');
+const yaml = require('js-yaml');
+const { databaseInfoFromYamlModel, DatabaseAnalyser } = require('dbgate-tools');
+const { startsWith } = require('lodash');
+const { archivedir, resolveArchiveFolder } = require('./directories');
+const loadFilesRecursive = require('./loadFilesRecursive');
+
+async function importDbModel(inputDir) {
+  const files = [];
+
+  const dir = inputDir.startsWith('archive:') ? resolveArchiveFolder(inputDir.substring('archive:'.length)) : inputDir;
+
+  for (const name of await loadFilesRecursive(dir)) {
+    if (name.endsWith('.table.yaml') || name.endsWith('.sql')) {
+      const text = await fs.readFile(path.join(dir, name), { encoding: 'utf-8' });
+
+      files.push({
+        name: path.parse(name).base,
+        text,
+        json: name.endsWith('.yaml') ? yaml.load(text) : null,
+      });
+    }
+  }
+
+  return databaseInfoFromYamlModel(files);
+}
+
+module.exports = importDbModel;

packages/api/src/utility/loadFilesRecursive.js

@@ -0,0 +1,20 @@
+const fs = require('fs-extra');
+const stream = require('stream');
+const readline = require('readline');
+const path = require('path');
+
+async function loadFilesRecursive(dir, root = null) {
+  if (!root) root = dir;
+  if (!root.endsWith(path.sep)) root += path.sep;
+  const dirents = await fs.readdir(dir, { withFileTypes: true });
+  const files = await Promise.all(
+    dirents.map(dirent => {
+      const res = path.join(dir, dirent.name);
+      return dirent.isDirectory() ? loadFilesRecursive(res, root) : res;
+    })
+  );
+  const flatten = Array.prototype.concat(...files);
+  return flatten.map(file => (file.startsWith(root) ? file.substr(root.length) : file));
+}
+
+module.exports = loadFilesRecursive;

packages/api/src/utility/platformInfo.js

@@ -2,6 +2,7 @@ const fs = require('fs');
 const os = require('os');
 const path = require('path');
 const processArgs = require('./processArgs');
+const isElectron = require('is-electron');
 
 const platform = process.env.OS_OVERRIDE ? process.env.OS_OVERRIDE : process.platform;
 const isWindows = platform === 'win32';
@@ -10,6 +11,7 @@ const isLinux = platform === 'linux';
 const isDocker = fs.existsSync('/home/dbgate-docker/public');
 const isDevMode = process.env.DEVMODE == '1';
 const isNpmDist = !!global['dbgateApiModulePath'];
+const isForkedApi = processArgs.isForkedApi;
 
 // function moduleAvailable(name) {
 //   try {
@@ -20,14 +22,14 @@ const isNpmDist = !!global['dbgateApiModulePath'];
 //   }
 // }
 
-const isElectronBundle = processArgs.isElectronBundle;
-
 const platformInfo = {
   isWindows,
   isMac,
   isLinux,
   isDocker,
-  isElectronBundle,
+  isElectronBundle: isElectron() && !isDevMode,
+  isForkedApi,
+  isElectron: isElectron(),
   isDevMode,
   isNpmDist,
   isSnap: process.env.ELECTRON_SNAP == 'true',
@@ -37,7 +39,7 @@ const platformInfo = {
   environment: process.env.NODE_ENV,
   platform,
   runningInWebpack: !!process.env.WEBPACK_DEV_SERVER_URL,
-  defaultKeyFile: path.join(os.homedir(), '.ssh/id_rsa'),
+  defaultKeyfile: path.join(os.homedir(), '.ssh/id_rsa'),
 };
 
 module.exports = platformInfo;

packages/api/src/utility/processArgs.js

@@ -7,15 +7,17 @@ function getNamedArg(name) {
 }
 
 const checkParent = process.argv.includes('--checkParent');
-const dynport = process.argv.includes('--dynport');
-const nativeModules = getNamedArg('--native-modules');
 const startProcess = getNamedArg('--start-process');
-const isElectronBundle = process.argv.includes('--is-electron-bundle');
+const isForkedApi = process.argv.includes('--is-forked-api');
+
+function getPassArgs() {
+  if (global['NATIVE_MODULES']) return ['--native-modules', global['NATIVE_MODULES']];
+  return [];
+}
 
 module.exports = {
   checkParent,
-  nativeModules,
   startProcess,
-  dynport,
-  isElectronBundle,
+  isForkedApi,
+  getPassArgs,
 };

packages/api/src/utility/requireEngineDriver.js

@@ -1,5 +1,6 @@
 const _ = require('lodash');
 const requirePlugin = require('../shell/requirePlugin');
+const { pickSafeConnectionInfo } = require('./crypting');
 
 /** @returns {import('dbgate-types').EngineDriver} */
 function requireEngineDriver(connection) {
@@ -10,14 +11,14 @@ function requireEngineDriver(connection) {
     engine = connection.engine;
   }
   if (!engine) {
-    throw new Error('Could not get driver from connection');
+    throw new Error(`Could not get driver from connection ${JSON.stringify(pickSafeConnectionInfo(connection))}`);
   }
   if (engine.includes('@')) {
     const [shortName, packageName] = engine.split('@');
     const plugin = requirePlugin(packageName);
     return plugin.drivers.find(x => x.engine == engine);
   }
-  throw new Error(`Could not found engine driver ${engine}`);
+  throw new Error(`Could not find engine driver ${engine}`);
 }
 
 module.exports = requireEngineDriver;

packages/api/src/utility/socket.js

@@ -1,19 +1,29 @@
-let socket = null;
+let sseResponse = null;
+let electronSender = null;
+let init = '';
 
 module.exports = {
-  set(value) {
-    socket = value;
+  setSseResponse(value) {
+    sseResponse = value;
   },
-  get() {
-    return socket;
+  setElectronSender(value) {
+    electronSender = value;
   },
   emit(message, data) {
-    // console.log('EMIT:', message, data);
-    socket.emit(message, data);
+    if (electronSender) {
+      electronSender.send(message, data == null ? null : data);
+    } else if (sseResponse) {
+      if (init) {
+        sseResponse.write(init);
+        init = '';
+      }
+      sseResponse.write(`event: ${message}\ndata: ${JSON.stringify(data == null ? null : data)}\n\n`);
+    } else {
+      init += sseResponse;
+    }
   },
   emitChanged(key) {
-    // console.log('EMIT_CHANGED:', key);
-    socket.emit('clean-cache', key);
-    socket.emit(key);
+    this.emit('clean-cache', key);
+    this.emit(key);
   },
 };

packages/api/src/utility/sshTunnel.js

@@ -16,9 +16,9 @@ const CONNECTION_FIELDS = [
   'sshLogin',
   'sshPassword',
   'sshMode',
-  'sshKeyFile',
+  'sshKeyfile',
   'sshBastionHost',
-  'sshKeyFilePassword',
+  'sshKeyfilePassword',
 ];
 const TUNNEL_FIELDS = [...CONNECTION_FIELDS, 'server', 'port'];
 
@@ -31,7 +31,7 @@ async function getSshConnection(connection) {
     endPort: connection.sshPort || 22,
     bastionHost: connection.sshBastionHost || '',
     agentForward: connection.sshMode == 'agent',
-    passphrase: connection.sshMode == 'keyFile' ? connection.sshKeyFilePassword : undefined,
+    passphrase: connection.sshMode == 'keyFile' ? connection.sshKeyfilePassword : undefined,
     username: connection.sshLogin,
     password: connection.sshMode == 'userPassword' ? connection.sshPassword : undefined,
     agentSocket: connection.sshMode == 'agent' ? platformInfo.sshAuthSock : undefined,

packages/api/src/utility/useController.js

@@ -4,7 +4,7 @@ const express = require('express');
 /**
  * @param {string} route
  */
-module.exports = function useController(app, route, controller) {
+module.exports = function useController(app, electron, route, controller) {
   const router = express.Router();
 
   if (controller._init) {
@@ -23,24 +23,39 @@ module.exports = function useController(app, route, controller) {
     const meta = controller[`${key}_meta`];
     if (!meta) continue;
 
-    let method = 'get';
+    const routeAction = `/${_.kebabCase(key)}`;
+
+    if (electron) {
+      if (meta === true) {
+        const handler = `${route.substring(1)}-${_.kebabCase(key)}`;
+        // console.log('REGISTERING HANDLER', handler);
+        electron.ipcMain.handle(handler, async (event, args) => {
+          const data = await controller[key](args);
+          // console.log('HANDLED API', handler, data);
+          return data;
+        });
+      }
+
+      continue;
+    }
+
+    let method = 'post';
     let raw = false;
     let rawParams = false;
 
-    if (_.isString(meta)) {
-      method = meta;
-    }
+    // if (_.isString(meta)) {
+    //   method = meta;
+    // }
     if (_.isPlainObject(meta)) {
       method = meta.method;
       raw = meta.raw;
       rawParams = meta.rawParams;
     }
 
-    const route = `/${_.kebabCase(key)}`;
     if (raw) {
-      router[method](route, controller[key]);
+      router[method](routeAction, controller[key]);
     } else {
-      router[method](route, async (req, res) => {
+      router[method](routeAction, async (req, res) => {
         // if (controller._init && !controller._init_called) {
         //   await controller._init();
         //   controller._init_called = true;
@@ -58,5 +73,7 @@ module.exports = function useController(app, route, controller) {
     }
   }
 
-  app.use(route, router);
+  if (app) {
+    app.use(route, router);
+  }
 };

packages/datalib/package.json

@@ -17,6 +17,6 @@
   "devDependencies": {
     "dbgate-types": "^4.1.1",
     "@types/node": "^13.7.0",
-    "typescript": "^3.7.5"
+    "typescript": "^4.4.3"
   }
 }

packages/datalib/src/ChangeSet.ts

@@ -118,7 +118,11 @@ export function setChangeSetValue(
   };
 }
 
-export function setChangeSetRowData(changeSet: ChangeSet, definition: ChangeSetRowDefinition, document: any): ChangeSet {
+export function setChangeSetRowData(
+  changeSet: ChangeSet,
+  definition: ChangeSetRowDefinition,
+  document: any
+): ChangeSet {
   if (!changeSet || !definition) return changeSet;
   let [fieldName, existingItem] = findExistingChangeSetItem(changeSet, definition);
   if (fieldName == 'deletes') {
@@ -213,7 +217,7 @@ function extractFields(item: ChangeSetItem, allowNulls = true): UpdateField[] {
     }));
 }
 
-function insertToSql(
+function changeSetInsertToSql(
   item: ChangeSetItem,
   dbinfo: DatabaseInfo = null
 ): [AllowIdentityInsert, Insert, AllowIdentityInsert] {
@@ -224,7 +228,7 @@ function insertToSql(
     const table = dbinfo.tables.find(x => x.schemaName == item.schemaName && x.pureName == item.pureName);
     if (table) {
       const autoIncCol = table.columns.find(x => x.autoIncrement);
-      console.log('autoIncCol', autoIncCol);
+      // console.log('autoIncCol', autoIncCol);
       if (autoIncCol && fields.find(x => x.targetColumn == autoIncCol.columnName)) {
         autoInc = true;
       }
@@ -257,7 +261,7 @@ function insertToSql(
   ];
 }
 
-function extractCondition(item: ChangeSetItem): Condition {
+export function extractChangeSetCondition(item: ChangeSetItem, alias?: string): Condition {
   return {
     conditionType: 'and',
     conditions: _.keys(item.condition).map(columnName => ({
@@ -271,6 +275,7 @@ function extractCondition(item: ChangeSetItem): Condition {
             pureName: item.pureName,
             schemaName: item.schemaName,
           },
+          alias,
         },
       },
       right: {
@@ -281,7 +286,7 @@ function extractCondition(item: ChangeSetItem): Condition {
   };
 }
 
-function updateToSql(item: ChangeSetItem): Update {
+function changeSetUpdateToSql(item: ChangeSetItem): Update {
   return {
     from: {
       name: {
@@ -291,11 +296,11 @@ function updateToSql(item: ChangeSetItem): Update {
     },
     commandType: 'update',
     fields: extractFields(item),
-    where: extractCondition(item),
+    where: extractChangeSetCondition(item),
   };
 }
 
-function deleteToSql(item: ChangeSetItem): Delete {
+function changeSetDeleteToSql(item: ChangeSetItem): Delete {
   return {
     from: {
       name: {
@@ -304,16 +309,16 @@ function deleteToSql(item: ChangeSetItem): Delete {
       },
     },
     commandType: 'delete',
-    where: extractCondition(item),
+    where: extractChangeSetCondition(item),
   };
 }
 
 export function changeSetToSql(changeSet: ChangeSet, dbinfo: DatabaseInfo): Command[] {
   return _.compact(
     _.flatten([
-      ...(changeSet.inserts.map(item => insertToSql(item, dbinfo)) as any),
-      ...changeSet.updates.map(updateToSql),
-      ...changeSet.deletes.map(deleteToSql),
+      ...(changeSet.inserts.map(item => changeSetInsertToSql(item, dbinfo)) as any),
+      ...changeSet.updates.map(changeSetUpdateToSql),
+      ...changeSet.deletes.map(changeSetDeleteToSql),
     ])
   );
 }
@@ -370,7 +375,7 @@ export function getChangeSetInsertedRows(changeSet: ChangeSet, name?: NamedObjec
 }
 
 export function changeSetInsertNewRow(changeSet: ChangeSet, name?: NamedObjectInfo): ChangeSet {
-  console.log('INSERT', name);
+  // console.log('INSERT', name);
   const insertedRows = getChangeSetInsertedRows(changeSet, name);
   return {
     ...changeSet,
@@ -385,6 +390,25 @@ export function changeSetInsertNewRow(changeSet: ChangeSet, name?: NamedObjectIn
   };
 }
 
+export function changeSetInsertDocuments(
+  changeSet: ChangeSet,
+  documents: any[],
+  name?: NamedObjectInfo
+): ChangeSet {
+  const insertedRows = getChangeSetInsertedRows(changeSet, name);
+  return {
+    ...changeSet,
+    inserts: [
+      ...changeSet.inserts,
+      ...documents.map((doc, index) => ({
+        ...name,
+        insertedRowIndex: insertedRows.length + index,
+        fields: doc,
+      })),
+    ],
+  };
+}
+
 export function changeSetContainsChanges(changeSet: ChangeSet) {
   if (!changeSet) return false;
   return changeSet.deletes.length > 0 || changeSet.updates.length > 0 || changeSet.inserts.length > 0;

packages/datalib/src/CollectionGridDisplay.ts

@@ -70,9 +70,14 @@ function getDisplayColumn(basePath, columnName, display) {
 
 export function analyseCollectionDisplayColumns(rows, display) {
   const res = [];
+  const addedColumns = display?.config?.addedColumns;
   for (const row of rows || []) {
     getColumnsForObject([], row, res, display);
   }
+  for (const added of addedColumns || []) {
+    if (res.find(x => x.uniqueName == added)) continue;
+    res.push(getDisplayColumn([], added, display));
+  }
   return (
     res.map(col => ({
       ...col,
@@ -89,10 +94,13 @@ export class CollectionGridDisplay extends GridDisplay {
     setConfig: ChangeConfigFunc,
     cache: GridCache,
     setCache: ChangeCacheFunc,
-    loadedRows
+    loadedRows,
+    changeSet
   ) {
     super(config, setConfig, cache, setCache, driver);
-    this.columns = analyseCollectionDisplayColumns(loadedRows, this);
+    const changedDocs = _.compact(changeSet.updates.map(chs => chs.document));
+    const insertedDocs = _.compact(changeSet.inserts.map(chs => chs.fields));
+    this.columns = analyseCollectionDisplayColumns([...(loadedRows || []), ...changedDocs, ...insertedDocs], this);
     this.filterable = true;
     this.sortable = true;
     this.editable = true;

packages/datalib/src/FreeTableGridDisplay.ts

@@ -3,6 +3,7 @@ import { EngineDriver, ViewInfo, ColumnInfo } from 'dbgate-types';
 import { GridDisplay, ChangeCacheFunc, ChangeConfigFunc } from './GridDisplay';
 import { GridConfig, GridCache } from './GridConfig';
 import { FreeTableModel } from './FreeTableModel';
+import { analyseCollectionDisplayColumns } from '.';
 
 export class FreeTableGridDisplay extends GridDisplay {
   constructor(
@@ -13,19 +14,22 @@ export class FreeTableGridDisplay extends GridDisplay {
     setCache: ChangeCacheFunc
   ) {
     super(config, setConfig, cache, setCache);
-    this.columns = this.getDisplayColumns(model);
+    this.columns = model?.structure?.__isDynamicStructure
+      ? analyseCollectionDisplayColumns(model?.rows, this)
+      : this.getDisplayColumns(model);
     this.filterable = false;
     this.sortable = false;
   }
 
   getDisplayColumns(model: FreeTableModel) {
-    return (
+    return _.uniqBy(
       model?.structure?.columns
         ?.map(col => this.getDisplayColumn(col))
         ?.map(col => ({
           ...col,
           isChecked: this.isColumnChecked(col),
-        })) || []
+        })) || [],
+      col => col.uniqueName
     );
   }
 

packages/datalib/src/GridConfig.ts

@@ -20,7 +20,7 @@ export type GroupFunc = 'GROUP' | 'MAX' | 'MIN' | 'SUM' | 'AVG' | 'COUNT' | 'COU
 
 export interface GridConfig extends GridConfigColumns {
   filters: { [uniqueName: string]: string };
-  focusedColumn?: string;
+  focusedColumns?: string[];
   columnWidths: { [uniqueName: string]: number };
   sort: {
     uniqueName: string;
@@ -48,7 +48,7 @@ export function createGridConfig(): GridConfig {
     filters: {},
     columnWidths: {},
     sort: [],
-    focusedColumn: null,
+    focusedColumns: null,
     grouping: {},
     formFilterColumns: [],
   };

packages/datalib/src/GridDisplay.ts

@@ -9,6 +9,7 @@ import {
   DatabaseInfo,
   CollectionInfo,
   SqlDialect,
+  ViewInfo,
 } from 'dbgate-types';
 import { parseFilter, getFilterType } from 'dbgate-filterparser';
 import { filterName } from 'dbgate-tools';
@@ -29,7 +30,7 @@ export interface DisplayColumn {
   foreignKey?: ForeignKeyInfo;
   isExpandable?: boolean;
   isChecked?: boolean;
-  hintColumnName?: string;
+  hintColumnNames?: string[];
   dataType?: string;
   filterType?: boolean;
   isStructured?: boolean;
@@ -69,12 +70,20 @@ export abstract class GridDisplay {
   dialect: SqlDialect;
   columns: DisplayColumn[];
   baseTable?: TableInfo;
+  baseView?: ViewInfo;
   baseCollection?: CollectionInfo;
+  get baseTableOrSimilar(): NamedObjectInfo {
+    return this.baseTable || this.baseCollection || this.baseView;
+  }
   get baseTableOrCollection(): NamedObjectInfo {
     return this.baseTable || this.baseCollection;
   }
+  get baseTableOrView(): TableInfo | ViewInfo {
+    return this.baseTable || this.baseView;
+  }
   changeSetKeyFields: string[] = null;
   sortable = false;
+  groupable = false;
   filterable = false;
   editable = false;
   isLoadedCorrectly = true;
@@ -91,10 +100,14 @@ export abstract class GridDisplay {
     }
   }
 
-  focusColumn(uniqueName: string) {
+  addDynamicColumn(name: string) {
+    this.includeInColumnSet('addedColumns', name, true);
+  }
+
+  focusColumns(uniqueNames: string[]) {
     this.setConfig(cfg => ({
       ...cfg,
-      focusedColumn: uniqueName,
+      focusedColumns: uniqueNames,
     }));
   }
 
@@ -102,8 +115,8 @@ export abstract class GridDisplay {
     return false;
   }
 
-  get focusedColumn() {
-    return this.config.focusedColumn;
+  get focusedColumns() {
+    return this.config.focusedColumns;
   }
 
   get engine() {
@@ -114,6 +127,14 @@ export abstract class GridDisplay {
     return this.getColumns(null).filter(col => col.isChecked || col.uniquePath.length == 1);
   }
 
+  findColumn(uniqueName: string) {
+    return this.getColumns(null).find(x => x.uniqueName == uniqueName);
+  }
+
+  getFkTarget(column: DisplayColumn): TableInfo {
+    return null;
+  }
+
   reload() {
     this.setCache(reloadDataCacheFunc);
   }
@@ -148,7 +169,7 @@ export abstract class GridDisplay {
   }
 
   get hiddenColumnIndexes() {
-    return (this.config.hiddenColumns || []).map(x => _.findIndex(this.columns, y => y.uniqueName == x));
+    return (this.config.hiddenColumns || []).map(x => _.findIndex(this.allColumns, y => y.uniqueName == x));
   }
 
   isColumnChecked(column: DisplayColumn) {
@@ -364,7 +385,7 @@ export abstract class GridDisplay {
   getGrouping(uniqueName): GroupFunc {
     if (this.isGrouped) {
       if (this.config.grouping[uniqueName]) return this.config.grouping[uniqueName];
-      const column = this.baseTable.columns.find(x => x.columnName == uniqueName);
+      const column = (this.baseTable || this.baseView)?.columns?.find(x => x.columnName == uniqueName);
       if (isTypeLogical(column?.dataType)) return 'COUNT DISTINCT';
       if (column?.autoIncrement) return 'COUNT';
       return 'MAX';
@@ -404,9 +425,9 @@ export abstract class GridDisplay {
   getChangeSetField(row, uniqueName, insertedRowIndex): ChangeSetFieldDefinition {
     const col = this.columns.find(x => x.uniqueName == uniqueName);
     if (!col) return null;
-    const baseTableOrCollection = this.baseTableOrCollection;
-    if (!baseTableOrCollection) return null;
-    if (baseTableOrCollection.pureName != col.pureName || baseTableOrCollection.schemaName != col.schemaName) {
+    const baseObj = this.baseTableOrSimilar;
+    if (!baseObj) return null;
+    if (baseObj.pureName != col.pureName || baseObj.schemaName != col.schemaName) {
       return null;
     }
 
@@ -418,11 +439,11 @@ export abstract class GridDisplay {
   }
 
   getChangeSetRow(row, insertedRowIndex): ChangeSetRowDefinition {
-    const baseTableOrCollection = this.baseTableOrCollection;
-    if (!baseTableOrCollection) return null;
+    const baseObj = this.baseTableOrSimilar;
+    if (!baseObj) return null;
     return {
-      pureName: baseTableOrCollection.pureName,
-      schemaName: baseTableOrCollection.schemaName,
+      pureName: baseObj.pureName,
+      schemaName: baseObj.schemaName,
       insertedRowIndex,
       condition: insertedRowIndex == null ? this.getChangeSetCondition(row) : null,
     };

packages/datalib/src/JslGridDisplay.ts

@@ -1,3 +1,4 @@
+import _ from 'lodash';
 import { GridDisplay, ChangeCacheFunc, ChangeConfigFunc } from './GridDisplay';
 import { QueryResultColumn } from 'dbgate-types';
 import { GridConfig, GridCache } from './GridConfig';
@@ -18,21 +19,24 @@ export class JslGridDisplay extends GridDisplay {
     this.filterable = true;
 
     if (structure.columns) {
-      this.columns = structure.columns
-        .map(col => ({
-          columnName: col.columnName,
-          headerText: col.columnName,
-          uniqueName: col.columnName,
-          uniquePath: [col.columnName],
-          notNull: col.notNull,
-          autoIncrement: col.autoIncrement,
-          pureName: null,
-          schemaName: null,
-        }))
-        ?.map(col => ({
-          ...col,
-          isChecked: this.isColumnChecked(col),
-        }));
+      this.columns = _.uniqBy(
+        structure.columns
+          .map(col => ({
+            columnName: col.columnName,
+            headerText: col.columnName,
+            uniqueName: col.columnName,
+            uniquePath: [col.columnName],
+            notNull: col.notNull,
+            autoIncrement: col.autoIncrement,
+            pureName: null,
+            schemaName: null,
+          }))
+          ?.map(col => ({
+            ...col,
+            isChecked: this.isColumnChecked(col),
+          })),
+        col => col.uniqueName
+      );
     }
 
     if (structure.__isDynamicStructure) {

packages/datalib/src/TableFormViewDisplay.ts

@@ -15,6 +15,7 @@ import {
 import { TableGridDisplay } from './TableGridDisplay';
 import stableStringify from 'json-stable-stringify';
 import { ChangeSetFieldDefinition, ChangeSetRowDefinition } from './ChangeSet';
+import { DictionaryDescriptionFunc } from '.';
 
 export class TableFormViewDisplay extends FormViewDisplay {
   // use utility functions from GridDisplay and publish result in FromViewDisplay interface
@@ -29,7 +30,8 @@ export class TableFormViewDisplay extends FormViewDisplay {
     setCache: ChangeCacheFunc,
     dbinfo: DatabaseInfo,
     displayOptions,
-    serverVersion
+    serverVersion,
+    getDictionaryDescription: DictionaryDescriptionFunc = null
   ) {
     super(config, setConfig, cache, setCache, driver, dbinfo, serverVersion);
     this.gridDisplay = new TableGridDisplay(
@@ -41,7 +43,8 @@ export class TableFormViewDisplay extends FormViewDisplay {
       setCache,
       dbinfo,
       displayOptions,
-      serverVersion
+      serverVersion,
+      getDictionaryDescription
     );
     this.gridDisplay.addAllExpandedColumnsToSelected = true;
 

packages/datalib/src/TableGridDisplay.ts

@@ -1,9 +1,25 @@
 import _ from 'lodash';
 import { filterName } from 'dbgate-tools';
 import { GridDisplay, ChangeCacheFunc, DisplayColumn, DisplayedColumnInfo, ChangeConfigFunc } from './GridDisplay';
-import { TableInfo, EngineDriver, ViewInfo, ColumnInfo, NamedObjectInfo, DatabaseInfo } from 'dbgate-types';
+import {
+  TableInfo,
+  EngineDriver,
+  ViewInfo,
+  ColumnInfo,
+  NamedObjectInfo,
+  DatabaseInfo,
+  ForeignKeyInfo,
+} from 'dbgate-types';
 import { GridConfig, GridCache, createGridCache } from './GridConfig';
-import { Expression, Select, treeToSql, dumpSqlSelect } from 'dbgate-sqltree';
+import { Expression, Select, treeToSql, dumpSqlSelect, ColumnRefExpression } from 'dbgate-sqltree';
+
+export interface DictionaryDescription {
+  expression: string;
+  columns: string[];
+  delimiter: string;
+}
+
+export type DictionaryDescriptionFunc = (table: TableInfo) => DictionaryDescription;
 
 export class TableGridDisplay extends GridDisplay {
   public table: TableInfo;
@@ -19,7 +35,8 @@ export class TableGridDisplay extends GridDisplay {
     setCache: ChangeCacheFunc,
     dbinfo: DatabaseInfo,
     public displayOptions: any,
-    serverVersion
+    serverVersion,
+    public getDictionaryDescription: DictionaryDescriptionFunc = null
   ) {
     super(config, setConfig, cache, setCache, driver, dbinfo, serverVersion);
 
@@ -35,6 +52,7 @@ export class TableGridDisplay extends GridDisplay {
     this.columns = this.getDisplayColumns(this.table, []);
     this.filterable = true;
     this.sortable = true;
+    this.groupable = true;
     this.editable = true;
     this.supportsReload = true;
     this.baseTable = this.table;
@@ -60,7 +78,11 @@ export class TableGridDisplay extends GridDisplay {
         ?.map(col => ({
           ...col,
           isChecked: this.isColumnChecked(col),
-          hintColumnName: col.foreignKey ? `hint_${col.uniqueName}` : null,
+          hintColumnNames:
+            this.getFkDictionaryDescription(col.foreignKey)?.columns?.map(
+              columnName => `hint_${col.uniqueName}_${columnName}`
+            ) || null,
+          hintColumnDelimiter: this.getFkDictionaryDescription(col.foreignKey)?.delimiter,
           isExpandable: !!col.foreignKey,
         })) || []
     );
@@ -115,6 +137,19 @@ export class TableGridDisplay extends GridDisplay {
     }
   }
 
+  getFkDictionaryDescription(foreignKey: ForeignKeyInfo) {
+    if (!foreignKey) return null;
+    const pureName = foreignKey.refTableName;
+    const schemaName = foreignKey.refSchemaName;
+    const table = this.findTable({ schemaName, pureName });
+
+    if (table && table.columns && table.columns.length > 0 && table.primaryKey) {
+      const hintDescription = this.getDictionaryDescription(table);
+      return hintDescription;
+    }
+    return null;
+  }
+
   addHintsToSelect(select: Select): boolean {
     let res = false;
     const groupColumns = this.groupColumns;
@@ -125,17 +160,23 @@ export class TableGridDisplay extends GridDisplay {
         }
         const table = this.getFkTarget(column);
         if (table && table.columns && table.columns.length > 0 && table.primaryKey) {
-          const hintColumn = table.columns.find(x => x?.dataType?.toLowerCase()?.includes('char'));
-          if (hintColumn) {
+          // const hintColumn = table.columns.find(x => x?.dataType?.toLowerCase()?.includes('char'));
+          const hintDescription = this.getDictionaryDescription(table);
+          if (hintDescription) {
             const parentUniqueName = column.uniquePath.slice(0, -1).join('.');
             this.addReferenceToSelect(select, parentUniqueName ? `${parentUniqueName}_ref` : 'basetbl', column);
             const childAlias = `${column.uniqueName}_ref`;
-            select.columns.push({
-              exprType: 'column',
-              columnName: hintColumn.columnName,
-              alias: `hint_${column.uniqueName}`,
-              source: { alias: childAlias },
-            });
+            select.columns.push(
+              ...hintDescription.columns.map(
+                columnName =>
+                  ({
+                    exprType: 'column',
+                    columnName,
+                    alias: `hint_${column.uniqueName}_${columnName}`,
+                    source: { alias: childAlias },
+                  } as ColumnRefExpression)
+              )
+            );
             res = true;
           }
         }

packages/datalib/src/ViewGridDisplay.ts

@@ -17,8 +17,10 @@ export class ViewGridDisplay extends GridDisplay {
     this.columns = this.getDisplayColumns(view);
     this.filterable = true;
     this.sortable = true;
+    this.groupable = false;
     this.editable = false;
     this.supportsReload = true;
+    this.baseView = view;
   }
 
   getDisplayColumns(view: ViewInfo) {

packages/datalib/src/deleteCascade.ts

@@ -0,0 +1,140 @@
+import _ from 'lodash';
+import { Command, Insert, Update, Delete, UpdateField, Condition, AllowIdentityInsert } from 'dbgate-sqltree';
+import { NamedObjectInfo, DatabaseInfo, ForeignKeyInfo, TableInfo } from 'dbgate-types';
+import { ChangeSet, ChangeSetItem, extractChangeSetCondition } from './ChangeSet';
+
+export interface ChangeSetDeleteCascade {
+  title: string;
+  commands: Command[];
+}
+
+// function getDeleteScript()
+
+function processDependencies(
+  changeSet: ChangeSet,
+  result: ChangeSetDeleteCascade[],
+  allForeignKeys: ForeignKeyInfo[],
+  fkPath: ForeignKeyInfo[],
+  table: TableInfo,
+  baseCmd: ChangeSetItem,
+  dbinfo: DatabaseInfo,
+  usedTables: string[]
+) {
+  if (result.find(x => x.title == table.pureName)) return;
+
+  const dependencies = allForeignKeys.filter(
+    x => x.refSchemaName == table.schemaName && x.refTableName == table.pureName
+  );
+
+  for (const fk of dependencies) {
+    const depTable = dbinfo.tables.find(x => x.pureName == fk.pureName && x.schemaName == fk.schemaName);
+    const subFkPath = [...fkPath, fk];
+    if (depTable && !usedTables.includes(depTable.pureName)) {
+      processDependencies(changeSet, result, allForeignKeys, subFkPath, depTable, baseCmd, dbinfo, [
+        ...usedTables,
+        depTable.pureName,
+      ]);
+    }
+
+    const refCmd: Delete = {
+      commandType: 'delete',
+      from: {
+        name: {
+          pureName: fk.pureName,
+          schemaName: fk.schemaName,
+        },
+      },
+      where: {
+        conditionType: 'exists',
+        subQuery: {
+          commandType: 'select',
+          selectAll: true,
+          from: {
+            name: {
+              pureName: fk.pureName,
+              schemaName: fk.schemaName,
+            },
+            alias: 't0',
+            relations: subFkPath.map((fkItem, fkIndex) => ({
+              joinType: 'INNER JOIN',
+              alias: `t${fkIndex + 1}`,
+              name: {
+                pureName: fkItem.refTableName,
+                schemaName: fkItem.refSchemaName,
+              },
+              conditions: fkItem.columns.map(column => ({
+                conditionType: 'binary',
+                operator: '=',
+                left: {
+                  exprType: 'column',
+                  columnName: column.columnName,
+                  source: { alias: `t${fkIndex}` },
+                },
+                right: {
+                  exprType: 'column',
+                  columnName: column.refColumnName,
+                  source: { alias: `t${fkIndex + 1}` },
+                },
+              })),
+            })),
+          },
+          where: {
+            conditionType: 'and',
+            conditions: [
+              extractChangeSetCondition(baseCmd, `t${subFkPath.length}`),
+              // @ts-ignore
+              ...table.primaryKey.columns.map(column => ({
+                conditionType: 'binary',
+                operator: '=',
+                left: {
+                  exprType: 'column',
+                  columnName: column.columnName,
+                  source: { alias: 't0' },
+                },
+                right: {
+                  exprType: 'column',
+                  columnName: column.columnName,
+                  source: {
+                    name: fk,
+                  },
+                },
+              })),
+            ],
+          },
+        },
+      },
+    };
+    let resItem = result.find(x => x.title == fk.pureName);
+    if (!resItem) {
+      resItem = {
+        title: fk.pureName,
+        commands: [],
+      };
+      result.push(resItem);
+    }
+    resItem.commands.push(refCmd);
+  }
+}
+
+export function getDeleteCascades(changeSet: ChangeSet, dbinfo: DatabaseInfo): ChangeSetDeleteCascade[] {
+  const result: ChangeSetDeleteCascade[] = [];
+  const allForeignKeys = _.flatten(dbinfo.tables.map(x => x.foreignKeys));
+  for (const baseCmd of changeSet.deletes) {
+    const table = dbinfo.tables.find(x => x.pureName == baseCmd.pureName && x.schemaName == baseCmd.schemaName);
+    if (!table.primaryKey) continue;
+
+    processDependencies(changeSet, result, allForeignKeys, [], table, baseCmd, dbinfo, [table.pureName]);
+
+    // let resItem = result.find(x => x.title == baseCmd.pureName);
+    // if (!resItem) {
+    //   resItem = {
+    //     title: baseCmd.pureName,
+    //     commands: [],
+    //   };
+    //   result.push(resItem);
+    // }
+    // resItem.commands.push(changeSetDeleteToSql(baseCmd));
+  }
+
+  return result;
+}

packages/datalib/src/index.ts

@@ -11,3 +11,4 @@ export * from './runMacro';
 export * from './FormViewDisplay';
 export * from './TableFormViewDisplay';
 export * from './CollectionGridDisplay';
+export * from './deleteCascade';

packages/filterparser/package.json

@@ -18,7 +18,7 @@
     "@types/node": "^13.7.0",
     "jest": "^24.9.0",
     "ts-jest": "^25.2.1",
-    "typescript": "^3.7.5"
+    "typescript": "^4.4.3"
   },
   "dependencies": {
     "@types/parsimmon": "^1.10.1",

packages/filterparser/src/filterTool.ts

@@ -6,6 +6,9 @@ export type FilterMultipleValuesMode = 'is' | 'is_not' | 'contains' | 'begins' |
 export function getFilterValueExpression(value, dataType) {
   if (value == null) return 'NULL';
   if (isTypeDateTime(dataType)) return moment(value).format('YYYY-MM-DD HH:mm:ss');
+  if (value === true) return 'TRUE';
+  if (value === false) return 'FALSE';
+  if (value.$oid) return `ObjectId("${value.$oid}")`;
   return `="${value}"`;
 }
 

packages/filterparser/src/mongoParser.ts

@@ -28,6 +28,16 @@ const numberTestCondition = () => value => ({
   ],
 });
 
+const idRegex = /[('"]([0-9a-f]{24})['")]/;
+
+const objectIdTestCondition = () => value => ({
+  $or: [
+    {
+      __placeholder__: { $oid: value.match(idRegex)[1] },
+    },
+  ],
+});
+
 const testCondition = (operator, value) => () => ({
   __placeholder__: {
     [operator]: value,
@@ -64,9 +74,12 @@ const createParser = () => {
         .map(Number)
         .desc('number'),
 
+    objectid: () => token(P.regexp(/ObjectId\(['"]?[0-9a-f]{24}['"]?\)/)).desc('ObjectId'),
+
     noQuotedString: () => P.regexp(/[^\s^,^'^"]+/).desc('string unquoted'),
 
-    value: r => P.alt(r.string1, r.string2, r.number, r.noQuotedString),
+    value: r => P.alt(r.objectid, r.string1, r.string2, r.number, r.noQuotedString),
+    valueTestObjectId: r => r.objectid.map(objectIdTestCondition()),
     valueTestNum: r => r.number.map(numberTestCondition()),
     valueTest: r => r.value.map(regexCondition('.*#VALUE#.*')),
 
@@ -108,6 +121,7 @@ const createParser = () => {
         r.startsWithNot,
         r.endsWithNot,
         r.containsNot,
+        r.valueTestObjectId,
         r.valueTestNum,
         r.valueTest
       ).trim(whitespace),

packages/filterparser/src/parseFilter.ts

@@ -223,8 +223,8 @@ const createParser = (filterType: FilterType) => {
     null: () => word('NULL').map(unaryCondition('isNull')),
     empty: () => word('EMPTY').map(unaryCondition('isEmpty')),
     notEmpty: r => r.not.then(r.empty).map(unaryCondition('isNotEmpty')),
-    true: () => word('TRUE').map(binaryFixedValueCondition(1)),
-    false: () => word('FALSE').map(binaryFixedValueCondition(0)),
+    true: () => P.regexp(/true/i).map(binaryFixedValueCondition(1)),
+    false: () => P.regexp(/false/i).map(binaryFixedValueCondition(0)),
     trueNum: () => word('1').map(binaryFixedValueCondition(1)),
     falseNum: () => word('0').map(binaryFixedValueCondition(0)),
 
@@ -253,6 +253,7 @@ const createParser = (filterType: FilterType) => {
 
     eq: r => word('=').then(r.value).map(binaryCondition('=')),
     ne: r => word('!=').then(r.value).map(binaryCondition('<>')),
+    ne2: r => word('<>').then(r.value).map(binaryCondition('<>')),
     lt: r => word('<').then(r.value).map(binaryCondition('<')),
     gt: r => word('>').then(r.value).map(binaryCondition('>')),
     le: r => word('<=').then(r.value).map(binaryCondition('<=')),
@@ -273,7 +274,7 @@ const createParser = (filterType: FilterType) => {
   if (filterType == 'string') allowedValues.push('string1', 'string2', 'noQuotedString');
   if (filterType == 'number') allowedValues.push('string1Num', 'string2Num', 'number');
 
-  const allowedElements = ['null', 'notNull', 'eq', 'ne'];
+  const allowedElements = ['null', 'notNull', 'eq', 'ne', 'ne2'];
   if (filterType == 'number' || filterType == 'datetime') allowedElements.push('lt', 'gt', 'le', 'ge');
   if (filterType == 'string')
     allowedElements.push(

packages/query-splitter/README.md

@@ -1,15 +1,15 @@
 [![NPM version](https://img.shields.io/npm/v/dbgate-query-splitter.svg)](https://www.npmjs.com/package/dbgate-query-splitter)
 
-dbgate-query-splitter
-====================
+# dbgate-query-splitter
 
-Splits long SQL query into into particular statements. Designed to have zero dependencies and to be fast.
+Splits long SQL query into into particular statements. Designed to have zero dependencies and to be fast. Also supports nodejs-streams.
 
 Supports following SQL dialects:
-* MySQL
-* PostgreSQL
-* SQLite
-* Microsoft SQL Server
+
+- MySQL
+- PostgreSQL
+- SQLite
+- Microsoft SQL Server
 
 ## Usage
 
@@ -19,10 +19,26 @@ import { splitQuery, mysqlSplitterOptions, mssqlSplitterOptions, postgreSplitter
 const output = splitQuery('SELECT * FROM `table1`;SELECT * FROM `table2`;', mysqlSplitterOptions);
 
 // output is ['SELECT * FROM `table1`', 'SELECT * FROM `table2`']
+```
+
+## Streaming support in nodejs
+Function splitQueryStream accepts input stream and query options. Result is object stream, each object for one splitted query.
+Tokens must not be divided into more input chunks. This can be accomplished eg. when input stream emits one chunk per line (eg. using byline module)
+
+```js
+const { mysqlSplitterOptions, mssqlSplitterOptions, postgreSplitterOptions } = require('dbgate-query-splitter');
+const { splitQueryStream } = require('dbgate-query-splitter/lib/splitQueryStream');
+const fs = require('fs');
+const byline = require('byline');
+
+const fileStream = fs.createReadStream('INPUT_FILE_NAME', 'utf-8');
+const lineStream = byline(fileStream);
+const splittedStream = splitQueryStream(lineStream, mysqlSplitterOptions);
 
 ```
 
 ## Contributing
+
 Please run tests before pushing any changes.
 
 ```sh
@@ -30,7 +46,8 @@ yarn test
 ```
 
 ## Supported syntax
-* Comments
-* Dollar strings (PostgreSQL)
-* GO separators (MS SQL)
-* Custom delimiter, setby DELIMITER keyword (MySQL)
+
+- Comments
+- Dollar strings (PostgreSQL)
+- GO separators (MS SQL)
+- Custom delimiter, setby DELIMITER keyword (MySQL)

packages/query-splitter/package.json

@@ -32,6 +32,6 @@
     "@types/node": "^13.7.0",
     "jest": "^24.9.0",
     "ts-jest": "^25.2.1",
-    "typescript": "^3.7.5"
+    "typescript": "^4.4.3"
   }
 }

packages/query-splitter/src/index.ts

@@ -1,2 +1,2 @@
-export * from './splitQuery';
+export { splitQuery } from './splitQuery';
 export * from './options';

packages/query-splitter/src/options.ts

@@ -8,6 +8,11 @@ export interface SplitterOptions {
   allowGoDelimiter: boolean;
   allowDollarDollarString: boolean;
   noSplit: boolean;
+  doubleDashComments: boolean;
+  multilineComments: boolean;
+  javaScriptComments: boolean;
+
+  returnRichInfo: boolean;
 }
 
 export const defaultSplitterOptions: SplitterOptions = {
@@ -20,6 +25,12 @@ export const defaultSplitterOptions: SplitterOptions = {
   allowGoDelimiter: false,
   allowDollarDollarString: false,
   noSplit: false,
+
+  doubleDashComments: true,
+  multilineComments: true,
+  javaScriptComments: false,
+
+  returnRichInfo: false,
 };
 
 export const mysqlSplitterOptions: SplitterOptions = {
@@ -59,6 +70,14 @@ export const sqliteSplitterOptions: SplitterOptions = {
   stringEscapes: { "'": "'", '"': '"' },
 };
 
+export const mongoSplitterOptions: SplitterOptions = {
+  ...defaultSplitterOptions,
+
+  stringsBegins: ["'", '"'],
+  stringsEnds: { "'": "'", '"': '"' },
+  stringEscapes: { "'": '\\', '"': '\\' },
+};
+
 export const noSplitSplitterOptions: SplitterOptions = {
   ...defaultSplitterOptions,
 

packages/query-splitter/src/splitQuery.ts

@@ -2,12 +2,25 @@ import { SplitterOptions, defaultSplitterOptions } from './options';
 
 const SEMICOLON = ';';
 
-interface SplitExecutionContext {
+export interface SplitStreamContext {
   options: SplitterOptions;
+  currentDelimiter: string;
+  pushOutput: (item: SplitResultItem) => void;
+  commandPart: string;
+
+  line: number;
+  column: number;
+  streamPosition: number;
+
+  commandStartPosition: number;
+  commandStartLine: number;
+  commandStartColumn: number;
+}
+
+export interface SplitLineContext extends SplitStreamContext {
   source: string;
   position: number;
-  currentDelimiter: string;
-  output: string[];
+  // output: string[];
   end: number;
   wasDataOnLine: boolean;
   currentCommandStart: number;
@@ -17,6 +30,45 @@ interface SplitExecutionContext {
   //   semicolonKeyTokenRegex: RegExp;
 }
 
+export interface SplitPositionDefinition {
+  position: number;
+  line: number;
+  column: number;
+}
+
+export interface SplitResultItemRich {
+  text: string;
+  start: SplitPositionDefinition;
+  end: SplitPositionDefinition;
+  trimStart?: SplitPositionDefinition;
+  trimEnd?: SplitPositionDefinition;
+}
+
+export type SplitResultItem = string | SplitResultItemRich;
+
+function movePosition(context: SplitLineContext, count: number) {
+  if (context.options.returnRichInfo) {
+    let { source, position, line, column, streamPosition } = context;
+    while (count > 0) {
+      if (source[position] == '\n') {
+        line += 1;
+        column = 0;
+      } else {
+        column += 1;
+      }
+      position += 1;
+      streamPosition += 1;
+      count -= 1;
+    }
+    context.position = position;
+    context.streamPosition = streamPosition;
+    context.line = line;
+    context.column = column;
+  } else {
+    context.position += count;
+  }
+}
+
 function isStringEnd(s: string, pos: number, endch: string, escapech: string) {
   if (!escapech) {
     return s[pos] == endch;
@@ -47,7 +99,7 @@ const DATA_TOKEN: Token = {
   length: 1,
 };
 
-function scanDollarQuotedString(context: SplitExecutionContext): Token {
+function scanDollarQuotedString(context: SplitLineContext): Token {
   if (!context.options.allowDollarDollarString) return null;
 
   let pos = context.position;
@@ -71,7 +123,7 @@ function scanDollarQuotedString(context: SplitExecutionContext): Token {
   return null;
 }
 
-function scanToken(context: SplitExecutionContext): Token {
+function scanToken(context: SplitLineContext): Token {
   let pos = context.position;
   const s = context.source;
   const ch = s[pos];
@@ -108,7 +160,7 @@ function scanToken(context: SplitExecutionContext): Token {
     return EOLN_TOKEN;
   }
 
-  if (ch == '-' && s[pos + 1] == '-') {
+  if (context.options.doubleDashComments && ch == '-' && s[pos + 1] == '-') {
     while (pos < context.end && s[pos] != '\n') pos++;
     return {
       type: 'comment',
@@ -116,7 +168,7 @@ function scanToken(context: SplitExecutionContext): Token {
     };
   }
 
-  if (ch == '/' && s[pos + 1] == '*') {
+  if (context.options.multilineComments && ch == '/' && s[pos + 1] == '*') {
     pos += 2;
     while (pos < context.end) {
       if (s[pos] == '*' && s[pos + 1] == '/') break;
@@ -155,103 +207,202 @@ function scanToken(context: SplitExecutionContext): Token {
   return DATA_TOKEN;
 }
 
-function pushQuery(context) {
-  const sql = context.source.slice(context.currentCommandStart, context.position);
+function pushQuery(context: SplitLineContext) {
+  const sql = (context.commandPart || '') + context.source.slice(context.currentCommandStart, context.position);
   const trimmed = sql.trim();
-  if (trimmed) context.output.push(trimmed);
+  if (trimmed) {
+    if (context.options.returnRichInfo) {
+      context.pushOutput(
+        countTrimmedPositions(sql, {
+          text: trimmed,
+
+          start: {
+            position: context.commandStartPosition,
+            line: context.commandStartLine,
+            column: context.commandStartColumn,
+          },
+
+          end: {
+            position: context.streamPosition,
+            line: context.line,
+            column: context.column,
+          },
+        })
+      );
+    } else {
+      context.pushOutput(trimmed);
+    }
+  }
 }
 
-export function splitQuery(sql: string, options: SplitterOptions = null): string[] {
+function countTrimmedPositions(full: string, positions: SplitResultItemRich): SplitResultItemRich {
+  const startIndex = full.indexOf(positions.text);
+
+  const trimStart = { ...positions.start };
+  for (let i = 0; i < startIndex; i += 1) {
+    if (full[i] == '\n') {
+      trimStart.position += 1;
+      trimStart.line += 1;
+      trimStart.column = 0;
+    } else {
+      trimStart.position += 1;
+      trimStart.column += 1;
+    }
+  }
+
+  return {
+    ...positions,
+    trimStart,
+    trimEnd: positions.end,
+  };
+}
+
+function markStartCommand(context: SplitLineContext) {
+  if (context.options.returnRichInfo) {
+    context.commandStartPosition = context.streamPosition;
+    context.commandStartLine = context.line;
+    context.commandStartColumn = context.column;
+  }
+}
+
+export function splitQueryLine(context: SplitLineContext) {
+  while (context.position < context.end) {
+    const token = scanToken(context);
+    if (!token) {
+      // nothing special, move forward
+      movePosition(context, 1);
+      continue;
+    }
+    switch (token.type) {
+      case 'string':
+        movePosition(context, token.length);
+        context.wasDataOnLine = true;
+        break;
+      case 'comment':
+        movePosition(context, token.length);
+        context.wasDataOnLine = true;
+        break;
+      case 'eoln':
+        movePosition(context, token.length);
+        context.wasDataOnLine = false;
+        break;
+      case 'data':
+        movePosition(context, token.length);
+        context.wasDataOnLine = true;
+        break;
+      case 'whitespace':
+        movePosition(context, token.length);
+        break;
+      case 'set_delimiter':
+        pushQuery(context);
+        context.commandPart = '';
+        context.currentDelimiter = token.value;
+        movePosition(context, token.length);
+        context.currentCommandStart = context.position;
+        markStartCommand(context);
+        break;
+      case 'go_delimiter':
+        pushQuery(context);
+        context.commandPart = '';
+        movePosition(context, token.length);
+        context.currentCommandStart = context.position;
+        markStartCommand(context);
+        break;
+      case 'delimiter':
+        pushQuery(context);
+        context.commandPart = '';
+        movePosition(context, token.length);
+        context.currentCommandStart = context.position;
+        markStartCommand(context);
+        break;
+    }
+  }
+
+  if (context.end > context.currentCommandStart) {
+    context.commandPart += context.source.slice(context.currentCommandStart, context.position);
+  }
+}
+
+export function getInitialDelimiter(options: SplitterOptions) {
+  return options?.allowSemicolon === false ? null : SEMICOLON;
+}
+
+export function finishSplitStream(context: SplitStreamContext) {
+  const trimmed = context.commandPart.trim();
+  if (trimmed) {
+    if (context.options.returnRichInfo) {
+      context.pushOutput(
+        countTrimmedPositions(context.commandPart, {
+          text: trimmed,
+
+          start: {
+            position: context.commandStartPosition,
+            line: context.commandStartLine,
+            column: context.commandStartColumn,
+          },
+
+          end: {
+            position: context.streamPosition,
+            line: context.line,
+            column: context.column,
+          },
+        })
+      );
+    } else {
+      context.pushOutput(trimmed);
+    }
+  }
+}
+
+export function splitQuery(sql: string, options: SplitterOptions = null): SplitResultItem[] {
   const usedOptions = {
     ...defaultSplitterOptions,
     ...options,
   };
 
   if (usedOptions.noSplit) {
+    if (usedOptions.returnRichInfo) {
+      const lines = sql.split('\n');
+      return [
+        {
+          text: sql,
+          start: {
+            position: 0,
+            line: 0,
+            column: 0,
+          },
+          end: {
+            position: sql.length,
+            line: lines.length,
+            column: lines[lines.length - 1]?.length || 0,
+          },
+        },
+      ];
+    }
     return [sql];
   }
 
-  const context: SplitExecutionContext = {
+  const output = [];
+  const context: SplitLineContext = {
     source: sql,
     end: sql.length,
-    currentDelimiter: options?.allowSemicolon === false ? null : SEMICOLON,
+    currentDelimiter: getInitialDelimiter(options),
     position: 0,
+    column: 0,
+    line: 0,
     currentCommandStart: 0,
-    output: [],
+    commandStartLine: 0,
+    commandStartColumn: 0,
+    commandStartPosition: 0,
+    streamPosition: 0,
+    pushOutput: cmd => output.push(cmd),
     wasDataOnLine: false,
     options: usedOptions,
+    commandPart: '',
   };
 
-  while (context.position < context.end) {
-    const token = scanToken(context);
-    if (!token) {
-      // nothing special, move forward
-      context.position += 1;
-      continue;
-    }
-    switch (token.type) {
-      case 'string':
-        context.position += token.length;
-        context.wasDataOnLine = true;
-        break;
-      case 'comment':
-        context.position += token.length;
-        context.wasDataOnLine = true;
-        break;
-      case 'eoln':
-        context.position += token.length;
-        context.wasDataOnLine = false;
-        break;
-      case 'data':
-        context.position += token.length;
-        context.wasDataOnLine = true;
-        break;
-      case 'whitespace':
-        context.position += token.length;
-        break;
-      case 'set_delimiter':
-        pushQuery(context);
-        context.currentDelimiter = token.value;
-        context.position += token.length;
-        context.currentCommandStart = context.position;
-        break;
-      case 'go_delimiter':
-        pushQuery(context);
-        context.position += token.length;
-        context.currentCommandStart = context.position;
-        break;
-      case 'delimiter':
-        pushQuery(context);
-        context.position += token.length;
-        context.currentCommandStart = context.position;
-        break;
-    }
-  }
+  splitQueryLine(context);
+  finishSplitStream(context);
 
-  if (context.end > context.currentCommandStart) {
-    pushQuery(context);
-  }
-
-  //   context.semicolonKeyTokenRegex = buildKeyTokenRegex(SEMICOLON, context);
-  //   let findResult: FindExpResult = {
-  //     expIndex: -1,
-  //     exp: null,
-  //     nextIndex: 0,
-  //   };
-  //   let lastUnreadLength;
-  //   do {
-  //     // console.log('context.unread', context.unread);
-  //     lastUnreadLength = context.unread.length;
-  //     findResult = findKeyToken(context.unread, context.currentDelimiter, context);
-  //     handleKeyTokenFindResult(context, findResult);
-  //     // Prevent infinite loop by returning incorrect result
-  //     if (lastUnreadLength === context.unread.length) {
-  //       read(context, context.unread.length);
-  //     }
-  //   } while (context.unread !== '');
-  //   publishStatement(context);
-
-  // console.log('RESULT', context.output);
-
-  return context.output;
+  return output;
 }

packages/query-splitter/src/splitQueryStream.ts

@@ -0,0 +1,52 @@
+import stream from 'stream';
+import {
+  SplitStreamContext,
+  getInitialDelimiter,
+  SplitLineContext,
+  splitQueryLine,
+  finishSplitStream,
+} from './splitQuery';
+import { SplitterOptions } from './options';
+
+export class SplitQueryStream extends stream.Transform {
+  context: SplitStreamContext;
+
+  constructor(options: SplitterOptions) {
+    super({ objectMode: true });
+    this.context = {
+      commandPart: '',
+      commandStartLine: 0,
+      commandStartColumn: 0,
+      commandStartPosition: 0,
+      streamPosition: 0,
+      line: 0,
+      column: 0,
+      options,
+      currentDelimiter: getInitialDelimiter(options),
+      pushOutput: cmd => this.push(cmd),
+    };
+  }
+  _transform(chunk, encoding, done) {
+    const lineContext: SplitLineContext = {
+      ...this.context,
+      position: 0,
+      currentCommandStart: 0,
+      wasDataOnLine: false,
+      source: chunk,
+      end: chunk.length,
+    };
+    splitQueryLine(lineContext);
+    this.context.commandPart = lineContext.commandPart;
+    done();
+  }
+  _flush(done) {
+    finishSplitStream(this.context);
+    done();
+  }
+}
+
+export function splitQueryStream(sourceStream, options: SplitterOptions) {
+  const splitter = new SplitQueryStream(options);
+  sourceStream.pipe(splitter);
+  return splitter;
+}