v4.2.5-beta.2

fix(mysql): contentHash return null if modifyDate is not set

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: dbgate
+patreon: # Replace with a single Patreon username
+open_collective: dbgate
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/workflows/build-app-beta.yaml

@@ -1,4 +1,4 @@
-name: Electron app
+name: Electron app BETA
 
 on:
   push:
@@ -12,7 +12,7 @@ jobs:
     strategy:
       matrix:
         # os: [ubuntu-18.04, windows-2016]
-        os: [macOS-10.14, windows-2016, ubuntu-18.04]
+        os: [macOS-10.15, windows-2016, ubuntu-18.04]
 
     steps:
       - name: Context
@@ -35,10 +35,14 @@ jobs:
       - name: fillNativeModulesElectron
         run: |
           yarn fillNativeModulesElectron
+      - name: fillPackagedPlugins
+        run: |
+          yarn fillPackagedPlugins
       - name: Install Snapcraft
         if: matrix.os == 'ubuntu-18.04'
         uses: samuelmeuli/action-snapcraft@v1
       - name: Publish
+        if: matrix.os != 'macOS-10.15'
         run: |
           yarn run build:app
         env:
@@ -46,6 +50,13 @@ jobs:
           WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
           WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
 
+      - name: Publish Mac
+        if: matrix.os == 'macOS-10.15'
+        run: |
+          yarn run build:app:mac
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
+
       - name: Save snap login
         if: matrix.os == 'ubuntu-18.04'
         run: 'echo "$SNAPCRAFT_LOGIN" > snapcraft.login'
@@ -64,10 +75,13 @@ jobs:
           mkdir artifacts          
 
           cp app/dist/*.deb artifacts/dbgate-beta.deb || true
-          cp app/dist/*.AppImage artifacts/dbgate-beta.AppImage || true
+          cp app/dist/*x86*.AppImage artifacts/dbgate-beta.AppImage || true
+          cp app/dist/*arm64*.AppImage artifacts/dbgate-beta-arm64.AppImage || true
+          cp app/dist/*armv7l*.AppImage artifacts/dbgate-beta-armv7l.AppImage || true
           cp app/dist/*.exe artifacts/dbgate-beta.exe || true
-          cp app/dist/*windows*.zip artifacts/dbgate-windows-beta.zip || true
-          cp app/dist/*.dmg artifacts/dbgate-beta.dmg || true
+          cp app/dist/*win*.zip artifacts/dbgate-windows-beta.zip || true
+          cp app/dist/*-mac.dmg artifacts/dbgate-beta.dmg || true
+          cp app/dist/*-mac_arm64.dmg artifacts/dbgate-beta-arm64.dmg || true
 
           mv app/dist/*.exe artifacts/ || true
           mv app/dist/*.zip artifacts/ || true

.github/workflows/build-app.yaml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       matrix:
         # os: [ubuntu-18.04, windows-2016]
-        os: [macOS-10.14, windows-2016, ubuntu-18.04]
+        os: [macOS-10.15, windows-2016, ubuntu-18.04]
 
     steps:
       - name: Context
@@ -39,6 +39,9 @@ jobs:
       - name: fillNativeModulesElectron
         run: |
           yarn fillNativeModulesElectron
+      - name: fillPackagedPlugins
+        run: |
+          yarn fillPackagedPlugins
       - name: Install Snapcraft
         if: matrix.os == 'ubuntu-18.04'
         uses: samuelmeuli/action-snapcraft@v1
@@ -72,10 +75,13 @@ jobs:
           mkdir artifacts          
 
           cp app/dist/*.deb artifacts/dbgate-latest.deb || true
-          cp app/dist/*.AppImage artifacts/dbgate-latest.AppImage || true
+          cp app/dist/*x86*.AppImage artifacts/dbgate-latest.AppImage || true
+          cp app/dist/*arm64*.AppImage artifacts/dbgate-latest-arm64.AppImage || true
+          cp app/dist/*armv7l*.AppImage artifacts/dbgate-latest-armv7l.AppImage || true
           cp app/dist/*.exe artifacts/dbgate-latest.exe || true
-          cp app/dist/*windows*.zip artifacts/dbgate-windows-latest.zip || true
-          cp app/dist/*.dmg artifacts/dbgate-latest.dmg || true
+          cp app/dist/*win*.zip artifacts/dbgate-windows-latest.zip || true
+          cp app/dist/*-mac.dmg artifacts/dbgate-latest.dmg || true
+          cp app/dist/*-mac_arm64.dmg artifacts/dbgate-latest-arm64.dmg || true
 
           mv app/dist/*.exe artifacts/ || true
           mv app/dist/*.zip artifacts/ || true
@@ -120,7 +126,7 @@ jobs:
           mv app/dist/latest-linux.yml artifacts/latest-linux.yml || true
 
       - name: Copy latest-mac.yml
-        if: matrix.os == 'macOS-10.14'
+        if: matrix.os == 'macOS-10.15'
         run: |
           mv app/dist/latest-mac.yml artifacts/latest-mac.yml || true
 

.github/workflows/build-docker-beta.yaml

@@ -1,4 +1,4 @@
-name: Docker image
+name: Docker image BETA
 
 # on: [push]
 

.github/workflows/build-docker.yaml

@@ -21,7 +21,6 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-18.04]
-        # os: [macOS-10.14, windows-2016, ubuntu-18.04]
 
     steps:
       - name: Context

.github/workflows/build-npm.yaml

@@ -21,7 +21,6 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-18.04]
-        # os: [macOS-10.14, windows-2016, ubuntu-18.04]
 
     steps:
       - name: Context
@@ -80,6 +79,11 @@ jobs:
         run: |
           npm publish
 
+      - name: Publish query-splitter
+        working-directory: packages/query-splitter
+        run: |
+          npm publish
+          
       - name: Publish web
         working-directory: packages/web
         run: |
@@ -119,3 +123,8 @@ jobs:
         working-directory: plugins/dbgate-plugin-postgres
         run: |
           npm publish
+
+      - name: Publish dbgate-plugin-sqlite
+        working-directory: plugins/dbgate-plugin-sqlite
+        run: |
+          npm publish

.github/workflows/run-tests.yaml

@@ -0,0 +1,81 @@
+name: Run tests
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  test-runner:
+    runs-on: ubuntu-latest
+    container: node:10.18-jessie
+
+    steps:
+      - name: Context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - uses: actions/checkout@v1
+        with:
+          fetch-depth: 1
+      - name: yarn install
+        run: |
+          yarn install
+      - name: Integration tests
+        run: |
+          cd integration-tests
+          yarn test:ci
+#          yarn wait:ci
+      - name: Filter parser tests
+        if: always()
+        run: |
+          cd packages/filterparser
+          yarn test:ci
+      - name: Query spliiter tests
+        if: always()
+        run: |
+          cd packages/query-splitter
+          yarn test:ci
+      - uses: tanmen/jest-reporter@v1
+        if: always()
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          result-file: integration-tests/result.json
+          action-name: Integration tests
+      - uses: tanmen/jest-reporter@v1
+        if: always()
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          result-file: packages/filterparser/result.json
+          action-name: Filter parser test results
+      - uses: tanmen/jest-reporter@v1
+        if: always()
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          result-file: packages/query-splitter/result.json
+          action-name: Query splitter test results
+
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: Pwd2020Db
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+      mysql:
+        image: mysql:8.0.18
+        env:
+          MYSQL_ROOT_PASSWORD: Pwd2020Db
+
+      mssql:
+        image: mcr.microsoft.com/mssql/server
+        env:
+          ACCEPT_EULA: Y
+          SA_PASSWORD: Pwd2020Db
+          MSSQL_PID: Express
+          
+      # cockroachdb:
+      #   image: cockroachdb/cockroach

.gitignore

@@ -30,4 +30,5 @@ yarn-debug.log*
 yarn-error.log*
 app/src/nativeModulesContent.js
 packages/api/src/nativeModulesContent.js
+packages/api/src/packagedPluginsContent.js
 .VSCodeCounter

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "jestrunner.jestCommand": "node_modules/.bin/cross-env DEVMODE=1 LOCALTEST=1 node_modules/.bin/jest"
+}

CHANGELOG.md

@@ -1,5 +1,41 @@
 # ChangeLog
 
+### 4.2.4
+- ADDED: Query history
+- ADDED: One-click exports in desktop app
+- ADDED: JSON array export
+- FIXED: Procedures in PostgreSQL #122
+- ADDED: Support of materialized views for PostgreSQL #123
+- ADDED: Integration tests
+- FIXED: Fixes in DB structure analysis in PostgreSQL, SQLite, MySQL
+- FIXED: Save data in SQLite, PostgreSQL
+- CHANGED: Introduced package dbgate-query-splitter, instead of sql-query-identifier and @verycrazydog/mysql-parse
+
+### 4.2.3
+- ADDED: ARM builds for MacOS and Linux
+- ADDED: Filter by columns in form view
+
+### 4.2.2
+- CHANGED: Further startup optimalization (approx. 2 times quicker start of electron app)
+
+### 4.2.1
+- FIXED: Fixed+optimalized app startup (esp. on Windows)
+
+### 4.2.0
+- ADDED: Support of SQLite database
+- ADDED: Support of Amazon Redshift database
+- ADDED: Support of CockcroachDB
+- CHANGED: DB Model is not auto-refreshed by default, refresh could be invoked from statusbar
+- FIXED: Fixed race conditions on startup
+- FIXED: Fixed broken style in data grid under strange circumstances
+- ADDED: Configure connections with commandline arguments  #108
+- CHANGED: Optimalized algorithm of incremental DB model updates
+- CHANGED: Loading queries from PostgreSQL doesn't need cursors, using streamed query instead
+- ADDED: Disconnect command
+- ADDED: Query executed on server has tab marker (formerly it had only "No DB" marker)
+- ADDED: Horizontal scroll using shift+mouse wheel #113
+- ADDED: Cosmetic improvements of MariaDB support
+
 ### 4.1.11
 - FIX: Fixed crash of API process when using SSH tunnel connection (race condition)
 

README.md

@@ -17,28 +17,40 @@ Supported databases:
 * PostgreSQL
 * SQL Server
 * MongoDB
+* SQLite
+* Amazon Redshift
+* CockroachDB
+* MariaDB
 
 ![Screenshot](https://raw.githubusercontent.com/dbgate/dbgate/master/screenshot.png)
 
 ## Features
-* Connect to Microsoft SQL Server, Postgre SQL, MySQL, MongoDB
 * Table data editing, with SQL change script preview
+* Light and dark theme
 * Master/detail views
 * Query designer
 * Form view for comfortable work with tables with many columns
 * JSON view on MongoDB collections
 * Explore tables, views, procedures, functions, MongoDB collections
-* SQL editor, execute SQL script, SQL code formatter, SQL code completion, SQL join wizard
+* SQL editor
+  * execute SQL script
+  * SQL code formatter
+  * SQL code completion
+  * Add SQL LEFT/INNER/RIGHT join utility
 * Mongo JavaScript editor, execute Mongo script (with NodeJs syntax)
 * Runs as application for Windows, Linux and Mac. Or in Docker container on server and in web Browser on client.
 * Import, export from/to CSV, Excel, JSON
 * Free table editor - quick table data editing (cleanup data after import/before export, prototype tables etc.)
 * Archives - backup your data in JSON files on local filesystem (or on DbGate server, when using web application)
-* Light and dark theme
 * Charts
 * For detailed info, how to run DbGate in docker container, visit [docker hub](https://hub.docker.com/r/dbgate/dbgate)
 * Extensible plugin architecture
 
+## How to contribute
+Any contributions are welcome. If you want to contribute without coding, consider following:
+* Create issue, if you find problem in app, or you have idea to new feature. If issue already exists, you could leave comment on it, to prioritise most wanted issues.
+* Tell your friends about DbGate or share on social networks - when more people will use DbGate, it will grow to be better
+
 ## Why is DbGate different
 There are many database managers now, so why DbGate? 
 * Works everywhere - Windows, Linux, Mac, Web browser (+mobile web is planned), without compromises in features
@@ -52,7 +64,6 @@ There are many database managers now, so why DbGate?
     * Backend - NodeJs, ExpressJs, socket.io, database connection drivers
     * JavaScript + TypeScript
     * App - electron
-    * There is plan to incorporate SQLite to support work with local datasets
 * Platform independed - will run as web application in single docker container on server, or as application using Electron platform on Linux, Windows and Mac
 
 ## Plugins
@@ -101,15 +112,3 @@ yarn build:app:local
 yarn start:app:local
 ```
 
-## Packages
-Some dbgate packages can be used also without DbGate. You can find them on [NPM repository](https://www.npmjs.com/search?q=keywords:dbgate)
-
-* [api](https://github.com/dbgate/dbgate/tree/master/packages/api) - backend, Javascript, ExpressJS [![NPM version](https://img.shields.io/npm/v/dbgate-api.svg)](https://www.npmjs.com/package/dbgate-api)
-* [datalib](https://github.com/dbgate/dbgate/tree/master/packages/datalib) - TypeScript library for utility classes [![NPM version](https://img.shields.io/npm/v/dbgate-datalib.svg)](https://www.npmjs.com/package/dbgate-datalib)
-* [app](https://github.com/dbgate/dbgate/tree/master/app) - application (JavaScript) structure, creating specific queries (JavaScript)
-* [filterparser](https://github.com/dbgate/dbgate/tree/master/packages/filterparser) - TypeScript library for parsing data filter expressions using parsimmon [![NPM version](https://img.shields.io/npm/v/dbgate-filterparser.svg)](https://www.npmjs.com/package/dbgate-filterparser)
-* [sqltree](https://github.com/dbgate/dbgate/tree/master/packages/sqltree) - JSON representation of SQL query, functions converting to SQL (TypeScript) [![NPM version](https://img.shields.io/npm/v/dbgate-sqltree.svg)](https://www.npmjs.com/package/dbgate-sqltree)
-* [types](https://github.com/dbgate/dbgate/tree/master/packages/types) - common TypeScript definitions [![NPM version](https://img.shields.io/npm/v/dbgate-types.svg)](https://www.npmjs.com/package/dbgate-types)
-* [web](https://github.com/dbgate/dbgate/tree/master/packages/web) - frontend in Svelte (JavaScript) [![NPM version](https://img.shields.io/npm/v/dbgate-web.svg)](https://www.npmjs.com/package/dbgate-web)
-* [tools](https://github.com/dbgate/dbgate/tree/master/packages/tools) - various tools [![NPM version](https://img.shields.io/npm/v/dbgate-tools.svg)](https://www.npmjs.com/package/dbgate-tools)
-

app/package.json

@@ -8,42 +8,54 @@
     "better-sqlite3-with-prebuilds": "^7.1.8",
     "electron-log": "^4.3.1",
     "electron-store": "^5.1.1",
-    "electron-updater": "^4.3.5"
+    "electron-updater": "^4.3.5",
+    "patch-package": "^6.4.7"
   },
   "repository": {
     "type": "git",
     "url": "https://github.com/dbgate/dbgate.git"
   },
   "build": {
+    "artifactName": "${productName}-${version}-${os}_${arch}.${ext}",
     "appId": "org.dbgate",
     "mac": {
       "category": "database",
       "icon": "icon512.png",
-      "artifactName": "dbgate-mac-${version}.${ext}",
       "publish": [
         "github"
       ],
       "target": {
         "target": "default",
         "arch": [
+          "arm64",
           "x64"
         ]
       }
     },
     "linux": {
       "target": [
-        "AppImage",
         "deb",
-        "snap"
+        "snap",
+        {
+          "target": "AppImage",
+          "arch": [
+            "x64",
+            "armv7l",
+            "arm64"
+          ]
+        }
       ],
       "icon": "icon.png",
-      "artifactName": "dbgate-linux-${version}.${ext}",
       "category": "Development",
-      "synopsis": "Database administration tool for MS SQL, MySQL and PostgreSQL",
+      "synopsis": "Database manager for SQL Server, MySQL, PostgreSQL, MongoDB and SQLite",
       "publish": [
         "github"
       ]
     },
+    "appImage": {
+      "license": "./LICENSE",
+      "category": "Development"
+    },
     "snap": {
       "publish": [
         "github",
@@ -58,7 +70,6 @@
         "nsis",
         "zip"
       ],
-      "artifactName": "dbgate-windows-${version}.${ext}",
       "icon": "icon.ico",
       "publish": [
         "github"
@@ -77,8 +88,9 @@
     "start:local": "cross-env electron .",
     "dist": "electron-builder",
     "build": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn dist",
+    "build:mac": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && node setMacPlatform x64 && yarn dist && node setMacPlatform arm64 && yarn dist",
     "build:local": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn predist",
-    "postinstall": "electron-builder install-app-deps",
+    "postinstall": "electron-builder install-app-deps && patch-package",
     "predist": "copyfiles ../packages/api/dist/* packages && copyfiles \"../packages/web/public/*\" packages && copyfiles \"../packages/web/public/**/*\" packages && copyfiles --up 3 \"../plugins/dist/**/*\" packages/plugins"
   },
   "main": "src/electron.js",
@@ -86,9 +98,9 @@
     "copyfiles": "^2.2.0",
     "cross-env": "^6.0.3",
     "electron": "11.2.3",
-    "electron-builder": "22.9.1"
+    "electron-builder": "22.10.5"
   },
   "optionalDependencies": {
     "msnodesqlv8": "^2.0.10"
   }
-}
+}

app/setMacPlatform.js

@@ -0,0 +1,8 @@
+const fs = require('fs');
+
+const text = fs.readFileSync('package.json', { encoding: 'utf-8' });
+const json = JSON.parse(text);
+
+json.build.mac.target.arch = process.argv[2];
+
+fs.writeFileSync('package.json', JSON.stringify(json, null, 2), { encoding: 'utf-8' });

app/src/electron.js

@@ -19,7 +19,6 @@ const store = new Store();
 // Keep a global reference of the window object, if you don't, the window will
 // be closed automatically when the JavaScript object is garbage collected.
 let mainWindow;
-let splashWindow;
 let mainMenu;
 
 log.transports.file.level = 'debug';
@@ -29,14 +28,6 @@ autoUpdater.logger = log;
 
 let commands = {};
 
-function hideSplash() {
-  if (splashWindow) {
-    splashWindow.destroy();
-    splashWindow = null;
-  }
-  mainWindow.show();
-}
-
 function commandItem(id) {
   const command = commands[id];
   return {
@@ -156,7 +147,6 @@ function createWindow() {
     title: 'DbGate',
     ...bounds,
     icon: os.platform() == 'win32' ? 'icon.ico' : path.resolve(__dirname, '../icon.png'),
-    show: false,
     webPreferences: {
       nodeIntegration: true,
       enableRemoteModule: true,
@@ -175,7 +165,7 @@ function createWindow() {
         slashes: true,
       });
     mainWindow.webContents.on('did-finish-load', function () {
-      hideSplash();
+      // hideSplash();
     });
     mainWindow.on('close', () => {
       store.set('winBounds', mainWindow.getBounds());
@@ -186,20 +176,6 @@ function createWindow() {
     }
   }
 
-  splashWindow = new BrowserWindow({
-    width: 300,
-    height: 120,
-    transparent: true,
-    frame: false,
-  });
-  splashWindow.loadURL(
-    url.format({
-      pathname: path.join(__dirname, '../packages/web/build/splash.html'),
-      protocol: 'file:',
-      slashes: true,
-    })
-  );
-
   if (process.env.ELECTRON_START_URL) {
     loadMainWindow();
   } else {

fillPackagedPlugins.js

@@ -0,0 +1,23 @@
+const fs = require('fs');
+const path = require('path');
+
+function load() {
+  const plugins = {};
+
+  for (const packageName of fs.readdirSync('plugins')) {
+    if (!packageName.startsWith('dbgate-plugin-')) continue;
+    const dir = path.join('plugins', packageName);
+    const frontend = fs.readFileSync(path.join(dir, 'dist', 'frontend.js'), 'utf-8');
+    const readme = fs.readFileSync(path.join(dir, 'README.md'), 'utf-8');
+    const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'package.json'), 'utf-8'));
+    plugins[packageName] = {
+      manifest,
+      frontend,
+      readme,
+    };
+  }
+
+  return plugins;
+}
+
+fs.writeFileSync('packages/api/src/packagedPluginsContent.js', `module.exports = () => (${JSON.stringify(load())});`);

integration-tests/.gitignore

@@ -0,0 +1 @@
+dbtemp

integration-tests/__tests__/object-analyse.spec.js

@@ -0,0 +1,89 @@
+const { testWrapper } = require('../tools');
+const engines = require('../engines');
+const _ = require('lodash');
+
+const initSql = ['CREATE TABLE t1 (id int)', 'CREATE TABLE t2 (id int)'];
+
+function flatSource() {
+  return _.flatten(
+    engines.map(engine => (engine.objects || []).map(object => [engine.label, object.type, object, engine]))
+  );
+}
+
+const obj1Match = expect.objectContaining({
+  pureName: 'obj1',
+});
+const view1Match = expect.objectContaining({
+  pureName: 'obj1',
+  columns: expect.arrayContaining([
+    expect.objectContaining({
+      columnName: 'id',
+    }),
+  ]),
+});
+
+describe('Object analyse', () => {
+  test.each(flatSource())(
+    'Full analysis - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create1);
+      const structure = await driver.analyseFull(conn);
+
+      expect(structure[type].length).toEqual(1);
+      expect(structure[type][0]).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+
+  test.each(flatSource())(
+    'Incremental analysis - add - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create2);
+      const structure1 = await driver.analyseFull(conn);
+      await driver.query(conn, object.create1);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2[type].length).toEqual(2);
+      expect(structure2[type].find(x => x.pureName == 'obj1')).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+
+  test.each(flatSource())(
+    'Incremental analysis - drop - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create1);
+      await driver.query(conn, object.create2);
+      const structure1 = await driver.analyseFull(conn);
+      await driver.query(conn, object.drop2);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2[type].length).toEqual(1);
+      expect(structure2[type][0]).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+
+  test.each(flatSource())(
+    'Create SQL - add - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create1);
+      const structure1 = await driver.analyseFull(conn);
+      await driver.query(conn, object.drop1);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+      expect(structure2[type].length).toEqual(0);
+
+      await driver.query(conn, structure1[type][0].createSql);
+
+      const structure3 = await driver.analyseIncremental(conn, structure2);
+
+      expect(structure3[type].length).toEqual(1);
+      expect(structure3[type][0]).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+});

integration-tests/__tests__/query.spec.js

@@ -0,0 +1,163 @@
+const engines = require('../engines');
+const { splitQuery } = require('dbgate-query-splitter');
+const { testWrapper } = require('../tools');
+
+const initSql = ['CREATE TABLE t1 (id int)', 'INSERT INTO t1 (id) VALUES (1)', 'INSERT INTO t1 (id) VALUES (2)'];
+
+expect.extend({
+  dataRow(row, expected) {
+    for (const key in expected) {
+      if (row[key] != expected[key]) {
+        return {
+          pass: false,
+          message: () => `Different key: ${key}`,
+        };
+      }
+    }
+    return {
+      pass: true,
+      message: () => '',
+    };
+  },
+});
+
+class StreamHandler {
+  constructor(resolve) {
+    this.results = [];
+    this.resolve = resolve;
+    this.infoRows = [];
+  }
+  row(row) {
+    this.results[this.results.length - 1].rows.push(row);
+  }
+  recordset(columns) {
+    this.results.push({
+      columns,
+      rows: [],
+    });
+  }
+  done(result) {
+    this.resolve(this.results);
+  }
+  info(msg) {
+    this.infoRows.push(msg);
+  }
+}
+
+function executeStreamItem(driver, conn, sql) {
+  return new Promise(resolve => {
+    const handler = new StreamHandler(resolve);
+    driver.stream(conn, sql, handler);
+  });
+}
+
+async function executeStream(driver, conn, sql) {
+  const results = [];
+  for (const sqlItem of splitQuery(sql, driver.getQuerySplitterOptions('stream'))) {
+    const item = await executeStreamItem(driver, conn, sqlItem);
+    results.push(...item);
+  }
+  return results;
+}
+
+describe('Query', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Simple query - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      const res = await driver.query(conn, 'SELECT id FROM t1 ORDER BY id');
+      expect(res.columns).toEqual([
+        expect.objectContaining({
+          columnName: 'id',
+        }),
+      ]);
+
+      expect(res.rows).toEqual([
+        expect.dataRow({
+          id: 1,
+        }),
+        expect.dataRow({
+          id: 2,
+        }),
+      ]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Simple stream query - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+      const results = await executeStream(driver, conn, 'SELECT id FROM t1 ORDER BY id');
+      expect(results.length).toEqual(1);
+      const res = results[0];
+
+      expect(res.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res.rows).toEqual([expect.dataRow({ id: 1 }), expect.dataRow({ id: 2 })]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'More queries - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+      const results = await executeStream(
+        driver,
+        conn,
+        'SELECT id FROM t1 ORDER BY id; SELECT id FROM t1 ORDER BY id DESC'
+      );
+      expect(results.length).toEqual(2);
+
+      const res1 = results[0];
+      expect(res1.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res1.rows).toEqual([expect.dataRow({ id: 1 }), expect.dataRow({ id: 2 })]);
+
+      const res2 = results[1];
+      expect(res2.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res2.rows).toEqual([expect.dataRow({ id: 2 }), expect.dataRow({ id: 1 })]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Script - return data - %s',
+    testWrapper(async (conn, driver, engine) => {
+      const results = await executeStream(
+        driver,
+        conn,
+        'CREATE TABLE t1 (id int); INSERT INTO t1 (id) VALUES (1); INSERT INTO t1 (id) VALUES (2); SELECT id FROM t1 ORDER BY id; '
+      );
+      expect(results.length).toEqual(1);
+
+      const res1 = results[0];
+      expect(res1.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res1.rows).toEqual([expect.dataRow({ id: 1 }), expect.dataRow({ id: 2 })]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Script - no data - %s',
+    testWrapper(async (conn, driver, engine) => {
+      const results = await executeStream(
+        driver,
+        conn,
+        'CREATE TABLE t1 (id int); INSERT INTO t1 (id) VALUES (1); INSERT INTO t1 (id) VALUES (2) '
+      );
+      expect(results.length).toEqual(0);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Save data query - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.script(
+        conn,
+        'INSERT INTO t1 (id) VALUES (3);INSERT INTO t1 (id) VALUES (4);UPDATE t1 SET id=10 WHERE id=1;DELETE FROM t1 WHERE id=2;'
+      );
+      const res = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM t1');
+      // console.log(res);
+      expect(res.rows[0].cnt == 3).toBeTruthy();
+    })
+  );
+});

integration-tests/__tests__/table-analyse.spec.js

@@ -0,0 +1,112 @@
+const engines = require('../engines');
+const { testWrapper } = require('../tools');
+
+const t1Sql = 'CREATE TABLE t1 (id int not null primary key, val1 varchar(50) null)';
+const t2Sql = 'CREATE TABLE t2 (id int not null primary key, val2 varchar(50) null)';
+
+const txMatch = (tname, vcolname, nextcol) =>
+  expect.objectContaining({
+    pureName: tname,
+    columns: [
+      expect.objectContaining({
+        columnName: 'id',
+        notNull: true,
+        dataType: expect.stringContaining('int'),
+      }),
+      expect.objectContaining({
+        columnName: vcolname,
+        notNull: false,
+        dataType: expect.stringMatching(/.*char.*\(50\)/),
+      }),
+      ...(nextcol
+        ? [
+            expect.objectContaining({
+              columnName: 'nextcol',
+              notNull: false,
+              dataType: expect.stringMatching(/.*char.*\(50\)/),
+            }),
+          ]
+        : []),
+    ],
+    primaryKey: expect.objectContaining({
+      columns: [
+        expect.objectContaining({
+          columnName: 'id',
+        }),
+      ],
+    }),
+  });
+
+const t1Match = txMatch('t1', 'val1');
+const t2Match = txMatch('t2', 'val2');
+const t2NextColMatch = txMatch('t2', 'val2', true);
+
+describe('Table analyse', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table structure - full analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+
+      const structure = await driver.analyseFull(conn);
+
+      expect(structure.tables.length).toEqual(1);
+      expect(structure.tables[0]).toEqual(t1Match);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table add - incremental analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t2Sql);
+
+      const structure1 = await driver.analyseFull(conn);
+      expect(structure1.tables.length).toEqual(1);
+      expect(structure1.tables[0]).toEqual(t2Match);
+
+      await driver.query(conn, t1Sql);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2.tables.length).toEqual(2);
+      expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
+      expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2Match);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table remove - incremental analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+      await driver.query(conn, t2Sql);
+      const structure1 = await driver.analyseFull(conn);
+      expect(structure1.tables.length).toEqual(2);
+      expect(structure1.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
+      expect(structure1.tables.find(x => x.pureName == 't2')).toEqual(t2Match);
+
+      await driver.query(conn, 'DROP TABLE t2');
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2.tables.length).toEqual(1);
+      expect(structure2.tables[0]).toEqual(t1Match);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table change - incremental analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+      await driver.query(conn, t2Sql);
+      const structure1 = await driver.analyseFull(conn);
+
+      if (engine.dbSnapshotBySeconds) await new Promise(resolve => setTimeout(resolve, 1100));
+      
+      await driver.query(conn, 'ALTER TABLE t2 ADD nextcol varchar(50)');
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2).toBeTruthy(); // if falsy, no modification is detected
+
+      expect(structure2.tables.length).toEqual(2);
+      expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
+      expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2NextColMatch);
+    })
+  );
+});

integration-tests/docker-compose.yaml

@@ -0,0 +1,55 @@
+version: '3'
+services:
+  postgres:
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_PASSWORD: Pwd2020Db
+    ports: 
+      - 15000:5432
+
+  mysql:
+    image: mysql:8.0.18
+    command: --default-authentication-plugin=mysql_native_password
+    restart: always
+    ports: 
+      - 15001:3306
+    environment:
+      - MYSQL_ROOT_PASSWORD=Pwd2020Db
+
+  mssql:
+    image: mcr.microsoft.com/mssql/server
+    restart: always
+    ports: 
+      - 15002:1433
+    environment:
+      - ACCEPT_EULA=Y
+      - SA_PASSWORD=Pwd2020Db
+      - MSSQL_PID=Express
+      
+  cockroachdb:
+    image: cockroachdb/cockroach
+    ports:
+      - 15003:26257
+    command: start-single-node --insecure
+
+#   mongodb:
+#     image: mongo:4.0.12
+#     restart: always
+#     volumes:
+#       - mongo-data:/data/db
+#       - mongo-config:/data/configdb
+#     ports:
+#       - 27017:27017
+
+
+#   cockroachdb-init:
+#     image: cockroachdb/cockroach
+#     # build: cockroach
+#     # entrypoint: /cockroach/init.sh 
+#     entrypoint: ./cockroach sql --insecure --host="cockroachdb" --execute="CREATE DATABASE IF NOT EXISTS test;"
+
+#     depends_on:
+#       - cockroachdb
+#     restart: on-failure
+

integration-tests/engines.js

@@ -0,0 +1,108 @@
+const views = {
+  type: 'views',
+  create1: 'CREATE VIEW obj1 AS SELECT id FROM t1',
+  create2: 'CREATE VIEW obj2 AS SELECT id FROM t2',
+  drop1: 'DROP VIEW obj1',
+  drop2: 'DROP VIEW obj2',
+};
+const matviews = {
+  type: 'matviews',
+  create1: 'CREATE MATERIALIZED VIEW obj1 AS SELECT id FROM t1',
+  create2: 'CREATE MATERIALIZED VIEW obj2 AS SELECT id FROM t2',
+  drop1: 'DROP MATERIALIZED VIEW obj1',
+  drop2: 'DROP MATERIALIZED VIEW obj2',
+};
+
+const engines = [
+  {
+    label: 'MySQL',
+    connection: {
+      engine: 'mysql@dbgate-plugin-mysql',
+      password: 'Pwd2020Db',
+      user: 'root',
+      server: 'mysql',
+      port: 3306,
+    },
+    local: {
+      server: 'localhost',
+      port: 15001,
+    },
+    // skipOnCI: true,
+    objects: [views],
+    dbSnapshotBySeconds: true,
+  },
+  {
+    label: 'PostgreSQL',
+    connection: {
+      engine: 'postgres@dbgate-plugin-postgres',
+      password: 'Pwd2020Db',
+      user: 'postgres',
+      server: 'postgres',
+      port: 5432,
+    },
+    local: {
+      server: 'localhost',
+      port: 15000,
+    },
+    objects: [
+      views,
+      matviews,
+      {
+        type: 'procedures',
+        create1: 'CREATE PROCEDURE obj1() LANGUAGE SQL AS $$  select * from t1 $$',
+        create2: 'CREATE PROCEDURE obj2() LANGUAGE SQL AS $$  select * from t2 $$',
+        drop1: 'DROP PROCEDURE obj1',
+        drop2: 'DROP PROCEDURE obj2',
+      },
+    ],
+  },
+  {
+    label: 'SQL Server',
+    connection: {
+      engine: 'mssql@dbgate-plugin-mssql',
+      password: 'Pwd2020Db',
+      user: 'sa',
+      server: 'mssql',
+      port: 1433,
+    },
+    local: {
+      server: 'localhost',
+      port: 15002,
+    },
+    objects: [
+      views,
+      {
+        type: 'procedures',
+        create1: 'CREATE PROCEDURE obj1 AS SELECT id FROM t1',
+        create2: 'CREATE PROCEDURE obj2 AS SELECT id FROM t2',
+        drop1: 'DROP PROCEDURE obj1',
+        drop2: 'DROP PROCEDURE obj2',
+      },
+    ],
+  },
+  {
+    label: 'SQLite',
+    generateDbFile: true,
+    connection: {
+      engine: 'sqlite@dbgate-plugin-sqlite',
+    },
+    objects: [views],
+  },
+  {
+    label: 'CockroachDB',
+    connection: {
+      engine: 'cockroach@dbgate-plugin-postgres',
+      user: 'root',
+      server: 'cockroachdb',
+      port: 26257,
+    },
+    local: {
+      server: 'localhost',
+      port: 15003,
+    },
+    skipOnCI: true,
+    objects: [views, matviews],
+  },
+];
+
+module.exports = process.env.CITEST ? engines.filter(x => !x.skipOnCI) : engines;

integration-tests/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "dbgate-integration-tests",
+  "version": "4.1.1",
+  "homepage": "https://dbgate.org/",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dbgate/dbgate.git"
+  },
+  "author": "Jan Prochazka",
+  "license": "MIT",
+  "scripts": {
+    "wait:local": "cross-env DEVMODE=1 LOCALTEST=1 node wait.js",
+    "wait:ci": "cross-env DEVMODE=1 CITEST=1 node wait.js",
+
+    "test:local": "cross-env DEVMODE=1 LOCALTEST=1 jest",
+    "test:ci": "cross-env DEVMODE=1 CITEST=1 jest --runInBand --json --outputFile=result.json --testLocationInResults",
+    
+    "run:local": "docker-compose down && docker-compose up -d && yarn wait:local && yarn test:local"
+  },
+  "devDependencies": {
+    "cross-env": "^7.0.3",
+    "jest": "^27.0.1"
+  },
+  "dependencies": {}
+}

integration-tests/tools.js

@@ -0,0 +1,63 @@
+global.DBGATE_TOOLS = require('dbgate-tools');
+const requireEngineDriver = require('dbgate-api/src/utility/requireEngineDriver');
+const crypto = require('crypto');
+
+function randomDbName() {
+  const generatedKey = crypto.randomBytes(6);
+  const newKey = generatedKey.toString('hex');
+  return `db${newKey}`;
+}
+
+function extractConnection(engine) {
+  const { connection } = engine;
+
+  if (process.env.LOCALTEST && engine.local) {
+    return {
+      ...connection,
+      ...engine.local,
+    };
+  }
+
+  return connection;
+}
+
+async function connect(engine, database) {
+  const connection = extractConnection(engine);
+  const driver = requireEngineDriver(connection);
+
+  if (engine.generateDbFile) {
+    const conn = await driver.connect({
+      ...connection,
+      databaseFile: `dbtemp/${database}`,
+    });
+    return conn;
+  } else {
+    const conn = await driver.connect(connection);
+    await driver.query(conn, `CREATE DATABASE ${database}`);
+    await driver.close(conn);
+
+    const res = await driver.connect({
+      ...connection,
+      database,
+    });
+    return res;
+  }
+}
+
+const testWrapper = body => async (label, ...other) => {
+  const engine = other[other.length - 1];
+  const driver = requireEngineDriver(engine.connection);
+  const conn = await connect(engine, randomDbName());
+  try {
+    await body(conn, driver, ...other);
+  } finally {
+    await driver.close(conn);
+  }
+};
+
+module.exports = {
+  randomDbName,
+  connect,
+  extractConnection,
+  testWrapper,
+};

integration-tests/wait.js

@@ -0,0 +1,29 @@
+const requireEngineDriver = require('dbgate-api/src/utility/requireEngineDriver');
+const engines = require('./engines');
+const { extractConnection } = require('./tools');
+global.DBGATE_TOOLS = require('dbgate-tools');
+
+async function connectEngine(engine) {
+  const connection = extractConnection(engine);
+  const driver = requireEngineDriver(connection);
+  for (;;) {
+    try {
+      const conn = await driver.connect(connection);
+      await driver.getVersion(conn);
+      console.log(`Connect to ${engine.label} - OK`);
+      await driver.close(conn);
+      return;
+    } catch (err) {
+      console.log(`Waiting for ${engine.label}, error: ${err.message}`);
+      await new Promise(resolve => setTimeout(resolve, 2500));
+      continue;
+    }
+  }
+}
+
+async function run() {
+  await new Promise(resolve => setTimeout(resolve, 10000));
+  await Promise.all(engines.map(engine => connectEngine(engine)));
+}
+
+run();

package.json

@@ -1,10 +1,11 @@
 {
   "private": true,
-  "version": "4.2.0-beta.3",
+  "version": "4.2.5-beta.2",
   "name": "dbgate-all",
   "workspaces": [
     "packages/*",
-    "plugins/*"
+    "plugins/*",
+    "integration-tests"
   ],
   "scripts": {
     "start:api": "yarn workspace dbgate-api start",
@@ -15,12 +16,15 @@
     "start:tools": "yarn workspace dbgate-tools start",
     "start:datalib": "yarn workspace dbgate-datalib start",
     "start:filterparser": "yarn workspace dbgate-filterparser start",
+    "start:querysplitter": "yarn workspace dbgate-query-splitter start",
     "build:sqltree": "yarn workspace dbgate-sqltree build",
     "build:datalib": "yarn workspace dbgate-datalib build",
     "build:filterparser": "yarn workspace dbgate-filterparser build",
+    "build:querysplitter": "yarn workspace dbgate-query-splitter build",
     "build:tools": "yarn workspace dbgate-tools build",
-    "build:lib": "yarn build:tools && yarn build:sqltree && yarn build:filterparser && yarn build:datalib",
+    "build:lib": "yarn build:querysplitter && yarn build:tools && yarn build:sqltree && yarn build:filterparser && yarn build:datalib",
     "build:app": "yarn plugins:copydist && cd app && yarn install && yarn build",
+    "build:app:mac": "yarn plugins:copydist && cd app && yarn install && yarn build:mac",
     "build:api": "yarn workspace dbgate-api build",
     "build:web:docker": "yarn workspace dbgate-web build",
     "build:plugins:frontend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:frontend",
@@ -32,15 +36,17 @@
     "generatePadFile": "node generatePadFile",
     "fillNativeModules": "node fillNativeModules",
     "fillNativeModulesElectron": "node fillNativeModules --electron",
+    "fillPackagedPlugins": "node fillPackagedPlugins",
+    "resetPackagedPlugins": "node resetPackagedPlugins",
     "prettier": "prettier --write packages/api/src && prettier --write packages/datalib/src && prettier --write packages/filterparser/src && prettier --write packages/sqltree/src && prettier --write packages/tools/src && prettier --write packages/types && prettier --write packages/web/src && prettier --write app/src",
     "copy:docker:build": "copyfiles packages/api/dist/* docker -f && copyfiles packages/web/public/* docker -u 2 && copyfiles \"packages/web/public/**/*\" docker -u 2 && copyfiles \"plugins/dist/**/*\" docker/plugins -u 2",
     "prepare:docker": "yarn plugins:copydist && yarn build:web:docker && yarn build:api && yarn copy:docker:build",
     "start": "concurrently --kill-others-on-fail \"yarn start:api\" \"yarn start:web\"",
-    "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\" \"yarn build:plugins:frontend:watch\"",
+    "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\" \"yarn start:querysplitter\" \"yarn build:plugins:frontend:watch\"",
     "ts:api": "yarn workspace dbgate-api ts",
     "ts:web": "yarn workspace dbgate-web ts",
     "ts": "yarn ts:api && yarn ts:web",
-    "postinstall": "yarn build:lib && patch-package && yarn fillNativeModules && yarn build:plugins:frontend"
+    "postinstall": "yarn resetPackagedPlugins && yarn build:lib && patch-package && yarn fillNativeModules && yarn build:plugins:frontend"
   },
   "dependencies": {
     "concurrently": "^5.1.0",

packages/api/.env-covid

@@ -1,14 +0,0 @@
-DEVMODE=1
-
-CONNECTIONS=mysql
-
-LABEL_mysql=MySql
-SERVER_mysql=dbgate.org
-USER_mysql=reader
-PASSWORD_mysql=CovidReader2020
-PORT_mysql=3326
-ENGINE_mysql=mysql@dbgate-plugin-mysql
-SINGLE_CONNECTION=mysql
-SINGLE_DATABASE=covid
-
-PERMISSIONS=files/charts/read

packages/api/.env-singledb

@@ -0,0 +1,15 @@
+DEVMODE=1
+
+CONNECTIONS=mysql
+
+LABEL_mysql=MySql localhost
+SERVER_mysql=localhost
+USER_mysql=root
+PASSWORD_mysql=test
+PORT_mysql=3307
+ENGINE_mysql=mysql@dbgate-plugin-mysql
+
+SINGLE_CONNECTION=mysql
+SINGLE_DATABASE=Chinook
+
+PERMISSIONS=files/charts/read

packages/api/package.json

@@ -18,7 +18,7 @@
   ],
   "dependencies": {
     "async-lock": "^1.2.4",
-    "axios": "^0.19.0",
+    "axios": "^0.21.1",
     "better-sqlite3-with-prebuilds": "^7.1.8",
     "body-parser": "^1.19.0",
     "bufferutil": "^4.0.1",
@@ -33,11 +33,12 @@
     "express-basic-auth": "^1.2.0",
     "express-fileupload": "^1.2.0",
     "fs-extra": "^9.1.0",
+    "fs-reverse": "^0.0.3",
     "get-port": "^5.1.1",
     "http": "^0.0.0",
     "json-stable-stringify": "^1.0.1",
     "line-reader": "^0.4.0",
-    "lodash": "^4.17.15",
+    "lodash": "^4.17.21",
     "ncp": "^2.0.0",
     "nedb-promises": "^4.0.1",
     "node-cron": "^2.0.3",
@@ -51,7 +52,9 @@
   "scripts": {
     "start": "env-cmd node src/index.js",
     "start:portal": "env-cmd -f .env-portal node src/index.js",
-    "start:covid": "env-cmd -f .env-covid node src/index.js",
+    "start:singledb": "env-cmd -f .env-singledb node src/index.js",
+    "start:filedb": "env-cmd node src/index.js /home/jena/test/chinook/Chinook.db",
+    "start:singleconn": "env-cmd node src/index.js --server localhost --user root --port 3307 --engine mysql@dbgate-plugin-mysql --password test",
     "ts": "tsc",
     "build": "webpack"
   },

packages/api/src/controllers/config.js

@@ -7,6 +7,7 @@ const _ = require('lodash');
 
 const currentVersion = require('../currentVersion');
 const platformInfo = require('../utility/platformInfo');
+const connections = require('../controllers/connections');
 
 module.exports = {
   settingsValue: {},
@@ -21,30 +22,11 @@ module.exports = {
 
   get_meta: 'get',
   async get() {
-    // const toolbarButtons = process.env.TOOLBAR;
-    // const toolbar = toolbarButtons
-    //   ? toolbarButtons.split(',').map((name) => ({
-    //       name,
-    //       icon: process.env[`ICON_${name}`],
-    //       title: process.env[`TITLE_${name}`],
-    //       page: process.env[`PAGE_${name}`],
-    //     }))
-    //   : null;
-    // const startupPages = process.env.STARTUP_PAGES ? process.env.STARTUP_PAGES.split(',') : [];
     const permissions = process.env.PERMISSIONS ? process.env.PERMISSIONS.split(',') : null;
-    const singleDatabase =
-      process.env.SINGLE_CONNECTION && process.env.SINGLE_DATABASE
-        ? {
-            conid: process.env.SINGLE_CONNECTION,
-            database: process.env.SINGLE_DATABASE,
-          }
-        : null;
 
     return {
-      runAsPortal: !!process.env.CONNECTIONS,
-      // toolbar,
-      // startupPages,
-      singleDatabase,
+      runAsPortal: !!connections.portalConnections,
+      singleDatabase: connections.singleDatabase,
       permissions,
       ...currentVersion,
     };

packages/api/src/controllers/connections.js

@@ -8,6 +8,32 @@ const socket = require('../utility/socket');
 const { encryptConnection } = require('../utility/crypting');
 const { handleProcessCommunication } = require('../utility/processComm');
 
+function getNamedArgs() {
+  const res = {};
+  for (let i = 0; i < process.argv.length; i++) {
+    const name = process.argv[i];
+    if (name.startsWith('--')) {
+      let value = process.argv[i + 1];
+      if (value && value.startsWith('--')) value = null;
+      res[name.substring(2)] = value == null ? true : value;
+      i++;
+    } else {
+      if (name.endsWith('.db') || name.endsWith('.sqlite') || name.endsWith('.sqlite3')) {
+        res.databaseFile = name;
+        res.engine = 'sqlite@dbgate-plugin-sqlite';
+      }
+    }
+  }
+  return res;
+}
+
+function getDatabaseFileLabel(databaseFile) {
+  if (!databaseFile) return databaseFile;
+  const m = databaseFile.match(/[\/]([^\/]+)$/);
+  if (m) return m[1];
+  return databaseFile;
+}
+
 function getPortalCollections() {
   if (process.env.CONNECTIONS) {
     return _.compact(process.env.CONNECTIONS.split(',')).map(id => ({
@@ -17,16 +43,79 @@ function getPortalCollections() {
       user: process.env[`USER_${id}`],
       password: process.env[`PASSWORD_${id}`],
       port: process.env[`PORT_${id}`],
+      databaseUrl: process.env[`URL_${id}`],
+      databaseFile: process.env[`FILE_${id}`],
+      defaultDatabase: process.env[`DATABASE_${id}`],
+      singleDatabase: !!process.env[`DATABASE_${id}`],
       displayName: process.env[`LABEL_${id}`],
     }));
   }
+
+  const args = getNamedArgs();
+  if (args.databaseFile) {
+    return [
+      {
+        _id: 'argv',
+        databaseFile: args.databaseFile,
+        singleDatabase: true,
+        defaultDatabase: getDatabaseFileLabel(args.databaseFile),
+        engine: args.engine,
+      },
+    ];
+  }
+  if (args.databaseUrl) {
+    return [
+      {
+        _id: 'argv',
+        useDatabaseUrl: true,
+        ...args,
+      },
+    ];
+  }
+  if (args.server) {
+    return [
+      {
+        _id: 'argv',
+        ...args,
+      },
+    ];
+  }
+
   return null;
 }
 const portalConnections = getPortalCollections();
 
+function getSingleDatabase() {
+  if (process.env.SINGLE_CONNECTION && process.env.SINGLE_DATABASE) {
+    // @ts-ignore
+    const connection = portalConnections.find(x => x._id == process.env.SINGLE_CONNECTION);
+    return {
+      connection,
+      name: process.env.SINGLE_DATABASE,
+    };
+  }
+  // @ts-ignore
+  const arg0 = (portalConnections || []).find(x => x._id == 'argv');
+  if (arg0) {
+    // @ts-ignore
+    if (arg0.singleDatabase) {
+      return {
+        connection: arg0,
+        // @ts-ignore
+        name: arg0.defaultDatabase,
+      };
+    }
+  }
+  return null;
+}
+
+const singleDatabase = getSingleDatabase();
+
 module.exports = {
   datastore: null,
   opened: [],
+  singleDatabase,
+  portalConnections,
 
   async _init() {
     const dir = datadir();

packages/api/src/controllers/databaseConnections.js

@@ -18,6 +18,12 @@ module.exports = {
     existing.structure = structure;
     socket.emitChanged(`database-structure-changed-${conid}-${database}`);
   },
+  handle_structureTime(conid, database, { analysedTime }) {
+    const existing = this.opened.find(x => x.conid == conid && x.database == database);
+    if (!existing) return;
+    existing.analysedTime = analysedTime;
+    socket.emitChanged(`database-status-changed-${conid}-${database}`);
+  },
   handle_version(conid, database, { version }) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (!existing) return;
@@ -106,6 +112,14 @@ module.exports = {
     return res;
   },
 
+  runScript_meta: 'post',
+  async runScript({ conid, database, sql }) {
+    console.log(`Processing script, conid=${conid}, database=${database}, sql=${sql}`);
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'runScript', sql });
+    return res;
+  },
+
   collectionData_meta: 'post',
   async collectionData({ conid, database, options }) {
     const opened = await this.ensureOpened(conid, database);
@@ -123,9 +137,19 @@ module.exports = {
   status_meta: 'get',
   async status({ conid, database }) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
-    if (existing) return existing.status;
+    if (existing) {
+      return {
+        ...existing.status,
+        analysedTime: existing.analysedTime,
+      };
+    }
     const lastClosed = this.closed[`${conid}/${database}`];
-    if (lastClosed) return lastClosed.status;
+    if (lastClosed) {
+      return {
+        ...lastClosed.status,
+        analysedTime: lastClosed.analysedTime,
+      };
+    }
     return {
       name: 'error',
       message: 'Not connected',
@@ -156,6 +180,13 @@ module.exports = {
     return { status: 'ok' };
   },
 
+  syncModel_meta: 'post',
+  async syncModel({ conid, database }) {
+    const conn = await this.ensureOpened(conid, database);
+    conn.subprocess.send({ msgtype: 'syncModel' });
+    return { status: 'ok' };
+  },
+
   close(conid, database, kill = true) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) {

packages/api/src/controllers/plugins.js

@@ -9,10 +9,17 @@ const requirePlugin = require('../shell/requirePlugin');
 const downloadPackage = require('../utility/downloadPackage');
 const hasPermission = require('../utility/hasPermission');
 const _ = require('lodash');
+const packagedPluginsContent = require('../packagedPluginsContent');
 
 module.exports = {
   script_meta: 'get',
   async script({ packageName }) {
+    const packagedContent = packagedPluginsContent();
+
+    if (packagedContent && packagedContent[packageName]) {
+      return packagedContent[packageName].frontend;
+    }
+
     const file1 = path.join(packagedPluginsDir(), packageName, 'dist', 'frontend.js');
     const file2 = path.join(pluginsdir(), packageName, 'dist', 'frontend.js');
     // @ts-ignore
@@ -58,26 +65,37 @@ module.exports = {
 
   installed_meta: 'get',
   async installed() {
-    const files1 = await fs.readdir(packagedPluginsDir());
+    const packagedContent = packagedPluginsContent();
+
+    const files1 = packagedContent ? _.keys(packagedContent) : await fs.readdir(packagedPluginsDir());
     const files2 = await fs.readdir(pluginsdir());
 
     const res = [];
     for (const packageName of _.union(files1, files2)) {
       if (!/^dbgate-plugin-.*$/.test(packageName)) continue;
       try {
-        const isPackaged = files1.includes(packageName);
-        const manifest = await fs
-          .readFile(path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'package.json'), {
-            encoding: 'utf-8',
-          })
-          .then(x => JSON.parse(x));
-        const readmeFile = path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'README.md');
-        // @ts-ignore
-        if (await fs.exists(readmeFile)) {
-          manifest.readme = await fs.readFile(readmeFile, { encoding: 'utf-8' });
+        if (packagedContent && packagedContent[packageName]) {
+          const manifest = {
+            ...packagedContent[packageName].manifest,
+          };
+          manifest.isPackaged = true;
+          manifest.readme = packagedContent[packageName].readme;
+          res.push(manifest);
+        } else {
+          const isPackaged = files1.includes(packageName);
+          const manifest = await fs
+            .readFile(path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'package.json'), {
+              encoding: 'utf-8',
+            })
+            .then(x => JSON.parse(x));
+          const readmeFile = path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'README.md');
+          // @ts-ignore
+          if (await fs.exists(readmeFile)) {
+            manifest.readme = await fs.readFile(readmeFile, { encoding: 'utf-8' });
+          }
+          manifest.isPackaged = isPackaged;
+          res.push(manifest);
         }
-        manifest.isPackaged = isPackaged;
-        res.push(manifest);
       } catch (err) {
         console.log(`Skipped plugin ${packageName}, error:`, err.message);
       }
@@ -135,8 +153,9 @@ module.exports = {
   async authTypes({ engine }) {
     const packageName = extractPackageName(engine);
     const content = requirePlugin(packageName);
-    if (!content.driver || content.driver.engine != engine || !content.driver.getAuthTypes) return null;
-    return content.driver.getAuthTypes() || null;
+    const driver = content.drivers.find(x => x.engine == engine);
+    if (!driver || !driver.getAuthTypes) return null;
+    return driver.getAuthTypes() || null;
   },
 
   // async _init() {

packages/api/src/controllers/queryHistory.js

@@ -0,0 +1,54 @@
+const fsReverse = require('fs-reverse');
+const fs = require('fs-extra');
+const path = require('path');
+const { datadir } = require('../utility/directories');
+const _ = require('lodash');
+const { filterName } = require('dbgate-tools');
+const socket = require('../utility/socket');
+
+function readCore(reader, skip, limit, filter) {
+  return new Promise((resolve, reject) => {
+    const res = [];
+    let readed = 0;
+    reader.on('data', line => {
+      if (!line && !line.trim()) return;
+      try {
+        const json = JSON.parse(line);
+        if (filterName(filter, json.sql, json.database)) {
+          if (!skip || readed >= skip) {
+            res.push(json);
+          }
+          readed++;
+          if (limit && readed > (skip || 0) + limit) {
+            reader.destroy();
+            resolve(res);
+          }
+        }
+      } catch (err) {
+        reader.destroy();
+        reject(err);
+      }
+    });
+    reader.on('end', () => resolve(res));
+  });
+}
+
+module.exports = {
+  read_meta: 'get',
+  async read({ skip, limit, filter }) {
+    const fileName = path.join(datadir(), 'query-history.jsonl');
+    // @ts-ignore
+    if (!(await fs.exists(fileName))) return [];
+    const reader = fsReverse(fileName);
+    const res = await readCore(reader, skip, limit, filter);
+    return res;
+  },
+
+  write_meta: 'post',
+  async write({ data }) {
+    const fileName = path.join(datadir(), 'query-history.jsonl');
+    await fs.appendFile(fileName, JSON.stringify(data) + '\n');
+    socket.emitChanged('query-history-changed');
+    return 'OK';
+  },
+};

packages/api/src/index.js

@@ -1,5 +1,8 @@
 const shell = require('./shell');
 const processArgs = require('./utility/processArgs');
+const dbgateTools = require('dbgate-tools');
+
+global['DBGATE_TOOLS'] = dbgateTools;
 
 if (processArgs.startProcess) {
   const proc = require('./proc');

packages/api/src/main.js

@@ -27,6 +27,7 @@ const uploads = require('./controllers/uploads');
 const plugins = require('./controllers/plugins');
 const files = require('./controllers/files');
 const scheduler = require('./controllers/scheduler');
+const queryHistory = require('./controllers/queryHistory');
 
 const { rundir } = require('./utility/directories');
 const platformInfo = require('./utility/platformInfo');
@@ -102,6 +103,7 @@ function start() {
   useController(app, '/plugins', plugins);
   useController(app, '/files', files);
   useController(app, '/scheduler', scheduler);
+  useController(app, '/query-history', queryHistory);
 
   // if (process.env.PAGES_DIRECTORY) {
   //   app.use('/pages', express.static(process.env.PAGES_DIRECTORY));

packages/api/src/proc/databaseConnectionProcess.js

@@ -1,4 +1,5 @@
 const stableStringify = require('json-stable-stringify');
+const { splitQuery } = require('dbgate-query-splitter');
 const childProcessChecker = require('../utility/childProcessChecker');
 const { extractBoolSettingsValue, extractIntSettingsValue } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
@@ -12,6 +13,7 @@ let afterConnectCallbacks = [];
 let analysedStructure = null;
 let lastPing = null;
 let lastStatus = null;
+let analysedTime = 0;
 
 async function checkedAsyncCall(promise) {
   try {
@@ -28,23 +30,42 @@ async function checkedAsyncCall(promise) {
   }
 }
 
+let loadingModel = false;
+
 async function handleFullRefresh() {
+  loadingModel = true;
   const driver = requireEngineDriver(storedConnection);
   setStatusName('loadStructure');
   analysedStructure = await checkedAsyncCall(driver.analyseFull(systemConnection));
+  analysedTime = new Date().getTime();
   process.send({ msgtype: 'structure', structure: analysedStructure });
+  process.send({ msgtype: 'structureTime', analysedTime });
   setStatusName('ok');
+  loadingModel = false;
 }
 
-async function handleIncrementalRefresh() {
+async function handleIncrementalRefresh(forceSend) {
+  loadingModel = true;
   const driver = requireEngineDriver(storedConnection);
   setStatusName('checkStructure');
   const newStructure = await checkedAsyncCall(driver.analyseIncremental(systemConnection, analysedStructure));
+  analysedTime = new Date().getTime();
   if (newStructure != null) {
     analysedStructure = newStructure;
+  }
+
+  if (forceSend || newStructure != null) {
     process.send({ msgtype: 'structure', structure: analysedStructure });
   }
+
+  process.send({ msgtype: 'structureTime', analysedTime });
   setStatusName('ok');
+  loadingModel = false;
+}
+
+function handleSyncModel() {
+  if (loadingModel) return;
+  handleIncrementalRefresh();
 }
 
 function setStatus(status) {
@@ -75,12 +96,12 @@ async function handleConnect({ connection, structure, globalSettings }) {
   readVersion();
   if (structure) {
     analysedStructure = structure;
-    handleIncrementalRefresh();
+    handleIncrementalRefresh(true);
   } else {
     handleFullRefresh();
   }
 
-  if (extractBoolSettingsValue(globalSettings, 'connection.autoRefresh', true)) {
+  if (extractBoolSettingsValue(globalSettings, 'connection.autoRefresh', false)) {
     setInterval(
       handleIncrementalRefresh,
       extractIntSettingsValue(globalSettings, 'connection.autoRefreshInterval', 30, 3, 3600) * 1000
@@ -100,6 +121,17 @@ function waitConnected() {
   });
 }
 
+async function handleRunScript({ msgid, sql }) {
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+  try {
+    await driver.script(systemConnection, sql);
+    process.send({ msgtype: 'response', msgid });
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+  }
+}
+
 async function handleQueryData({ msgid, sql }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
@@ -168,10 +200,12 @@ function handlePing() {
 const messageHandlers = {
   connect: handleConnect,
   queryData: handleQueryData,
+  runScript: handleRunScript,
   updateCollection: handleUpdateCollection,
   collectionData: handleCollectionData,
   sqlPreview: handleSqlPreview,
   ping: handlePing,
+  syncModel: handleSyncModel,
   // runCommand: handleRunCommand,
 };
 

packages/api/src/proc/serverConnectionProcess.js

@@ -61,7 +61,7 @@ async function handleConnect(connection) {
     systemConnection = await connectUtility(driver, storedConnection);
     readVersion();
     handleRefresh();
-    if (extractBoolSettingsValue(globalSettings, 'connection.autoRefresh', true)) {
+    if (extractBoolSettingsValue(globalSettings, 'connection.autoRefresh', false)) {
       setInterval(handleRefresh, extractIntSettingsValue(globalSettings, 'connection.autoRefreshInterval', 30, 5, 3600) * 1000);
     }
   } catch (err) {

packages/api/src/proc/sessionProcess.js

@@ -3,7 +3,7 @@ const path = require('path');
 const fs = require('fs');
 const _ = require('lodash');
 const childProcessChecker = require('../utility/childProcessChecker');
-const goSplit = require('../utility/goSplit');
+const { splitQuery } = require('dbgate-query-splitter');
 
 const { jsldir } = require('../utility/directories');
 const requireEngineDriver = require('../utility/requireEngineDriver');
@@ -166,7 +166,7 @@ async function handleExecuteQuery({ sql }) {
   const resultIndexHolder = {
     value: 0,
   };
-  for (const sqlItem of goSplit(sql)) {
+  for (const sqlItem of splitQuery(sql, driver.getQuerySplitterOptions('stream'))) {
     await handleStream(driver, resultIndexHolder, sqlItem);
     // const handler = new StreamHandler(resultIndex);
     // const stream = await driver.stream(systemConnection, sqlItem, handler);

packages/api/src/shell/executeQuery.js

@@ -1,6 +1,5 @@
-const goSplit = require('../utility/goSplit');
+const { splitQuery } = require('dbgate-query-splitter');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const { decryptConnection } = require('../utility/crypting');
 const connectUtility = require('../utility/connectUtility');
 
 async function executeQuery({ connection, sql }) {
@@ -10,9 +9,9 @@ async function executeQuery({ connection, sql }) {
   const pool = await connectUtility(driver, connection);
   console.log(`Connected.`);
 
-  for (const sqlItem of goSplit(sql)) {
+  for (const sqlItem of splitQuery(sql, driver.getQuerySplitterOptions('script'))) {
     console.log('Executing query', sqlItem);
-    await driver.query(pool, sqlItem);
+    await driver.query(pool, sqlItem, { discardResult: true });
   }
 
   console.log(`Query finished`);

packages/api/src/shell/index.js

@@ -6,6 +6,7 @@ const copyStream = require('./copyStream');
 const fakeObjectReader = require('./fakeObjectReader');
 const consoleObjectWriter = require('./consoleObjectWriter');
 const jsonLinesWriter = require('./jsonLinesWriter');
+const jsonArrayWriter = require('./jsonArrayWriter');
 const jsonLinesReader = require('./jsonLinesReader');
 const jslDataReader = require('./jslDataReader');
 const archiveWriter = require('./archiveWriter');
@@ -26,6 +27,7 @@ const dbgateApi = {
   tableReader,
   copyStream,
   jsonLinesWriter,
+  jsonArrayWriter,
   jsonLinesReader,
   fakeObjectReader,
   consoleObjectWriter,

packages/api/src/shell/jsonArrayWriter.js

@@ -0,0 +1,52 @@
+const fs = require('fs');
+const stream = require('stream');
+
+class StringifyStream extends stream.Transform {
+  constructor() {
+    super({ objectMode: true });
+    this.wasHeader = false;
+    this.wasRecord = false;
+  }
+  _transform(chunk, encoding, done) {
+    let skip = false;
+
+    if (!this.wasHeader) {
+      skip =
+        chunk.__isStreamHeader ||
+        // TODO remove isArray test
+        Array.isArray(chunk.columns);
+      this.wasHeader = true;
+    }
+    if (!skip) {
+      if (!this.wasRecord) {
+        this.push('[\n');
+      } else {
+        this.push(',\n');
+      }
+      this.wasRecord = true;
+
+      this.push(JSON.stringify(chunk));
+    }
+    done();
+  }
+
+  _flush(done) {
+    if (!this.wasRecord) {
+      this.push('[]\n');
+    } else {
+      this.push('\n]\n');
+    }
+    done();
+  }
+}
+
+async function jsonArrayWriter({ fileName, encoding = 'utf-8' }) {
+  console.log(`Writing file ${fileName}`);
+  const stringify = new StringifyStream();
+  const fileStream = fs.createWriteStream(fileName, encoding);
+  stringify.pipe(fileStream);
+  stringify['finisher'] = fileStream;
+  return stringify;
+}
+
+module.exports = jsonArrayWriter;

packages/api/src/shell/requirePlugin.js

@@ -22,7 +22,7 @@ function requirePlugin(packageName, requiredPlugin = null) {
       // @ts-ignore
       module = __non_webpack_require__(modulePath);
     } catch (err) {
-      console.log('Failed load webpacked module', err.message);
+      // console.log('Failed load webpacked module', err.message);
       module = require(modulePath);
     }
     requiredPlugin = module.__esModule ? module.default : module;

packages/api/src/utility/goSplit.js

@@ -1,18 +0,0 @@
-function goSplit(sql) {
-  if (!sql) return [];
-  const lines = sql.split('\n');
-  const res = [];
-  let buffer = '';
-  for (const line of lines) {
-    if (/^\s*go\s*$/i.test(line)) {
-      if (buffer.trim()) res.push(buffer);
-      buffer = '';
-    } else {
-      buffer += line + '\n';
-    }
-  }
-  if (buffer.trim()) res.push(buffer);
-  return res;
-}
-
-module.exports = goSplit;

packages/api/src/utility/requireEngineDriver.js

@@ -15,7 +15,7 @@ function requireEngineDriver(connection) {
   if (engine.includes('@')) {
     const [shortName, packageName] = engine.split('@');
     const plugin = requirePlugin(packageName);
-    return plugin.driver;
+    return plugin.drivers.find(x => x.engine == engine);
   }
   throw new Error(`Could not found engine driver ${engine}`);
 }

packages/datalib/src/FormViewDisplay.ts

@@ -1,19 +1,7 @@
 import _ from 'lodash';
 import { GridConfig, GridCache, GridConfigColumns, createGridCache, GroupFunc } from './GridConfig';
-import {
-  ForeignKeyInfo,
-  TableInfo,
-  ColumnInfo,
-  EngineDriver,
-  NamedObjectInfo,
-  DatabaseInfo,
-  SqlDialect,
-} from 'dbgate-types';
-import { parseFilter, getFilterType, getFilterValueExpression } from 'dbgate-filterparser';
-import { filterName } from './filterName';
-import { ChangeSetFieldDefinition, ChangeSetRowDefinition } from './ChangeSet';
-import { Expression, Select, treeToSql, dumpSqlSelect, Condition } from 'dbgate-sqltree';
-import { isTypeLogical } from 'dbgate-tools';
+import { TableInfo, EngineDriver, DatabaseInfo, SqlDialect } from 'dbgate-types';
+import { getFilterValueExpression } from 'dbgate-filterparser';
 import { ChangeCacheFunc, ChangeConfigFunc, DisplayColumn } from './GridDisplay';
 
 export class FormViewDisplay {

packages/datalib/src/GridConfig.ts

@@ -33,6 +33,7 @@ export interface GridConfig extends GridConfigColumns {
   formViewKey?: { [uniqueName: string]: string };
   formViewKeyRequested?: { [uniqueName: string]: string };
   formFilterColumns: string[];
+  formColumnFilterText?: string;
 }
 
 export interface GridCache {

packages/datalib/src/GridDisplay.ts

@@ -11,7 +11,7 @@ import {
   SqlDialect,
 } from 'dbgate-types';
 import { parseFilter, getFilterType } from 'dbgate-filterparser';
-import { filterName } from './filterName';
+import { filterName } from 'dbgate-tools';
 import { ChangeSetFieldDefinition, ChangeSetRowDefinition } from './ChangeSet';
 import { Expression, Select, treeToSql, dumpSqlSelect, Condition } from 'dbgate-sqltree';
 import { isTypeLogical } from 'dbgate-tools';
@@ -288,8 +288,8 @@ export abstract class GridDisplay {
     return this.config.expandedColumns.includes(uniqueName);
   }
 
-  toggleExpandedColumn(uniqueName: string) {
-    this.includeInColumnSet('expandedColumns', uniqueName, !this.isExpandedColumn(uniqueName));
+  toggleExpandedColumn(uniqueName: string, value?: boolean) {
+    this.includeInColumnSet('expandedColumns', uniqueName, value == null ? !this.isExpandedColumn(uniqueName) : value);
   }
 
   getFilter(uniqueName: string) {

packages/datalib/src/TableFormViewDisplay.ts

@@ -12,7 +12,6 @@ import {
   Condition,
   OrderByExpression,
 } from 'dbgate-sqltree';
-import { filterName } from './filterName';
 import { TableGridDisplay } from './TableGridDisplay';
 import stableStringify from 'json-stable-stringify';
 import { ChangeSetFieldDefinition, ChangeSetRowDefinition } from './ChangeSet';
@@ -265,8 +264,8 @@ export class TableFormViewDisplay extends FormViewDisplay {
     };
   }
 
-  toggleExpandedColumn(uniqueName: string) {
-    this.gridDisplay.toggleExpandedColumn(uniqueName);
+  toggleExpandedColumn(uniqueName: string, value?: boolean) {
+    this.gridDisplay.toggleExpandedColumn(uniqueName, value);
     this.gridDisplay.reload();
   }
 

packages/datalib/src/TableGridDisplay.ts

@@ -1,9 +1,9 @@
 import _ from 'lodash';
+import { filterName } from 'dbgate-tools';
 import { GridDisplay, ChangeCacheFunc, DisplayColumn, DisplayedColumnInfo, ChangeConfigFunc } from './GridDisplay';
 import { TableInfo, EngineDriver, ViewInfo, ColumnInfo, NamedObjectInfo, DatabaseInfo } from 'dbgate-types';
 import { GridConfig, GridCache, createGridCache } from './GridConfig';
 import { Expression, Select, treeToSql, dumpSqlSelect } from 'dbgate-sqltree';
-import { filterName } from './filterName';
 
 export class TableGridDisplay extends GridDisplay {
   public table: TableInfo;

packages/datalib/src/index.ts

@@ -4,7 +4,6 @@ export * from './TableGridDisplay';
 export * from './ViewGridDisplay';
 export * from './JslGridDisplay';
 export * from './ChangeSet';
-export * from './filterName';
 export * from './FreeTableGridDisplay';
 export * from './FreeTableModel';
 export * from './MacroDefinition';

packages/filterparser/package.json

@@ -6,7 +6,8 @@
   "scripts": {
     "build": "tsc",
     "start": "tsc --watch",
-    "test": "jest"
+    "test": "jest",
+    "test:ci": "jest --json --outputFile=result.json --testLocationInResults"
   },
   "files": [
     "lib"
@@ -22,7 +23,7 @@
   "dependencies": {
     "@types/parsimmon": "^1.10.1",
     "dbgate-tools": "^4.1.1",
-    "lodash": "^4.17.15",
+    "lodash": "^4.17.21",
     "moment": "^2.24.0",
     "parsimmon": "^1.13.0"
   }

packages/filterparser/src/parserFilter.test.ts

@@ -3,5 +3,9 @@ import { parseFilter } from './parseFilter';
 test('parse string', () => {
   const ast = parseFilter('"123"', 'string');
   console.log(JSON.stringify(ast));
-  expect(ast).toBe(3);
+  expect(ast).toEqual({
+    conditionType: 'like',
+    left: { exprType: 'placeholder' },
+    right: { exprType: 'value', value: '%123%' },
+  });
 });

packages/query-splitter/.gitignore

@@ -0,0 +1 @@
+lib

packages/query-splitter/README.md

@@ -0,0 +1,36 @@
+[![NPM version](https://img.shields.io/npm/v/dbgate-query-splitter.svg)](https://www.npmjs.com/package/dbgate-query-splitter)
+
+dbgate-query-splitter
+====================
+
+Splits long SQL query into into particular statements. Designed to have zero dependencies and to be fast.
+
+Supports following SQL dialects:
+* MySQL
+* PostgreSQL
+* SQLite
+* Microsoft SQL Server
+
+## Usage
+
+```js
+import { splitQuery, mysqlSplitterOptions, mssqlSplitterOptions, postgreSplitterOptions } from 'dbgate-query-splitter';
+
+const output = splitQuery('SELECT * FROM `table1`;SELECT * FROM `table2`;', mysqlSplitterOptions);
+
+// output is ['SELECT * FROM `table1`', 'SELECT * FROM `table2`']
+
+```
+
+## Contributing
+Please run tests before pushing any changes.
+
+```sh
+yarn test
+```
+
+## Supported syntax
+* Comments
+* Dollar strings (PostgreSQL)
+* GO separators (MS SQL)
+* Custom delimiter, setby DELIMITER keyword (MySQL)

packages/query-splitter/jest.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  moduleFileExtensions: ['js'],
+};

packages/query-splitter/package.json

@@ -0,0 +1,37 @@
+{
+  "version": "4.1.1",
+  "name": "dbgate-query-splitter",
+  "main": "lib/index.js",
+  "typings": "lib/index.d.ts",
+  "description": "SQL Query splitter for verious database engines",
+  "homepage": "https://github.com/dbgate/dbgate/tree/master/packages/query-splitter",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dbgate/dbgate"
+  },
+  "author": "Jan Prochazka",
+  "license": "MIT",
+  "keywords": [
+    "SQL",
+    "query",
+    "split",
+    "parse"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "start": "tsc --watch",
+    "test": "jest",
+    "test:ci": "jest --json --outputFile=result.json --testLocationInResults"
+  },
+  "files": [
+    "lib"
+  ],
+  "devDependencies": {
+    "dbgate-types": "^4.1.1",
+    "@types/jest": "^25.1.4",
+    "@types/node": "^13.7.0",
+    "jest": "^24.9.0",
+    "ts-jest": "^25.2.1",
+    "typescript": "^3.7.5"
+  }
+}

packages/query-splitter/src/index.ts

@@ -0,0 +1,2 @@
+export * from './splitQuery';
+export * from './options';

packages/query-splitter/src/options.ts

@@ -0,0 +1,66 @@
+export interface SplitterOptions {
+  stringsBegins: string[];
+  stringsEnds: { [begin: string]: string };
+  stringEscapes: { [begin: string]: string };
+
+  allowSemicolon: boolean;
+  allowCustomDelimiter: boolean;
+  allowGoDelimiter: boolean;
+  allowDollarDollarString: boolean;
+  noSplit: boolean;
+}
+
+export const defaultSplitterOptions: SplitterOptions = {
+  stringsBegins: ["'"],
+  stringsEnds: { "'": "'" },
+  stringEscapes: { "'": "'" },
+
+  allowSemicolon: true,
+  allowCustomDelimiter: false,
+  allowGoDelimiter: false,
+  allowDollarDollarString: false,
+  noSplit: false,
+};
+
+export const mysqlSplitterOptions: SplitterOptions = {
+  ...defaultSplitterOptions,
+
+  allowCustomDelimiter: true,
+  stringsBegins: ["'", '`'],
+  stringsEnds: { "'": "'", '`': '`' },
+  stringEscapes: { "'": '\\', '`': '`' },
+};
+
+export const mssqlSplitterOptions: SplitterOptions = {
+  ...defaultSplitterOptions,
+  allowSemicolon: false,
+  allowGoDelimiter: true,
+
+  stringsBegins: ["'", '['],
+  stringsEnds: { "'": "'", '[': ']' },
+  stringEscapes: { "'": "'" },
+};
+
+export const postgreSplitterOptions: SplitterOptions = {
+  ...defaultSplitterOptions,
+
+  allowDollarDollarString: true,
+
+  stringsBegins: ["'", '"'],
+  stringsEnds: { "'": "'", '"': '"' },
+  stringEscapes: { "'": "'", '"': '"' },
+};
+
+export const sqliteSplitterOptions: SplitterOptions = {
+  ...defaultSplitterOptions,
+
+  stringsBegins: ["'", '"'],
+  stringsEnds: { "'": "'", '"': '"' },
+  stringEscapes: { "'": "'", '"': '"' },
+};
+
+export const noSplitSplitterOptions: SplitterOptions = {
+  ...defaultSplitterOptions,
+
+  noSplit: true,
+};

packages/query-splitter/src/splitQuery.ts

@@ -0,0 +1,257 @@
+import { SplitterOptions, defaultSplitterOptions } from './options';
+
+const SEMICOLON = ';';
+
+interface SplitExecutionContext {
+  options: SplitterOptions;
+  source: string;
+  position: number;
+  currentDelimiter: string;
+  output: string[];
+  end: number;
+  wasDataOnLine: boolean;
+  currentCommandStart: number;
+
+  //   unread: string;
+  //   currentStatement: string;
+  //   semicolonKeyTokenRegex: RegExp;
+}
+
+function isStringEnd(s: string, pos: number, endch: string, escapech: string) {
+  if (!escapech) {
+    return s[pos] == endch;
+  }
+  if (endch == escapech) {
+    return s[pos] == endch && s[pos + 1] != endch;
+  } else {
+    return s[pos] == endch && s[pos - 1] != escapech;
+  }
+}
+
+interface Token {
+  type: 'string' | 'delimiter' | 'whitespace' | 'eoln' | 'data' | 'set_delimiter' | 'comment' | 'go_delimiter';
+  length: number;
+  value?: string;
+}
+
+const WHITESPACE_TOKEN: Token = {
+  type: 'whitespace',
+  length: 1,
+};
+const EOLN_TOKEN: Token = {
+  type: 'eoln',
+  length: 1,
+};
+const DATA_TOKEN: Token = {
+  type: 'data',
+  length: 1,
+};
+
+function scanDollarQuotedString(context: SplitExecutionContext): Token {
+  if (!context.options.allowDollarDollarString) return null;
+
+  let pos = context.position;
+  const s = context.source;
+
+  const match = /^(\$[a-zA-Z0-9_]*\$)/.exec(s.slice(pos));
+  if (!match) return null;
+  const label = match[1];
+  pos += label.length;
+
+  while (pos < context.end) {
+    if (s.slice(pos).startsWith(label)) {
+      return {
+        type: 'string',
+        length: pos + label.length - context.position,
+      };
+    }
+    pos++;
+  }
+
+  return null;
+}
+
+function scanToken(context: SplitExecutionContext): Token {
+  let pos = context.position;
+  const s = context.source;
+  const ch = s[pos];
+
+  if (context.options.stringsBegins.includes(ch)) {
+    pos++;
+    const endch = context.options.stringsEnds[ch];
+    const escapech = context.options.stringEscapes[ch];
+    while (pos < context.end && !isStringEnd(s, pos, endch, escapech)) {
+      if (endch == escapech && s[pos] == endch && s[pos + 1] == endch) {
+        pos += 2;
+      } else {
+        pos++;
+      }
+    }
+    return {
+      type: 'string',
+      length: pos - context.position + 1,
+    };
+  }
+
+  if (context.currentDelimiter && s.slice(pos).startsWith(context.currentDelimiter)) {
+    return {
+      type: 'delimiter',
+      length: context.currentDelimiter.length,
+    };
+  }
+
+  if (ch == ' ' || ch == '\t' || ch == '\r') {
+    return WHITESPACE_TOKEN;
+  }
+
+  if (ch == '\n') {
+    return EOLN_TOKEN;
+  }
+
+  if (ch == '-' && s[pos + 1] == '-') {
+    while (pos < context.end && s[pos] != '\n') pos++;
+    return {
+      type: 'comment',
+      length: pos - context.position,
+    };
+  }
+
+  if (ch == '/' && s[pos + 1] == '*') {
+    pos += 2;
+    while (pos < context.end) {
+      if (s[pos] == '*' && s[pos + 1] == '/') break;
+      pos++;
+    }
+    return {
+      type: 'comment',
+      length: pos - context.position + 2,
+    };
+  }
+
+  if (context.options.allowCustomDelimiter && !context.wasDataOnLine) {
+    const m = s.slice(pos).match(/^DELIMITER[ \t]+([^\n]+)/i);
+    if (m) {
+      return {
+        type: 'set_delimiter',
+        value: m[1].trim(),
+        length: m[0].length,
+      };
+    }
+  }
+
+  if (context.options.allowGoDelimiter && !context.wasDataOnLine) {
+    const m = s.slice(pos).match(/^GO[\t\r ]*(\n|$)/i);
+    if (m) {
+      return {
+        type: 'go_delimiter',
+        length: m[0].length - 1,
+      };
+    }
+  }
+
+  const dollarString = scanDollarQuotedString(context);
+  if (dollarString) return dollarString;
+
+  return DATA_TOKEN;
+}
+
+function pushQuery(context) {
+  const sql = context.source.slice(context.currentCommandStart, context.position);
+  const trimmed = sql.trim();
+  if (trimmed) context.output.push(trimmed);
+}
+
+export function splitQuery(sql: string, options: SplitterOptions = null): string[] {
+  const usedOptions = {
+    ...defaultSplitterOptions,
+    ...options,
+  };
+
+  if (usedOptions.noSplit) {
+    return [sql];
+  }
+
+  const context: SplitExecutionContext = {
+    source: sql,
+    end: sql.length,
+    currentDelimiter: options?.allowSemicolon === false ? null : SEMICOLON,
+    position: 0,
+    currentCommandStart: 0,
+    output: [],
+    wasDataOnLine: false,
+    options: usedOptions,
+  };
+
+  while (context.position < context.end) {
+    const token = scanToken(context);
+    if (!token) {
+      // nothing special, move forward
+      context.position += 1;
+      continue;
+    }
+    switch (token.type) {
+      case 'string':
+        context.position += token.length;
+        context.wasDataOnLine = true;
+        break;
+      case 'comment':
+        context.position += token.length;
+        context.wasDataOnLine = true;
+        break;
+      case 'eoln':
+        context.position += token.length;
+        context.wasDataOnLine = false;
+        break;
+      case 'data':
+        context.position += token.length;
+        context.wasDataOnLine = true;
+        break;
+      case 'whitespace':
+        context.position += token.length;
+        break;
+      case 'set_delimiter':
+        pushQuery(context);
+        context.currentDelimiter = token.value;
+        context.position += token.length;
+        context.currentCommandStart = context.position;
+        break;
+      case 'go_delimiter':
+        pushQuery(context);
+        context.position += token.length;
+        context.currentCommandStart = context.position;
+        break;
+      case 'delimiter':
+        pushQuery(context);
+        context.position += token.length;
+        context.currentCommandStart = context.position;
+        break;
+    }
+  }
+
+  if (context.end > context.currentCommandStart) {
+    pushQuery(context);
+  }
+
+  //   context.semicolonKeyTokenRegex = buildKeyTokenRegex(SEMICOLON, context);
+  //   let findResult: FindExpResult = {
+  //     expIndex: -1,
+  //     exp: null,
+  //     nextIndex: 0,
+  //   };
+  //   let lastUnreadLength;
+  //   do {
+  //     // console.log('context.unread', context.unread);
+  //     lastUnreadLength = context.unread.length;
+  //     findResult = findKeyToken(context.unread, context.currentDelimiter, context);
+  //     handleKeyTokenFindResult(context, findResult);
+  //     // Prevent infinite loop by returning incorrect result
+  //     if (lastUnreadLength === context.unread.length) {
+  //       read(context, context.unread.length);
+  //     }
+  //   } while (context.unread !== '');
+  //   publishStatement(context);
+
+  // console.log('RESULT', context.output);
+
+  return context.output;
+}

packages/query-splitter/src/splitter.test.ts

@@ -0,0 +1,79 @@
+import { mysqlSplitterOptions, mssqlSplitterOptions, postgreSplitterOptions, noSplitSplitterOptions } from './options';
+import { splitQuery } from './splitQuery';
+
+test('simple query', () => {
+  const output = splitQuery('select * from A');
+  expect(output).toEqual(['select * from A']);
+});
+
+test('correct split 2 queries', () => {
+  const output = splitQuery('SELECT * FROM `table1`;SELECT * FROM `table2`;', mysqlSplitterOptions);
+  expect(output).toEqual(['SELECT * FROM `table1`', 'SELECT * FROM `table2`']);
+});
+
+test('correct split 2 queries - no end semicolon', () => {
+  const output = splitQuery('SELECT * FROM `table1`;SELECT * FROM `table2`', mysqlSplitterOptions);
+  expect(output).toEqual(['SELECT * FROM `table1`', 'SELECT * FROM `table2`']);
+});
+
+test('delete empty query', () => {
+  const output = splitQuery(';;;\n;;SELECT * FROM `table1`;;;;;SELECT * FROM `table2`;;; ;;;', mysqlSplitterOptions);
+  expect(output).toEqual(['SELECT * FROM `table1`', 'SELECT * FROM `table2`']);
+});
+
+test('should handle double backtick', () => {
+  const input = ['CREATE TABLE `a``b` (`c"d` INT)', 'CREATE TABLE `a````b` (`c"d` INT)'];
+  const output = splitQuery(input.join(';\n') + ';', mysqlSplitterOptions);
+  expect(output).toEqual(input);
+});
+
+test('semicolon inside string', () => {
+  const input = ['CREATE TABLE a', "INSERT INTO a (x) VALUES ('1;2;3;4')"];
+  const output = splitQuery(input.join(';\n') + ';', mysqlSplitterOptions);
+  expect(output).toEqual(input);
+});
+
+test('semicolon inside identyifier - mssql', () => {
+  const input = ['CREATE TABLE [a;1]', "INSERT INTO [a;1] (x) VALUES ('1')"];
+  const output = splitQuery(input.join(';\n') + ';', {
+    ...mssqlSplitterOptions,
+    allowSemicolon: true,
+  });
+  expect(output).toEqual(input);
+});
+
+test('delimiter test', () => {
+  const input = 'SELECT 1;\n DELIMITER $$\n SELECT 2; SELECT 3; \n DELIMITER ;';
+  const output = splitQuery(input, mysqlSplitterOptions);
+  expect(output).toEqual(['SELECT 1', 'SELECT 2; SELECT 3;']);
+});
+
+test('one line comment test', () => {
+  const input = 'SELECT 1 -- comment1;comment2\n;SELECT 2';
+  const output = splitQuery(input, mysqlSplitterOptions);
+  expect(output).toEqual(['SELECT 1 -- comment1;comment2', 'SELECT 2']);
+});
+
+test('multi line comment test', () => {
+  const input = 'SELECT 1 /* comment1;comment2\ncomment3*/;SELECT 2';
+  const output = splitQuery(input, mysqlSplitterOptions);
+  expect(output).toEqual(['SELECT 1 /* comment1;comment2\ncomment3*/', 'SELECT 2']);
+});
+
+test('dollar string', () => {
+  const input = 'CREATE PROC $$ SELECT 1; SELECT 2; $$ ; SELECT 3';
+  const output = splitQuery(input, postgreSplitterOptions);
+  expect(output).toEqual(['CREATE PROC $$ SELECT 1; SELECT 2; $$', 'SELECT 3']);
+});
+
+test('go delimiter', () => {
+  const input = 'SELECT 1\ngo\nSELECT 2';
+  const output = splitQuery(input, mssqlSplitterOptions);
+  expect(output).toEqual(['SELECT 1', 'SELECT 2']);
+});
+
+test('no split', () => {
+  const input = 'SELECT 1;SELECT 2';
+  const output = splitQuery(input, noSplitSplitterOptions);
+  expect(output).toEqual(['SELECT 1;SELECT 2']);
+});

packages/query-splitter/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "target": "ES2015",
+    "module": "commonjs",
+    "declaration": true,
+    "skipLibCheck": true,
+    "outDir": "lib",
+    "preserveWatchOutput": true,
+    "esModuleInterop": true
+  },
+  "include": [
+    "src/**/*"
+  ]
+}

packages/sqltree/package.json

@@ -31,6 +31,6 @@
     "typescript": "^3.7.5"
   },
   "dependencies": {
-    "lodash": "^4.17.15"
+    "lodash": "^4.17.21"
   }
 }

packages/tools/package.json

@@ -31,6 +31,6 @@
     "typescript": "^3.7.5"
   },
   "dependencies": {
-    "lodash": "^4.17.15"
+    "lodash": "^4.17.21"
   }
 }

packages/tools/src/DatabaseAnalyser.ts

@@ -2,12 +2,14 @@ import { DatabaseInfo, DatabaseModification, EngineDriver } from 'dbgate-types';
 import _sortBy from 'lodash/sortBy';
 import _groupBy from 'lodash/groupBy';
 import _pick from 'lodash/pick';
+import _compact from 'lodash/compact';
 
 const fp_pick = arg => array => _pick(array, arg);
 export class DatabaseAnalyser {
   structure: DatabaseInfo;
   modifications: DatabaseModification[];
   singleObjectFilter: any;
+  singleObjectId: string = null;
 
   constructor(public pool, public driver: EngineDriver) {}
 
@@ -15,15 +17,30 @@ export class DatabaseAnalyser {
     return DatabaseAnalyser.createEmptyStructure();
   }
 
-  /** @returns {Promise<import('dbgate-types').DatabaseModification[]>} */
-  async getModifications() {
-    if (this.structure == null) throw new Error('DatabaseAnalyse.getModifications - structure must be filled');
-
+  async _getFastSnapshot(): Promise<DatabaseInfo> {
     return null;
   }
 
+  async _computeSingleObjectId() {}
+
   async fullAnalysis() {
-    return this._runAnalysis();
+    const res = await this._runAnalysis();
+    // console.log('FULL ANALYSIS', res);
+    return res;
+  }
+
+  async singleObjectAnalysis(name, typeField) {
+    // console.log('Analysing SINGLE OBJECT', name, typeField);
+    this.singleObjectFilter = { ...name, typeField };
+    await this._computeSingleObjectId();
+    const res = await this._runAnalysis();
+    // console.log('SINGLE OBJECT RES', res);
+    const obj =
+      res[typeField]?.length == 1
+        ? res[typeField][0]
+        : res[typeField]?.find(x => x.pureName == name.pureName && x.schemaName == name.schemaName);
+    // console.log('SINGLE OBJECT', obj);
+    return obj;
   }
 
   async incrementalAnalysis(structure) {
@@ -37,7 +54,7 @@ export class DatabaseAnalyser {
     }
     if (this.modifications.length == 0) return null;
     console.log('DB modifications detected:', this.modifications);
-    return this._runAnalysis();
+    return this.mergeAnalyseResult(await this._runAnalysis());
   }
 
   mergeAnalyseResult(newlyAnalysed) {
@@ -49,7 +66,7 @@ export class DatabaseAnalyser {
     }
 
     const res = {};
-    for (const field of ['tables', 'collections', 'views', 'functions', 'procedures', 'triggers']) {
+    for (const field of ['tables', 'collections', 'views', 'matviews', 'functions', 'procedures', 'triggers']) {
       const removedIds = this.modifications
         .filter(x => x.action == 'remove' && x.objectTypeField == field)
         .map(x => x.objectId);
@@ -57,9 +74,19 @@ export class DatabaseAnalyser {
       const addedChangedIds = newArray.map(x => x.objectId);
       const removeAllIds = [...removedIds, ...addedChangedIds];
       res[field] = _sortBy(
-        [...this.structure[field].filter(x => !removeAllIds.includes(x.objectId)), ...newArray],
+        [...(this.structure[field] || []).filter(x => !removeAllIds.includes(x.objectId)), ...newArray],
         x => x.pureName
       );
+
+      // merge missing data from old structure
+      for (const item of res[field]) {
+        const original = (this.structure[field] || []).find(x => x.objectId == item.objectId);
+        if (original) {
+          for (const key in original) {
+            if (!item[key]) item[key] = original[key];
+          }
+        }
+      }
     }
 
     return res;
@@ -77,7 +104,10 @@ export class DatabaseAnalyser {
       if (typeField == objectTypeField) return [pureName];
     }
     if (this.modifications) {
-      return this.modifications.filter(x => x.objectTypeField == objectTypeField).map(x => x.newName.pureName);
+      return this.modifications
+        .filter(x => x.objectTypeField == objectTypeField)
+        .filter(x => x.newName)
+        .map(x => x.newName.pureName);
     }
     return allPureNames;
   }
@@ -86,11 +116,107 @@ export class DatabaseAnalyser {
   //   return this.structure.tables.find((x) => x.objectId == id);
   // }
 
+  createQuery(template, typeFields) {
+    // let res = template;
+    if (this.singleObjectFilter) {
+      const { typeField } = this.singleObjectFilter;
+      if (!this.singleObjectId) return null;
+      if (!typeFields || !typeFields.includes(typeField)) return null;
+      return template.replace(/=OBJECT_ID_CONDITION/g, ` = '${this.singleObjectId}'`);
+    }
+    if (!this.modifications || !typeFields || this.modifications.length == 0) {
+      return template.replace(/=OBJECT_ID_CONDITION/g, ' is not null');
+    }
+    if (this.modifications.some(x => typeFields.includes(x.objectTypeField) && x.action == 'all')) {
+      // do not filter objects
+      return template.replace(/=OBJECT_ID_CONDITION/g, ' is not null');
+    }
+    const filterIds = this.modifications
+      .filter(x => typeFields.includes(x.objectTypeField) && (x.action == 'add' || x.action == 'change'))
+      .map(x => x.objectId);
+    if (filterIds.length == 0) {
+      return template.replace(/=OBJECT_ID_CONDITION/g, " = '0'");
+    }
+    return template.replace(/=OBJECT_ID_CONDITION/g, ` in (${filterIds.map(x => `'${x}'`).join(',')})`);
+  }
+
+  getDeletedObjectsForField(snapshot, objectTypeField) {
+    const items = snapshot[objectTypeField];
+    if (!items) return [];
+    if (!this.structure[objectTypeField]) return [];
+    return this.structure[objectTypeField]
+      .filter(x => !items.find(y => x.objectId == y.objectId))
+      .map(x => ({
+        oldName: _pick(x, ['schemaName', 'pureName']),
+        objectId: x.objectId,
+        action: 'remove',
+        objectTypeField,
+      }));
+  }
+
+  getDeletedObjects(snapshot) {
+    return [
+      ...this.getDeletedObjectsForField(snapshot, 'tables'),
+      ...this.getDeletedObjectsForField(snapshot, 'collections'),
+      ...this.getDeletedObjectsForField(snapshot, 'views'),
+      ...this.getDeletedObjectsForField(snapshot, 'matviews'),
+      ...this.getDeletedObjectsForField(snapshot, 'procedures'),
+      ...this.getDeletedObjectsForField(snapshot, 'functions'),
+      ...this.getDeletedObjectsForField(snapshot, 'triggers'),
+    ];
+  }
+
+  async getModifications() {
+    const snapshot = await this._getFastSnapshot();
+    if (!snapshot) return null;
+
+    // console.log('STRUCTURE', this.structure);
+    // console.log('SNAPSHOT', snapshot);
+
+    const res = [];
+    for (const field in snapshot) {
+      const items = snapshot[field];
+      if (items === null) {
+        res.push({ objectTypeField: field, action: 'all' });
+        continue;
+      }
+      if (items === undefined) {
+        // skip - undefined meens, that field is not supported
+        continue;
+      }
+      for (const item of items) {
+        const { objectId, schemaName, pureName, contentHash } = item;
+        const obj = this.structure[field].find(x => x.objectId == objectId);
+
+        if (obj && contentHash && obj.contentHash == contentHash) continue;
+
+        const action = obj
+          ? {
+              newName: { schemaName, pureName },
+              oldName: _pick(obj, ['schemaName', 'pureName']),
+              action: 'change',
+              objectTypeField: field,
+              objectId,
+            }
+          : {
+              newName: { schemaName, pureName },
+              action: 'add',
+              objectTypeField: field,
+              objectId,
+            };
+        res.push(action);
+      }
+    }
+
+    return [..._compact(res), ...this.getDeletedObjects(snapshot)];
+  }
+
   static createEmptyStructure(): DatabaseInfo {
     return {
       tables: [],
       collections: [],
       views: [],
+      matviews: [],
       functions: [],
       procedures: [],
       triggers: [],

packages/tools/src/SqlDumper.ts

@@ -293,6 +293,20 @@ export class SqlDumper {
   changeViewSchema(obj: ViewInfo, newSchema: string) {}
   renameView(obj: ViewInfo, newSchema: string) {}
 
+  createMatview(obj: ViewInfo) {
+    this.putRaw(obj.createSql);
+    this.endCommand();
+  }
+  dropMatview(obj: ViewInfo, { testIfExists = false }) {
+    this.putCmd('^drop ^materialized ^view  %f', obj);
+  }
+  alterMatview(obj: ViewInfo) {
+    this.putRaw(obj.createSql.replace(/create\s+view/i, 'ALTER VIEW'));
+    this.endCommand();
+  }
+  changeMatviewSchema(obj: ViewInfo, newSchema: string) {}
+  renameMatview(obj: ViewInfo, newSchema: string) {}
+
   createProcedure(obj: ProcedureInfo) {
     this.putRaw(obj.createSql);
     this.endCommand();

packages/tools/src/SqlGenerator.ts

@@ -7,7 +7,8 @@ import {
   TriggerInfo,
   ViewInfo,
 } from 'dbgate-types';
-import _ from 'lodash';
+import _flatten from 'lodash/flatten';
+import _uniqBy from 'lodash/uniqBy'
 import { SqlDumper } from './SqlDumper';
 import { extendDatabaseInfo } from './structureTools';
 
@@ -29,6 +30,10 @@ interface SqlGeneratorOptions {
   checkIfViewExists: boolean;
   createViews: boolean;
 
+  dropMatviews: boolean;
+  checkIfMatviewExists: boolean;
+  createMatviews: boolean;
+
   dropProcedures: boolean;
   checkIfProcedureExists: boolean;
   createProcedures: boolean;
@@ -51,6 +56,7 @@ interface SqlGeneratorObject {
 export class SqlGenerator {
   private tables: TableInfo[];
   private views: ViewInfo[];
+  private matviews: ViewInfo[];
   private procedures: ProcedureInfo[];
   private functions: FunctionInfo[];
   private triggers: TriggerInfo[];
@@ -69,6 +75,7 @@ export class SqlGenerator {
     this.dbinfo = extendDatabaseInfo(dbinfo);
     this.tables = this.extract('tables');
     this.views = this.extract('views');
+    this.matviews = this.extract('matviews');
     this.procedures = this.extract('procedures');
     this.functions = this.extract('functions');
     this.triggers = this.extract('triggers');
@@ -89,6 +96,8 @@ export class SqlGenerator {
       if (this.checkDumper()) return;
       this.dropObjects(this.views, 'View');
       if (this.checkDumper()) return;
+      this.dropObjects(this.matviews, 'Matview');
+      if (this.checkDumper()) return;
       this.dropObjects(this.triggers, 'Trigger');
       if (this.checkDumper()) return;
 
@@ -113,6 +122,8 @@ export class SqlGenerator {
       if (this.checkDumper()) return;
       this.createObjects(this.views, 'View');
       if (this.checkDumper()) return;
+      this.createObjects(this.matviews, 'Matview');
+      if (this.checkDumper()) return;
       this.createObjects(this.triggers, 'Trigger');
       if (this.checkDumper()) return;
     } finally {
@@ -122,9 +133,9 @@ export class SqlGenerator {
 
   createForeignKeys() {
     const fks = [];
-    if (this.options.createForeignKeys) fks.push(..._.flatten(this.tables.map(x => x.foreignKeys || [])));
-    if (this.options.createReferences) fks.push(..._.flatten(this.tables.map(x => x.dependencies || [])));
-    for (const fk of _.uniqBy(fks, 'constraintName')) {
+    if (this.options.createForeignKeys) fks.push(..._flatten(this.tables.map(x => x.foreignKeys || [])));
+    if (this.options.createReferences) fks.push(..._flatten(this.tables.map(x => x.dependencies || [])));
+    for (const fk of _uniqBy(fks, 'constraintName')) {
       this.dmp.createForeignKey(fk);
       if (this.checkDumper()) return;
     }
@@ -152,7 +163,7 @@ export class SqlGenerator {
       }
     }
     if (this.options.createIndexes) {
-      for (const index of _.flatten(this.tables.map(x => x.indexes || []))) {
+      for (const index of _flatten(this.tables.map(x => x.indexes || []))) {
         this.dmp.createIndex(index);
       }
     }
@@ -204,7 +215,7 @@ export class SqlGenerator {
 
   dropTables() {
     if (this.options.dropReferences) {
-      for (const fk of _.flatten(this.tables.map(x => x.dependencies || []))) {
+      for (const fk of _flatten(this.tables.map(x => x.dependencies || []))) {
         this.dmp.dropForeignKey(fk);
       }
     }

packages/tools/src/driverBase.ts

@@ -1,4 +1,5 @@
 import { SqlDumper } from './SqlDumper';
+import { splitQuery } from 'dbgate-query-splitter';
 
 const dialect = {
   limitSelect: true,
@@ -22,12 +23,7 @@ export const driverBase = {
   },
   async analyseSingleObject(pool, name, typeField = 'tables') {
     const analyser = new this.analyserClass(pool, this);
-    analyser.singleObjectFilter = { ...name, typeField };
-    const res = await analyser.fullAnalysis();
-    if (res[typeField].length == 1) return res[typeField][0];
-    const obj = res[typeField].find(x => x.pureName == name.pureName && x.schemaName == name.schemaName);
-    // console.log('FIND', name, obj);
-    return obj;
+    return analyser.singleObjectAnalysis(name, typeField);
   },
   analyseSingleTable(pool, name) {
     return this.analyseSingleObject(pool, name, 'tables');
@@ -39,4 +35,9 @@ export const driverBase = {
   createDumper() {
     return new this.dumperClass(this);
   },
+  async script(pool, sql) {
+    for (const sqlItem of splitQuery(sql, this.getQuerySplitterOptions('script'))) {
+      await this.query(pool, sqlItem, { discardResult: true });
+    }
+  },
 };

packages/tools/src/filterName.ts

@@ -1,4 +1,4 @@
-import _ from 'lodash';
+import _compact from 'lodash/compact';
 
 // original C# variant
 // public bool Match(string value)
@@ -39,6 +39,12 @@ export function filterName(filter: string, ...names: string[]) {
   // const camelVariants = [name.replace(/[^A-Z]/g, '')]
   const tokens = filter.split(' ').map(x => x.trim());
 
-  return !!_.compact(names).find(name => !tokens.find(token => !name.toUpperCase().includes(token.toUpperCase())));
-  // return name.toUpperCase().includes(filter.toUpperCase());
+  const namesCompacted = _compact(names);
+  for (const token of tokens) {
+    const tokenUpper = token.toUpperCase();
+    const found = namesCompacted.find(name => name.toUpperCase().includes(tokenUpper));
+    if (!found) return false;
+  }
+
+  return true;
 }

packages/tools/src/index.ts

@@ -10,3 +10,4 @@ export * from './testPermission';
 export * from './SqlGenerator';
 export * from './structureTools';
 export * from './settingsExtractors';
+export * from './filterName';

packages/tools/src/settingsExtractors.ts

@@ -1,11 +1,12 @@
-import _ from 'lodash';
+import _isNaN from 'lodash/isNaN';
+import _isNumber from 'lodash/isNumber';
 
 export function extractIntSettingsValue(settings, name, defaultValue, min = null, max = null) {
   const parsed = parseInt(settings[name]);
-  if (_.isNaN(parsed)) {
+  if (_isNaN(parsed)) {
     return defaultValue;
   }
-  if (_.isNumber(parsed)) {
+  if (_isNumber(parsed)) {
     if (min != null && parsed < min) return min;
     if (max != null && parsed > max) return max;
     return parsed;

packages/tools/src/structureTools.ts

@@ -1,8 +1,8 @@
 import { DatabaseInfo } from 'dbgate-types';
-import _ from 'lodash';
+import _flatten from 'lodash/flatten';
 
 export function addTableDependencies(db: DatabaseInfo): DatabaseInfo {
-  const allForeignKeys = _.flatten(db.tables.map(x => x.foreignKeys || []));
+  const allForeignKeys = _flatten(db.tables.map(x => x.foreignKeys || []));
   return {
     ...db,
     tables: db.tables.map(table => ({
@@ -64,6 +64,10 @@ function fillTableExtendedInfo(db: DatabaseInfo): DatabaseInfo {
       ...obj,
       objectTypeField: 'views',
     })),
+    matviews: (db.matviews || []).map(obj => ({
+      ...obj,
+      objectTypeField: 'matviews',
+    })),
     procedures: (db.procedures || []).map(obj => ({
       ...obj,
       objectTypeField: 'procedures',

packages/types/dbinfo.d.ts

@@ -95,6 +95,7 @@ export interface DatabaseInfoObjects {
   tables: TableInfo[];
   collections: CollectionInfo[];
   views: ViewInfo[];
+  matviews: ViewInfo[];
   procedures: ProcedureInfo[];
   functions: FunctionInfo[];
   triggers: TriggerInfo[];

packages/types/engines.d.ts

@@ -12,6 +12,10 @@ export interface StreamOptions {
   info?: (info) => void;
 }
 
+export interface QueryOptions {
+  discardResult?: boolean;
+}
+
 export interface WriteTableOptions {
   dropIfExists?: boolean;
   truncate?: boolean;
@@ -38,10 +42,14 @@ export interface EngineDriver {
   title: string;
   defaultPort?: number;
   supportsDatabaseUrl?: boolean;
-  isFileDatabase?: boolean;
+  isElectronOnly?: boolean;
+  showConnectionField?: (field: string, values: any) => boolean;
+  showConnectionTab?: (tab: 'ssl' | 'sshTunnel', values: any) => boolean;
+  beforeConnectionSave?: (values: any) => any;
   databaseUrlPlaceholder?: string;
-  connect({ server, port, user, password, database }): any;
-  query(pool: any, sql: string): Promise<QueryResult>;
+  connect({ server, port, user, password, database }): Promise<any>;
+  close(pool): Promise<any>;
+  query(pool: any, sql: string, options?: QueryOptions): Promise<QueryResult>;
   stream(pool: any, sql: string, options: StreamOptions);
   readQuery(pool: any, sql: string, structure?: TableInfo): Promise<stream.Readable>;
   writeTable(pool: any, name: NamedObjectInfo, options: WriteTableOptions): Promise<stream.Writeable>;
@@ -69,6 +77,8 @@ export interface EngineDriver {
   updateCollection(pool: any, changeSet: any): Promise<any>;
   getCollectionUpdateScript(changeSet: any): string;
   createDatabase(pool: any, name: string): Promise;
+  getQuerySplitterOptions(usage: 'stream' | 'script'): any;
+  script(pool: any, sql: string): Promise;
 
   analyserClass?: any;
   dumperClass?: any;
@@ -78,6 +88,6 @@ export interface DatabaseModification {
   oldName?: NamedObjectInfo;
   newName?: NamedObjectInfo;
   objectId?: string;
-  action: 'add' | 'remove' | 'change';
+  action: 'add' | 'remove' | 'change' | 'all';
   objectTypeField: keyof DatabaseInfo;
 }

packages/types/extensions.d.ts

@@ -33,9 +33,16 @@ export interface PluginDefinition {
   content: any;
 }
 
+export interface QuickExportDefinition {
+  label: string;
+  createWriter: (fileName: string) => { functionName: string; props: any };
+  extension: string;
+}
+
 export interface ExtensionsDirectory {
   plugins: PluginDefinition[];
   fileFormats: FileFormatDefinition[];
+  quickExports: QuickExportDefinition[];
   drivers: EngineDriver[];
   themes: ThemeDefinition[];
 }

packages/types/index.d.ts

@@ -4,6 +4,7 @@ export interface OpenedDatabaseConnection {
   conid: string;
   database: string;
   structure: DatabaseInfo;
+  analysedTime?: number;
   serverVersion?: any;
   subprocess: ChildProcess;
   disconnected?: boolean;

packages/web/package.json

@@ -29,7 +29,7 @@
     "file-selector": "^0.2.4",
     "json-stable-stringify": "^1.0.1",
     "localforage": "^1.9.0",
-    "lodash": "^4.17.15",
+    "lodash": "^4.17.21",
     "randomcolor": "^0.6.2",
     "resize-observer-polyfill": "^1.5.1",
     "rollup": "^2.3.4",

packages/web/public/index.html

@@ -21,9 +21,25 @@
 	<link rel='stylesheet' href='build/fonts/materialdesignicons.css'>
 
 	<script defer src='build/bundle.js'></script>
+
+	<style>
+		#starting_dbgate_zero {
+			position: absolute;
+			left: 0;
+			top: 0;
+			right: 0;
+			bottom: 0;
+			display: flex;
+			align-items: center;
+			justify-content: space-around;
+		}
+	</style>
 </head>
 
 <body>
+	<div id='starting_dbgate_zero'>
+		Loading DbGate App ...
+	</div>
 </body>
 
 </html>

packages/web/src/App.svelte

@@ -1,25 +1,68 @@
 <script lang="ts">
+  import { onMount } from 'svelte';
+
   import CommandListener from './commands/CommandListener.svelte';
   import DataGridRowHeightMeter from './datagrid/DataGridRowHeightMeter.svelte';
   import LoadingInfo from './elements/LoadingInfo.svelte';
 
   import PluginsProvider from './plugins/PluginsProvider.svelte';
   import Screen from './Screen.svelte';
+  import { loadingPluginStore } from './stores';
+  import { setAppLoaded } from './utility/appLoadManager';
+  import axiosInstance from './utility/axiosInstance';
   import ErrorHandler from './utility/ErrorHandler.svelte';
-  import { useSettings } from './utility/metadataLoaders';
   import OpenTabsOnStartup from './utility/OpenTabsOnStartup.svelte';
 
-  const settings = useSettings();
+  let loadedApi = false;
+
+  async function loadApi() {
+    try {
+      const settings = await axiosInstance.get('config/get-settings');
+      const connections = await axiosInstance.get('connections/list');
+      const config = await axiosInstance.get('config/get');
+      loadedApi = settings?.data && connections?.data && config?.data;
+      if (!loadedApi) {
+        console.log('API not initialized correctly, trying again in 1s');
+        setTimeout(loadApi, 1000);
+      }
+    } catch (err) {
+      console.log('Error calling API, trying again in 1s');
+      setTimeout(loadApi, 1000);
+    }
+  }
+
+  onMount(loadApi);
+
+  onMount(() => {
+    const removed = document.getElementById('starting_dbgate_zero');
+    if (removed) removed.remove();
+  });
+
+  $: {
+    if (loadedApi && $loadingPluginStore?.loaded) {
+      setAppLoaded();
+    }
+  }
+
 </script>
 
 <DataGridRowHeightMeter />
 <ErrorHandler />
-<PluginsProvider />
 <CommandListener />
-<OpenTabsOnStartup />
 
-{#if $settings}
-  <Screen />
+{#if loadedApi}
+  <PluginsProvider />
+  {#if $loadingPluginStore?.loaded}
+    <OpenTabsOnStartup />
+    <Screen />
+  {:else}
+    <LoadingInfo
+      message={$loadingPluginStore.loadingPackageName
+        ? `Loading plugin ${$loadingPluginStore.loadingPackageName} ...`
+        : 'Preparing plugins ...'}
+      wrapper
+    />
+  {/if}
 {:else}
-  <LoadingInfo message="Loading settings..." wrapper />
+  <LoadingInfo message="Starting DbGate ..." wrapper />
 {/if}

packages/web/src/Screen.svelte

@@ -6,6 +6,7 @@
     currentThemeDefinition,
     isFileDragActive,
     leftPanelWidth,
+    openedSnackbars,
     selectedWidget,
     visibleCommandPalette,
     visibleToolbar,
@@ -17,11 +18,13 @@
   import splitterDrag from './utility/splitterDrag';
   import CurrentDropDownMenu from './modals/CurrentDropDownMenu.svelte';
   import StatusBar from './widgets/StatusBar.svelte';
+  import Snackbar from './widgets/Snackbar.svelte';
   import ModalLayer from './modals/ModalLayer.svelte';
   import DragAndDropFileTarget from './DragAndDropFileTarget.svelte';
   import dragDropFileTarget from './utility/dragDropFileTarget';
 
   $: currentThemeType = $currentThemeDefinition?.themeType == 'dark' ? 'theme-type-dark' : 'theme-type-light';
+
 </script>
 
 <div class={`${$currentTheme} ${currentThemeType} root`} use:dragDropFileTarget>
@@ -64,6 +67,11 @@
   {#if $isFileDragActive}
     <DragAndDropFileTarget />
   {/if}
+  <div class="snackbar-container">
+    {#each $openedSnackbars as snackbar(snackbar.id)}
+      <Snackbar {...snackbar} />
+    {/each}
+  </div>
 </div>
 
 <style>
@@ -139,4 +147,11 @@
     bottom: var(--dim-statusbar-height);
     left: calc(var(--dim-widget-icon-size) + var(--dim-left-panel-width));
   }
+
+  .snackbar-container {
+    position: fixed;
+    right: 0;
+    bottom: var(--dim-statusbar-height);
+  }
+
 </style>

packages/web/src/appobj/AppObjectList.svelte

@@ -9,7 +9,7 @@
   export let subItemsComponent = undefined;
   export let expandOnClick = false;
   export let isExpandable = undefined;
-  export let filter;
+  export let filter = undefined;
   export let expandIconFunc = undefined;
   export let checkedObjectsStore = null;
   export let disableContextMenu = false;
@@ -34,10 +34,11 @@
     : null;
 
   $: groups = groupFunc ? _.groupBy(listGrouped, 'group') : null;
+
 </script>
 
 {#if groupFunc}
-  {#each _.keys(groups) as group (group)}
+  {#each _.keys(groups) as group}
     <AppObjectGroup
       {group}
       {module}
@@ -51,7 +52,7 @@
     />
   {/each}
 {:else}
-  {#each filtered as data (module.extractKey(data))}
+  {#each filtered as data}
     <AppObjectListItem
       {module}
       {subItemsComponent}

packages/web/src/appobj/ArchiveFileAppObject.svelte

@@ -14,14 +14,19 @@
 
   export const extractKey = data => data.fileName;
   export const createMatcher = ({ fileName }) => filter => filterName(filter, fileName);
+
 </script>
 
 <script lang="ts">
-  import { filterName } from 'dbgate-datalib';
+  import { filterName } from 'dbgate-tools';
+  import ImportExportModal from '../modals/ImportExportModal.svelte';
+  import { showModal } from '../modals/modalTools';
 
-  import { currentArchive } from '../stores';
+  import { currentArchive, extensions } from '../stores';
 
   import axiosInstance from '../utility/axiosInstance';
+  import createQuickExportMenu from '../utility/createQuickExportMenu';
+  import { exportElectronFile } from '../utility/exportElectronFile';
   import openNewTab from '../utility/openNewTab';
   import AppObjectCore from './AppObjectCore.svelte';
 
@@ -60,8 +65,34 @@
       { text: 'Open (readonly)', onClick: handleOpenRead },
       { text: 'Open in free table editor', onClick: handleOpenWrite },
       { text: 'Delete', onClick: handleDelete },
+      createQuickExportMenu($extensions, fmt => async () => {
+        exportElectronFile(
+          data.fileName,
+          {
+            functionName: 'archiveReader',
+            props: {
+              fileName: data.fileName,
+              folderName: data.folderName,
+            },
+          },
+          fmt
+        );
+      }),
+      {
+        text: 'Export',
+        onClick: () => {
+          showModal(ImportExportModal, {
+            initialValues: {
+              sourceStorageType: 'archive',
+              sourceArchiveFolder: data.folderName,
+              sourceList: [data.fileName],
+            },
+          });
+        },
+      },
     ];
   }
+
 </script>
 
 <AppObjectCore

packages/web/src/appobj/ArchiveFolderAppObject.svelte

@@ -4,7 +4,7 @@
 </script>
 
 <script lang="ts">
-  import { filterName } from 'dbgate-datalib';
+  import { filterName } from 'dbgate-tools';
 
   import { currentArchive } from '../stores';
 

packages/web/src/appobj/ConnectionAppObject.svelte

@@ -8,7 +8,7 @@
   import AppObjectCore from './AppObjectCore.svelte';
   import { currentDatabase, extensions, getCurrentConfig, openedConnections } from '../stores';
   import axiosInstance from '../utility/axiosInstance';
-  import { filterName } from 'dbgate-datalib';
+  import { filterName } from 'dbgate-tools';
   import { showModal } from '../modals/modalTools';
   import ConnectionModal from '../modals/ConnectionModal.svelte';
   import ConfirmModal from '../modals/ConfirmModal.svelte';

packages/web/src/appobj/DatabaseObjectAppObject.svelte

@@ -1,11 +1,13 @@
 <script lang="ts" context="module">
   export const extractKey = ({ schemaName, pureName }) => (schemaName ? `${schemaName}.${pureName}` : pureName);
   export const createMatcher = ({ pureName }) => filter => filterName(filter, pureName);
+  const electron = getElectron();
 
   const icons = {
     tables: 'img table',
     collections: 'img collection',
     views: 'img view',
+    matviews: 'img view',
     procedures: 'img procedure',
     functions: 'img function',
   };
@@ -14,6 +16,7 @@
     tables: 'TableDataTab',
     collections: 'CollectionDataTab',
     views: 'ViewDataTab',
+    matviews: 'ViewDataTab',
   };
 
   const menus = {
@@ -44,6 +47,10 @@
       {
         divider: true,
       },
+      {
+        isQuickExport: true,
+        functionName: 'tableReader',
+      },
       {
         label: 'Export',
         isExport: true,
@@ -106,6 +113,10 @@
       {
         divider: true,
       },
+      {
+        isQuickExport: true,
+        functionName: 'tableReader',
+      },
       {
         label: 'Export',
         isExport: true,
@@ -146,6 +157,67 @@
         },
       },
     ],
+    matviews: [
+      {
+        label: 'Open data',
+        tab: 'ViewDataTab',
+        forceNewTab: true,
+      },
+      {
+        label: 'Open structure',
+        tab: 'TableStructureTab',
+      },
+      {
+        label: 'Query designer',
+        isQueryDesigner: true,
+      },
+      {
+        divider: true,
+      },
+      {
+        isQuickExport: true,
+        functionName: 'tableReader',
+      },
+      {
+        label: 'Export',
+        isExport: true,
+      },
+      {
+        label: 'Open in free table editor',
+        isOpenFreeTable: true,
+      },
+      {
+        label: 'Open active chart',
+        isActiveChart: true,
+      },
+      {
+        divider: true,
+      },
+      {
+        label: 'SQL: CREATE MATERIALIZED VIEW',
+        scriptTemplate: 'CREATE OBJECT',
+      },
+      {
+        label: 'SQL: CREATE TABLE',
+        scriptTemplate: 'CREATE TABLE',
+      },
+      {
+        label: 'SQL: SELECT',
+        scriptTemplate: 'SELECT',
+      },
+      {
+        label: 'SQL Generator: CREATE MATERIALIZED VIEW',
+        sqlGeneratorProps: {
+          createMatviews: true,
+        },
+      },
+      {
+        label: 'SQL Generator: DROP MATERIALIZED VIEW',
+        sqlGeneratorProps: {
+          dropMatviews: true,
+        },
+      },
+    ],
     procedures: [
       {
         label: 'SQL: CREATE PROCEDURE',
@@ -202,6 +274,10 @@
           },
         },
       },
+      {
+        isQuickExport: true,
+        functionName: 'tableReader',
+      },
       {
         label: 'Export',
         isExport: true,
@@ -252,6 +328,7 @@
       { forceNewTab }
     );
   }
+
 </script>
 
 <script lang="ts">
@@ -259,7 +336,7 @@
   import AppObjectCore from './AppObjectCore.svelte';
   import { currentDatabase, extensions, openedConnections } from '../stores';
   import openNewTab from '../utility/openNewTab';
-  import { filterName } from 'dbgate-datalib';
+  import { filterName } from 'dbgate-tools';
   import { getConnectionInfo } from '../utility/metadataLoaders';
   import fullDisplayName from '../utility/fullDisplayName';
   import ImportExportModal from '../modals/ImportExportModal.svelte';
@@ -267,7 +344,10 @@
   import { findEngineDriver } from 'dbgate-tools';
   import uuidv1 from 'uuid/v1';
   import SqlGeneratorModal from '../modals/SqlGeneratorModal.svelte';
-import getConnectionLabel from '../utility/getConnectionLabel';
+  import getConnectionLabel from '../utility/getConnectionLabel';
+  import getElectron from '../utility/getElectron';
+  import { exportElectronFile } from '../utility/exportElectronFile';
+  import createQuickExportMenu from '../utility/createQuickExportMenu';
 
   export let data;
 
@@ -311,104 +391,129 @@ import getConnectionLabel from '../utility/getConnectionLabel';
 
   function createMenu() {
     const { objectTypeField } = data;
-    return menus[objectTypeField].map(menu => {
-      if (menu.divider) return menu;
-      return {
-        text: menu.label,
-        onClick: async () => {
-          if (menu.isExport) {
-            showModal(ImportExportModal, {
-              initialValues: {
-                sourceStorageType: 'database',
-                sourceConnectionId: data.conid,
-                sourceDatabaseName: data.database,
-                sourceSchemaName: data.schemaName,
-                sourceList: [data.pureName],
-              },
-            });
-          } else if (menu.isOpenFreeTable) {
+    return menus[objectTypeField]
+      .filter(x => x)
+      .map(menu => {
+        if (menu.divider) return menu;
+
+        if (menu.isQuickExport) {
+          return createQuickExportMenu($extensions, fmt => async () => {
             const coninfo = await getConnectionInfo(data);
-            openNewTab({
-              title: data.pureName,
-              icon: 'img free-table',
-              tabComponent: 'FreeTableTab',
-              props: {
-                initialArgs: {
-                  functionName: 'tableReader',
-                  props: {
-                    connection: {
-                      ...coninfo,
-                      database: data.database,
+            exportElectronFile(
+              data.pureName,
+              {
+                functionName: menu.functionName,
+                props: {
+                  connection: {
+                    ..._.omit(coninfo, ['_id', 'displayName']),
+                    ..._.pick(data, ['database']),
+                  },
+                  ..._.pick(data, ['pureName', 'schemaName']),
+                },
+              },
+              fmt
+            );
+          });
+        }
+
+        return {
+          text: menu.label,
+          onClick: async () => {
+            if (menu.isExport) {
+              showModal(ImportExportModal, {
+                initialValues: {
+                  sourceStorageType: 'database',
+                  sourceConnectionId: data.conid,
+                  sourceDatabaseName: data.database,
+                  sourceSchemaName: data.schemaName,
+                  sourceList: [data.pureName],
+                },
+              });
+            } else if (menu.isOpenFreeTable) {
+              const coninfo = await getConnectionInfo(data);
+              openNewTab({
+                title: data.pureName,
+                icon: 'img free-table',
+                tabComponent: 'FreeTableTab',
+                props: {
+                  initialArgs: {
+                    functionName: 'tableReader',
+                    props: {
+                      connection: {
+                        ...coninfo,
+                        database: data.database,
+                      },
+                      schemaName: data.schemaName,
+                      pureName: data.pureName,
                     },
-                    schemaName: data.schemaName,
-                    pureName: data.pureName,
                   },
                 },
-              },
-            });
-          } else if (menu.isActiveChart) {
-            const driver = await getDriver();
-            const dmp = driver.createDumper();
-            dmp.put('^select * from %f', data);
-            openNewTab(
-              {
-                title: data.pureName,
-                icon: 'img chart',
-                tabComponent: 'ChartTab',
-                props: {
-                  conid: data.conid,
-                  database: data.database,
+              });
+            } else if (menu.isActiveChart) {
+              const driver = await getDriver();
+              const dmp = driver.createDumper();
+              dmp.put('^select * from %f', data);
+              openNewTab(
+                {
+                  title: data.pureName,
+                  icon: 'img chart',
+                  tabComponent: 'ChartTab',
+                  props: {
+                    conid: data.conid,
+                    database: data.database,
+                  },
                 },
-              },
-              {
-                editor: {
-                  config: { chartType: 'bar' },
-                  sql: dmp.s,
+                {
+                  editor: {
+                    config: { chartType: 'bar' },
+                    sql: dmp.s,
+                  },
+                }
+              );
+            } else if (menu.isQueryDesigner) {
+              openNewTab(
+                {
+                  title: 'Query #',
+                  icon: 'img query-design',
+                  tabComponent: 'QueryDesignTab',
+                  props: {
+                    conid: data.conid,
+                    database: data.database,
+                  },
                 },
-              }
-            );
-          } else if (menu.isQueryDesigner) {
-            openNewTab(
-              {
-                title: 'Query #',
-                icon: 'img query-design',
-                tabComponent: 'QueryDesignTab',
-                props: {
-                  conid: data.conid,
-                  database: data.database,
-                },
-              },
-              {
-                editor: {
-                  tables: [
-                    {
-                      ...data,
-                      designerId: uuidv1(),
-                      left: 50,
-                      top: 50,
-                    },
-                  ],
-                },
-              }
-            );
-          } else if (menu.sqlGeneratorProps) {
-            showModal(SqlGeneratorModal, {
-              initialObjects: [data],
-              initialConfig: menu.sqlGeneratorProps,
-              conid: data.conid,
-              database: data.database,
-            });
-          } else {
-            openDatabaseObjectDetail(menu.tab, menu.scriptTemplate, data, menu.forceNewTab, menu.initialData);
-          }
-        },
-      };
-    });
+                {
+                  editor: {
+                    tables: [
+                      {
+                        ...data,
+                        designerId: uuidv1(),
+                        left: 50,
+                        top: 50,
+                      },
+                    ],
+                  },
+                }
+              );
+            } else if (menu.sqlGeneratorProps) {
+              showModal(SqlGeneratorModal, {
+                initialObjects: [data],
+                initialConfig: menu.sqlGeneratorProps,
+                conid: data.conid,
+                database: data.database,
+              });
+            } else {
+              openDatabaseObjectDetail(menu.tab, menu.scriptTemplate, data, menu.forceNewTab, menu.initialData);
+            }
+          },
+        };
+      });
   }
+
 </script>
 
 <AppObjectCore
   {...$$restProps}
+  module={$$props.module}
   {data}
   title={data.schemaName ? `${data.schemaName}.${data.pureName}` : data.pureName}
   icon={icons[data.objectTypeField]}

packages/web/src/appobj/MacroAppObject.svelte

@@ -4,7 +4,7 @@
 </script>
 
 <script lang="ts">
-  import { filterName } from 'dbgate-datalib';
+  import { filterName } from 'dbgate-tools';
   import { getContext } from 'svelte';
 
   import AppObjectCore from './AppObjectCore.svelte';

packages/web/src/commands/CommandPalette.svelte

@@ -15,7 +15,7 @@
 </script>
 
 <script>
-  import { filterName } from 'dbgate-datalib';
+  import { filterName } from 'dbgate-tools';
 
   import _ from 'lodash';
   import { onMount } from 'svelte';

packages/web/src/datagrid/CollectionDataGridCore.svelte

@@ -112,6 +112,7 @@
 
     return response.data.count;
   }
+
 </script>
 
 <script lang="ts">
@@ -125,10 +126,14 @@
   import ErrorMessageModal from '../modals/ErrorMessageModal.svelte';
   import ImportExportModal from '../modals/ImportExportModal.svelte';
   import { showModal } from '../modals/modalTools';
+  import { extensions } from '../stores';
 
   import axiosInstance from '../utility/axiosInstance';
   import { registerMenu } from '../utility/contextMenu';
   import createActivator, { getActiveComponent } from '../utility/createActivator';
+  import createQuickExportMenu from '../utility/createQuickExportMenu';
+  import { exportElectronFile } from '../utility/exportElectronFile';
+  import { getConnectionInfo } from '../utility/metadataLoaders';
   import openNewTab from '../utility/openNewTab';
   import ChangeSetGrider from './ChangeSetGrider';
 
@@ -201,8 +206,30 @@
 
   registerMenu(
     { command: 'collectionDataGrid.openQuery', tag: 'export' },
+    {
+      ...createQuickExportMenu($extensions, fmt => async () => {
+        const coninfo = await getConnectionInfo({ conid });
+        exportElectronFile(
+          pureName || 'Data',
+          {
+            functionName: 'queryReader',
+            props: {
+              connection: {
+                ..._.omit(coninfo, ['_id', 'displayName']),
+                database,
+              },
+              sql: getExportQuery(),
+            },
+          },
+          fmt
+        );
+      }),
+      tag: 'export',
+    },
+
     { command: 'collectionDataGrid.export', tag: 'export' }
   );
+
 </script>
 
 <LoadingDataGridCore

packages/web/src/datagrid/ColumnManager.svelte

@@ -1,5 +1,6 @@
 <script lang="ts">
-  import { filterName, GridDisplay } from 'dbgate-datalib';
+  import { GridDisplay } from 'dbgate-datalib';
+  import { filterName } from 'dbgate-tools';
 
   import InlineButton from '../elements/InlineButton.svelte';
   import ManagerInnerContainer from '../elements/ManagerInnerContainer.svelte';
@@ -13,6 +14,7 @@
   export let isJsonView = false;
 
   let filter;
+
 </script>
 
 <SearchBoxWrapper>

packages/web/src/datagrid/DataGridCore.svelte

@@ -171,6 +171,7 @@
     if (_.isPlainObject(value) || _.isArray(value)) return JSON.stringify(value);
     return value;
   }
+
 </script>
 
 <script lang="ts">
@@ -581,6 +582,7 @@
     if (event.target.closest('.collapseButtonMarker')) return;
     if (event.target.closest('input')) return;
 
+    shiftDragStartCell = null;
     // event.target.closest('table').focus();
     event.preventDefault();
     if (domFocusField) domFocusField.focus();
@@ -663,23 +665,42 @@
   }
 
   function handleGridWheel(event) {
-    let newFirstVisibleRowScrollIndex = firstVisibleRowScrollIndex;
-    if (event.deltaY > 0) {
-      newFirstVisibleRowScrollIndex += wheelRowCount;
-    }
-    if (event.deltaY < 0) {
-      newFirstVisibleRowScrollIndex -= wheelRowCount;
-    }
-    let rowCount = grider.rowCount;
-    if (newFirstVisibleRowScrollIndex + visibleRowCountLowerBound > rowCount) {
-      newFirstVisibleRowScrollIndex = rowCount - visibleRowCountLowerBound + 1;
-    }
-    if (newFirstVisibleRowScrollIndex < 0) {
-      newFirstVisibleRowScrollIndex = 0;
-    }
-    firstVisibleRowScrollIndex = newFirstVisibleRowScrollIndex;
+    if (event.shiftKey) {
+      let newFirstVisibleColumnScrollIndex = firstVisibleColumnScrollIndex;
+      if (event.deltaY > 0) {
+        newFirstVisibleColumnScrollIndex++;
+      }
+      if (event.deltaY < 0) {
+        newFirstVisibleColumnScrollIndex--;
+      }
+      if (newFirstVisibleColumnScrollIndex > maxScrollColumn) {
+        newFirstVisibleColumnScrollIndex = maxScrollColumn;
+      }
+      if (newFirstVisibleColumnScrollIndex < 0) {
+        newFirstVisibleColumnScrollIndex = 0;
+      }
+      firstVisibleColumnScrollIndex = newFirstVisibleColumnScrollIndex;
 
-    domVerticalScroll.scroll(newFirstVisibleRowScrollIndex);
+      domHorizontalScroll.scroll(newFirstVisibleColumnScrollIndex);
+    } else {
+      let newFirstVisibleRowScrollIndex = firstVisibleRowScrollIndex;
+      if (event.deltaY > 0) {
+        newFirstVisibleRowScrollIndex += wheelRowCount;
+      }
+      if (event.deltaY < 0) {
+        newFirstVisibleRowScrollIndex -= wheelRowCount;
+      }
+      let rowCount = grider.rowCount;
+      if (newFirstVisibleRowScrollIndex + visibleRowCountLowerBound > rowCount) {
+        newFirstVisibleRowScrollIndex = rowCount - visibleRowCountLowerBound + 1;
+      }
+      if (newFirstVisibleRowScrollIndex < 0) {
+        newFirstVisibleRowScrollIndex = 0;
+      }
+      firstVisibleRowScrollIndex = newFirstVisibleRowScrollIndex;
+
+      domVerticalScroll.scroll(newFirstVisibleRowScrollIndex);
+    }
   }
 
   function getSelectedRowIndexes() {
@@ -735,7 +756,7 @@
 
     handleCursorMove(event);
 
-    if (event.shiftKey) {
+    if (event.shiftKey && event.keyCode != keycodes.shift) {
       selectedCells = getCellRange(shiftDragStartCell || currentCell, currentCell);
     }
   }
@@ -958,6 +979,7 @@
   );
 
   const menu = getContextMenu();
+
 </script>
 
 {#if !display || (!isDynamicStructure && (!columns || columns.length == 0))}
@@ -1164,4 +1186,5 @@
     right: 40px;
     bottom: 20px;
   }
+
 </style>

packages/web/src/datagrid/JslDataGridCore.svelte

@@ -39,6 +39,7 @@
     });
     return response.data.rowCount;
   }
+
 </script>
 
 <script lang="ts">
@@ -46,10 +47,13 @@
   import registerCommand from '../commands/registerCommand';
   import ImportExportModal from '../modals/ImportExportModal.svelte';
   import { showModal } from '../modals/modalTools';
+  import { extensions } from '../stores';
 
   import axiosInstance from '../utility/axiosInstance';
   import { registerMenu } from '../utility/contextMenu';
   import createActivator, { getActiveComponent } from '../utility/createActivator';
+  import createQuickExportMenu from '../utility/createQuickExportMenu';
+  import { exportElectronFile } from '../utility/exportElectronFile';
   import socket from '../utility/socket';
   import useEffect from '../utility/useEffect';
 
@@ -103,7 +107,40 @@
     showModal(ImportExportModal, { initialValues });
   }
 
-  registerMenu({ command: 'jslTableGrid.export', tag: 'export' });
+  registerMenu(
+    {
+      ...createQuickExportMenu($extensions, fmt => async () => {
+        const archiveMatch = jslid.match(/^archive:\/\/([^/]+)\/(.*)$/);
+        if (archiveMatch) {
+          exportElectronFile(
+            archiveMatch[2],
+            {
+              functionName: 'archiveReader',
+              props: {
+                folderName: archiveMatch[1],
+                fileName: archiveMatch[2],
+              },
+            },
+            fmt
+          );
+        } else {
+          exportElectronFile(
+            'Query',
+            {
+              functionName: 'jslDataReader',
+              props: {
+                jslid,
+              },
+            },
+            fmt
+          );
+        }
+      }),
+      tag: 'export',
+    },
+    { command: 'jslTableGrid.export', tag: 'export' }
+  );
+
 </script>
 
 <LoadingDataGridCore

packages/web/src/datagrid/ReferenceManager.svelte

@@ -1,5 +1,6 @@
 <script lang="ts">
-  import { filterName, GridDisplay } from 'dbgate-datalib';
+  import { GridDisplay } from 'dbgate-datalib';
+  import { filterName } from 'dbgate-tools';
   import { createEventDispatcher } from 'svelte';
 
   import InlineButton from '../elements/InlineButton.svelte';
@@ -12,7 +13,7 @@
 
   export let managerSize;
   export let display: GridDisplay;
-  export let onReferenceClick = (ref) => {};
+  export let onReferenceClick = ref => {};
 
   const dispatch = createEventDispatcher();
 
@@ -20,6 +21,7 @@
 
   $: foreignKeys = display?.baseTable?.foreignKeys || [];
   $: dependencies = display?.baseTable?.dependencies || [];
+
 </script>
 
 <SearchBoxWrapper>
@@ -86,4 +88,5 @@
   .link:hover {
     text-decoration: underline;
   }
+
 </style>

packages/web/src/datagrid/SqlDataGridCore.svelte

@@ -68,20 +68,23 @@
 
     return parseInt(response.data.rows[0].count);
   }
+
 </script>
 
 <script lang="ts">
-  import { changeSetToSql, createChangeSet } from 'dbgate-datalib';
-  import { scriptToSql } from 'dbgate-sqltree';
+  import _ from 'lodash';
+
   import registerCommand from '../commands/registerCommand';
-  import ConfirmSqlModal from '../modals/ConfirmSqlModal.svelte';
-  import ErrorMessageModal from '../modals/ErrorMessageModal.svelte';
   import ImportExportModal from '../modals/ImportExportModal.svelte';
   import { showModal } from '../modals/modalTools';
+  import { extensions } from '../stores';
 
   import axiosInstance from '../utility/axiosInstance';
   import { registerMenu } from '../utility/contextMenu';
   import createActivator, { getActiveComponent } from '../utility/createActivator';
+  import createQuickExportMenu from '../utility/createQuickExportMenu';
+  import { exportElectronFile } from '../utility/exportElectronFile';
+  import { getConnectionInfo } from '../utility/metadataLoaders';
   import openNewTab from '../utility/openNewTab';
   import ChangeSetGrider from './ChangeSetGrider';
 
@@ -173,8 +176,29 @@
   registerMenu(
     { command: 'sqlDataGrid.openActiveChart', tag: 'chart' },
     { command: 'sqlDataGrid.openQuery', tag: 'export' },
+    {
+      ...createQuickExportMenu($extensions, fmt => async () => {
+        const coninfo = await getConnectionInfo({ conid });
+        exportElectronFile(
+          pureName || 'Data',
+          {
+            functionName: 'queryReader',
+            props: {
+              connection: {
+                ..._.omit(coninfo, ['_id', 'displayName']),
+                database,
+              },
+              sql: display.getExportQuery(),
+            },
+          },
+          fmt
+        );
+      }),
+      tag: 'export',
+    },
     { command: 'sqlDataGrid.export', tag: 'export' }
   );
+
 </script>
 
 <LoadingDataGridCore

packages/web/src/elements/ObjectListControl.svelte

@@ -6,6 +6,7 @@
   export let collection;
   export let columns;
   export let showIfEmpty = false;
+
 </script>
 
 {#if collection?.length > 0 || showIfEmpty}
@@ -55,24 +56,24 @@
       </TableControl>
     </div>
   </div>
-
-  <style>
-    .wrapper {
-      margin-bottom: 20px;
-    }
-
-    .header {
-      background-color: var(--theme-bg-1);
-      padding: 5px;
-    }
-
-    .title {
-      font-weight: bold;
-      margin-left: 5px;
-    }
-
-    .body {
-      margin: 20px;
-    }
-  </style>
 {/if}
+
+<style>
+  .wrapper {
+    margin-bottom: 20px;
+  }
+
+  .header {
+    background-color: var(--theme-bg-1);
+    padding: 5px;
+  }
+
+  .title {
+    font-weight: bold;
+    margin-left: 5px;
+  }
+
+  .body {
+    margin: 20px;
+  }
+</style>

packages/web/src/formview/FormView.svelte

@@ -152,9 +152,12 @@
   function isDataCell(cell) {
     return cell[1] % 2 == 1;
   }
+
 </script>
 
 <script lang="ts">
+  import { filterName } from 'dbgate-tools';
+
   import _ from 'lodash';
 
   import { getContext } from 'svelte';
@@ -210,7 +213,10 @@
 
   $: rowCount = Math.floor((wrapperHeight - 22) / (rowHeight + 2));
 
-  $: columnChunks = _.chunk(formDisplay.columns, rowCount) as any[][];
+  $: columnChunks = _.chunk(
+    formDisplay.columns.filter(x => filterName(formDisplay.config.formColumnFilterText, x.columnName)),
+    rowCount
+  ) as any[][];
 
   $: rowCountInfo = getRowCountInfo(rowCountBefore, allRowCount);
 
@@ -380,10 +386,38 @@
         (event.keyCode >= keycodes.n0 && event.keyCode <= keycodes.n9) ||
         event.keyCode == keycodes.dash)
     ) {
-      // @ts-ignore
-      event.preventDefault();
-      dispatchInsplaceEditor({ type: 'show', text: event.key, cell: currentCell });
-      // console.log('event', event.nativeEvent);
+      if (currentCell[1] % 2 == 0) {
+        setConfig(x => ({
+          ...x,
+          // @ts-ignore
+          formColumnFilterText: (x.formColumnFilterText || '') + event.key,
+        }));
+      } else {
+        // @ts-ignore
+        event.preventDefault();
+        dispatchInsplaceEditor({ type: 'show', text: event.key, cell: currentCell });
+      }
+    }
+
+    if (event.keyCode == keycodes.escape) {
+      setConfig(x => ({
+        ...x,
+        formColumnFilterText: '',
+      }));
+    }
+
+    if (event.keyCode == keycodes.numPadAdd) {
+      const col = getCellColumn(currentCell);
+      if (col.foreignKey) {
+        formDisplay.toggleExpandedColumn(col.uniqueName, true);
+      }
+    }
+
+    if (event.keyCode == keycodes.numPadSub) {
+      const col = getCellColumn(currentCell);
+      if (col.foreignKey) {
+        formDisplay.toggleExpandedColumn(col.uniqueName, false);
+      }
     }
 
     if (event.keyCode == keycodes.f2) {
@@ -441,6 +475,7 @@
   function handleSetFormView(rowData, column) {
     openReferenceForm(rowData, column, conid, database);
   }
+
 </script>
 
 <div class="outer">
@@ -594,4 +629,5 @@
     right: 40px;
     bottom: 20px;
   }
+
 </style>

packages/web/src/formview/FormViewFilters.svelte

@@ -2,19 +2,46 @@
   import _ from 'lodash';
 
   import ManagerInnerContainer from '../elements/ManagerInnerContainer.svelte';
+import keycodes from '../utility/keycodes';
   import FormViewFilterColumn from './FormViewFilterColumn.svelte';
   import PrimaryKeyFilterEditor from './PrimaryKeyFilterEditor.svelte';
 
   export let managerSize;
   export let formDisplay;
+  export let setConfig;
 
   $: baseTable = formDisplay?.baseTable;
   $: formFilterColumns = formDisplay?.config?.formFilterColumns;
   $: filters = formDisplay?.config?.filters;
 
   $: allFilterNames = _.union(_.keys(filters || {}), formFilterColumns || []);
+
 </script>
 
+<div class="m-1">
+  <div>Column filter</div>
+  <div class="flex">
+    <input
+      type="text"
+      value={formDisplay?.config?.formColumnFilterText || ''}
+      on:keydown={e => {
+        if (e.keyCode == keycodes.escape) {
+          setConfig(x => ({
+            ...x,
+            formColumnFilterText: '',
+          }));
+        }
+      }}
+      on:input={e =>
+        setConfig(x => ({
+          ...x,
+          // @ts-ignore
+          formColumnFilterText: e.target.value,
+        }))}
+    />
+  </div>
+</div>
+
 {#if baseTable?.primaryKey}
   <ManagerInnerContainer width={managerSize}>
     {#each baseTable.primaryKey.columns as col}

packages/web/src/formview/SqlFormView.svelte

@@ -20,14 +20,6 @@
 </script>
 
 <script lang="ts">
-  import { changeSetToSql, createChangeSet } from 'dbgate-datalib';
-  import { scriptToSql } from 'dbgate-sqltree';
-  import ConfirmSqlModal from '../modals/ConfirmSqlModal.svelte';
-
-  import ErrorMessageModal from '../modals/ErrorMessageModal.svelte';
-
-  import { showModal } from '../modals/modalTools';
-
   import axiosInstance from '../utility/axiosInstance';
   import ChangeSetFormer from './ChangeSetFormer';
   import FormView from './FormView.svelte';

packages/web/src/icons/FontIcon.svelte

@@ -40,6 +40,7 @@
     'icon home': 'mdi mdi-home',
     'icon query-design': 'mdi mdi-vector-polyline-edit',
     'icon form': 'mdi mdi-form-select',
+    'icon history': 'mdi mdi-history',
 
     'icon edit': 'mdi mdi-pencil',
     'icon delete': 'mdi mdi-delete',
@@ -66,6 +67,7 @@
     'icon chevron-left': 'mdi mdi-chevron-left',
     'icon chevron-right': 'mdi mdi-chevron-right',
     'icon chevron-up': 'mdi mdi-chevron-up',
+    'icon menu-right': 'mdi mdi-menu-right',
     'icon plugin': 'mdi mdi-toy-brick',
     'icon menu': 'mdi mdi-menu',
 
@@ -75,6 +77,7 @@
     'img error': 'mdi mdi-close-circle color-icon-red',
     'img error-inv': 'mdi mdi-close-circle color-icon-inv-red',
     'img warn': 'mdi mdi-alert color-icon-gold',
+    'img info': 'mdi mdi-information color-icon-blue',
     // 'img statusbar-ok': 'mdi mdi-check-circle color-on-statusbar-green',
 
     'img archive': 'mdi mdi-table color-icon-gold',

packages/web/src/impexp/FormTablesSelect.svelte

@@ -5,6 +5,7 @@
 
   import { getFormContext } from '../forms/FormProviderCore.svelte';
   import FormSelectField from '../forms/FormSelectField.svelte';
+import { getObjectTypeFieldLabel } from '../utility/common';
   import { useDatabaseInfo, useDatabaseList } from '../utility/metadataLoaders';
 
   export let conidName;
@@ -15,19 +16,25 @@
   const { values, setFieldValue } = getFormContext();
   $: dbinfo = useDatabaseInfo({ conid: $values[conidName], database: $values[databaseName] });
 
-  $: tablesOptions = _.compact([...($dbinfo?.tables || []), ...($dbinfo?.views || []), ...($dbinfo?.collections || [])])
+  $: tablesOptions = _.compact([
+    ...($dbinfo?.tables || []),
+    ...($dbinfo?.views || []),
+    ...($dbinfo?.matviews || []),
+    ...($dbinfo?.collections || []),
+  ])
     .filter(x => !$values[schemaName] || x.schemaName == $values[schemaName])
     .map(x => ({
       value: x.pureName,
       label: x.pureName,
     }));
+
 </script>
 
 <div class="wrapper">
   <FormSelectField {...$$restProps} {name} options={tablesOptions} isMulti templateProps={{ noMargin: true }} />
 
   <div>
-    {#each ['tables', 'views', 'collections'] as field}
+    {#each ['tables', 'views', 'matviews', 'collections'] as field}
       {#if $dbinfo && $dbinfo[field]?.length > 0}
         <FormStyledButton
           type="button"
@@ -49,4 +56,5 @@
   .wrapper {
     margin: var(--dim-large-form-margin);
   }
+
 </style>