v5.5.3-beta.3

This reverts commit 0238e6a7f1.

.github/workflows/build-app-beta.yaml

@@ -30,6 +30,9 @@ jobs:
       - name: yarn adjustPackageJson
         run: |
           yarn adjustPackageJson
+      - name: setUpdaterChannel beta
+        run: |
+          node setUpdaterChannel beta
       - name: yarn set timeout
         run: |
           yarn config set network-timeout 100000
@@ -99,9 +102,13 @@ jobs:
           mv app/dist/*.deb artifacts/ || true
           mv app/dist/*.snap artifacts/ || true
           mv app/dist/*.dmg artifacts/ || true
+          mv app/dist/*.blockmap artifacts/ || true
+
+          mv app/dist/*.yml artifacts/ || true
+          rm artifacts/builder-debug.yml
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.os }}
           path: artifacts

.github/workflows/build-app-pro-beta.yaml

@@ -1,9 +1,9 @@
-name: Electron app BETA PREMIUM
+name: Electron app PREMIUM BETA
 
 on:
   push:
     tags:
-      - 'v[0-9]+.[0-9]+.[0-9]+-pro.[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+-premium-beta.[0-9]+'
 
 jobs:
   build:
@@ -12,9 +12,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04]
-        # vos: [windows-2022, ubuntu-22.04]
-        # os: [macos-12, windows-2022, ubuntu-22.04]
+        # os: [windows-2022]
+        # os: [ubuntu-22.04]
+        # os: [windows-2022, ubuntu-22.04]
+        os: [macos-12, windows-2022, ubuntu-22.04]
         # os: [macOS-10.15]
 
     steps:
@@ -54,11 +55,16 @@ jobs:
           cd ..
           cd dbgate-merged
           yarn adjustPackageJson
-      - name: yarn adjustPackageJsonPremium
+      - name: adjustPackageJsonPremium
         run: |
           cd ..
           cd dbgate-merged
           node adjustPackageJsonPremium
+      - name: setUpdaterChannel premium-beta
+        run: |
+          cd ..
+          cd dbgate-merged
+          node setUpdaterChannel premium-beta
       - name: yarn set timeout
         run: |
           cd ..
@@ -126,9 +132,13 @@ jobs:
           mv ../dbgate-merged/app/dist/*.deb artifacts/ || true
           mv ../dbgate-merged/app/dist/*.snap artifacts/ || true
           mv ../dbgate-merged/app/dist/*.dmg artifacts/ || true
+          mv ../dbgate-merged/app/dist/*.blockmap artifacts/ || true
+
+          mv ../dbgate-merged/app/dist/*.yml artifacts/ || true
+          rm artifacts/builder-debug.yml
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.os }}
           path: artifacts

.github/workflows/build-app-pro.yaml

@@ -0,0 +1,155 @@
+name: Electron app PREMIUM
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+      # - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
+
+    # branches:
+    #   - production
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # os: [ubuntu-22.04, windows-2016]
+        os: [macos-12, windows-2022, ubuntu-22.04]
+
+    steps:
+      - name: Context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 18.x
+
+      - name: Checkout dbgate/dbgate-pro
+        uses: actions/checkout@v2
+        with:
+          repository: dbgate/dbgate-pro
+          token: ${{ secrets.GH_TOKEN }}
+          path: dbgate-pro
+
+      - name: Merge dbgate/dbgate-pro
+        run: |
+          mkdir ../dbgate-pro
+          mv dbgate-pro/* ../dbgate-pro/
+          cd ..
+          mkdir dbgate-merged
+          cd dbgate-pro
+          cd sync
+          yarn
+          node sync.js --nowatch
+          cd ..
+
+      - name: yarn adjustPackageJson
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn adjustPackageJson
+      - name: yarn adjustPackageJsonPremium
+        run: |
+          cd ..
+          cd dbgate-merged
+          node adjustPackageJsonPremium
+      - name: setUpdaterChannel premium
+        run: |
+          cd ..
+          cd dbgate-merged
+          node setUpdaterChannel premium
+      - name: yarn set timeout
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn config set network-timeout 100000
+      - name: yarn install
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn install
+      - name: setCurrentVersion
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn setCurrentVersion
+      - name: printSecrets
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn printSecrets
+        env:
+          GIST_UPLOAD_SECRET : ${{secrets.GIST_UPLOAD_SECRET}}
+      - name: fillNativeModulesElectron
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn fillNativeModulesElectron
+      - name: fillPackagedPlugins
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn fillPackagedPlugins
+      - name: Publish
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn run build:app
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
+
+          WIN_CSC_LINK: ${{ secrets.WINCERT_2025 }}
+          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_2025_PASSWORD }}
+          # WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
+          # WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
+
+          CSC_LINK: ${{ secrets.APPLECERT_CERTIFICATE }}
+          CSC_KEY_PASSWORD: ${{ secrets.APPLECERT_PASSWORD }}
+
+          APPLE_ID: ${{ secrets.APPLE_ID }}
+          APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
+          APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
+
+      - name: Copy artifacts
+        run: |
+          mkdir artifacts          
+
+          cp ../dbgate-merged/app/dist/*x86*.AppImage artifacts/dbgate-premium-latest.AppImage || true
+          cp ../dbgate-merged/app/dist/*.exe artifacts/dbgate-premium-latest.exe || true
+          cp ../dbgate-merged/app/dist/*win_x64.zip artifacts/dbgate-premium-windows-latest.zip || true
+          cp ../dbgate-merged/app/dist/*win_arm64.zip artifacts/dbgate-premium-windows-latest-arm64.zip || true
+          cp ../dbgate-merged/app/dist/*-mac_universal.dmg artifacts/dbgate-premium-latest.dmg || true
+          cp ../dbgate-merged/app/dist/*-mac_x64.dmg artifacts/dbgate-premium-latest-x64.dmg || true
+
+          mv ../dbgate-merged/app/dist/*.exe artifacts/ || true
+          mv ../dbgate-merged/app/dist/*.zip artifacts/ || true
+          mv ../dbgate-merged/app/dist/*.tar.gz artifacts/ || true
+          mv ../dbgate-merged/app/dist/*.AppImage artifacts/ || true
+          mv ../dbgate-merged/app/dist/*.deb artifacts/ || true
+          mv ../dbgate-merged/app/dist/*.dmg artifacts/ || true
+          mv ../dbgate-merged/app/dist/*.blockmap artifacts/ || true
+
+          mv ../dbgate-merged/app/dist/*.yml artifacts/ || true
+          rm artifacts/builder-debug.yml
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.os }}
+          path: artifacts
+
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          files: 'artifacts/**'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/build-app.yaml

@@ -109,6 +109,10 @@ jobs:
           mv app/dist/*.deb artifacts/ || true
           mv app/dist/*.dmg artifacts/ || true
           mv app/dist/*.snap artifacts/dbgate-latest.snap || true
+          mv app/dist/*.blockmap artifacts/ || true
+
+          mv app/dist/*.yml artifacts/ || true
+          rm artifacts/builder-debug.yml
 
       # - name: Copy artifacts Linux, MacOs
       #   if: matrix.os != 'windows-2016'
@@ -134,24 +138,13 @@ jobs:
 
       #     mv app/dist/latest.yml artifacts/latest.yml || true
 
-      - name: Copy latest.yml (windows)
+      - name: Copy PAD file
         if: matrix.os == 'windows-2022'
         run: |
-          mv app/dist/latest.yml artifacts/latest.yml || true
           mv app/dist/dbgate-pad.xml artifacts/ || true
 
-      - name: Copy latest-linux.yml
-        if: matrix.os == 'ubuntu-22.04'
-        run: |
-          mv app/dist/latest-linux.yml artifacts/latest-linux.yml || true
-
-      - name: Copy latest-mac.yml
-        if: matrix.os == 'macos-12'
-        run: |
-          mv app/dist/latest-mac.yml artifacts/latest-mac.yml || true
-
       - name: Upload artifacts
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.os }}
           path: artifacts

.github/workflows/build-docker-pro.yaml

@@ -0,0 +1,104 @@
+name: Docker image PREMIUM
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+-premium-beta.[0-9]+'
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ubuntu-22.04]
+
+    steps:
+      - name: Context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            dbgate/dbgate-premium
+          flavor: |
+            latest=false
+          tags: |
+            type=raw,value=beta,enable=${{ contains(github.ref_name, '-docker.') || contains(github.ref_name, '-beta.') }}
+
+            type=match,pattern=\d+.\d+.\d+,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
+            type=raw,value=latest,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
+
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 18.x
+
+      - name: Checkout dbgate/dbgate-pro
+        uses: actions/checkout@v2
+        with:
+          repository: dbgate/dbgate-pro
+          token: ${{ secrets.GH_TOKEN }}
+          path: dbgate-pro
+
+      - name: Merge dbgate/dbgate-pro
+        run: |
+          mkdir ../dbgate-pro
+          mv dbgate-pro/* ../dbgate-pro/
+          cd ..
+          mkdir dbgate-merged
+          cd dbgate-pro
+          cd sync
+          yarn
+          node sync.js --nowatch
+          cd ..
+
+      - name: yarn install
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn install
+
+      - name: setCurrentVersion
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn setCurrentVersion
+          
+      - name: printSecrets
+        run: |
+          cd ..
+          cd dbgate-merged
+            yarn printSecrets
+        env:
+          GIST_UPLOAD_SECRET : ${{secrets.GIST_UPLOAD_SECRET}}
+      - name: Prepare docker image
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn run prepare:docker
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          push: true
+          context: ../dbgate-merged/docker
+          tags: ${{ steps.meta.outputs.tags }}
+          platforms: linux/amd64,linux/arm64

.github/workflows/build-docker.yaml

@@ -5,7 +5,6 @@ on:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+'
       - 'v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+'
-      - 'v[0-9]+.[0-9]+.[0-9]+-docker.[0-9]+'
 
 jobs:
   build:

.github/workflows/build-npm.yaml

@@ -154,3 +154,8 @@ jobs:
         working-directory: plugins/dbgate-plugin-oracle
         run: |
           npm publish
+
+      - name: Publish dbgate-plugin-clickhouse
+        working-directory: plugins/dbgate-plugin-clickhouse
+        run: |
+          npm publish

.github/workflows/run-tests.yaml

@@ -4,6 +4,7 @@ on:
     branches:
       - master
       - develop
+      - 'feature/**'
 
 jobs:
   test-runner:
@@ -77,6 +78,11 @@ jobs:
           ACCEPT_EULA: Y
           SA_PASSWORD: Pwd2020Db
           MSSQL_PID: Express
-          
+
+      clickhouse:
+        image: bitnami/clickhouse:24.8.4
+        env:
+          CLICKHOUSE_ADMIN_PASSWORD: Pwd2020Db
+
       # cockroachdb:
       #   image: cockroachdb/cockroach

.gitignore

@@ -31,4 +31,6 @@ yarn-error.log*
 app/src/nativeModulesContent.js
 packages/api/src/nativeModulesContent.js
 packages/api/src/packagedPluginsContent.js
-.VSCodeCounter
+.VSCodeCounter
+
+packages/web/public/*.html

CHANGELOG.md

@@ -8,6 +8,68 @@ Builds:
  - linux - application for linux
  - win - application for Windows
 
+### 5.5.2
+- FIXED: MySQL, PostgreSQL readonly conections #900
+
+### 5.5.1
+- ADDED: Clickhouse support (#532)
+- ADDED: MySQL - specify table engine, show table engine in table list
+- FIXED: Hidden primary key name in PK editor for DB engines with anonymous PK (MySQL)
+- CHANGED: Import/export dialog is now tacub instead of modal
+- ADDED: Saving import/export job
+- REMOVED: Ability to reopen export/import wizard from generated script. This was a bit hack, now you could save import/export job instead
+- ADDED: Autodetect CSV delimited
+- FIXED: Import CSV files with spaces around quotes
+- ADDED: JSON file import
+- ADDED: JSON export can export objects with ID field used as object key
+- ADDED: JSON and JSON lines imports supports importing from web URL
+- FIXED: Editing imported URL in job editor
+- ADDED: Quick export from table data grid (#892)
+- CHANGED: Create table workflow is reworked, you can specify schema and table name in table editor
+- FIXED: After saving new table, table editor is reset to empty state
+- ADDED: (PostgreSQL, SQL Server) - ability to filter objects by schema
+- ADDED: (PostgreSQL, SQL Server) - Use separate schemas option - for databases with lot of schemas, only selected schema is loaded
+- FIXED: Internal refactor of drivers, client objects are not more messed up with auxiliary fields
+- ADDED: Copy connection error to clipboard after clicking on error icon
+- FIXED: (MySQL) Fixed importing SQL dump exported from mysqldump (#702)
+- FIXED: (PostgreSQL) Fixed filtering JSONB fields (#889)
+- FIXED: OIDC authentication not working anymore (#891)
+- ADDED: Added tests for import from CSV and JSON
+- FIXED: multiple shortcuts handling #898
+- ADDED: (Premium) MS Entra authentization for Azure SQL databases
+
+### 5.4.4
+- CHANGED: Improved autoupdate, notification is now in app
+- CHANGED: Default behaviour of autoupdate, new version is downloaded after click of "Download" button
+- ADDED: Ability to configure autoupdate (check only, check+download, don't check)
+- ADDED: Option to run check for new version manually
+- FIXED: Fixed autoupgrade channel for premium edition
+- FIXED: Fixes following issues: #886, #865, #782, #375
+
+### 5.4.2
+- FIXED: DbGate now works correctly with Oracle 10g
+- FIXED: Fixed update channel for premium edition
+
+### 5.4.1
+- FIXED: Broken older plugins #881
+- ADDED: Premium edition - "Start trial" button
+
+### 5.4.0
+- ADDED: Support for CosmosDB (Premium only)
+- ADDED: Administration UI (Premium only)
+- ADDED: New application icon
+- ADDED: MongoDB type support in data editing
+- ADDED: MongoDB - posibility to remove field
+- ADDED: Oracle - posibility to connect via SID
+- FIXED: Many improvements in MongoDB filtering
+- FIXED: Switch to form and back to table rows missing #343
+- ADDED: Posibility to deactivate MongoDB Profiler #745
+- ADDED: Ability to use Oracle thick driver - neccessary for connecting older Oracle servers #843
+- FIXED: Connection permissions configuration is broken #860
+- ADDED: ssh key file authentication option missing #876
+- ADDED: Ability to reset layout #878
+- FIXED: Script with escaped backslash causes erro #880
+
 ### 5.3.4
 - FIXED: On blank system does not start (window does not appear) #862
 - FIXED: Missing Execute, Export bar #861

README.md

@@ -29,6 +29,8 @@ DbGate is licensed under GPL-3.0 license and is free to use for any purpose.
 * Amazon Redshift
 * CockroachDB
 * MariaDB
+* CosmosDB (Premium)
+* ClickHouse
 
 <!-- Learn more about DbGate features at the [DbGate website](https://dbgate.org/), or try our online [demo application](https://demo.dbgate.org) -->
 
@@ -68,8 +70,8 @@ DbGate is licensed under GPL-3.0 license and is free to use for any purpose.
 * Redis tree view, generate script from keys, run Redis script
 * Runs as application for Windows, Linux and Mac. Or in Docker container on server and in web Browser on client.
 * Import, export from/to CSV, Excel, JSON, NDJSON, XML
-* Free table editor - quick table data editing (cleanup data after import/before export, prototype tables etc.)
 * Archives - backup your data in NDJSON files on local filesystem (or on DbGate server, when using web application)
+* NDJSON data viewer and editor - browse NDJSON data, edit data and structure directly on NDJSON files. Works also for big NDSON files
 * Charts, export chart to HTML page
 * For detailed info, how to run DbGate in docker container, visit [docker hub](https://hub.docker.com/r/dbgate/dbgate)
 * Extensible plugin architecture

app/package.json

@@ -6,7 +6,7 @@
   "description": "Opensource database administration tool",
   "dependencies": {
     "electron-log": "^4.4.1",
-    "electron-updater": "^4.6.1",
+    "electron-updater": "^6.3.4",
     "jsonwebtoken": "^9.0.2",
     "lodash.clonedeepwith": "^4.5.0",
     "patch-package": "^6.4.7"
@@ -28,7 +28,11 @@
       "entitlements": "entitlements.mac.plist",
       "entitlementsInherit": "entitlements.mac.plist",
       "publish": [
-        "github"
+        {
+          "provider": "github",
+          "owner": "dbgate",
+          "repo": "dbgate"
+        }
       ],
       "target": {
         "target": "default",
@@ -56,7 +60,11 @@
       "category": "Development",
       "synopsis": "Database manager for SQL Server, MySQL, PostgreSQL, MongoDB and SQLite",
       "publish": [
-        "github"
+        {
+          "provider": "github",
+          "owner": "dbgate",
+          "repo": "dbgate"
+        }
       ]
     },
     "appImage": {
@@ -91,7 +99,11 @@
       ],
       "icon": "icon.ico",
       "publish": [
-        "github"
+        {
+          "provider": "github",
+          "owner": "dbgate",
+          "repo": "dbgate"
+        }
       ]
     },
     "files": [

app/src/electron.js

@@ -16,8 +16,8 @@ const BrowserWindow = electron.BrowserWindow;
 const path = require('path');
 const url = require('url');
 const mainMenuDefinition = require('./mainMenuDefinition');
-const { isProApp, checkLicense } = require('./proTools');
-let disableAutoUpgrade = false;
+const { isProApp } = require('./proTools');
+const updaterChannel = require('./updaterChannel');
 
 // require('@electron/remote/main').initialize();
 
@@ -28,6 +28,8 @@ let apiLoaded = false;
 let mainModule;
 // let getLogger;
 // let loadLogsContent;
+let appUpdateStatus = '';
+let settingsJson = {};
 
 process.on('uncaughtException', function (error) {
   console.error('uncaughtException', error);
@@ -51,21 +53,11 @@ const isMac = () => os.platform() == 'darwin';
 
 try {
   initialConfig = JSON.parse(fs.readFileSync(configRootPath, { encoding: 'utf-8' }));
-  disableAutoUpgrade = initialConfig['disableAutoUpgrade'] || false;
 } catch (err) {
   console.log('Error loading config-root:', err.message);
   initialConfig = {};
 }
 
-if (process.argv.includes('--disable-auto-upgrade')) {
-  console.log('Disabling auto-upgrade');
-  disableAutoUpgrade = true;
-}
-if (process.argv.includes('--enable-auto-upgrade')) {
-  console.log('Enabling auto-upgrade');
-  disableAutoUpgrade = false;
-}
-
 // Keep a global reference of the window object, if you don't, the window will
 // be closed automatically when the JavaScript object is garbage collected.
 let mainWindow;
@@ -74,6 +66,10 @@ let runCommandOnLoad = null;
 
 log.transports.file.level = 'debug';
 autoUpdater.logger = log;
+if (updaterChannel) {
+  autoUpdater.channel = updaterChannel;
+  autoUpdater.allowPrerelease = updaterChannel.includes('beta');
+}
 // TODO - create settings for this
 // appUpdater.channel = 'beta';
 
@@ -207,6 +203,15 @@ ipcMain.on('app-started', async (event, arg) => {
   if (initialConfig['winIsMaximized']) {
     mainWindow.webContents.send('setIsMaximized', true);
   }
+  if (autoUpdater.isUpdaterActive()) {
+    mainWindow.webContents.send('setAppUpdaterActive');
+  }
+  if (!process.env.DEVMODE) {
+    if (settingsJson['app.autoUpdateMode'] != 'skip') {
+      autoUpdater.autoDownload = settingsJson['app.autoUpdateMode'] == 'download';
+      autoUpdater.checkForUpdates();
+    }
+  }
 });
 ipcMain.on('window-action', async (event, arg) => {
   if (!mainWindow) {
@@ -280,6 +285,20 @@ ipcMain.handle('showItemInFolder', async (event, path) => {
 ipcMain.handle('openExternal', async (event, url) => {
   electron.shell.openExternal(url);
 });
+ipcMain.on('downloadUpdate', async (event, url) => {
+  autoUpdater.downloadUpdate();
+  changeAppUpdateStatus({
+    icon: 'icon loading',
+    message: `Downloading update...`,
+  });
+});
+ipcMain.on('applyUpdate', async (event, url) => {
+  autoUpdater.quitAndInstall(false, true);
+});
+ipcMain.on('check-for-updates', async (event, url) => {
+  autoUpdater.autoDownload = false;
+  autoUpdater.checkForUpdates();
+});
 
 function fillMissingSettings(value) {
   const res = {
@@ -317,8 +336,6 @@ function ensureBoundsVisible(bounds) {
 function createWindow() {
   const datadir = path.join(os.homedir(), '.dbgate');
 
-  let settingsJson = {};
-  let licenseKey = null;
   try {
     settingsJson = fillMissingSettings(
       JSON.parse(fs.readFileSync(path.join(datadir, 'settings.json'), { encoding: 'utf-8' }))
@@ -327,32 +344,22 @@ function createWindow() {
     console.log('Error loading settings.json:', err.message);
     settingsJson = fillMissingSettings({});
   }
-  if (isProApp()) {
-    try {
-      licenseKey = fs.readFileSync(path.join(datadir, 'license.key'), { encoding: 'utf-8' });
-    } catch (err) {
-      console.log('Error loading license.key:', err.message);
-      licenseKey = null;
-    }
-  }
-
-  const licenseOk = !isProApp() || checkLicense(licenseKey) == 'premium';
 
   let bounds = initialConfig['winBounds'];
   if (bounds) {
     bounds = ensureBoundsVisible(bounds);
   }
-  useNativeMenu = settingsJson['app.useNativeMenu'] || !licenseOk;
+  useNativeMenu = settingsJson['app.useNativeMenu'];
 
   mainWindow = new BrowserWindow({
     width: 1200,
     height: 800,
-    title: 'DbGate',
+    title: isProApp() ? 'DbGate Premium' : 'DbGate',
     frame: useNativeMenu,
     titleBarStyle: useNativeMenu ? undefined : 'hidden',
     ...bounds,
     icon: os.platform() == 'win32' ? 'icon.ico' : path.resolve(__dirname, '../icon.png'),
-    partition: 'persist:dbgate',
+    partition: isProApp() ? 'persist:dbgate-premium' : 'persist:dbgate',
     webPreferences: {
       nodeIntegration: true,
       contextIsolation: false,
@@ -386,7 +393,6 @@ function createWindow() {
             JSON.stringify({
               winBounds: mainWindow.getBounds(),
               winIsMaximized: mainWindow.isMaximized(),
-              disableAutoUpgrade,
             }),
             'utf-8'
           );
@@ -456,13 +462,61 @@ function createWindow() {
   });
 }
 
+function changeAppUpdateStatus(status) {
+  appUpdateStatus = status;
+  mainWindow.webContents.send('app-update-status', appUpdateStatus);
+}
+
+autoUpdater.on('checking-for-update', () => {
+  console.log('Checking for updates');
+  changeAppUpdateStatus({
+    icon: 'icon loading',
+    message: 'Checking for updates...',
+  });
+});
+
+autoUpdater.on('update-available', info => {
+  console.log('Update available', info);
+  if (autoUpdater.autoDownload) {
+    changeAppUpdateStatus({
+      icon: 'icon loading',
+      message: `Downloading update...`,
+    });
+  } else {
+    mainWindow.webContents.send('update-available', info.version);
+    changeAppUpdateStatus({
+      icon: 'icon download',
+      message: `Update available`,
+    });
+  }
+});
+
+autoUpdater.on('update-not-available', info => {
+  console.log('Update not available', info);
+  changeAppUpdateStatus({
+    icon: 'icon check',
+    message: `No new updates`,
+  });
+});
+
+autoUpdater.on('update-downloaded', info => {
+  console.log('Update downloaded from', info);
+  changeAppUpdateStatus({
+    icon: 'icon download',
+    message: `Downloaded ${info.version}`,
+  });
+  mainWindow.webContents.send('downloaded-new-version', info.version);
+});
+
+autoUpdater.on('error', error => {
+  changeAppUpdateStatus({
+    icon: 'icon error',
+    message: `Autoupdate error`,
+  });
+  console.error('Update error', error);
+});
+
 function onAppReady() {
-  if (disableAutoUpgrade) {
-    console.log('Auto-upgrade is disabled, run dbgate --enable-auto-upgrade to enable');
-  }
-  if (!process.env.DEVMODE && !disableAutoUpgrade) {
-    autoUpdater.checkForUpdatesAndNotify();
-  }
   createWindow();
 }
 

app/src/mainMenuDefinition.js

@@ -105,6 +105,8 @@ module.exports = ({ editMenu }) => [
       { command: 'settings.commands', hideDisabled: true },
       { command: 'tabs.changelog', hideDisabled: true },
       { command: 'about.show', hideDisabled: true },
+      { divider: true },
+      { command: 'file.checkForUpdates', hideDisabled: true },
     ],
   },
 ];

app/src/updaterChannel.js

@@ -0,0 +1 @@
+module.exports = null;

app/yarn.lock

@@ -193,11 +193,6 @@
   dependencies:
     "@types/node" "*"
 
-"@types/semver@^7.3.6":
-  version "7.5.8"
-  resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.8.tgz#8268a8c57a3e4abd25c165ecd36237db7948a55e"
-  integrity sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==
-
 "@types/verror@^1.10.3":
   version "1.10.10"
   resolved "https://registry.yarnpkg.com/@types/verror/-/verror-1.10.10.tgz#d5a4b56abac169bfbc8b23d291363a682e6fa087"
@@ -504,14 +499,6 @@ buffer@^5.1.0, buffer@^5.5.0:
     base64-js "^1.3.1"
     ieee754 "^1.1.13"
 
-builder-util-runtime@8.9.2:
-  version "8.9.2"
-  resolved "https://registry.yarnpkg.com/builder-util-runtime/-/builder-util-runtime-8.9.2.tgz#a9669ae5b5dcabfe411ded26678e7ae997246c28"
-  integrity sha512-rhuKm5vh7E0aAmT6i8aoSfEjxzdYEFX7zDApK+eNgOhjofnWb74d9SRJv0H/8nsgOkos0TZ4zxW0P8J4N7xQ2A==
-  dependencies:
-    debug "^4.3.2"
-    sax "^1.2.4"
-
 builder-util-runtime@9.0.2:
   version "9.0.2"
   resolved "https://registry.yarnpkg.com/builder-util-runtime/-/builder-util-runtime-9.0.2.tgz#dc54f8581bbcf1e0428da4483fa46d09524be857"
@@ -520,6 +507,14 @@ builder-util-runtime@9.0.2:
     debug "^4.3.4"
     sax "^1.2.4"
 
+builder-util-runtime@9.2.5:
+  version "9.2.5"
+  resolved "https://registry.yarnpkg.com/builder-util-runtime/-/builder-util-runtime-9.2.5.tgz#0afdffa0adb5c84c14926c7dd2cf3c6e96e9be83"
+  integrity sha512-HjIDfhvqx/8B3TDN4GbABQcgpewTU4LMRTQPkVpKYV3lsuxEJoIfvg09GyWTNmfVNSUAYf+fbTN//JX4TH20pg==
+  dependencies:
+    debug "^4.3.4"
+    sax "^1.2.4"
+
 builder-util@23.0.9:
   version "23.0.9"
   resolved "https://registry.yarnpkg.com/builder-util/-/builder-util-23.0.9.tgz#8b1aeeeee679060e39ad2bd0f50f5b3f3cb53a59"
@@ -791,7 +786,7 @@ crypto-random-string@^2.0.0:
   resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-2.0.0.tgz#ef2a7a966ec11083388369baa02ebead229b30d5"
   integrity sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==
 
-debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4:
+debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.4:
   version "4.3.4"
   resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865"
   integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==
@@ -1012,19 +1007,19 @@ electron-publish@23.0.9:
     lazy-val "^1.0.5"
     mime "^2.5.2"
 
-electron-updater@^4.6.1:
-  version "4.6.5"
-  resolved "https://registry.yarnpkg.com/electron-updater/-/electron-updater-4.6.5.tgz#e9a75458bbfd6bb41a58a829839e150ad2eb2d3d"
-  integrity sha512-kdTly8O9mSZfm9fslc1mnCY+mYOeaYRy7ERa2Fed240u01BKll3aiupzkd07qKw69KvhBSzuHroIW3mF0D8DWA==
+electron-updater@^6.3.4:
+  version "6.3.4"
+  resolved "https://registry.yarnpkg.com/electron-updater/-/electron-updater-6.3.4.tgz#3934bc89875bb524c2cbbd11041114e97c0c2496"
+  integrity sha512-uZUo7p1Y53G4tl6Cgw07X1yF8Jlz6zhaL7CQJDZ1fVVkOaBfE2cWtx80avwDVi8jHp+I/FWawrMgTAeCCNIfAg==
   dependencies:
-    "@types/semver" "^7.3.6"
-    builder-util-runtime "8.9.2"
-    fs-extra "^10.0.0"
+    builder-util-runtime "9.2.5"
+    fs-extra "^10.1.0"
     js-yaml "^4.1.0"
     lazy-val "^1.0.5"
     lodash.escaperegexp "^4.1.2"
     lodash.isequal "^4.5.0"
-    semver "^7.3.5"
+    semver "^7.6.3"
+    tiny-typed-emitter "^2.1.0"
 
 electron@30.0.2:
   version "30.0.2"
@@ -2407,7 +2402,7 @@ semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7:
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.1.tgz#60bfe090bf907a25aa8119a72b9f90ef7ca281b2"
   integrity sha512-f/vbBsu+fOiYt+lmwZV0rVwJScl46HppnOA1ZvIuBWKOTlllpyJ3bfVax76/OrhCH38dyxoDIA8K7uB963IYgA==
 
-semver@^7.5.4:
+semver@^7.5.4, semver@^7.6.3:
   version "7.6.3"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143"
   integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==
@@ -2650,6 +2645,11 @@ through2@^2.0.1:
     readable-stream "~2.3.6"
     xtend "~4.0.1"
 
+tiny-typed-emitter@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/tiny-typed-emitter/-/tiny-typed-emitter-2.1.0.tgz#b3b027fdd389ff81a152c8e847ee2f5be9fad7b5"
+  integrity sha512-qVtvMxeXbVej0cQWKqVSSAHmKZEHAvxdF8HEUBFWts8h+xEo5m/lEiPakuyZ3BnCBjOD8i24kzNOiOLLgsSxhA==
+
 tmp-promise@^3.0.2:
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/tmp-promise/-/tmp-promise-3.0.3.tgz#60a1a1cc98c988674fcbfd23b6e3367bdeac4ce7"

integration-tests/__tests__/alter-database.spec.js

@@ -7,7 +7,9 @@ const { getAlterDatabaseScript, extendDatabaseInfo, generateDbPairingId } = requ
 
 function flatSource() {
   return _.flatten(
-    engines.map(engine => (engine.objects || []).map(object => [engine.label, object.type, object, engine]))
+    engines
+      .filter(x => !x.skipReferences)
+      .map(engine => (engine.objects || []).map(object => [engine.label, object.type, object, engine]))
   );
 }
 
@@ -41,7 +43,7 @@ async function testDatabaseDiff(conn, driver, mangle, createObject = null) {
 }
 
 describe('Alter database', () => {
-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipReferences).map(engine => [engine.label, engine]))(
     'Drop referenced table - %s',
     testWrapper(async (conn, driver, engine) => {
       await testDatabaseDiff(conn, driver, db => {
@@ -65,3 +67,4 @@ describe('Alter database', () => {
     })
   );
 });
+

integration-tests/__tests__/alter-table.spec.js

@@ -6,39 +6,44 @@ const engines = require('../engines');
 const crypto = require('crypto');
 const { getAlterTableScript, extendDatabaseInfo, generateDbPairingId } = require('dbgate-tools');
 
-function pickImportantTableInfo(table) {
+function pickImportantTableInfo(engine, table) {
+  const props = ['columnName'];
+  if (!engine.skipNullability) props.push('notNull');
+  if (!engine.skipAutoIncrement) props.push('autoIncrement');
   return {
     pureName: table.pureName,
-    columns: table.columns
-      .filter(x => x.columnName != 'rowid')
-      .map(fp.pick(['columnName', 'notNull', 'autoIncrement'])),
+    columns: table.columns.filter(x => x.columnName != 'rowid').map(fp.pick(props)),
   };
 }
 
-function checkTableStructure(t1, t2) {
+function checkTableStructure(engine, t1, t2) {
   // expect(t1.pureName).toEqual(t2.pureName)
-  expect(pickImportantTableInfo(t1)).toEqual(pickImportantTableInfo(t2));
+  expect(pickImportantTableInfo(engine, t1)).toEqual(pickImportantTableInfo(engine, t2));
 }
 
-async function testTableDiff(conn, driver, mangle) {
+async function testTableDiff(engine, conn, driver, mangle) {
   await driver.query(conn, `create table t0 (id int not null primary key)`);
 
   await driver.query(
     conn,
     `create table t1 (
     col_pk int not null primary key, 
-    col_std int null, 
-    col_def int null default 12,
-    col_fk int null references t0(id),
-    col_idx int null,
-    col_uq int null unique,
-    col_ref int null unique
+    col_std int, 
+    col_def int default 12,
+    ${engine.skipReferences ? '' : 'col_fk int references t0(id),'}
+    col_idx int,
+    col_uq int ${engine.skipUnique ? '' : 'unique'} ,
+    col_ref int ${engine.skipUnique ? '' : 'unique'}
   )`
   );
 
-  await driver.query(conn, `create index idx1 on t1(col_idx)`);
+  if (!engine.skipIndexes) {
+    await driver.query(conn, `create index idx1 on t1(col_idx)`);
+  }
 
-  await driver.query(conn, `create table t2 (id int not null primary key, fkval int null references t1(col_ref))`);
+  if (!engine.skipReferences) {
+    await driver.query(conn, `create table t2 (id int not null primary key, fkval int null references t1(col_ref))`);
+  }
 
   const tget = x => x.tables.find(y => y.pureName == 't1');
   const structure1 = generateDbPairingId(extendDatabaseInfo(await driver.analyseFull(conn)));
@@ -53,7 +58,7 @@ async function testTableDiff(conn, driver, mangle) {
 
   const structure2Real = extendDatabaseInfo(await driver.analyseFull(conn));
 
-  checkTableStructure(tget(structure2Real), tget(structure2));
+  checkTableStructure(engine, tget(structure2Real), tget(structure2));
   // expect(stableStringify(structure2)).toEqual(stableStringify(structure2Real));
 }
 
@@ -65,14 +70,22 @@ const TESTED_COLUMNS = ['col_pk', 'col_std', 'col_def', 'col_fk', 'col_ref', 'co
 // const TESTED_COLUMNS = ['col_ref'];
 
 function engines_columns_source() {
-  return _.flatten(engines.map(engine => TESTED_COLUMNS.map(column => [engine.label, column, engine])));
+  return _.flatten(
+    engines.map(engine =>
+      TESTED_COLUMNS.filter(col => !col.endsWith('_pk') || !engine.skipPkColumnTesting).map(column => [
+        engine.label,
+        column,
+        engine,
+      ])
+    )
+  );
 }
 
 describe('Alter table', () => {
   test.each(engines.map(engine => [engine.label, engine]))(
     'Add column - %s',
     testWrapper(async (conn, driver, engine) => {
-      await testTableDiff(conn, driver, tbl => {
+      await testTableDiff(engine, conn, driver, tbl => {
         tbl.columns.push({
           columnName: 'added',
           dataType: 'int',
@@ -87,7 +100,7 @@ describe('Alter table', () => {
   test.each(engines_columns_source())(
     'Drop column - %s - %s',
     testWrapper(async (conn, driver, column, engine) => {
-      await testTableDiff(conn, driver, tbl => (tbl.columns = tbl.columns.filter(x => x.columnName != column)));
+      await testTableDiff(engine, conn, driver, tbl => (tbl.columns = tbl.columns.filter(x => x.columnName != column)));
     })
   );
 
@@ -95,6 +108,7 @@ describe('Alter table', () => {
     'Change nullability - %s - %s',
     testWrapper(async (conn, driver, column, engine) => {
       await testTableDiff(
+        engine,
         conn,
         driver,
         tbl => (tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, notNull: true } : x)))
@@ -106,6 +120,7 @@ describe('Alter table', () => {
     'Rename column - %s - %s',
     testWrapper(async (conn, driver, column, engine) => {
       await testTableDiff(
+        engine,
         conn,
         driver,
         tbl => (tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, columnName: 'col_renamed' } : x)))
@@ -116,7 +131,7 @@ describe('Alter table', () => {
   test.each(engines.map(engine => [engine.label, engine]))(
     'Drop index - %s',
     testWrapper(async (conn, driver, engine) => {
-      await testTableDiff(conn, driver, tbl => {
+      await testTableDiff(engine, conn, driver, tbl => {
         tbl.indexes = [];
       });
     })

integration-tests/__tests__/data-duplicator.spec.js

@@ -5,7 +5,7 @@ const dataDuplicator = require('dbgate-api/src/shell/dataDuplicator');
 const { runCommandOnDriver } = require('dbgate-tools');
 
 describe('Data duplicator', () => {
-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipDataDuplicator).map(engine => [engine.label, engine]))(
     'Insert simple data - %s',
     testWrapper(async (conn, driver, engine) => {
       runCommandOnDriver(conn, driver, dmp =>

integration-tests/__tests__/deploy-database.spec.js

@@ -167,7 +167,7 @@ describe('Deploy database', () => {
     })
   );
 
-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipReferences).map(engine => [engine.label, engine]))(
     'Foreign keys - %s',
     testWrapper(async (conn, driver, engine) => {
       await testDatabaseDeploy(
@@ -222,7 +222,7 @@ describe('Deploy database', () => {
     })
   );
 
-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipDataModifications).map(engine => [engine.label, engine]))(
     'Deploy preloaded data - %s',
     testWrapper(async (conn, driver, engine) => {
       await testDatabaseDeploy(conn, driver, [
@@ -251,7 +251,7 @@ describe('Deploy database', () => {
     })
   );
 
-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipDataModifications).map(engine => [engine.label, engine]))(
     'Deploy preloaded data - update - %s',
     testWrapper(async (conn, driver, engine) => {
       await testDatabaseDeploy(conn, driver, [

integration-tests/__tests__/import-formats.spec.js

@@ -0,0 +1,151 @@
+const dbgateApi = require('dbgate-api/src/shell');
+// const jsonLinesWriter = require('dbgate-api/src/shell/jsonLinesWriter');
+const tmp = require('tmp');
+// const dbgatePluginCsv = require('dbgate-plugin-csv/src/backend');
+const fs = require('fs');
+const requirePlugin = require('dbgate-api/src/shell/requirePlugin');
+
+const CSV_DATA = `Issue Number; Title; Github URL; Labels; State; Created At; Updated At; Reporter; Assignee
+801; "Does it 'burst' the database on startup or first lUI load ? "; https://github.com/dbgate/dbgate/issues/801; ""; open; 05/23/2024; 05/23/2024; rgarrigue; 
+799; "BUG: latest AppImage crashes on opening in Fedora 39"; https://github.com/dbgate/dbgate/issues/799; ""; open; 05/21/2024; 05/24/2024; BenGraham-Git; 
+798; "MongoDB write operations fail"; https://github.com/dbgate/dbgate/issues/798; "bug,solved"; open; 05/21/2024; 05/24/2024; mahmed0715; 
+797; "BUG: Unable to open SQL files"; https://github.com/dbgate/dbgate/issues/797; "bug"; open; 05/20/2024; 05/21/2024; cesarValdivia; 
+795; "BUG: MS SQL Server connection error (KEY_USAGE_BIT_INCORRECT)"; https://github.com/dbgate/dbgate/issues/795; ""; open; 05/20/2024; 05/20/2024; keskinonur; 
+794; "GLIBC_2.29' not found and i have 2.31"; https://github.com/dbgate/dbgate/issues/794; ""; closed; 05/20/2024; 05/21/2024; MFdanGM; 
+793; "BUG: PostgresSQL doesn't show tables when connected"; https://github.com/dbgate/dbgate/issues/793; ""; open; 05/20/2024; 05/22/2024; stomper013; 
+792; "FEAT: Wayland support"; https://github.com/dbgate/dbgate/issues/792; ""; closed; 05/19/2024; 05/21/2024; VosaXalo; 
+`;
+
+async function getReaderRows(reader) {
+  const jsonLinesFileName = tmp.tmpNameSync();
+
+  const writer = await dbgateApi.jsonLinesWriter({
+    fileName: jsonLinesFileName,
+  });
+  await dbgateApi.copyStream(reader, writer);
+
+  const jsonData = fs.readFileSync(jsonLinesFileName, 'utf-8');
+  const rows = jsonData
+    .split('\n')
+    .filter(x => x.trim() !== '')
+    .map(x => JSON.parse(x));
+
+  return rows;
+}
+
+test('csv import test', async () => {
+  const dbgatePluginCsv = requirePlugin('dbgate-plugin-csv');
+
+  const csvFileName = tmp.tmpNameSync();
+
+  fs.writeFileSync(csvFileName, CSV_DATA);
+
+  const reader = await dbgatePluginCsv.shellApi.reader({
+    fileName: csvFileName,
+  });
+
+  const rows = await getReaderRows(reader);
+
+  expect(rows[0].columns).toEqual([
+    { columnName: 'Issue Number' },
+    { columnName: 'Title' },
+    { columnName: 'Github URL' },
+    { columnName: 'Labels' },
+    { columnName: 'State' },
+    { columnName: 'Created At' },
+    { columnName: 'Updated At' },
+    { columnName: 'Reporter' },
+    { columnName: 'Assignee' },
+  ]);
+  expect(rows.length).toEqual(9);
+  expect(rows[1]).toEqual({
+    'Issue Number': '801',
+    Title: "Does it 'burst' the database on startup or first lUI load ? ",
+    'Github URL': 'https://github.com/dbgate/dbgate/issues/801',
+    Labels: '',
+    State: 'open',
+    'Created At': '05/23/2024',
+    'Updated At': '05/23/2024',
+    Reporter: 'rgarrigue',
+    Assignee: '',
+  });
+});
+
+test('JSON array import test', async () => {
+  const jsonFileName = tmp.tmpNameSync();
+
+  fs.writeFileSync(
+    jsonFileName,
+    JSON.stringify([
+      { id: 1, val: 'v1' },
+      { id: 2, val: 'v2' },
+    ])
+  );
+
+  const reader = await dbgateApi.jsonReader({
+    fileName: jsonFileName,
+  });
+
+  const rows = await getReaderRows(reader);
+
+  expect(rows.length).toEqual(2);
+  expect(rows).toEqual([
+    { id: 1, val: 'v1' },
+    { id: 2, val: 'v2' },
+  ]);
+});
+
+test('JSON object import test', async () => {
+  const jsonFileName = tmp.tmpNameSync();
+
+  fs.writeFileSync(
+    jsonFileName,
+    JSON.stringify({
+      k1: { id: 1, val: 'v1' },
+      k2: { id: 2, val: 'v2' },
+    })
+  );
+
+  const reader = await dbgateApi.jsonReader({
+    fileName: jsonFileName,
+    jsonStyle: 'object',
+    keyField: 'mykey',
+  });
+
+  const rows = await getReaderRows(reader);
+
+  expect(rows.length).toEqual(2);
+  expect(rows).toEqual([
+    { mykey: 'k1', id: 1, val: 'v1' },
+    { mykey: 'k2', id: 2, val: 'v2' },
+  ]);
+});
+
+test('JSON filtered object import test', async () => {
+  const jsonFileName = tmp.tmpNameSync();
+
+  fs.writeFileSync(
+    jsonFileName,
+    JSON.stringify({
+      filtered: {
+        k1: { id: 1, val: 'v1' },
+        k2: { id: 2, val: 'v2' },
+      },
+    })
+  );
+
+  const reader = await dbgateApi.jsonReader({
+    fileName: jsonFileName,
+    jsonStyle: 'object',
+    keyField: 'mykey',
+    rootField: 'filtered',
+  });
+
+  const rows = await getReaderRows(reader);
+
+  expect(rows.length).toEqual(2);
+  expect(rows).toEqual([
+    { mykey: 'k1', id: 1, val: 'v1' },
+    { mykey: 'k2', id: 2, val: 'v2' },
+  ]);
+});

integration-tests/__tests__/object-analyse.spec.js

@@ -2,7 +2,7 @@ const { testWrapper } = require('../tools');
 const engines = require('../engines');
 const _ = require('lodash');
 
-const initSql = ['CREATE TABLE t1 (id int)', 'CREATE TABLE t2 (id int)'];
+const initSql = ['CREATE TABLE t1 (id int primary key)', 'CREATE TABLE t2 (id int primary key)'];
 
 function flatSource() {
   return _.flatten(
@@ -26,9 +26,9 @@ describe('Object analyse', () => {
   test.each(flatSource())(
     'Full analysis - %s - %s',
     testWrapper(async (conn, driver, type, object, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql);
+      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
 
-      await driver.query(conn, object.create1);
+      await driver.query(conn, object.create1, { discardResult: true });
       const structure = await driver.analyseFull(conn);
 
       expect(structure[type].length).toEqual(1);
@@ -39,11 +39,11 @@ describe('Object analyse', () => {
   test.each(flatSource())(
     'Incremental analysis - add - %s - %s',
     testWrapper(async (conn, driver, type, object, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql);
+      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
 
-      await driver.query(conn, object.create2);
+      await driver.query(conn, object.create2, { discardResult: true });
       const structure1 = await driver.analyseFull(conn);
-      await driver.query(conn, object.create1);
+      await driver.query(conn, object.create1, { discardResult: true });
       const structure2 = await driver.analyseIncremental(conn, structure1);
 
       expect(structure2[type].length).toEqual(2);
@@ -54,12 +54,12 @@ describe('Object analyse', () => {
   test.each(flatSource())(
     'Incremental analysis - drop - %s - %s',
     testWrapper(async (conn, driver, type, object, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql);
+      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
 
-      await driver.query(conn, object.create1);
-      await driver.query(conn, object.create2);
+      await driver.query(conn, object.create1, { discardResult: true });
+      await driver.query(conn, object.create2, { discardResult: true });
       const structure1 = await driver.analyseFull(conn);
-      await driver.query(conn, object.drop2);
+      await driver.query(conn, object.drop2, { discardResult: true });
       const structure2 = await driver.analyseIncremental(conn, structure1);
 
       expect(structure2[type].length).toEqual(1);
@@ -70,15 +70,15 @@ describe('Object analyse', () => {
   test.each(flatSource())(
     'Create SQL - add - %s - %s',
     testWrapper(async (conn, driver, type, object, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql);
+      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
 
-      await driver.query(conn, object.create1);
+      await driver.query(conn, object.create1, { discardResult: true });
       const structure1 = await driver.analyseFull(conn);
-      await driver.query(conn, object.drop1);
+      await driver.query(conn, object.drop1, { discardResult: true });
       const structure2 = await driver.analyseIncremental(conn, structure1);
       expect(structure2[type].length).toEqual(0);
 
-      await driver.query(conn, structure1[type][0].createSql);
+      await driver.query(conn, structure1[type][0].createSql, { discardResult: true });
 
       const structure3 = await driver.analyseIncremental(conn, structure2);
 

integration-tests/__tests__/query.spec.js

@@ -2,7 +2,11 @@ const engines = require('../engines');
 const { splitQuery } = require('dbgate-query-splitter');
 const { testWrapper } = require('../tools');
 
-const initSql = ['CREATE TABLE t1 (id int)', 'INSERT INTO t1 (id) VALUES (1)', 'INSERT INTO t1 (id) VALUES (2)'];
+const initSql = [
+  'CREATE TABLE t1 (id int primary key)',
+  'INSERT INTO t1 (id) VALUES (1)',
+  'INSERT INTO t1 (id) VALUES (2)',
+];
 
 expect.extend({
   dataRow(row, expected) {
@@ -64,7 +68,7 @@ describe('Query', () => {
   test.each(engines.map(engine => [engine.label, engine]))(
     'Simple query - %s',
     testWrapper(async (conn, driver, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql);
+      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
 
       const res = await driver.query(conn, 'SELECT id FROM t1 ORDER BY id');
       expect(res.columns).toEqual([
@@ -87,7 +91,7 @@ describe('Query', () => {
   test.each(engines.map(engine => [engine.label, engine]))(
     'Simple stream query - %s',
     testWrapper(async (conn, driver, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql);
+      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
       const results = await executeStream(driver, conn, 'SELECT id FROM t1 ORDER BY id');
       expect(results.length).toEqual(1);
       const res = results[0];
@@ -100,7 +104,7 @@ describe('Query', () => {
   test.each(engines.map(engine => [engine.label, engine]))(
     'More queries - %s',
     testWrapper(async (conn, driver, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql);
+      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
       const results = await executeStream(
         driver,
         conn,
@@ -124,7 +128,7 @@ describe('Query', () => {
       const results = await executeStream(
         driver,
         conn,
-        'CREATE TABLE t1 (id int); INSERT INTO t1 (id) VALUES (1); INSERT INTO t1 (id) VALUES (2); SELECT id FROM t1 ORDER BY id; '
+        'CREATE TABLE t1 (id int primary key); INSERT INTO t1 (id) VALUES (1); INSERT INTO t1 (id) VALUES (2); SELECT id FROM t1 ORDER BY id; '
       );
       expect(results.length).toEqual(1);
 
@@ -146,14 +150,15 @@ describe('Query', () => {
     })
   );
 
-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipDataModifications).map(engine => [engine.label, engine]))(
     'Save data query - %s',
     testWrapper(async (conn, driver, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql);
+      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
 
       await driver.script(
         conn,
-        'INSERT INTO t1 (id) VALUES (3);INSERT INTO t1 (id) VALUES (4);UPDATE t1 SET id=10 WHERE id=1;DELETE FROM t1 WHERE id=2;'
+        'INSERT INTO t1 (id) VALUES (3);INSERT INTO t1 (id) VALUES (4);UPDATE t1 SET id=10 WHERE id=1;DELETE FROM t1 WHERE id=2;',
+        { discardResult: true }
       );
       const res = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM t1');
       // console.log(res);

integration-tests/__tests__/schema-tests.spec.js

@@ -0,0 +1,90 @@
+const stableStringify = require('json-stable-stringify');
+const _ = require('lodash');
+const fp = require('lodash/fp');
+const { testWrapper, extractConnection } = require('../tools');
+const engines = require('../engines');
+const { runCommandOnDriver } = require('dbgate-tools');
+
+async function baseStructure(conn, driver) {
+  await driver.query(conn, `create table t1 (id int not null primary key)`);
+
+  await driver.query(
+    conn,
+    `create table t2 (
+        id int not null primary key, 
+        t1_id int
+      )`
+  );
+}
+
+describe('Schema tests', () => {
+  test.each(engines.filter(x => x.supportSchemas).map(engine => [engine.label, engine]))(
+    'Create schema - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await baseStructure(conn, driver);
+      const structure1 = await driver.analyseFull(conn);
+      const schemas1 = await driver.listSchemas(conn);
+      expect(schemas1.find(x => x.schemaName == 'myschema')).toBeFalsy();
+      const count = schemas1.length;
+      expect(structure1.tables.length).toEqual(2);
+      await runCommandOnDriver(conn, driver, dmp => dmp.createSchema('myschema'));
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+      const schemas2 = await driver.listSchemas(conn);
+      expect(schemas2.find(x => x.schemaName == 'myschema')).toBeTruthy();
+      expect(schemas2.length).toEqual(count + 1);
+      expect(schemas2.find(x => x.isDefault).schemaName).toEqual(engine.defaultSchemaName);
+      expect(structure2).toBeNull();
+    })
+  );
+
+  test.each(engines.filter(x => x.supportSchemas).map(engine => [engine.label, engine]))(
+    'Drop schema - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await baseStructure(conn, driver);
+      await runCommandOnDriver(conn, driver, dmp => dmp.createSchema('myschema'));
+
+      const structure1 = await driver.analyseFull(conn);
+      const schemas1 = await driver.listSchemas(conn);
+      expect(schemas1.find(x => x.schemaName == 'myschema')).toBeTruthy();
+      expect(structure1.tables.length).toEqual(2);
+      await runCommandOnDriver(conn, driver, dmp => dmp.dropSchema('myschema'));
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+      const schemas2 = await driver.listSchemas(conn);
+      expect(schemas2.find(x => x.schemaName == 'myschema')).toBeFalsy();
+      expect(structure2).toBeNull();
+    })
+  );
+
+  test.each(engines.filter(x => x.supportSchemas && !x.skipSeparateSchemas).map(engine => [engine.label, engine]))(
+    'Table inside schema - %s',
+    testWrapper(async (handle, driver, engine) => {
+      await baseStructure(handle, driver);
+      await runCommandOnDriver(handle, driver, dmp => dmp.createSchema('myschema'));
+
+      const schemaConnDef = {
+        ...extractConnection(engine),
+        database: `${handle.database}::myschema`,
+      };
+
+      const schemaConn = await driver.connect(schemaConnDef);
+      await driver.query(schemaConn, `create table myschema.myt1 (id int not null primary key)`);
+      const structure1 = await driver.analyseFull(schemaConn);
+      expect(structure1.tables.length).toEqual(1);
+      expect(structure1.tables[0].pureName).toEqual('myt1');
+    })
+  );
+});
+
+describe('Base analyser test', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Structure without change - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await baseStructure(conn, driver);
+
+      const structure1 = await driver.analyseFull(conn);
+      expect(structure1.tables.length).toEqual(2);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+      expect(structure2).toBeNull();
+    })
+  );
+});

integration-tests/__tests__/table-analyse.spec.js

@@ -1,32 +1,37 @@
 const engines = require('../engines');
 const { testWrapper } = require('../tools');
 
-const t1Sql = 'CREATE TABLE t1 (id int not null primary key, val1 varchar(50) null)';
+const t1Sql = 'CREATE TABLE t1 (id int not null primary key, val1 varchar(50))';
 const ix1Sql = 'CREATE index ix1 ON t1(val1, id)';
-const t2Sql = 'CREATE TABLE t2 (id int not null primary key, val2 varchar(50) null unique)';
+const t2Sql = engine =>
+  `CREATE TABLE t2 (id int not null primary key, val2 varchar(50) ${engine.skipUnique ? '' : 'unique'})`;
 const t3Sql = 'CREATE TABLE t3 (id int not null primary key, valfk int, foreign key (valfk) references t2(id))';
 // const fkSql = 'ALTER TABLE t3 ADD FOREIGN KEY (valfk) REFERENCES t2(id)'
 
-const txMatch = (tname, vcolname, nextcol) =>
+const txMatch = (engine, tname, vcolname, nextcol) =>
   expect.objectContaining({
     pureName: tname,
     columns: [
       expect.objectContaining({
         columnName: 'id',
-        notNull: true,
-        dataType: expect.stringMatching(/int/i),
+        dataType: expect.stringMatching(/int.*/i),
+        ...(engine.skipNullability ? {} : { notNull: true }),
       }),
       expect.objectContaining({
         columnName: vcolname,
-        notNull: false,
-        dataType: expect.stringMatching(/.*char.*\(50\)/),
+        ...(engine.skipNullability ? {} : { notNull: false }),
+        dataType: engine.skipStringLength
+          ? expect.stringMatching(/.*string|char.*/i)
+          : expect.stringMatching(/.*char.*\(50\)/i),
       }),
       ...(nextcol
         ? [
             expect.objectContaining({
               columnName: 'nextcol',
-              notNull: false,
-              dataType: expect.stringMatching(/.*char.*\(50\)/),
+              ...(engine.skipNullability ? {} : { notNull: false }),
+              dataType: engine.skipStringLength
+                ? expect.stringMatching(/.*string.*|char.*/i)
+                : expect.stringMatching(/.*char.*\(50\).*/i),
             }),
           ]
         : []),
@@ -40,9 +45,9 @@ const txMatch = (tname, vcolname, nextcol) =>
     }),
   });
 
-const t1Match = txMatch('t1', 'val1');
-const t2Match = txMatch('t2', 'val2');
-const t2NextColMatch = txMatch('t2', 'val2', true);
+const t1Match = engine => txMatch(engine, 't1', 'val1');
+const t2Match = engine => txMatch(engine, 't2', 'val2');
+const t2NextColMatch = engine => txMatch(engine, 't2', 'val2', true);
 
 describe('Table analyse', () => {
   test.each(engines.map(engine => [engine.label, engine]))(
@@ -53,25 +58,25 @@ describe('Table analyse', () => {
       const structure = await driver.analyseFull(conn);
 
       expect(structure.tables.length).toEqual(1);
-      expect(structure.tables[0]).toEqual(t1Match);
+      expect(structure.tables[0]).toEqual(t1Match(engine));
     })
   );
 
   test.each(engines.map(engine => [engine.label, engine]))(
     'Table add - incremental analysis - %s',
     testWrapper(async (conn, driver, engine) => {
-      await driver.query(conn, t2Sql);
+      await driver.query(conn, t2Sql(engine));
 
       const structure1 = await driver.analyseFull(conn);
       expect(structure1.tables.length).toEqual(1);
-      expect(structure1.tables[0]).toEqual(t2Match);
+      expect(structure1.tables[0]).toEqual(t2Match(engine));
 
       await driver.query(conn, t1Sql);
       const structure2 = await driver.analyseIncremental(conn, structure1);
 
       expect(structure2.tables.length).toEqual(2);
-      expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
-      expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2Match);
+      expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match(engine));
+      expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2Match(engine));
     })
   );
 
@@ -79,17 +84,17 @@ describe('Table analyse', () => {
     'Table remove - incremental analysis - %s',
     testWrapper(async (conn, driver, engine) => {
       await driver.query(conn, t1Sql);
-      await driver.query(conn, t2Sql);
+      await driver.query(conn, t2Sql(engine));
       const structure1 = await driver.analyseFull(conn);
       expect(structure1.tables.length).toEqual(2);
-      expect(structure1.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
-      expect(structure1.tables.find(x => x.pureName == 't2')).toEqual(t2Match);
+      expect(structure1.tables.find(x => x.pureName == 't1')).toEqual(t1Match(engine));
+      expect(structure1.tables.find(x => x.pureName == 't2')).toEqual(t2Match(engine));
 
       await driver.query(conn, 'DROP TABLE t2');
       const structure2 = await driver.analyseIncremental(conn, structure1);
 
       expect(structure2.tables.length).toEqual(1);
-      expect(structure2.tables[0]).toEqual(t1Match);
+      expect(structure2.tables[0]).toEqual(t1Match(engine));
     })
   );
 
@@ -97,23 +102,26 @@ describe('Table analyse', () => {
     'Table change - incremental analysis - %s',
     testWrapper(async (conn, driver, engine) => {
       await driver.query(conn, t1Sql);
-      await driver.query(conn, t2Sql);
+      await driver.query(conn, t2Sql(engine));
       const structure1 = await driver.analyseFull(conn);
 
       if (engine.dbSnapshotBySeconds) await new Promise(resolve => setTimeout(resolve, 1100));
 
-      await driver.query(conn, 'ALTER TABLE t2 ADD nextcol varchar(50)');
+      await driver.query(
+        conn,
+        `ALTER TABLE t2 ADD ${engine.alterTableAddColumnSyntax ? 'COLUMN' : ''} nextcol varchar(50)`
+      );
       const structure2 = await driver.analyseIncremental(conn, structure1);
 
       expect(structure2).toBeTruthy(); // if falsy, no modification is detected
 
       expect(structure2.tables.length).toEqual(2);
-      expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
-      expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2NextColMatch);
+      expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match(engine));
+      expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2NextColMatch(engine));
     })
   );
 
-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipIndexes).map(engine => [engine.label, engine]))(
     'Index - full analysis - %s',
     testWrapper(async (conn, driver, engine) => {
       await driver.query(conn, t1Sql);
@@ -128,10 +136,10 @@ describe('Table analyse', () => {
     })
   );
 
-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipUnique).map(engine => [engine.label, engine]))(
     'Unique - full analysis - %s',
     testWrapper(async (conn, driver, engine) => {
-      await driver.query(conn, t2Sql);
+      await driver.query(conn, t2Sql(engine));
       const structure = await driver.analyseFull(conn);
 
       const t2 = structure.tables.find(x => x.pureName == 't2');
@@ -142,10 +150,10 @@ describe('Table analyse', () => {
     })
   );
 
-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipReferences).map(engine => [engine.label, engine]))(
     'Foreign key - full analysis - %s',
     testWrapper(async (conn, driver, engine) => {
-      await driver.query(conn, t2Sql);
+      await driver.query(conn, t2Sql(engine));
       await driver.query(conn, t3Sql);
       // await driver.query(conn, fkSql);
 

integration-tests/__tests__/table-create.spec.js

@@ -62,7 +62,7 @@ describe('Table create', () => {
     })
   );
 
-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipIndexes).map(engine => [engine.label, engine]))(
     'Table with index - %s',
     testWrapper(async (conn, driver, engine) => {
       await testTableCreate(conn, driver, {
@@ -92,7 +92,7 @@ describe('Table create', () => {
     })
   );
 
-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipReferences).map(engine => [engine.label, engine]))(
     'Table with foreign key - %s',
     testWrapper(async (conn, driver, engine) => {
       await testTableCreate(conn, driver, {
@@ -122,7 +122,7 @@ describe('Table create', () => {
     })
   );
 
-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipUnique).map(engine => [engine.label, engine]))(
     'Table with unique - %s',
     testWrapper(async (conn, driver, engine) => {
       await testTableCreate(conn, driver, {

integration-tests/docker-compose.yaml

@@ -1,12 +1,12 @@
 version: '3'
 services:
-  # postgres:
-  #   image: postgres
-  #   restart: always
-  #   environment:
-  #     POSTGRES_PASSWORD: Pwd2020Db
-  #   ports: 
-  #     - 15000:5432
+  postgres:
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_PASSWORD: Pwd2020Db
+    ports: 
+      - 15000:5432
 
   # mariadb:
   #   image: mariadb
@@ -26,15 +26,23 @@ services:
   #   environment:
   #     - MYSQL_ROOT_PASSWORD=Pwd2020Db
 
-  mssql:
-    image: mcr.microsoft.com/mssql/server
-    restart: always
-    ports: 
-      - 15002:1433
-    environment:
-      - ACCEPT_EULA=Y
-      - SA_PASSWORD=Pwd2020Db
-      - MSSQL_PID=Express
+  # clickhouse:
+  #   image: bitnami/clickhouse:24.8.4
+  #   restart: always
+  #   ports: 
+  #     - 15005:8123
+  #   environment:
+  #     - CLICKHOUSE_ADMIN_PASSWORD=Pwd2020Db
+
+  # mssql:
+  #   image: mcr.microsoft.com/mssql/server
+  #   restart: always
+  #   ports: 
+  #     - 15002:1433
+  #   environment:
+  #     - ACCEPT_EULA=Y
+  #     - SA_PASSWORD=Pwd2020Db
+  #     - MSSQL_PID=Express
       
   # cockroachdb:
   #   image: cockroachdb/cockroach

integration-tests/engines.js

@@ -81,6 +81,8 @@ const engines = [
         drop2: 'DROP FUNCTION obj2',
       },
     ],
+    supportSchemas: true,
+    defaultSchemaName: 'public',
   },
   {
     label: 'SQL Server',
@@ -105,6 +107,9 @@ const engines = [
         drop2: 'DROP PROCEDURE obj2',
       },
     ],
+    supportSchemas: true,
+    defaultSchemaName: 'dbo',
+    // skipSeparateSchemas: true,
   },
   {
     label: 'SQLite',
@@ -113,6 +118,7 @@ const engines = [
       engine: 'sqlite@dbgate-plugin-sqlite',
     },
     objects: [views],
+    skipOnCI: false,
   },
   {
     label: 'CockroachDB',
@@ -129,6 +135,30 @@ const engines = [
     skipOnCI: true,
     objects: [views, matviews],
   },
+  {
+    label: 'ClickHouse',
+    connection: {
+      engine: 'clickhouse@dbgate-plugin-clickhouse',
+      databaseUrl: 'http://clickhouse:8123',
+      password: 'Pwd2020Db',
+    },
+    local: {
+      databaseUrl: 'http://localhost:15005',
+    },
+    skipOnCI: false,
+    objects: [views],
+    skipDataModifications: true,
+    skipReferences: true,
+    skipIndexes: true,
+    skipNullability: true,
+    skipUnique: true,
+    skipAutoIncrement: true,
+    skipPkColumnTesting: true,
+    skipDataDuplicator: true,
+    skipStringLength: true,
+    alterTableAddColumnSyntax: true,
+    dbSnapshotBySeconds: true,
+  },
 ];
 
 const filterLocal = [
@@ -139,6 +169,7 @@ const filterLocal = [
   '-SQL Server',
   'SQLite',
   '-CockroachDB',
+  '-ClickHouse',
 ];
 
 const enginesPostgre = engines.filter(x => x.label == 'PostgreSQL');

integration-tests/jest.config.js

@@ -0,0 +1,3 @@
+module.exports = {
+  setupFilesAfterEnv: ['<rootDir>/setupTests.js'],
+};

integration-tests/package.json

@@ -11,12 +11,9 @@
   "scripts": {
     "wait:local": "cross-env DEVMODE=1 LOCALTEST=1 node wait.js",
     "wait:ci": "cross-env DEVMODE=1 CITEST=1 node wait.js",
-
     "test:local": "cross-env DEVMODE=1 LOCALTEST=1 jest",
     "test:local:path": "cross-env DEVMODE=1 LOCALTEST=1 jest --runTestsByPath __tests__/data-duplicator.spec.js",
-    
-    "test:ci": "cross-env DEVMODE=1 CITEST=1 jest --runInBand --json --outputFile=result.json --testLocationInResults",
-    
+    "test:ci": "cross-env DEVMODE=1 CITEST=1 jest --runInBand --json --outputFile=result.json --testLocationInResults --detectOpenHandles --forceExit",
     "run:local": "docker-compose down && docker-compose up -d && yarn wait:local && yarn test:local"
   },
   "jest": {
@@ -24,7 +21,8 @@
   },
   "devDependencies": {
     "cross-env": "^7.0.3",
-    "jest": "^27.0.1"
-  },
-  "dependencies": {}
+    "jest": "^27.0.1",
+    "pino-pretty": "^11.2.2",
+    "tmp": "^0.2.3"
+  }
 }

integration-tests/setupTests.js

@@ -0,0 +1,30 @@
+global.DBGATE_PACKAGES = {
+  'dbgate-tools': require('dbgate-tools'),
+  'dbgate-sqltree': require('dbgate-sqltree'),
+};
+
+const { prettyFactory } = require('pino-pretty');
+const tmp = require('tmp');
+
+const pretty = prettyFactory({
+  colorize: true,
+  translateTime: 'SYS:standard',
+  ignore: 'pid,hostname',
+});
+
+global.console = {
+  ...console,
+  log: (...messages) => {
+    try {
+      const parsedMessage = JSON.parse(messages[0]);
+      process.stdout.write(pretty(parsedMessage));
+    } catch (error) {
+      process.stdout.write(messages.join(' ') + '\n');
+    }
+  },
+  debug: (...messages) => {
+    process.stdout.write(messages.join(' ') + '\n');
+  },
+};
+
+tmp.setGracefulCleanup();

integration-tests/tools.js

@@ -1,8 +1,3 @@
-global.DBGATE_PACKAGES = {
-  'dbgate-tools': require('dbgate-tools'),
-  'dbgate-sqltree': require('dbgate-sqltree'),
-};
-
 const requireEngineDriver = require('dbgate-api/src/utility/requireEngineDriver');
 const crypto = require('crypto');
 

package.json

@@ -1,6 +1,6 @@
 {
   "private": true,
-  "version": "5.3.5-pro.19",
+  "version": "5.5.3-beta.3",
   "name": "dbgate-all",
   "workspaces": [
     "packages/*",
@@ -36,6 +36,7 @@
     "build:api": "yarn workspace dbgate-api build",
     "build:web:docker": "yarn workspace dbgate-web build",
     "build:plugins:frontend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:frontend",
+    "build:plugins:backend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:backend",
     "build:plugins:frontend:watch": "workspaces-run --parallel --only=\"dbgate-plugin-*\" -- yarn build:frontend:watch",
     "storage-json": "dbmodel model-to-json storage-db packages/api/src/storageModel.js --commonjs",
     "plugins:copydist": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn copydist",

packages/api/package.json

@@ -57,6 +57,7 @@
     "rimraf": "^3.0.0",
     "simple-encryptor": "^4.0.0",
     "ssh2": "^1.11.0",
+    "stream-json": "^1.8.0",
     "tar": "^6.0.5"
   },
   "scripts": {

packages/api/src/controllers/config.js

@@ -7,6 +7,7 @@ const { hasPermission } = require('../utility/hasPermission');
 const socket = require('../utility/socket');
 const _ = require('lodash');
 const AsyncLock = require('async-lock');
+const jwt = require('jsonwebtoken');
 
 const currentVersion = require('../currentVersion');
 const platformInfo = require('../utility/platformInfo');
@@ -14,6 +15,8 @@ const connections = require('../controllers/connections');
 const { getAuthProviderFromReq } = require('../auth/authProvider');
 const { checkLicense, checkLicenseKey } = require('../utility/checkLicense');
 const storage = require('./storage');
+const { getAuthProxyUrl } = require('../utility/authProxy');
+const { getPublicHardwareFingerprint } = require('../utility/hardwareFingerprint');
 
 const lock = new AsyncLock();
 
@@ -150,6 +153,22 @@ module.exports = {
 
   saveLicenseKey_meta: true,
   async saveLicenseKey({ licenseKey }) {
+    const decoded = jwt.decode(licenseKey);
+    if (!decoded) {
+      return {
+        status: 'error',
+        errorMessage: 'Invalid license key',
+      };
+    }
+
+    const { exp } = decoded;
+    if (exp * 1000 < Date.now()) {
+      return {
+        status: 'error',
+        errorMessage: 'License key is expired',
+      };
+    }
+
     try {
       if (process.env.STORAGE_DATABASE) {
         await storage.writeConfig({ group: 'license', config: { licenseKey } });
@@ -158,8 +177,33 @@ module.exports = {
         await fs.writeFile(path.join(datadir(), 'license.key'), licenseKey);
       }
       socket.emitChanged(`config-changed`);
+      return { status: 'ok' };
     } catch (err) {
-      return null;
+      return {
+        status: 'error',
+        errorMessage: err.message,
+      };
+    }
+  },
+
+  startTrial_meta: true,
+  async startTrial() {
+    try {
+      const fingerprint = await getPublicHardwareFingerprint();
+
+      const resp = await axios.default.post(`${getAuthProxyUrl()}/trial-license`, {
+        type: 'premium-trial',
+        days: 30,
+        fingerprint,
+      });
+      const { token } = resp.data;
+
+      return await this.saveLicenseKey({ licenseKey: token });
+    } catch (err) {
+      return {
+        status: 'error',
+        errorMessage: err.message,
+      };
     }
   },
 

packages/api/src/controllers/connections.js

@@ -76,6 +76,7 @@ function getPortalCollections() {
       allowedDatabases: process.env[`ALLOWED_DATABASES_${id}`]?.replace(/\|/g, '\n'),
       allowedDatabasesRegex: process.env[`ALLOWED_DATABASES_REGEX_${id}`],
       parent: process.env[`PARENT_${id}`] || undefined,
+      useSeparateSchemas: !!process.env[`USE_SEPARATE_SCHEMAS_${id}`],
 
       // SSH tunnel
       useSshTunnel: process.env[`USE_SSH_${id}`],

packages/api/src/controllers/databaseConnections.js

@@ -213,6 +213,17 @@ module.exports = {
     return res.result || null;
   },
 
+  schemaList_meta: true,
+  async schemaList({ conid, database }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('schemaList', { conid, database });
+  },
+
+  dispatchDatabaseChangedEvent_meta: true,
+  dispatchDatabaseChangedEvent({ event, conid, database }) {
+    socket.emitChanged(event, { conid, database });
+  },
+
   loadKeys_meta: true,
   async loadKeys({ conid, database, root, filter }, req) {
     testConnectionPermission(conid, req);

packages/api/src/controllers/jsldata.js

@@ -18,11 +18,14 @@ function readFirstLine(file) {
       }
       if (reader.hasNextLine()) {
         reader.nextLine((err, line) => {
-          if (err) reject(err);
-          resolve(line);
+          if (err) {
+            reader.close(() => reject(err)); // Ensure reader is closed on error
+            return;
+          }
+          reader.close(() => resolve(line)); // Ensure reader is closed after reading
         });
       } else {
-        resolve(null);
+        reader.close(() => resolve(null)); // Properly close if no lines are present
       }
     });
   });

packages/api/src/controllers/plugins.js

@@ -42,13 +42,14 @@ module.exports = {
 
   info_meta: true,
   async info({ packageName }) {
+    // @ts-ignore
+    const isPackaged = await fs.exists(path.join(packagedPluginsDir(), packageName));
+
     try {
       const infoResp = await axios.default.get(`https://registry.npmjs.org/${packageName}`);
       const { latest } = infoResp.data['dist-tags'];
       const manifest = infoResp.data.versions[latest];
       const { readme } = infoResp.data;
-      // @ts-ignore
-      const isPackaged = await fs.exists(path.join(packagedPluginsDir(), packageName));
 
       return {
         readme,
@@ -57,6 +58,7 @@ module.exports = {
       };
     } catch (err) {
       return {
+        isPackaged,
         state: 'error',
         error: err.message,
       };

packages/api/src/proc/databaseConnectionProcess.js

@@ -1,7 +1,13 @@
 const stableStringify = require('json-stable-stringify');
 const { splitQuery } = require('dbgate-query-splitter');
 const childProcessChecker = require('../utility/childProcessChecker');
-const { extractBoolSettingsValue, extractIntSettingsValue, getLogger } = require('dbgate-tools');
+const {
+  extractBoolSettingsValue,
+  extractIntSettingsValue,
+  getLogger,
+  isCompositeDbName,
+  dbNameLogCategory,
+} = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
@@ -11,7 +17,7 @@ const { dumpSqlSelect } = require('dbgate-sqltree');
 
 const logger = getLogger('dbconnProcess');
 
-let systemConnection;
+let dbhan;
 let storedConnection;
 let afterConnectCallbacks = [];
 let afterAnalyseCallbacks = [];
@@ -28,6 +34,25 @@ function getStatusCounter() {
   return statusCounter;
 }
 
+function extractErrorMessage(err, defaultMessage) {
+  if (!err) {
+    return defaultMessage;
+  }
+  if (err.errors) {
+    try {
+      return err.errors.map(x => x.message).join('\n');
+    } catch (e2) {}
+  }
+  if (err.message) {
+    return err.message;
+  }
+  const s = `${err}`;
+  if (s && (!s.endsWith('Error') || s.includes(' '))) {
+    return s;
+  }
+  return defaultMessage;
+}
+
 async function checkedAsyncCall(promise) {
   try {
     const res = await promise;
@@ -35,7 +60,7 @@ async function checkedAsyncCall(promise) {
   } catch (err) {
     setStatus({
       name: 'error',
-      message: err.message,
+      message: extractErrorMessage(err, 'Checked call error'),
     });
     // console.error(err);
     setTimeout(() => process.exit(1), 1000);
@@ -46,10 +71,16 @@ async function checkedAsyncCall(promise) {
 let loadingModel = false;
 
 async function handleFullRefresh() {
+  if (storedConnection.useSeparateSchemas && !isCompositeDbName(dbhan?.database)) {
+    resolveAnalysedPromises();
+    // skip loading DB structure
+    return;
+  }
+
   loadingModel = true;
   const driver = requireEngineDriver(storedConnection);
   setStatusName('loadStructure');
-  analysedStructure = await checkedAsyncCall(driver.analyseFull(systemConnection, serverVersion));
+  analysedStructure = await checkedAsyncCall(driver.analyseFull(dbhan, serverVersion));
   analysedTime = new Date().getTime();
   process.send({ msgtype: 'structure', structure: analysedStructure });
   process.send({ msgtype: 'structureTime', analysedTime });
@@ -60,12 +91,15 @@ async function handleFullRefresh() {
 }
 
 async function handleIncrementalRefresh(forceSend) {
+  if (storedConnection.useSeparateSchemas && !isCompositeDbName(dbhan?.database)) {
+    resolveAnalysedPromises();
+    // skip loading DB structure
+    return;
+  }
   loadingModel = true;
   const driver = requireEngineDriver(storedConnection);
   setStatusName('checkStructure');
-  const newStructure = await checkedAsyncCall(
-    driver.analyseIncremental(systemConnection, analysedStructure, serverVersion)
-  );
+  const newStructure = await checkedAsyncCall(driver.analyseIncremental(dbhan, analysedStructure, serverVersion));
   analysedTime = new Date().getTime();
   if (newStructure != null) {
     analysedStructure = newStructure;
@@ -103,7 +137,8 @@ function setStatusName(name) {
 
 async function readVersion() {
   const driver = requireEngineDriver(storedConnection);
-  const version = await driver.getVersion(systemConnection);
+  const version = await driver.getVersion(dbhan);
+  logger.debug(`Got server version: ${version.version}`);
   process.send({ msgtype: 'version', version });
   serverVersion = version;
 }
@@ -114,8 +149,13 @@ async function handleConnect({ connection, structure, globalSettings }) {
 
   if (!structure) setStatusName('pending');
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await checkedAsyncCall(connectUtility(driver, storedConnection, 'app'));
-  systemConnection.feedback = feedback => setStatus({ feedback });
+  dbhan = await checkedAsyncCall(connectUtility(driver, storedConnection, 'app'));
+  logger.debug(
+    `Connected to database, driver: ${storedConnection.engine}, separate schemas: ${
+      storedConnection.useSeparateSchemas ? 'YES' : 'NO'
+    }, 'DB: ${dbNameLogCategory(dbhan.database)} }`
+  );
+  dbhan.feedback = feedback => setStatus({ feedback });
   await checkedAsyncCall(readVersion());
   if (structure) {
     analysedStructure = structure;
@@ -138,7 +178,7 @@ async function handleConnect({ connection, structure, globalSettings }) {
 }
 
 function waitConnected() {
-  if (systemConnection) return Promise.resolve();
+  if (dbhan) return Promise.resolve();
   return new Promise((resolve, reject) => {
     afterConnectCallbacks.push([resolve, reject]);
   });
@@ -163,10 +203,14 @@ async function handleRunScript({ msgid, sql, useTransaction }, skipReadonlyCheck
   const driver = requireEngineDriver(storedConnection);
   try {
     if (!skipReadonlyCheck) ensureExecuteCustomScript(driver);
-    await driver.script(systemConnection, sql, { useTransaction });
+    await driver.script(dbhan, sql, { useTransaction });
     process.send({ msgtype: 'response', msgid });
   } catch (err) {
-    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+    process.send({
+      msgtype: 'response',
+      msgid,
+      errorMessage: extractErrorMessage(err, 'Error executing SQL script'),
+    });
   }
 }
 
@@ -175,10 +219,14 @@ async function handleRunOperation({ msgid, operation, useTransaction }, skipRead
   const driver = requireEngineDriver(storedConnection);
   try {
     if (!skipReadonlyCheck) ensureExecuteCustomScript(driver);
-    await driver.operation(systemConnection, operation, { useTransaction });
+    await driver.operation(dbhan, operation, { useTransaction });
     process.send({ msgtype: 'response', msgid });
   } catch (err) {
-    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+    process.send({
+      msgtype: 'response',
+      msgid,
+      errorMessage: extractErrorMessage(err, 'Error executing DB operation'),
+    });
   }
 }
 
@@ -188,10 +236,14 @@ async function handleQueryData({ msgid, sql }, skipReadonlyCheck = false) {
   try {
     if (!skipReadonlyCheck) ensureExecuteCustomScript(driver);
     // console.log(sql);
-    const res = await driver.query(systemConnection, sql);
+    const res = await driver.query(dbhan, sql);
     process.send({ msgtype: 'response', msgid, ...res });
   } catch (err) {
-    process.send({ msgtype: 'response', msgid, errorMessage: err.message || 'Error executing SQL script' });
+    process.send({
+      msgtype: 'response',
+      msgid,
+      errorMessage: extractErrorMessage(err, 'Error executing SQL script'),
+    });
   }
 }
 
@@ -202,52 +254,64 @@ async function handleSqlSelect({ msgid, select }) {
   return handleQueryData({ msgid, sql: dmp.s }, true);
 }
 
-async function handleDriverDataCore(msgid, callMethod) {
+async function handleDriverDataCore(msgid, callMethod, { logName }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
   try {
     const result = await callMethod(driver);
     process.send({ msgtype: 'response', msgid, result });
   } catch (err) {
-    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+    logger.error(err, `Error when handling message ${logName}`);
+    process.send({ msgtype: 'response', msgid, errorMessage: extractErrorMessage(err, 'Error executing DB data') });
   }
 }
 
+async function handleSchemaList({ msgid }) {
+  logger.debug('Loading schema list');
+  return handleDriverDataCore(msgid, driver => driver.listSchemas(dbhan), { logName: 'listSchemas' });
+}
+
 async function handleCollectionData({ msgid, options }) {
-  return handleDriverDataCore(msgid, driver => driver.readCollection(systemConnection, options));
+  return handleDriverDataCore(msgid, driver => driver.readCollection(dbhan, options), { logName: 'readCollection' });
 }
 
 async function handleLoadKeys({ msgid, root, filter }) {
-  return handleDriverDataCore(msgid, driver => driver.loadKeys(systemConnection, root, filter));
+  return handleDriverDataCore(msgid, driver => driver.loadKeys(dbhan, root, filter), { logName: 'loadKeys' });
 }
 
 async function handleExportKeys({ msgid, options }) {
-  return handleDriverDataCore(msgid, driver => driver.exportKeys(systemConnection, options));
+  return handleDriverDataCore(msgid, driver => driver.exportKeys(dbhan, options), { logName: 'exportKeys' });
 }
 
 async function handleLoadKeyInfo({ msgid, key }) {
-  return handleDriverDataCore(msgid, driver => driver.loadKeyInfo(systemConnection, key));
+  return handleDriverDataCore(msgid, driver => driver.loadKeyInfo(dbhan, key), { logName: 'loadKeyInfo' });
 }
 
 async function handleCallMethod({ msgid, method, args }) {
-  return handleDriverDataCore(msgid, driver => {
-    if (storedConnection.isReadOnly) {
-      throw new Error('Connection is read only, cannot call custom methods');
-    }
+  return handleDriverDataCore(
+    msgid,
+    driver => {
+      if (storedConnection.isReadOnly) {
+        throw new Error('Connection is read only, cannot call custom methods');
+      }
 
-    ensureExecuteCustomScript(driver);
-    return driver.callMethod(systemConnection, method, args);
-  });
+      ensureExecuteCustomScript(driver);
+      return driver.callMethod(dbhan, method, args);
+    },
+    { logName: `callMethod:${method}` }
+  );
 }
 
 async function handleLoadKeyTableRange({ msgid, key, cursor, count }) {
-  return handleDriverDataCore(msgid, driver => driver.loadKeyTableRange(systemConnection, key, cursor, count));
+  return handleDriverDataCore(msgid, driver => driver.loadKeyTableRange(dbhan, key, cursor, count), {
+    logName: 'loadKeyTableRange',
+  });
 }
 
 async function handleLoadFieldValues({ msgid, schemaName, pureName, field, search }) {
-  return handleDriverDataCore(msgid, driver =>
-    driver.loadFieldValues(systemConnection, { schemaName, pureName }, field, search)
-  );
+  return handleDriverDataCore(msgid, driver => driver.loadFieldValues(dbhan, { schemaName, pureName }, field, search), {
+    logName: 'loadFieldValues',
+  });
 }
 
 function ensureExecuteCustomScript(driver) {
@@ -264,10 +328,10 @@ async function handleUpdateCollection({ msgid, changeSet }) {
   const driver = requireEngineDriver(storedConnection);
   try {
     ensureExecuteCustomScript(driver);
-    const result = await driver.updateCollection(systemConnection, changeSet);
+    const result = await driver.updateCollection(dbhan, changeSet);
     process.send({ msgtype: 'response', msgid, result });
   } catch (err) {
-    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+    process.send({ msgtype: 'response', msgid, errorMessage: extractErrorMessage(err, 'Error updating collection') });
   }
 }
 
@@ -277,7 +341,7 @@ async function handleSqlPreview({ msgid, objects, options }) {
 
   try {
     const dmp = driver.createDumper();
-    const generator = new SqlGenerator(analysedStructure, options, objects, dmp, driver, systemConnection);
+    const generator = new SqlGenerator(analysedStructure, options, objects, dmp, driver, dbhan);
 
     await generator.dump();
     process.send({ msgtype: 'response', msgid, sql: dmp.s, isTruncated: generator.isTruncated });
@@ -288,7 +352,12 @@ async function handleSqlPreview({ msgid, objects, options }) {
       }, 500);
     }
   } catch (err) {
-    process.send({ msgtype: 'response', msgid, isError: true, errorMessage: err.message });
+    process.send({
+      msgtype: 'response',
+      msgid,
+      isError: true,
+      errorMessage: extractErrorMessage(err, 'Error generating SQL preview'),
+    });
   }
 }
 
@@ -297,14 +366,19 @@ async function handleGenerateDeploySql({ msgid, modelFolder }) {
 
   try {
     const res = await generateDeploySql({
-      systemConnection,
+      systemConnection: dbhan,
       connection: storedConnection,
       analysedStructure,
       modelFolder,
     });
     process.send({ ...res, msgtype: 'response', msgid });
   } catch (err) {
-    process.send({ msgtype: 'response', msgid, isError: true, errorMessage: err.message });
+    process.send({
+      msgtype: 'response',
+      msgid,
+      isError: true,
+      errorMessage: extractErrorMessage(err, 'Error generating deploy SQL'),
+    });
   }
 }
 
@@ -337,6 +411,7 @@ const messageHandlers = {
   loadFieldValues: handleLoadFieldValues,
   sqlSelect: handleSqlSelect,
   exportKeys: handleExportKeys,
+  schemaList: handleSchemaList,
   // runCommand: handleRunCommand,
 };
 
@@ -362,7 +437,7 @@ function start() {
       await handleMessage(message);
     } catch (err) {
       logger.error({ err }, 'Error in DB connection');
-      process.send({ msgtype: 'error', error: err.message });
+      process.send({ msgtype: 'error', error: extractErrorMessage(err, 'Error processing message') });
     }
   });
 }

packages/api/src/proc/serverConnectionProcess.js

@@ -115,7 +115,7 @@ async function handleDatabaseOp(op, { msgid, name }) {
       const dmp = driver.createDumper();
       dmp[op](name);
       logger.info({ sql: dmp.s }, 'Running script');
-      await driver.query(systemConnection, dmp.s);
+      await driver.query(systemConnection, dmp.s, { discardResult: true });
     }
     await handleRefresh();
 

packages/api/src/shell/importDatabase.js

@@ -16,7 +16,7 @@ class ImportStream extends stream.Transform {
   }
   async _transform(chunk, encoding, cb) {
     try {
-      await this.driver.script(this.pool, chunk);
+      await this.driver.script(this.pool, chunk, { queryOptions: { importSqlDump: true } });
     } catch (err) {
       this.emit('error', err.message);
     }
@@ -47,7 +47,9 @@ async function importDatabase({ connection = undefined, systemConnection = undef
   const pool = systemConnection || (await connectUtility(driver, connection, 'write'));
   logger.info(`Connected.`);
 
+  logger.info(`Input file: ${inputFile}`);
   const downloadedFile = await download(inputFile);
+  logger.info(`Downloaded file: ${downloadedFile}`);
 
   const fileStream = fs.createReadStream(downloadedFile, 'utf-8');
   const splittedStream = splitQueryStream(fileStream, driver.getQuerySplitterOptions('script'));

packages/api/src/shell/index.js

@@ -6,7 +6,7 @@ const copyStream = require('./copyStream');
 const fakeObjectReader = require('./fakeObjectReader');
 const consoleObjectWriter = require('./consoleObjectWriter');
 const jsonLinesWriter = require('./jsonLinesWriter');
-const jsonArrayWriter = require('./jsonArrayWriter');
+const jsonWriter = require('./jsonWriter');
 const jsonLinesReader = require('./jsonLinesReader');
 const sqlDataWriter = require('./sqlDataWriter');
 const jslDataReader = require('./jslDataReader');
@@ -29,6 +29,7 @@ const modifyJsonLinesReader = require('./modifyJsonLinesReader');
 const dataDuplicator = require('./dataDuplicator');
 const dbModelToJson = require('./dbModelToJson');
 const jsonToDbModel = require('./jsonToDbModel');
+const jsonReader = require('./jsonReader');
 
 const dbgateApi = {
   queryReader,
@@ -37,8 +38,9 @@ const dbgateApi = {
   tableReader,
   copyStream,
   jsonLinesWriter,
-  jsonArrayWriter,
   jsonLinesReader,
+  jsonReader,
+  jsonWriter,
   sqlDataWriter,
   fakeObjectReader,
   consoleObjectWriter,

packages/api/src/shell/jsonArrayWriter.js

@@ -1,52 +0,0 @@
-const { getLogger } = require('dbgate-tools');
-const fs = require('fs');
-const stream = require('stream');
-
-const logger = getLogger('jsonArrayWriter');
-
-class StringifyStream extends stream.Transform {
-  constructor() {
-    super({ objectMode: true });
-    this.wasHeader = false;
-    this.wasRecord = false;
-  }
-  _transform(chunk, encoding, done) {
-    let skip = false;
-
-    if (!this.wasHeader) {
-      skip = chunk.__isStreamHeader;
-      this.wasHeader = true;
-    }
-    if (!skip) {
-      if (!this.wasRecord) {
-        this.push('[\n');
-      } else {
-        this.push(',\n');
-      }
-      this.wasRecord = true;
-
-      this.push(JSON.stringify(chunk));
-    }
-    done();
-  }
-
-  _flush(done) {
-    if (!this.wasRecord) {
-      this.push('[]\n');
-    } else {
-      this.push('\n]\n');
-    }
-    done();
-  }
-}
-
-async function jsonArrayWriter({ fileName, encoding = 'utf-8' }) {
-  logger.info(`Writing file ${fileName}`);
-  const stringify = new StringifyStream();
-  const fileStream = fs.createWriteStream(fileName, encoding);
-  stringify.pipe(fileStream);
-  stringify['finisher'] = fileStream;
-  return stringify;
-}
-
-module.exports = jsonArrayWriter;

packages/api/src/shell/jsonLinesReader.js

@@ -2,6 +2,7 @@ const fs = require('fs');
 const stream = require('stream');
 const byline = require('byline');
 const { getLogger } = require('dbgate-tools');
+const download = require('./download');
 const logger = getLogger('jsonLinesReader');
 
 class ParseStream extends stream.Transform {
@@ -35,8 +36,10 @@ class ParseStream extends stream.Transform {
 async function jsonLinesReader({ fileName, encoding = 'utf-8', limitRows = undefined }) {
   logger.info(`Reading file ${fileName}`);
 
+  const downloadedFile = await download(fileName);
+
   const fileStream = fs.createReadStream(
-    fileName,
+    downloadedFile,
     // @ts-ignore
     encoding
   );

packages/api/src/shell/jsonReader.js

@@ -0,0 +1,84 @@
+const fs = require('fs');
+const stream = require('stream');
+const byline = require('byline');
+const { getLogger } = require('dbgate-tools');
+const { parser } = require('stream-json');
+const { pick } = require('stream-json/filters/Pick');
+const { streamArray } = require('stream-json/streamers/StreamArray');
+const { streamObject } = require('stream-json/streamers/StreamObject');
+const download = require('./download');
+
+const logger = getLogger('jsonReader');
+
+
+class ParseStream extends stream.Transform {
+  constructor({ limitRows, jsonStyle, keyField }) {
+    super({ objectMode: true });
+    this.wasHeader = false;
+    this.limitRows = limitRows;
+    this.jsonStyle = jsonStyle;
+    this.keyField = keyField || '_key';
+    this.rowsWritten = 0;
+  }
+  _transform(chunk, encoding, done) {
+    if (!this.wasHeader) {
+      this.push({
+        __isStreamHeader: true,
+        __isDynamicStructure: true,
+      });
+
+      this.wasHeader = true;
+    }
+    if (!this.limitRows || this.rowsWritten < this.limitRows) {
+      if (this.jsonStyle === 'object') {
+        this.push({
+          ...chunk.value,
+          [this.keyField]: chunk.key,
+        });
+      } else {
+        this.push(chunk.value);
+      }
+
+      this.rowsWritten += 1;
+    }
+    done();
+  }
+}
+
+async function jsonReader({
+  fileName,
+  jsonStyle,
+  keyField = '_key',
+  rootField = null,
+  encoding = 'utf-8',
+  limitRows = undefined,
+}) {
+  logger.info(`Reading file ${fileName}`);
+
+  const downloadedFile = await download(fileName);
+  const fileStream = fs.createReadStream(
+    downloadedFile,
+    // @ts-ignore
+    encoding
+  );
+  const parseJsonStream = parser();
+  fileStream.pipe(parseJsonStream);
+
+  const parseStream = new ParseStream({ limitRows, jsonStyle, keyField });
+
+  const tramsformer = jsonStyle === 'object' ? streamObject() : streamArray();
+
+  if (rootField) {
+    const filterStream = pick({ filter: rootField });
+    parseJsonStream.pipe(filterStream);
+    filterStream.pipe(tramsformer);
+  } else {
+    parseJsonStream.pipe(tramsformer);
+  }
+
+  tramsformer.pipe(parseStream);
+
+  return parseStream;
+}
+
+module.exports = jsonReader;

packages/api/src/shell/jsonWriter.js

@@ -0,0 +1,97 @@
+const { getLogger } = require('dbgate-tools');
+const fs = require('fs');
+const stream = require('stream');
+const _ = require('lodash');
+
+const logger = getLogger('jsonArrayWriter');
+
+class StringifyStream extends stream.Transform {
+  constructor({ jsonStyle, keyField, rootField }) {
+    super({ objectMode: true });
+    this.wasHeader = false;
+    this.wasRecord = false;
+    this.jsonStyle = jsonStyle;
+    this.keyField = keyField || '_key';
+    this.rootField = rootField;
+  }
+  _transform(chunk, encoding, done) {
+    let skip = false;
+
+    if (!this.wasHeader) {
+      skip = chunk.__isStreamHeader;
+      this.wasHeader = true;
+    }
+    if (!skip) {
+      if (!this.wasRecord) {
+        if (this.rootField) {
+          if (this.jsonStyle === 'object') {
+            this.push(`{"${this.rootField}": {\n`);
+          } else {
+            this.push(`{"${this.rootField}": [\n`);
+          }
+        } else {
+          if (this.jsonStyle === 'object') {
+            this.push('{\n');
+          } else {
+            this.push('[\n');
+          }
+        }
+      } else {
+        this.push(',\n');
+      }
+      this.wasRecord = true;
+
+      if (this.jsonStyle === 'object') {
+        const key = chunk[this.keyField] ?? chunk[Object.keys(chunk)[0]];
+        this.push(`"${key}": ${JSON.stringify(_.omit(chunk, [this.keyField]))}`);
+      } else {
+        this.push(JSON.stringify(chunk));
+      }
+    }
+    done();
+  }
+
+  _flush(done) {
+    if (!this.wasRecord) {
+      if (this.rootField) {
+        if (this.jsonStyle === 'object') {
+          this.push(`{"${this.rootField}": {}}\n`);
+        } else {
+          this.push(`{"${this.rootField}": []}\n`);
+        }
+      } else {
+        if (this.jsonStyle === 'object') {
+          this.push('{}\n');
+        } else {
+          this.push('[]\n');
+        }
+      }
+    } else {
+      if (this.rootField) {
+        if (this.jsonStyle === 'object') {
+          this.push('\n}}\n');
+        } else {
+          this.push('\n]}\n');
+        }
+      } else {
+        if (this.jsonStyle === 'object') {
+          this.push('\n}\n');
+        } else {
+          this.push('\n]\n');
+        }
+      }
+    }
+    done();
+  }
+}
+
+async function jsonWriter({ fileName, jsonStyle, keyField = '_key', rootField, encoding = 'utf-8' }) {
+  logger.info(`Writing file ${fileName}`);
+  const stringify = new StringifyStream({ jsonStyle, keyField, rootField });
+  const fileStream = fs.createWriteStream(fileName, encoding);
+  stringify.pipe(fileStream);
+  stringify['finisher'] = fileStream;
+  return stringify;
+}
+
+module.exports = jsonWriter;

packages/api/src/shell/modifyJsonLinesReader.js

@@ -66,7 +66,7 @@ class ParseStream extends stream.Transform {
                 ...obj,
                 ...update.fields,
               },
-              (v, k) => v.$$undefined$$
+              (v, k) => v?.$$undefined$$
             );
           }
         }

packages/api/src/shell/runScript.js

@@ -11,7 +11,7 @@ async function runScript(func) {
     await func();
     process.exit(0);
   } catch (err) {
-    logger.error({ err }, `Error running script: ${err.message}`);
+    logger.error({ err }, `Error running script: ${err.message || err}`);
     process.exit(1);
   }
 }

packages/api/src/utility/authProxy.js

@@ -12,9 +12,14 @@ async function authProxyGetTokenFromCode(options) {
 
 function startTokenChecking(sid, callback) {}
 
+function getAuthProxyUrl() {
+  return 'https://auth.dbgate.eu';
+}
+
 module.exports = {
   isAuthProxySupported,
   authProxyGetRedirectUrl,
   authProxyGetTokenFromCode,
   startTokenChecking,
+  getAuthProxyUrl,
 };

packages/api/src/utility/checkLicense.js

@@ -5,6 +5,14 @@ function checkLicense() {
   };
 }
 
+function checkLicenseKey(key) {
+  return {
+    status: 'ok',
+    type: 'community',
+  };
+}
+
 module.exports = {
   checkLicense,
+  checkLicenseKey,
 };

packages/api/src/utility/hardwareFingerprint.js

@@ -0,0 +1,88 @@
+const axios = require('axios');
+const os = require('os');
+const crypto = require('crypto');
+const platformInfo = require('./platformInfo');
+
+async function getPublicIpInfo() {
+  try {
+    const resp = await axios.default.get('https://ipinfo.io/json');
+    if (!resp.data?.ip) {
+      return { ip: 'unknown-ip' };
+    }
+    return resp.data;
+  } catch (err) {
+    return { ip: 'unknown-ip' };
+  }
+}
+
+function getMacAddress() {
+  try {
+    const interfaces = os.networkInterfaces();
+    for (let iface of Object.values(interfaces)) {
+      for (let config of iface) {
+        if (config.mac && config.mac !== '00:00:00:00:00:00') {
+          return config.mac;
+        }
+      }
+    }
+    return '00:00:00:00:00:00';
+  } catch (err) {
+    return '00:00:00:00:00:00';
+  }
+}
+
+async function getHardwareFingerprint() {
+  const publicIpInfo = await getPublicIpInfo();
+  const macAddress = getMacAddress();
+  const platform = os.platform();
+  const release = os.release();
+  const hostname = os.hostname();
+  const totalMemory = os.totalmem();
+
+  return {
+    publicIp: publicIpInfo.ip,
+    country: publicIpInfo.country,
+    region: publicIpInfo.region,
+    city: publicIpInfo.city,
+    macAddress,
+    platform,
+    release,
+    hostname,
+    totalMemory,
+  };
+}
+
+async function getHardwareFingerprintHash(data = undefined) {
+  if (!data) {
+    data = await getHardwareFingerprint();
+  }
+  const fingerprintData = JSON.stringify(data);
+  const hash = crypto.createHash('sha256').update(fingerprintData).digest('hex');
+  return hash;
+}
+
+async function getPublicHardwareFingerprint() {
+  const fingerprint = await getHardwareFingerprint();
+  const hash = await getHardwareFingerprintHash(fingerprint);
+  return {
+    hash,
+    payload: {
+      platform: fingerprint.platform,
+      city: fingerprint.city,
+      country: fingerprint.country,
+      region: fingerprint.region,
+      isDocker: platformInfo.isDocker,
+      isElectron: platformInfo.isElectron,
+    },
+  };
+}
+
+// getHardwareFingerprint().then(console.log);
+// getHardwareFingerprintHash().then(console.log);
+// getPublicHardwareFingerprint().then(console.log);
+
+module.exports = {
+  getHardwareFingerprint,
+  getHardwareFingerprintHash,
+  getPublicHardwareFingerprint,
+};

packages/datalib/src/GridDisplay.ts

@@ -225,7 +225,7 @@ export abstract class GridDisplay {
           conditions.push(
             _.cloneDeepWith(condition, (expr: Expression) => {
               if (expr.exprType == 'placeholder') {
-                return this.createColumnExpression(column, { alias: column.sourceAlias });
+                return this.createColumnExpression(column, { alias: column.sourceAlias }, undefined, 'filter');
               }
               // return {
               //   exprType: 'column',
@@ -253,7 +253,7 @@ export abstract class GridDisplay {
             orCondition.conditions.push(
               _.cloneDeepWith(condition, (expr: Expression) => {
                 if (expr.exprType == 'placeholder') {
-                  return this.createColumnExpression(column, { alias: 'basetbl' });
+                  return this.createColumnExpression(column, { alias: 'basetbl' }, undefined, 'filter');
                 }
               })
             );
@@ -278,7 +278,7 @@ export abstract class GridDisplay {
 
   applySortOnSelect(select: Select, displayedColumnInfo: DisplayedColumnInfo) {
     if (this.config.sort?.length > 0) {
-      select.orderBy = this.config.sort
+      const orderByColumns = this.config.sort
         .map(col => ({ ...col, dispInfo: displayedColumnInfo[col.uniqueName] }))
         .map(col => ({ ...col, expr: select.columns.find(x => x.alias == col.uniqueName) }))
         .filter(col => col.dispInfo && col.expr)
@@ -286,6 +286,10 @@ export abstract class GridDisplay {
           ...col.expr,
           direction: col.order,
         }));
+
+      if (orderByColumns.length > 0) {
+        select.orderBy = orderByColumns;
+      }
     }
   }
 
@@ -570,10 +574,10 @@ export abstract class GridDisplay {
 
   processReferences(select: Select, displayedColumnInfo: DisplayedColumnInfo, options) {}
 
-  createColumnExpression(col, source, alias?) {
+  createColumnExpression(col, source, alias?, purpose: 'view' | 'filter' = 'view') {
     let expr = null;
     if (this.dialect.createColumnViewExpression) {
-      expr = this.dialect.createColumnViewExpression(col.columnName, col.dataType, source, alias);
+      expr = this.dialect.createColumnViewExpression(col.columnName, col.dataType, source, alias, purpose);
       if (expr) {
         return expr;
       }
@@ -595,7 +599,7 @@ export abstract class GridDisplay {
         name: _.pick(name, ['schemaName', 'pureName']),
         alias: 'basetbl',
       },
-      columns: columns.map(col => this.createColumnExpression(col, { alias: 'basetbl' })),
+      columns: columns.map(col => this.createColumnExpression(col, { alias: 'basetbl' }, undefined, 'view')),
       orderBy: this.driver?.requiresDefaultSortCriteria
         ? [
             {
@@ -625,7 +629,7 @@ export abstract class GridDisplay {
       columns: [
         ...select.columns,
         {
-          alias: '_rowNumber',
+          alias: '_RowNumber',
           exprType: 'rowNumber',
           orderBy: select.orderBy
             ? select.orderBy.map(x =>
@@ -683,7 +687,7 @@ export abstract class GridDisplay {
     let select = this.createSelect();
     if (!select) return null;
     if (this.dialect.rangeSelect) select.range = { offset: offset, limit: count };
-    else if (this.dialect.rowNumberOverPaging && offset > 0)
+    else if (this.dialect.rowNumberOverPaging && (offset > 0 || !this.dialect.topRecords))
       select = this.getRowNumberOverSelect(select, offset, count);
     else if (this.dialect.limitSelect) select.topRecords = count;
     return select;

packages/datalib/src/TableGridDisplay.ts

@@ -63,7 +63,7 @@ export class TableGridDisplay extends GridDisplay {
         ? this.table.primaryKey.columns.map(x => x.columnName)
         : this.table.columns.map(x => x.columnName);
     }
-    
+
     if (this.config.isFormView) {
       this.addAllExpandedColumnsToSelected = true;
       this.hintBaseColumns = this.formColumns;
@@ -287,7 +287,7 @@ export class TableGridDisplay extends GridDisplay {
     for (const column of columns) {
       if (this.addAllExpandedColumnsToSelected || this.config.addedColumns.includes(column.uniqueName)) {
         select.columns.push(
-          this.createColumnExpression(column, { name: column, alias: parentAlias }, column.uniqueName)
+          this.createColumnExpression(column, { name: column, alias: parentAlias }, column.uniqueName, 'view')
         );
         displayedColumnInfo[column.uniqueName] = {
           ...column,

packages/datalib/src/ViewGridDisplay.ts

@@ -1,6 +1,6 @@
 import _ from 'lodash';
 import { GridDisplay, ChangeCacheFunc, ChangeConfigFunc } from './GridDisplay';
-import type { EngineDriver, ViewInfo, ColumnInfo } from 'dbgate-types';
+import type { EngineDriver, ViewInfo, ColumnInfo, DatabaseInfo } from 'dbgate-types';
 import { GridConfig, GridCache } from './GridConfig';
 
 export class ViewGridDisplay extends GridDisplay {
@@ -11,9 +11,10 @@ export class ViewGridDisplay extends GridDisplay {
     setConfig: ChangeConfigFunc,
     cache: GridCache,
     setCache: ChangeCacheFunc,
+    dbinfo: DatabaseInfo,
     serverVersion
   ) {
-    super(config, setConfig, cache, setCache, driver, serverVersion);
+    super(config, setConfig, cache, setCache, driver, dbinfo, serverVersion);
     this.columns = this.getDisplayColumns(view);
     this.formColumns = this.columns;
     this.filterable = true;

packages/sqltree/src/dumpSqlCommand.ts

@@ -62,10 +62,13 @@ export function dumpSqlSelect(dmp: SqlDumper, cmd: Select) {
 }
 
 export function dumpSqlUpdate(dmp: SqlDumper, cmd: Update) {
-  dmp.put('^update ');
-  dumpSqlSourceRef(dmp, cmd.from);
-
-  dmp.put('&n^set ');
+  if (cmd.alterTableUpdateSyntax) {
+    dmp.put('^alter ^table %f &n^update ', cmd.from?.name);
+  } else {
+    dmp.put('^update ');
+    dumpSqlSourceRef(dmp, cmd.from);
+    dmp.put('&n^set ');
+  }
   dmp.put('&>');
   dmp.putCollection(', ', cmd.fields, col => {
     dmp.put('%i=', col.targetColumn);
@@ -81,8 +84,14 @@ export function dumpSqlUpdate(dmp: SqlDumper, cmd: Update) {
 }
 
 export function dumpSqlDelete(dmp: SqlDumper, cmd: Delete) {
-  dmp.put('^delete ^from ');
-  dumpSqlSourceRef(dmp, cmd.from);
+  if (cmd.alterTableDeleteSyntax) {
+    dmp.put('^alter ^table ');
+    dumpSqlSourceRef(dmp, cmd.from);
+    dmp.put(' ^delete ');
+  } else {
+    dmp.put('^delete ^from ');
+    dumpSqlSourceRef(dmp, cmd.from);
+  }
 
   if (cmd.where) {
     dmp.put('&n^where ');

packages/sqltree/src/types.ts

@@ -26,12 +26,17 @@ export interface Update {
   fields: UpdateField[];
   from: FromDefinition;
   where?: Condition;
+  // ALTER TABLE xxx UPDATE col1=val1 - syntax for ClickHouse
+  alterTableUpdateSyntax?: boolean;
 }
 
 export interface Delete {
   commandType: 'delete';
   from: FromDefinition;
   where?: Condition;
+
+  // ALTER TABLE xxx DELETE  - syntax for ClickHouse
+  alterTableDeleteSyntax?: boolean;
 }
 
 export interface Insert {

packages/tools/src/DatabaseAnalyser.ts

@@ -1,10 +1,11 @@
-import { DatabaseInfo, DatabaseModification, EngineDriver, SqlDialect } from 'dbgate-types';
+import { DatabaseHandle, DatabaseInfo, DatabaseModification, EngineDriver, SqlDialect } from 'dbgate-types';
 import _sortBy from 'lodash/sortBy';
 import _groupBy from 'lodash/groupBy';
 import _pick from 'lodash/pick';
 import _compact from 'lodash/compact';
 import { getLogger } from './getLogger';
 import { type Logger } from 'pinomin';
+import { dbNameLogCategory, isCompositeDbName, splitCompositeDbName } from './schemaInfoTools';
 
 const logger = getLogger('dbAnalyser');
 
@@ -40,7 +41,7 @@ export class DatabaseAnalyser {
   dialect: SqlDialect;
   logger: Logger;
 
-  constructor(public pool, public driver: EngineDriver, version) {
+  constructor(public dbhan: DatabaseHandle, public driver: EngineDriver, version) {
     this.dialect = (driver?.dialectByVersion && driver?.dialectByVersion(version)) || driver?.dialect;
     this.logger = logger;
   }
@@ -68,6 +69,7 @@ export class DatabaseAnalyser {
   }
 
   async fullAnalysis() {
+    logger.info(`Performing full analysis, DB=${dbNameLogCategory(this.dbhan.database)}, engine=${this.driver.engine}`);
     const res = this.addEngineField(await this._runAnalysis());
     // console.log('FULL ANALYSIS', res);
     return res;
@@ -88,6 +90,7 @@ export class DatabaseAnalyser {
   }
 
   async incrementalAnalysis(structure) {
+    logger.info(`Performing incremental analysis, DB=${dbNameLogCategory(this.dbhan.database)}, engine=${this.driver.engine}`);
     this.structure = structure;
 
     const modifications = await this.getModifications();
@@ -180,8 +183,19 @@ export class DatabaseAnalyser {
   //   return this.createQueryCore('=OBJECT_ID_CONDITION', typeFields) != ' is not null';
   // }
 
+  getDefaultSchemaNameCondition() {
+    return 'is not null';
+  }
+
   createQuery(template, typeFields, replacements = {}) {
-    return this.createQueryCore(this.processQueryReplacements(template, replacements), typeFields);
+    let query = this.createQueryCore(this.processQueryReplacements(template, replacements), typeFields);
+
+    const dbname = this.dbhan.database;
+    const schemaCondition = isCompositeDbName(dbname)
+      ? `= '${splitCompositeDbName(dbname).schema}' `
+      : ` ${this.getDefaultSchemaNameCondition()} `;
+
+    return query?.replace(/=SCHEMA_NAME_CONDITION/g, schemaCondition);
   }
 
   processQueryReplacements(query, replacements) {
@@ -242,8 +256,8 @@ export class DatabaseAnalyser {
   }
 
   feedback(obj) {
-    if (this.pool.feedback) {
-      this.pool.feedback(obj);
+    if (this.dbhan.feedback) {
+      this.dbhan.feedback(obj);
     }
     if (obj && obj.analysingMessage) {
       logger.debug(obj.analysingMessage);
@@ -318,11 +332,11 @@ export class DatabaseAnalyser {
       };
     }
     try {
-      const res = await this.driver.query(this.pool, sql);
+      const res = await this.driver.query(this.dbhan, sql);
       this.logger.debug({ rows: res.rows.length, template }, `Loaded analyser query`);
       return res;
     } catch (err) {
-      logger.error({ err }, 'Error running analyser query');
+      logger.error({ err, message: err.message, template }, 'Error running analyser query');
       return {
         rows: [],
       };
@@ -338,7 +352,6 @@ export class DatabaseAnalyser {
       functions: [],
       procedures: [],
       triggers: [],
-      schemas: [],
     };
   }
 

packages/tools/src/SqlDumper.ts

@@ -214,6 +214,14 @@ export class SqlDumper implements AlterProcessor {
     this.putCmd('^drop ^database %i', name);
   }
 
+  createSchema(name: string) {
+    this.putCmd('^create ^schema %i', name);
+  }
+
+  dropSchema(name: string) {
+    this.putCmd('^drop ^schema %i', name);
+  }
+
   specialColumnOptions(column) {}
 
   selectScopeIdentity(table: TableInfo) {}
@@ -246,7 +254,7 @@ export class SqlDumper implements AlterProcessor {
 
     this.putRaw(' ');
     this.specialColumnOptions(column);
-    if (includeNullable) {
+    if (includeNullable && !this.dialect?.specificNullabilityImplementation) {
       this.put(column.notNull ? '^not ^null' : '^null');
     }
     if (includeDefault && column.defaultValue?.trim()) {
@@ -294,12 +302,25 @@ export class SqlDumper implements AlterProcessor {
     });
 
     this.put('&<&n)');
+
+    this.tableOptions(table);
+
     this.endCommand();
     (table.indexes || []).forEach(ix => {
       this.createIndex(ix);
     });
   }
 
+  tableOptions(table: TableInfo) {
+    const options = this.driver?.dialect?.getTableFormOptions?.('sqlCreateTable') || [];
+    for (const option of options) {
+      if (table[option.name]) {
+        this.put('&n');
+        this.put(option.sqlFormatString, table[option.name]);
+      }
+    }
+  }
+
   createTablePrimaryKeyCore(table: TableInfo) {
     if (table.primaryKey) {
       this.put(',&n');
@@ -531,7 +552,9 @@ export class SqlDumper implements AlterProcessor {
   renameConstraint(constraint: ConstraintInfo, newName: string) {}
 
   createColumn(column: ColumnInfo, constraints: ConstraintInfo[]) {
-    this.put('^alter ^table %f ^add %i ', column, column.columnName);
+    this.put('^alter ^table %f ^add ', column);
+    if (this.dialect.createColumnWithColumnKeyword) this.put('^column ');
+    this.put(' %i ', column.columnName);
     this.columnDefinition(column);
     this.inlineConstraints(constraints);
     this.endCommand();
@@ -607,10 +630,8 @@ export class SqlDumper implements AlterProcessor {
     if (!oldTable.pairingId || !newTable.pairingId || oldTable.pairingId != newTable.pairingId) {
       throw new Error('Recreate is not possible: oldTable.paringId != newTable.paringId');
     }
-    const tmpTable = `temp_${uuidv1()}`;
 
-    // console.log('oldTable', oldTable);
-    // console.log('newTable', newTable);
+    const tmpTable = `temp_${uuidv1()}`;
 
     const columnPairs = oldTable.columns
       .map(oldcol => ({
@@ -619,33 +640,49 @@ export class SqlDumper implements AlterProcessor {
       }))
       .filter(x => x.newcol);
 
-    this.dropConstraints(oldTable, true);
-    this.renameTable(oldTable, tmpTable);
+    if (this.driver.supportsTransactions) {
+      this.dropConstraints(oldTable, true);
+      this.renameTable(oldTable, tmpTable);
 
-    this.createTable(newTable);
+      this.createTable(newTable);
 
-    const autoinc = newTable.columns.find(x => x.autoIncrement);
-    if (autoinc) {
-      this.allowIdentityInsert(newTable, true);
+      const autoinc = newTable.columns.find(x => x.autoIncrement);
+      if (autoinc) {
+        this.allowIdentityInsert(newTable, true);
+      }
+
+      this.putCmd(
+        '^insert ^into %f (%,i) select %,s ^from %f',
+        newTable,
+        columnPairs.map(x => x.newcol.columnName),
+        columnPairs.map(x => x.oldcol.columnName),
+        { ...oldTable, pureName: tmpTable }
+      );
+
+      if (autoinc) {
+        this.allowIdentityInsert(newTable, false);
+      }
+
+      if (this.dialect.dropForeignKey) {
+        newTable.dependencies.forEach(cnt => this.createConstraint(cnt));
+      }
+
+      this.dropTable({ ...oldTable, pureName: tmpTable });
+    } else {
+      // we have to preserve old table as long as possible
+      this.createTable({ ...newTable, pureName: tmpTable });
+
+      this.putCmd(
+        '^insert ^into %f (%,i) select %,s ^from %f',
+        { ...newTable, pureName: tmpTable },
+        columnPairs.map(x => x.newcol.columnName),
+        columnPairs.map(x => x.oldcol.columnName),
+        oldTable
+      );
+
+      this.dropTable(oldTable);
+      this.renameTable({ ...newTable, pureName: tmpTable }, newTable.pureName);
     }
-
-    this.putCmd(
-      '^insert ^into %f (%,i) select %,s ^from %f',
-      newTable,
-      columnPairs.map(x => x.newcol.columnName),
-      columnPairs.map(x => x.oldcol.columnName),
-      { ...oldTable, pureName: tmpTable }
-    );
-
-    if (autoinc) {
-      this.allowIdentityInsert(newTable, false);
-    }
-
-    if (this.dialect.dropForeignKey) {
-      newTable.dependencies.forEach(cnt => this.createConstraint(cnt));
-    }
-
-    this.dropTable({ ...oldTable, pureName: tmpTable });
   }
 
   createSqlObject(obj: SqlObjectInfo) {
@@ -671,6 +708,23 @@ export class SqlDumper implements AlterProcessor {
     this.putCmd('^drop %s %f', this.getSqlObjectSqlName(obj.objectTypeField), obj);
   }
 
+  setTableOption(table: TableInfo, optionName: string, optionValue: string) {
+    const options = this?.dialect?.getTableFormOptions?.('sqlAlterTable');
+    const option = options?.find(x => x.name == optionName && !x.disabled);
+    if (!option) {
+      return;
+    }
+
+    this.setTableOptionCore(table, optionName, optionValue, option.sqlFormatString);
+
+    this.endCommand();
+  }
+
+  setTableOptionCore(table: TableInfo, optionName: string, optionValue: string, formatString: string) {
+    this.put('^alter ^table %f ', table);
+    this.put(formatString, optionValue);
+  }
+
   fillPreloadedRows(
     table: NamedObjectInfo,
     oldRows: any[],

packages/tools/src/alterPlan.ts

@@ -97,6 +97,13 @@ interface AlterOperation_FillPreloadedRows {
   autoIncrementColumn: string;
 }
 
+interface AlterOperation_SetTableOption {
+  operationType: 'setTableOption';
+  table: TableInfo;
+  optionName: string;
+  optionValue: string;
+}
+
 type AlterOperation =
   | AlterOperation_CreateColumn
   | AlterOperation_ChangeColumn
@@ -112,7 +119,8 @@ type AlterOperation =
   | AlterOperation_CreateSqlObject
   | AlterOperation_DropSqlObject
   | AlterOperation_RecreateTable
-  | AlterOperation_FillPreloadedRows;
+  | AlterOperation_FillPreloadedRows
+  | AlterOperation_SetTableOption;
 
 export class AlterPlan {
   recreates = {
@@ -253,6 +261,15 @@ export class AlterPlan {
     });
   }
 
+  setTableOption(table: TableInfo, optionName: string, optionValue: string) {
+    this.operations.push({
+      operationType: 'setTableOption',
+      table,
+      optionName,
+      optionValue,
+    });
+  }
+
   run(processor: AlterProcessor) {
     for (const op of this.operations) {
       runAlterOperation(op, processor);
@@ -267,6 +284,7 @@ export class AlterPlan {
       : [];
     const constraints = _.compact([
       dependencyDefinition?.includes('primaryKey') ? table.primaryKey : null,
+      dependencyDefinition?.includes('sortingKey') ? table.sortingKey : null,
       ...(dependencyDefinition?.includes('foreignKeys') ? table.foreignKeys : []),
       ...(dependencyDefinition?.includes('indexes') ? table.indexes : []),
       ...(dependencyDefinition?.includes('uniques') ? table.uniques : []),
@@ -297,35 +315,40 @@ export class AlterPlan {
         return res;
       }
 
-      if (op.operationType == 'changeColumn') {
-        const constraints = this._getDependendColumnConstraints(op.oldObject, this.dialect.changeColumnDependencies);
+      for (const [testedOperationType, testedDependencies, testedObject] of [
+        ['changeColumn', this.dialect.changeColumnDependencies, (op as AlterOperation_ChangeColumn).oldObject],
+        ['renameColumn', this.dialect.renameColumnDependencies, (op as AlterOperation_RenameColumn).object],
+      ]) {
+        if (op.operationType == testedOperationType) {
+          const constraints = this._getDependendColumnConstraints(testedObject as ColumnInfo, testedDependencies);
 
-        if (constraints.length > 0 && this.opts.noDropConstraint) {
-          return [];
+          if (constraints.length > 0 && this.opts.noDropConstraint) {
+            return [];
+          }
+
+          const res: AlterOperation[] = [
+            ...constraints.map(oldObject => {
+              const opRes: AlterOperation = {
+                operationType: 'dropConstraint',
+                oldObject,
+              };
+              return opRes;
+            }),
+            op,
+            ..._.reverse([...constraints]).map(newObject => {
+              const opRes: AlterOperation = {
+                operationType: 'createConstraint',
+                newObject,
+              };
+              return opRes;
+            }),
+          ];
+
+          if (constraints.length > 0) {
+            this.recreates.constraints += 1;
+          }
+          return res;
         }
-
-        const res: AlterOperation[] = [
-          ...constraints.map(oldObject => {
-            const opRes: AlterOperation = {
-              operationType: 'dropConstraint',
-              oldObject,
-            };
-            return opRes;
-          }),
-          op,
-          ..._.reverse([...constraints]).map(newObject => {
-            const opRes: AlterOperation = {
-              operationType: 'createConstraint',
-              newObject,
-            };
-            return opRes;
-          }),
-        ];
-
-        if (constraints.length > 0) {
-          this.recreates.constraints += 1;
-        }
-        return res;
       }
 
       if (op.operationType == 'dropTable') {
@@ -374,7 +397,8 @@ export class AlterPlan {
         this._testTableRecreate(op, 'dropColumn', this.dialect.dropColumn, 'oldObject') ||
         this._testTableRecreate(op, 'createConstraint', obj => this._canCreateConstraint(obj), 'newObject') ||
         this._testTableRecreate(op, 'dropConstraint', obj => this._canDropConstraint(obj), 'oldObject') ||
-        this._testTableRecreate(op, 'changeColumn', this.dialect.changeColumn, 'newObject') || [op]
+        this._testTableRecreate(op, 'changeColumn', this.dialect.changeColumn, 'newObject') ||
+        this._testTableRecreate(op, 'renameColumn', true, 'object') || [op]
       );
     });
 
@@ -383,6 +407,7 @@ export class AlterPlan {
 
   _canCreateConstraint(cnt: ConstraintInfo) {
     if (cnt.constraintType == 'primaryKey') return this.dialect.createPrimaryKey;
+    if (cnt.constraintType == 'sortingKey') return this.dialect.createPrimaryKey;
     if (cnt.constraintType == 'foreignKey') return this.dialect.createForeignKey;
     if (cnt.constraintType == 'index') return this.dialect.createIndex;
     if (cnt.constraintType == 'unique') return this.dialect.createUnique;
@@ -392,6 +417,7 @@ export class AlterPlan {
 
   _canDropConstraint(cnt: ConstraintInfo) {
     if (cnt.constraintType == 'primaryKey') return this.dialect.dropPrimaryKey;
+    if (cnt.constraintType == 'sortingKey') return this.dialect.dropPrimaryKey;
     if (cnt.constraintType == 'foreignKey') return this.dialect.dropForeignKey;
     if (cnt.constraintType == 'index') return this.dialect.dropIndex;
     if (cnt.constraintType == 'unique') return this.dialect.dropUnique;
@@ -453,7 +479,7 @@ export class AlterPlan {
         }
       } else {
         // @ts-ignore
-        const oldObject: TableInfo = op.oldObject;
+        const oldObject: TableInfo = op.oldObject || op.object;
         if (oldObject) {
           const recreated = recreates[`${oldObject.schemaName}||${oldObject.pureName}`];
           if (recreated) {
@@ -575,6 +601,9 @@ export function runAlterOperation(op: AlterOperation, processor: AlterProcessor)
     case 'dropSqlObject':
       processor.dropSqlObject(op.oldObject);
       break;
+    case 'setTableOption':
+      processor.setTableOption(op.table, op.optionName, op.optionValue);
+      break;
     case 'fillPreloadedRows':
       processor.fillPreloadedRows(op.table, op.oldRows, op.newRows, op.key, op.insertOnly, op.autoIncrementColumn);
       break;

packages/tools/src/createBulkInsertStreamBase.ts

@@ -5,7 +5,7 @@ import { prepareTableForImport } from './tableTransforms';
 
 const logger = getLogger('bulkStreamBase');
 
-export function createBulkInsertStreamBase(driver: EngineDriver, stream, pool, name, options: WriteTableOptions): any {
+export function createBulkInsertStreamBase(driver: EngineDriver, stream, dbhan, name, options: WriteTableOptions): any {
   const fullNameQuoted = name.schemaName
     ? `${driver.dialect.quoteIdentifier(name.schemaName)}.${driver.dialect.quoteIdentifier(name.pureName)}`
     : driver.dialect.quoteIdentifier(name.pureName);
@@ -29,21 +29,22 @@ export function createBulkInsertStreamBase(driver: EngineDriver, stream, pool, n
   };
 
   writable.checkStructure = async () => {
-    let structure = await driver.analyseSingleTable(pool, name);
+    let structure = await driver.analyseSingleTable(dbhan, name);
     // console.log('ANALYSING', name, structure);
     if (structure && options.dropIfExists) {
       logger.info(`Dropping table ${fullNameQuoted}`);
-      await driver.script(pool, `DROP TABLE ${fullNameQuoted}`);
+      await driver.script(dbhan, `DROP TABLE ${fullNameQuoted}`);
     }
     if (options.createIfNotExists && (!structure || options.dropIfExists)) {
       const dmp = driver.createDumper();
-      dmp.createTable(prepareTableForImport({ ...writable.structure, ...name }));
+      const createdTableInfo = driver.adaptTableInfo(prepareTableForImport({ ...writable.structure, ...name }));
+      dmp.createTable(createdTableInfo);
       logger.info({ sql: dmp.s }, `Creating table ${fullNameQuoted}`);
-      await driver.script(pool, dmp.s);
-      structure = await driver.analyseSingleTable(pool, name);
+      await driver.script(dbhan, dmp.s);
+      structure = await driver.analyseSingleTable(dbhan, name);
     }
     if (options.truncate) {
-      await driver.script(pool, `TRUNCATE TABLE ${fullNameQuoted}`);
+      await driver.script(dbhan, `TRUNCATE TABLE ${fullNameQuoted}`);
     }
 
     writable.columnNames = _intersection(
@@ -73,7 +74,7 @@ export function createBulkInsertStreamBase(driver: EngineDriver, stream, pool, n
       dmp.putRaw(';');
       // require('fs').writeFileSync('/home/jena/test.sql', dmp.s);
       // console.log(dmp.s);
-      await driver.query(pool, dmp.s, { discardResult: true });
+      await driver.query(dbhan, dmp.s, { discardResult: true });
     } else {
       for (const row of rows) {
         const dmp = driver.createDumper();
@@ -84,13 +85,13 @@ export function createBulkInsertStreamBase(driver: EngineDriver, stream, pool, n
         dmp.putRaw('(');
         dmp.putCollection(',', writable.columnNames, col => dmp.putValue(row[col as string]));
         dmp.putRaw(')');
-        await driver.query(pool, dmp.s, { discardResult: true });
+        await driver.query(dbhan, dmp.s, { discardResult: true });
       }
     }
     if (options.commitAfterInsert) {
       const dmp = driver.createDumper();
       dmp.commitTransaction();
-      await driver.query(pool, dmp.s, { discardResult: true });
+      await driver.query(dbhan, dmp.s, { discardResult: true });
     }
   };
 

packages/tools/src/database-info-alter-processor.ts

@@ -11,6 +11,7 @@ import {
   UniqueInfo,
   SqlObjectInfo,
   NamedObjectInfo,
+  ColumnsConstraintInfo,
 } from '../../types';
 
 export class DatabaseInfoAlterProcessor {
@@ -59,6 +60,9 @@ export class DatabaseInfoAlterProcessor {
       case 'primaryKey':
         table.primaryKey = constraint as PrimaryKeyInfo;
         break;
+      case 'sortingKey':
+        table.sortingKey = constraint as ColumnsConstraintInfo;
+        break;
       case 'foreignKey':
         table.foreignKeys.push(constraint as ForeignKeyInfo);
         break;
@@ -86,6 +90,9 @@ export class DatabaseInfoAlterProcessor {
       case 'primaryKey':
         table.primaryKey = null;
         break;
+      case 'sortingKey':
+        table.sortingKey = null;
+        break;
       case 'foreignKey':
         table.foreignKeys = table.foreignKeys.filter(x => x.constraintName != constraint.constraintName);
         break;
@@ -129,4 +136,9 @@ export class DatabaseInfoAlterProcessor {
     tableInfo.preloadedRowsKey = key;
     tableInfo.preloadedRowsInsertOnly = insertOnly;
   }
+
+  setTableOption(table: TableInfo, optionName: string, optionValue: string) {
+    const tableInfo = this.db.tables.find(x => x.pureName == table.pureName && x.schemaName == table.schemaName);
+    tableInfo[optionName] = optionValue;
+  }
 }

packages/tools/src/diffTools.ts

@@ -46,6 +46,14 @@ export function generateTablePairingId(table: TableInfo): TableInfo {
   if (!table.pairingId) {
     return {
       ...table,
+      primaryKey: table.primaryKey && {
+        ...table.primaryKey,
+        pairingId: table.primaryKey.pairingId || uuidv1(),
+      },
+      sortingKey: table.sortingKey && {
+        ...table.sortingKey,
+        pairingId: table.sortingKey.pairingId || uuidv1(),
+      },
       columns: table.columns?.map(col => ({
         ...col,
         pairingId: col.pairingId || uuidv1(),
@@ -335,6 +343,7 @@ export function testEqualTypes(a: ColumnInfo, b: ColumnInfo, opts: DbDiffOptions
 function getTableConstraints(table: TableInfo) {
   const res = [];
   if (table.primaryKey) res.push(table.primaryKey);
+  if (table.sortingKey) res.push(table.sortingKey);
   if (table.foreignKeys) res.push(...table.foreignKeys);
   if (table.indexes) res.push(...table.indexes);
   if (table.uniques) res.push(...table.uniques);
@@ -345,7 +354,9 @@ function getTableConstraints(table: TableInfo) {
 function createPairs(oldList, newList, additionalCondition = null) {
   const res = [];
   for (const a of oldList) {
-    const b = newList.find(x => x.pairingId == a.pairingId || (additionalCondition && additionalCondition(a, x)));
+    const b = newList.find(
+      x => (a.pairingId && x.pairingId == a.pairingId) || (additionalCondition && additionalCondition(a, x))
+    );
     if (b) {
       res.push([a, b]);
     } else {
@@ -381,9 +392,14 @@ function planAlterTable(plan: AlterPlan, oldTable: TableInfo, newTable: TableInf
   const constraintPairs = createPairs(
     getTableConstraints(oldTable),
     getTableConstraints(newTable),
-    (a, b) => a.constraintType == 'primaryKey' && b.constraintType == 'primaryKey'
+    (a, b) =>
+      (a.constraintType == 'primaryKey' && b.constraintType == 'primaryKey') ||
+      (a.constraintType == 'sortingKey' && b.constraintType == 'sortingKey')
   );
-  // console.log('constraintPairs SOURCE', getTableConstraints(oldTable), getTableConstraints(newTable));
+  // console.log('constraintPairs OLD TABLE', oldTable);
+  // console.log('constraintPairs NEW TABLE', newTable);
+  // console.log('constraintPairs SOURCE OLD', getTableConstraints(oldTable));
+  // console.log('constraintPairs SOURCE NEW', getTableConstraints(newTable));
   // console.log('constraintPairs', constraintPairs);
 
   if (!opts.noDropConstraint) {
@@ -407,7 +423,7 @@ function planAlterTable(plan: AlterPlan, oldTable: TableInfo, newTable: TableInf
           // console.log('PLAN RENAME COLUMN')
           plan.renameColumn(x[0], x[1].columnName);
         } else {
-          // console.log('PLAN CHANGE COLUMN')
+          // console.log('PLAN CHANGE COLUMN', x[0], x[1]);
           plan.changeColumn(x[0], x[1]);
         }
       }
@@ -425,6 +441,28 @@ function planAlterTable(plan: AlterPlan, oldTable: TableInfo, newTable: TableInf
   constraintPairs.filter(x => x[0] == null).forEach(x => plan.createConstraint(x[1]));
 
   planTablePreload(plan, oldTable, newTable);
+
+  planChangeTableOptions(plan, oldTable, newTable, opts);
+
+  // console.log('oldTable', oldTable);
+  // console.log('newTable', newTable);
+  // console.log('plan.operations', plan.operations);
+}
+
+function planChangeTableOptions(plan: AlterPlan, oldTable: TableInfo, newTable: TableInfo, opts: DbDiffOptions) {
+  for (const option of plan.dialect?.getTableFormOptions?.('sqlAlterTable') || []) {
+    if (option.disabled) {
+      continue;
+    }
+    const name = option.name;
+    if (
+      oldTable[name] != newTable[name] &&
+      (oldTable[name] || newTable[name]) &&
+      (newTable[name] || option.allowEmptyValue)
+    ) {
+      plan.setTableOption(newTable, name, newTable[name]);
+    }
+  }
 }
 
 export function testEqualTables(

packages/tools/src/driverBase.ts

@@ -8,6 +8,7 @@ import { detectSqlFilterBehaviour } from './detectSqlFilterBehaviour';
 const dialect = {
   limitSelect: true,
   rangeSelect: true,
+  topRecords: false,
   offsetFetchRangeSyntax: true,
   stringEscapeChar: "'",
   fallbackDataType: 'nvarchar(max)',
@@ -65,25 +66,35 @@ export const driverBase = {
     return new this.dumperClass(this, options);
   },
   async script(pool, sql, options: RunScriptOptions) {
-    if (options?.useTransaction) {
+    if (options?.useTransaction && this.supportsTransactions) {
       runCommandOnDriver(pool, this, dmp => dmp.beginTransaction());
     }
     for (const sqlItem of splitQuery(sql, this.getQuerySplitterOptions('script'))) {
       try {
-        await this.query(pool, sqlItem, { discardResult: true });
+        await this.query(pool, sqlItem, { discardResult: true, ...options?.queryOptions });
       } catch (err) {
-        if (options?.useTransaction) {
+        if (options?.useTransaction && this.supportsTransactions) {
           runCommandOnDriver(pool, this, dmp => dmp.rollbackTransaction());
         }
         throw err;
       }
     }
-    if (options?.useTransaction) {
+    if (options?.useTransaction && this.supportsTransactions) {
       runCommandOnDriver(pool, this, dmp => dmp.commitTransaction());
     }
   },
   async operation(pool, operation, options: RunScriptOptions) {
-    throw new Error('Operation not defined in target driver');
+    const { type } = operation;
+    switch (type) {
+      case 'createSchema':
+        await runCommandOnDriver(pool, this, dmp => dmp.createSchema(operation.schemaName));
+        break;
+      case 'dropSchema':
+        await runCommandOnDriver(pool, this, dmp => dmp.dropSchema(operation.schemaName));
+        break;
+      default:
+        throw new Error(`Operation type ${type} not supported`);
+    }
   },
   getNewObjectTemplates() {
     if (this.databaseEngineTypes.includes('sql')) {
@@ -172,4 +183,16 @@ export const driverBase = {
     parseSqlNull: true,
     parseHexAsBuffer: true,
   },
+
+  createSaveChangeSetScript(changeSet, dbinfo, defaultCreator) {
+    return defaultCreator(changeSet, dbinfo);
+  },
+
+  adaptTableInfo(table) {
+    return table;
+  },
+
+  async listSchemas(pool) {
+    return null;
+  },
 };

packages/tools/src/index.ts

@@ -23,3 +23,4 @@ export * from './getLogger';
 export * from './getConnectionLabel';
 export * from './detectSqlFilterBehaviour';
 export * from './filterBehaviours';
+export * from './schemaInfoTools';

packages/tools/src/schemaEditorTools.ts

@@ -2,6 +2,7 @@ import uuidv1 from 'uuid/v1';
 import _omit from 'lodash/omit';
 import type {
   ColumnInfo,
+  ColumnsConstraintInfo,
   ConstraintInfo,
   ForeignKeyInfo,
   IndexInfo,
@@ -195,6 +196,13 @@ export function editorAddConstraint(table: TableInfo, constraint: ConstraintInfo
     } as PrimaryKeyInfo;
   }
 
+  if (constraint.constraintType == 'sortingKey') {
+    res.sortingKey = {
+      pairingId: uuidv1(),
+      ...constraint,
+    } as ColumnsConstraintInfo;
+  }
+
   if (constraint.constraintType == 'foreignKey') {
     res.foreignKeys = [
       ...(res.foreignKeys || []),
@@ -240,6 +248,13 @@ export function editorModifyConstraint(table: TableInfo, constraint: ConstraintI
     };
   }
 
+  if (constraint.constraintType == 'sortingKey') {
+    res.sortingKey = {
+      ...res.sortingKey,
+      ...constraint,
+    };
+  }
+
   if (constraint.constraintType == 'foreignKey') {
     res.foreignKeys = table.foreignKeys.map(fk =>
       fk.pairingId == constraint.pairingId ? { ...fk, ...constraint } : fk
@@ -266,6 +281,10 @@ export function editorDeleteConstraint(table: TableInfo, constraint: ConstraintI
     res.primaryKey = null;
   }
 
+  if (constraint.constraintType == 'sortingKey') {
+    res.sortingKey = null;
+  }
+
   if (constraint.constraintType == 'foreignKey') {
     res.foreignKeys = table.foreignKeys.filter(x => x.pairingId != constraint.pairingId);
   }

packages/tools/src/schemaInfoTools.ts

@@ -0,0 +1,48 @@
+import { SchemaInfo, SqlDialect } from 'dbgate-types';
+
+export function findDefaultSchema(schemaList: SchemaInfo[], dialect: SqlDialect, schemaInStorage: string = null) {
+  if (!schemaList) {
+    return null;
+  }
+
+  if (schemaInStorage && schemaList.find(x => x.schemaName == schemaInStorage)) {
+    return schemaInStorage;
+  }
+
+  const dynamicDefaultSchema = schemaList.find(x => x.isDefault);
+  if (dynamicDefaultSchema) {
+    return dynamicDefaultSchema.schemaName;
+  }
+  if (dialect?.defaultSchemaName && schemaList.find(x => x.schemaName == dialect.defaultSchemaName)) {
+    return dialect.defaultSchemaName;
+  }
+  return schemaList[0]?.schemaName;
+}
+
+export function isCompositeDbName(name: string) {
+  return name?.includes('::');
+}
+
+export function splitCompositeDbName(name: string) {
+  if (!isCompositeDbName(name)) return null;
+  const [database, schema] = name.split('::');
+  return { database, schema };
+}
+
+export function extractDbNameFromComposite(name: string) {
+  return isCompositeDbName(name) ? splitCompositeDbName(name).database : name;
+}
+
+export function extractSchemaNameFromComposite(name: string) {
+  return splitCompositeDbName(name)?.schema;
+}
+
+export function dbNameLogCategory(database: string): string {
+  if (isCompositeDbName(database)) {
+    return '~composite';
+  }
+  if (database) {
+    return '~simple';
+  }
+  return '~nodb';
+}

packages/tools/src/structureTools.ts

@@ -5,7 +5,7 @@ export function addTableDependencies(db: DatabaseInfo): DatabaseInfo {
   if (!db.tables) {
     return db;
   }
-  
+
   const allForeignKeys = _flatten(db.tables.map(x => x.foreignKeys || []));
   return {
     ...db,
@@ -33,6 +33,14 @@ export function extendTableInfo(table: TableInfo): TableInfo {
           constraintType: 'primaryKey',
         }
       : undefined,
+    sortingKey: table.sortingKey
+      ? {
+          ...table.sortingKey,
+          pureName: table.pureName,
+          schemaName: table.schemaName,
+          constraintType: 'sortingKey',
+        }
+      : undefined,
     foreignKeys: (table.foreignKeys || []).map(cnt => ({
       ...cnt,
       pureName: table.pureName,

packages/tools/src/tableTransforms.ts

@@ -10,6 +10,7 @@ export function prepareTableForImport(table: TableInfo): TableInfo {
   res.uniques = [];
   res.checks = [];
   if (res.primaryKey) res.primaryKey.constraintName = null;
+  res.tableEngine = null;
   return res;
 }
 

packages/tools/src/yamlModelConv.ts

@@ -27,6 +27,7 @@ export interface TableInfoYaml {
   // schema?: string;
   columns: ColumnInfoYaml[];
   primaryKey?: string[];
+  sortingKey?: string[];
 
   insertKey?: string[];
   insertOnly?: string[];
@@ -91,6 +92,9 @@ export function tableInfoToYaml(table: TableInfo): TableInfoYaml {
   if (tableCopy.primaryKey && !tableCopy.primaryKey['_dumped']) {
     res.primaryKey = tableCopy.primaryKey.columns.map(x => x.columnName);
   }
+  if (tableCopy.sortingKey && !tableCopy.sortingKey['_dumped']) {
+    res.sortingKey = tableCopy.sortingKey.columns.map(x => x.columnName);
+  }
   // const foreignKeys = (tableCopy.foreignKeys || []).filter(x => !x['_dumped']).map(foreignKeyInfoToYaml);
   return res;
 }
@@ -132,6 +136,13 @@ export function tableInfoFromYaml(table: TableInfoYaml, allTables: TableInfoYaml
       columns: table.primaryKey.map(columnName => ({ columnName })),
     };
   }
+  if (table.sortingKey) {
+    res.sortingKey = {
+      pureName: table.name,
+      constraintType: 'sortingKey',
+      columns: table.sortingKey.map(columnName => ({ columnName })),
+    };
+  }
   res.preloadedRows = table.data;
   res.preloadedRowsKey = table.insertKey;
   res.preloadedRowsInsertOnly = table.insertOnly;

packages/types/alter-processor.d.ts

@@ -15,6 +15,7 @@ export interface AlterProcessor {
   recreateTable(oldTable: TableInfo, newTable: TableInfo);
   createSqlObject(obj: SqlObjectInfo);
   dropSqlObject(obj: SqlObjectInfo);
+  setTableOption(table: TableInfo, optionName: string, optionValue: string);
   fillPreloadedRows(
     table: NamedObjectInfo,
     oldRows: any[],

packages/types/dbinfo.d.ts

@@ -15,7 +15,7 @@ export interface ColumnReference {
 export interface ConstraintInfo extends NamedObjectInfo {
   pairingId?: string;
   constraintName?: string;
-  constraintType: 'primaryKey' | 'foreignKey' | 'index' | 'check' | 'unique';
+  constraintType: 'primaryKey' | 'foreignKey' | 'sortingKey' | 'index' | 'check' | 'unique';
 }
 
 export interface ColumnsConstraintInfo extends ConstraintInfo {
@@ -49,6 +49,7 @@ export interface ColumnInfo extends NamedObjectInfo {
   notNull?: boolean;
   autoIncrement?: boolean;
   dataType: string;
+  displayedDataType?: string;
   precision?: number;
   scale?: number;
   length?: number;
@@ -61,7 +62,7 @@ export interface ColumnInfo extends NamedObjectInfo {
   isUnsigned?: boolean;
   isZerofill?: boolean;
   options?: [];
-  canSelectMultipleOptions?: boolean,
+  canSelectMultipleOptions?: boolean;
 }
 
 export interface DatabaseObjectInfo extends NamedObjectInfo {
@@ -82,6 +83,7 @@ export interface SqlObjectInfo extends DatabaseObjectInfo {
 export interface TableInfo extends DatabaseObjectInfo {
   columns: ColumnInfo[];
   primaryKey?: PrimaryKeyInfo;
+  sortingKey?: ColumnsConstraintInfo;
   foreignKeys: ForeignKeyInfo[];
   dependencies?: ForeignKeyInfo[];
   indexes?: IndexInfo[];
@@ -91,6 +93,7 @@ export interface TableInfo extends DatabaseObjectInfo {
   preloadedRowsKey?: string[];
   preloadedRowsInsertOnly?: string[];
   tableRowCount?: number | string;
+  tableEngine?: string;
   __isDynamicStructure?: boolean;
 }
 
@@ -102,10 +105,10 @@ export interface CollectionInfo extends DatabaseObjectInfo {
   uniqueKey?: ColumnReference[];
 
   // partition key columns
-  partitionKey?: ColumnReference[]
+  partitionKey?: ColumnReference[];
 
   // unique key inside partition
-  clusterKey?: ColumnReference[];  
+  clusterKey?: ColumnReference[];
 }
 
 export interface ViewInfo extends SqlObjectInfo {
@@ -123,6 +126,7 @@ export interface TriggerInfo extends SqlObjectInfo {}
 export interface SchemaInfo {
   objectId?: string;
   schemaName: string;
+  isDefault?: boolean;
 }
 
 export interface DatabaseInfoObjects {
@@ -136,7 +140,5 @@ export interface DatabaseInfoObjects {
 }
 
 export interface DatabaseInfo extends DatabaseInfoObjects {
-  schemas?: SchemaInfo[];
   engine?: string;
-  defaultSchema?: string;
 }

packages/types/dialect.d.ts

@@ -3,6 +3,7 @@ export interface SqlDialect {
   limitSelect?: boolean;
   ilike?: boolean;
   rowNumberOverPaging?: boolean;
+  topRecords?: boolean;
   stringEscapeChar: string;
   offsetFetchRangeSyntax?: boolean;
   quoteIdentifier(s: string): string;
@@ -16,6 +17,7 @@ export interface SqlDialect {
 
   dropColumnDependencies?: string[];
   changeColumnDependencies?: string[];
+  renameColumnDependencies?: string[];
 
   dropIndexContainsTableSpec?: boolean;
 
@@ -33,11 +35,33 @@ export interface SqlDialect {
   createCheck?: boolean;
   dropCheck?: boolean;
 
+  specificNullabilityImplementation?: boolean;
+  omitForeignKeys?: boolean;
+  omitUniqueConstraints?: boolean;
+  omitIndexes?: boolean;
+  sortingKeys?: boolean;
+
+  // syntax for create column: ALTER TABLE table ADD COLUMN column
+  createColumnWithColumnKeyword?: boolean;
+
   dropReferencesWhenDropTable?: boolean;
   requireFromDual?: boolean;
 
   predefinedDataTypes: string[];
 
   // create sql-tree expression
-  createColumnViewExpression(columnName: string, dataType: string, source: { alias: string }, alias?: string): any;
+  createColumnViewExpression(
+    columnName: string,
+    dataType: string,
+    source: { alias: string },
+    alias?: string,
+    purpose: 'view' | 'filter' = 'view'
+  ): any;
+
+  getTableFormOptions(intent: 'newTableForm' | 'editTableForm' | 'sqlCreateTable' | 'sqlAlterTable'): {
+    name: string;
+    sqlFormatString: string;
+    disabled?: boolean;
+    allowEmptyValue?: boolean;
+  }[];
 }

packages/types/engines.d.ts

@@ -11,6 +11,7 @@ import {
   FunctionInfo,
   TriggerInfo,
   CollectionInfo,
+  SchemaInfo,
 } from './dbinfo';
 import { FilterBehaviour } from './filter-type';
 
@@ -24,10 +25,12 @@ export interface StreamOptions {
 
 export interface RunScriptOptions {
   useTransaction: boolean;
+  queryOptions?: QueryOptions;
 }
 
 export interface QueryOptions {
   discardResult?: boolean;
+  importSqlDump?: boolean;
 }
 
 export interface WriteTableOptions {
@@ -129,6 +132,15 @@ export interface FilterBehaviourProvider {
   getFilterBehaviour(dataType: string, standardFilterBehaviours: { [id: string]: FilterBehaviour }): FilterBehaviour;
 }
 
+export interface DatabaseHandle {
+  client: any;
+  database?: string;
+  feedback?: (message: any) => void;
+  getDatabase?: () => any;
+  connectionType?: string;
+  treeKeySeparator?: string;
+}
+
 export interface EngineDriver extends FilterBehaviourProvider {
   engine: string;
   title: string;
@@ -148,6 +160,7 @@ export interface EngineDriver extends FilterBehaviourProvider {
   profilerChartAggregateFunction?: string;
   profilerChartMeasures?: { label: string; field: string }[];
   isElectronOnly?: boolean;
+  supportsTransactions?: boolean;
 
   collectionSingularLabel?: string;
   collectionPluralLabel?: string;
@@ -169,52 +182,52 @@ export interface EngineDriver extends FilterBehaviourProvider {
   defaultSocketPath?: string;
   authTypeLabel?: string;
   importExportArgs?: any[];
-  connect({ server, port, user, password, database }): Promise<any>;
-  close(pool): Promise<any>;
-  query(pool: any, sql: string, options?: QueryOptions): Promise<QueryResult>;
-  stream(pool: any, sql: string, options: StreamOptions);
-  readQuery(pool: any, sql: string, structure?: TableInfo): Promise<stream.Readable>;
-  readJsonQuery(pool: any, query: any, structure?: TableInfo): Promise<stream.Readable>;
-  writeTable(pool: any, name: NamedObjectInfo, options: WriteTableOptions): Promise<stream.Writable>;
+  connect({ server, port, user, password, database }): Promise<DatabaseHandle>;
+  close(dbhan: DatabaseHandle): Promise<any>;
+  query(dbhan: DatabaseHandle, sql: string, options?: QueryOptions): Promise<QueryResult>;
+  stream(dbhan: DatabaseHandle, sql: string, options: StreamOptions);
+  readQuery(dbhan: DatabaseHandle, sql: string, structure?: TableInfo): Promise<stream.Readable>;
+  readJsonQuery(dbhan: DatabaseHandle, query: any, structure?: TableInfo): Promise<stream.Readable>;
+  writeTable(dbhan: DatabaseHandle, name: NamedObjectInfo, options: WriteTableOptions): Promise<stream.Writable>;
   analyseSingleObject(
-    pool: any,
+    dbhan: DatabaseHandle,
     name: NamedObjectInfo,
     objectTypeField: keyof DatabaseInfo
   ): Promise<TableInfo | ViewInfo | ProcedureInfo | FunctionInfo | TriggerInfo>;
-  analyseSingleTable(pool: any, name: NamedObjectInfo): Promise<TableInfo>;
-  getVersion(pool: any): Promise<{ version: string }>;
-  listDatabases(pool: any): Promise<
+  analyseSingleTable(dbhan: DatabaseHandle, name: NamedObjectInfo): Promise<TableInfo>;
+  getVersion(dbhan: DatabaseHandle): Promise<{ version: string }>;
+  listDatabases(dbhan: DatabaseHandle): Promise<
     {
       name: string;
     }[]
   >;
-  loadKeys(pool, root: string, filter?: string): Promise;
-  exportKeys(pool, options: {}): Promise;
-  loadKeyInfo(pool, key): Promise;
-  loadKeyTableRange(pool, key, cursor, count): Promise;
-  loadFieldValues(pool: any, name: NamedObjectInfo, field: string, search: string): Promise;
-  analyseFull(pool: any, serverVersion): Promise<DatabaseInfo>;
-  analyseIncremental(pool: any, structure: DatabaseInfo, serverVersion): Promise<DatabaseInfo>;
+  loadKeys(dbhan: DatabaseHandle, root: string, filter?: string): Promise;
+  exportKeys(dbhan: DatabaseHandle, options: {}): Promise;
+  loadKeyInfo(dbhan: DatabaseHandle, key): Promise;
+  loadKeyTableRange(dbhan: DatabaseHandle, key, cursor, count): Promise;
+  loadFieldValues(dbhan: DatabaseHandle, name: NamedObjectInfo, field: string, search: string): Promise;
+  analyseFull(dbhan: DatabaseHandle, serverVersion): Promise<DatabaseInfo>;
+  analyseIncremental(dbhan: DatabaseHandle, structure: DatabaseInfo, serverVersion): Promise<DatabaseInfo>;
   dialect: SqlDialect;
   dialectByVersion(version): SqlDialect;
   createDumper(options = null): SqlDumper;
-  createBackupDumper(pool: any, options): Promise<SqlBackupDumper>;
+  createBackupDumper(dbhan: DatabaseHandle, options): Promise<SqlBackupDumper>;
   getAuthTypes(): EngineAuthType[];
-  readCollection(pool: any, options: ReadCollectionOptions): Promise<any>;
-  updateCollection(pool: any, changeSet: any): Promise<any>;
+  readCollection(dbhan: DatabaseHandle, options: ReadCollectionOptions): Promise<any>;
+  updateCollection(dbhan: DatabaseHandle, changeSet: any): Promise<any>;
   getCollectionUpdateScript(changeSet: any, collectionInfo: CollectionInfo): string;
-  createDatabase(pool: any, name: string): Promise;
-  dropDatabase(pool: any, name: string): Promise;
+  createDatabase(dbhan: DatabaseHandle, name: string): Promise;
+  dropDatabase(dbhan: DatabaseHandle, name: string): Promise;
   getQuerySplitterOptions(usage: 'stream' | 'script' | 'editor'): any;
-  script(pool: any, sql: string, options?: RunScriptOptions): Promise;
-  operation(pool: any, operation: {}, options?: RunScriptOptions): Promise;
+  script(dbhan: DatabaseHandle, sql: string, options?: RunScriptOptions): Promise;
+  operation(dbhan: DatabaseHandle, operation: {}, options?: RunScriptOptions): Promise;
   getNewObjectTemplates(): NewObjectTemplate[];
-  // direct call of pool method, only some methods could be supported, on only some drivers
-  callMethod(pool, method, args);
-  serverSummary(pool): Promise<ServerSummary>;
-  summaryCommand(pool, command, row): Promise<void>;
-  startProfiler(pool, options): Promise<any>;
-  stopProfiler(pool, profiler): Promise<void>;
+  // direct call of dbhan.client method, only some methods could be supported, on only some drivers
+  callMethod(dbhan: DatabaseHandle, method, args);
+  serverSummary(dbhan: DatabaseHandle): Promise<ServerSummary>;
+  summaryCommand(dbhan: DatabaseHandle, command, row): Promise<void>;
+  startProfiler(dbhan: DatabaseHandle, options): Promise<any>;
+  stopProfiler(dbhan: DatabaseHandle, profiler): Promise<void>;
   getRedirectAuthUrl(connection, options): Promise<{ url: string; sid: string }>;
   getAuthTokenFromCode(connection, options): Promise<string>;
   getAccessTokenFromAuth(connection, req): Promise<string | null>;
@@ -222,6 +235,14 @@ export interface EngineDriver extends FilterBehaviourProvider {
   getCollectionExportQueryJson(collection: string, condition: any, sort?: CollectionSortDefinition): {};
   getScriptTemplates(objectTypeField: keyof DatabaseInfo): { label: string; scriptTemplate: string }[];
   getScriptTemplateContent(scriptTemplate: string, props: any): Promise<string>;
+  createSaveChangeSetScript(
+    changeSet: any,
+    dbinfo: DatabaseInfo,
+    defaultCreator: (changeSet: any, dbinfo: DatabaseInfo) => any
+  ): any[];
+  // adapts table info from different source (import, other database) to be suitable for this database
+  adaptTableInfo(table: TableInfo): TableInfo;
+  listSchemas(dbhan: DatabaseHandle): SchemaInfo[];
 
   analyserClass?: any;
   dumperClass?: any;

packages/web/build-index.js

@@ -0,0 +1,8 @@
+const fs = require('fs');
+
+const template = fs.readFileSync('./index.html.tpl', 'utf-8');
+
+for (const page of ['', 'not-logged', 'error', 'admin-login', 'login', 'admin', 'license']) {
+  const text = template.replace(/{{page}}/g, page);
+  fs.writeFileSync(`public/${page || 'index'}.html`, text);
+}

packages/web/index.html.tpl

@@ -22,6 +22,11 @@
     <link rel="stylesheet" href="build/fonts/materialdesignicons.css" />
     <link rel="stylesheet" href="build/diff2html.min.css" />
 
+    <script lang="javascript">
+      window.dbgate_page = '{{page}}';
+    </script>
+
+
     <script defer src="build/bundle.js"></script>
 
     <style>

packages/web/package.json

@@ -2,10 +2,11 @@
   "name": "dbgate-web",
   "version": "5.0.0-alpha.1",
   "scripts": {
-    "build": "rollup -c",
-    "dev": "cross-env API_URL=http://localhost:3000 rollup -c -w",
+    "build": "yarn build:index && rollup -c",
+    "dev": "yarn build:index && cross-env API_URL=http://localhost:3000 rollup -c -w",
     "start": "sirv public --port 5001",
     "validate": "svelte-check",
+    "build:index": "node build-index.js",
     "prepublishOnly": "yarn build"
   },
   "files": [

packages/web/src/App.svelte

@@ -21,6 +21,7 @@
   import AppStartInfo from './widgets/AppStartInfo.svelte';
   import SettingsListener from './utility/SettingsListener.svelte';
   import { handleAuthOnStartup } from './clientAuth';
+  import { initializeAppUpdates } from './utility/appUpdate';
 
   export let isAdminPage = false;
 
@@ -49,6 +50,7 @@
         subscribeConnectionPingers();
         subscribePermissionCompiler();
         installNewVolatileConnectionListener();
+        initializeAppUpdates();
       }
 
       loadedApi = loadedApiValue;

packages/web/src/EnterLicensePage.svelte

@@ -10,12 +10,13 @@
   import FormTextAreaField from './forms/FormTextAreaField.svelte';
   import FormSubmit from './forms/FormSubmit.svelte';
   import { apiCall } from './utility/api';
+  import FormStyledButton from './buttons/FormStyledButton.svelte';
+  import getElectron from './utility/getElectron';
 
   const config = useConfig();
   const values = writable({ amoid: null, databaseServer: null });
 
-  const params = new URLSearchParams(location.search);
-  const error = params.get('error');
+  let errorMessage = '';
 
   onMount(() => {
     const removed = document.getElementById('starting_dbgate_zero');
@@ -32,18 +33,58 @@
       </div>
       <div class="box">
         <div class="heading">License</div>
-        <FormTextAreaField label="License key" name="licenseKey" rows={5} />
+        <FormTextAreaField label="Enter your license key" name="licenseKey" rows={5} />
 
         <div class="submit">
           <FormSubmit
             value="Save license"
             on:click={async e => {
               const { licenseKey } = e.detail;
-              await apiCall('config/save-license-key', { licenseKey });
-              internalRedirectTo('/');
+              const resp = await apiCall('config/save-license-key', { licenseKey });
+              if (resp?.status == 'ok') {
+                internalRedirectTo('/index.html');
+              } else {
+                errorMessage = resp?.errorMessage || 'Error saving license key';
+              }
             }}
           />
         </div>
+
+        <div class="submit">
+          <FormStyledButton
+            value="Start 30-day trial"
+            on:click={async e => {
+              errorMessage = '';
+              const license = await apiCall('config/start-trial');
+              if (license?.status == 'ok') {
+                internalRedirectTo('/index.html');
+              } else {
+                errorMessage = license?.errorMessage || 'Error starting trial';
+              }
+            }}
+          />
+        </div>
+
+        {#if getElectron()}
+          <div class="submit">
+            <FormStyledButton
+              value="Exit"
+              on:click={e => {
+                getElectron().send('quit-app');
+              }}
+            />
+          </div>
+        {/if}
+
+        {#if errorMessage}
+          <div class="error">{errorMessage}</div>
+        {/if}
+
+        <div class="purchase-info">
+          If you want to purchase Premium license, please contact us at <Link href="mailto:sales@dbgate.eu"
+            >sales@dbgate.eu</Link
+          >
+        </div>
       </div>
     </div>
   </div>
@@ -111,4 +152,13 @@
     flex: 1;
     font-size: larger;
   }
+
+  .error {
+    margin: var(--dim-large-form-margin);
+    color: red;
+  }
+
+  .purchase-info {
+    margin: var(--dim-large-form-margin);
+  }
 </style>

packages/web/src/LoginPage.svelte

@@ -45,6 +45,12 @@
     }
   }
 
+  function extractRedirectUri() {
+    const res = (location.origin + location.pathname).replace(/\/login.html$/, '/');
+    console.log('Using redirect URI:', res);
+    return res;
+  }
+
   async function processSingleProvider(provider) {
     if (provider.workflowType == 'redirect') {
       await processRedirectLogin(provider.amoid);
@@ -86,7 +92,7 @@
     const resp = await apiCall('auth/redirect', {
       amoid: amoid,
       state,
-      redirectUri: location.origin + location.pathname,
+      redirectUri: extractRedirectUri(),
     });
 
     const { uri } = resp;
@@ -103,7 +109,7 @@
     });
     if (resp.error) {
       internalRedirectTo(
-        `/?page=not-logged&error=${encodeURIComponent(resp.error)}&is-admin=${isAdminPage ? 'true' : ''}`
+        `/not-logged.html?error=${encodeURIComponent(resp.error)}&is-admin=${isAdminPage ? 'true' : ''}`
       );
       return;
     }
@@ -111,13 +117,13 @@
     if (accessToken) {
       localStorage.setItem(isAdminPage ? 'adminAccessToken' : 'accessToken', accessToken);
       if (isAdminPage) {
-        internalRedirectTo('/?page=admin');
+        internalRedirectTo('/admin.html');
       } else {
         internalRedirectTo('/');
       }
       return;
     }
-    internalRedirectTo(`/?page=not-logged`);
+    internalRedirectTo(`/not-logged.html`);
   }
 </script>
 
@@ -194,9 +200,7 @@
                 //   }`
                 // );
                 internalRedirectTo(
-                  `/connections/dblogin-web?conid=${selectedConnection?.conid}&state=${encodeURIComponent(state)}&redirectUri=${
-                    location.origin + location.pathname
-                  }`
+                  `/connections/dblogin-web?conid=${selectedConnection?.conid}&state=${encodeURIComponent(state)}&redirectUri=${extractRedirectUri()}`
                 );
               }}
             />

packages/web/src/appobj/AppObjectCore.svelte

@@ -3,6 +3,8 @@
   import contextMenu from '../utility/contextMenu';
   import { createEventDispatcher } from 'svelte';
   import CheckboxField from '../forms/CheckboxField.svelte';
+  import { copyTextToClipboard } from '../utility/clipboard';
+  import { showSnackbarSuccess } from '../utility/snackbar';
 
   const dispatch = createEventDispatcher();
 
@@ -16,6 +18,7 @@
   export let statusIcon = undefined;
   export let statusIconBefore = undefined;
   export let statusTitle = undefined;
+  export let statusTitleToCopy = undefined;
   export let extInfo = undefined;
   export let menu = undefined;
   export let expandIcon = undefined;
@@ -114,7 +117,16 @@
   {/if}
   {#if statusIcon}
     <span class="status">
-      <FontIcon icon={statusIcon} title={statusTitle} />
+      <FontIcon
+        icon={statusIcon}
+        title={statusTitle}
+        on:click={() => {
+          if (statusTitleToCopy) {
+            copyTextToClipboard(statusTitleToCopy);
+            showSnackbarSuccess('Copied to clipboard');
+          }
+        }}
+      />
     </span>
   {/if}
   {#if extInfo}

packages/web/src/appobj/ArchiveFileAppObject.svelte

@@ -65,18 +65,18 @@
 <script lang="ts">
   import _ from 'lodash';
   import { filterName } from 'dbgate-tools';
-  import ImportExportModal from '../modals/ImportExportModal.svelte';
   import { showModal } from '../modals/modalTools';
 
   import { getExtensions } from '../stores';
 
   import createQuickExportMenu from '../utility/createQuickExportMenu';
-  import { exportQuickExportFile } from '../utility/exportFileTools';
+  import { exportQuickExportFile,  } from '../utility/exportFileTools';
   import openNewTab from '../utility/openNewTab';
   import AppObjectCore from './AppObjectCore.svelte';
   import InputTextModal from '../modals/InputTextModal.svelte';
   import ConfirmModal from '../modals/ConfirmModal.svelte';
   import { apiCall } from '../utility/api';
+  import { openImportExportTab } from '../utility/importExportTools';
 
   export let data;
 
@@ -156,13 +156,19 @@
           {
             text: 'Export',
             onClick: () => {
-              showModal(ImportExportModal, {
-                initialValues: {
-                  sourceStorageType: 'archive',
-                  sourceArchiveFolder: data.folderName,
-                  sourceList: [data.fileName],
-                },
+              openImportExportTab({
+                sourceStorageType: 'archive',
+                sourceArchiveFolder: data.folderName,
+                sourceList: [data.fileName],
               });
+
+              // showModal(ImportExportModal, {
+              //   initialValues: {
+              //     sourceStorageType: 'archive',
+              //     sourceArchiveFolder: data.folderName,
+              //     sourceList: [data.fileName],
+              //   },
+              // });
             },
           }
         ),

packages/web/src/appobj/ColumnAppObject.svelte

@@ -31,7 +31,7 @@
     return [
       { text: 'Rename column', onClick: handleRenameColumn },
       { text: 'Drop column', onClick: handleDropColumn },
-      { text: 'Copy name', onClick: () => navigator.clipboard.writeText(data.columnName)},
+      { text: 'Copy name', onClick: () => navigator.clipboard.writeText(data.columnName) },
     ];
   }
 

packages/web/src/appobj/ConnectionAppObject.svelte

@@ -14,7 +14,7 @@
   export function openConnection(connection) {
     const config = getCurrentConfig();
     if (connection.singleDatabase) {
-      currentDatabase.set({ connection, name: connection.defaultDatabase });
+      switchCurrentDatabase({ connection, name: connection.defaultDatabase });
       apiCall('database-connections/refresh', {
         conid: connection._id,
         database: connection.defaultDatabase,
@@ -60,7 +60,7 @@
       if (electron) {
         apiCall('database-connections/disconnect', { conid, database: currentDb.name });
       }
-      currentDatabase.set(null);
+      switchCurrentDatabase(null);
     }
     closeMultipleTabs(closeCondition);
     // if (data.unsaved) {
@@ -107,6 +107,7 @@
   import { tick } from 'svelte';
   import { getConnectionLabel } from 'dbgate-tools';
   import hasPermission from '../utility/hasPermission';
+  import { switchCurrentDatabase } from '../utility/common';
 
   export let data;
   export let passProps;
@@ -142,7 +143,7 @@
       return;
     }
     if ($openedSingleDatabaseConnections.includes(data._id)) {
-      currentDatabase.set({ connection: data, name: data.defaultDatabase });
+      switchCurrentDatabase({ connection: data, name: data.defaultDatabase });
       return;
     }
     if ($openedConnections.includes(data._id)) {
@@ -322,6 +323,7 @@
     : _.get($currentDatabase, 'connection._id') == data._id}
   statusIcon={statusIcon || engineStatusIcon}
   statusTitle={statusTitle || engineStatusTitle}
+  statusTitleToCopy={statusTitle || engineStatusTitle}
   statusIconBefore={data.isReadOnly ? 'icon lock' : null}
   {extInfo}
   colorMark={passProps?.connectionColorFactory && passProps?.connectionColorFactory({ conid: data._id })}

packages/web/src/appobj/DatabaseAppObject.svelte

@@ -26,7 +26,7 @@
       apiCall('database-connections/disconnect', { conid, database });
     }
     if (getCurrentDatabase()?.connection?._id == conid && getCurrentDatabase()?.name == database) {
-      currentDatabase.set(null);
+      switchCurrentDatabase(null);
     }
     openedSingleDatabaseConnections.update(list => list.filter(x => x != conid));
     closeMultipleTabs(closeCondition);
@@ -56,27 +56,7 @@
     };
 
     const handleNewTable = () => {
-      const tooltip = `${getConnectionLabel(connection)}\n${name}`;
-      openNewTab(
-        {
-          title: 'Table #',
-          tooltip,
-          icon: 'img table-structure',
-          tabComponent: 'TableStructureTab',
-          props: {
-            conid: connection._id,
-            database: name,
-          },
-        },
-        {
-          editor: {
-            columns: [],
-          },
-        },
-        {
-          forceNewTab: true,
-        }
-      );
+      newTable(connection, name);
     };
 
     const handleDropDatabase = () => {
@@ -98,25 +78,39 @@
     };
 
     const handleImport = () => {
-      showModal(ImportExportModal, {
-        initialValues: {
-          sourceStorageType: getDefaultFileFormat($extensions).storageType,
-          targetStorageType: 'database',
-          targetConnectionId: connection._id,
-          targetDatabaseName: name,
-        },
+      openImportExportTab({
+        sourceStorageType: getDefaultFileFormat($extensions).storageType,
+        targetStorageType: 'database',
+        targetConnectionId: connection._id,
+        targetDatabaseName: name,
       });
+
+      // showModal(ImportExportModal, {
+      //   initialValues: {
+      //     sourceStorageType: getDefaultFileFormat($extensions).storageType,
+      //     targetStorageType: 'database',
+      //     targetConnectionId: connection._id,
+      //     targetDatabaseName: name,
+      //   },
+      // });
     };
 
     const handleExport = () => {
-      showModal(ImportExportModal, {
-        initialValues: {
-          targetStorageType: getDefaultFileFormat($extensions).storageType,
-          sourceStorageType: 'database',
-          sourceConnectionId: connection._id,
-          sourceDatabaseName: name,
-        },
+      openImportExportTab({
+        targetStorageType: getDefaultFileFormat($extensions).storageType,
+        sourceStorageType: 'database',
+        sourceConnectionId: connection._id,
+        sourceDatabaseName: name,
       });
+
+      // showModal(ImportExportModal, {
+      //   initialValues: {
+      //     targetStorageType: getDefaultFileFormat($extensions).storageType,
+      //     sourceStorageType: 'database',
+      //     sourceConnectionId: connection._id,
+      //     sourceDatabaseName: name,
+      //   },
+      // });
     };
 
     const handleSqlGenerator = () => {
@@ -268,6 +262,17 @@
       });
     };
 
+    const handleRefreshSchemas = () => {
+      const conid = connection._id;
+      const database = name;
+      apiCall('database-connections/dispatch-database-changed-event', {
+        event: 'schema-list-changed',
+        conid,
+        database,
+      });
+      loadSchemaList(conid, database);
+    };
+
     async function handleConfirmSql(sql) {
       saveScriptToDatabase({ conid: connection._id, database: name }, sql, false);
     }
@@ -296,11 +301,13 @@
         onClick: handleNewPerspective,
         text: 'Design perspective query',
       },
+      connection.useSeparateSchemas && { onClick: handleRefreshSchemas, text: 'Refresh schemas' },
+
       { divider: true },
       isSqlOrDoc &&
         !connection.isReadOnly &&
-        hasPermission(`dbops/import`) && { onClick: handleImport, text: 'Import wizard' },
-      isSqlOrDoc && hasPermission(`dbops/export`) && { onClick: handleExport, text: 'Export wizard' },
+        hasPermission(`dbops/import`) && { onClick: handleImport, text: 'Import' },
+      isSqlOrDoc && hasPermission(`dbops/export`) && { onClick: handleExport, text: 'Export' },
       driver?.databaseEngineTypes?.includes('sql') &&
         hasPermission(`dbops/sql-dump/import`) &&
         !connection.isReadOnly && { onClick: handleSqlRestore, text: 'Restore/import SQL dump' },
@@ -360,7 +367,6 @@
   import uuidv1 from 'uuid/v1';
 
   import _, { find } from 'lodash';
-  import ImportExportModal from '../modals/ImportExportModal.svelte';
   import { showModal } from '../modals/modalTools';
   import SqlGeneratorModal from '../modals/SqlGeneratorModal.svelte';
   import { getDefaultFileFormat } from '../plugins/fileformats';
@@ -371,6 +377,7 @@
     getCurrentDatabase,
     getExtensions,
     getOpenedTabs,
+    loadingSchemaLists,
     openedConnections,
     openedSingleDatabaseConnections,
     pinnedDatabases,
@@ -381,7 +388,7 @@
   import openNewTab from '../utility/openNewTab';
   import AppObjectCore from './AppObjectCore.svelte';
   import { showSnackbarError, showSnackbarSuccess } from '../utility/snackbar';
-  import { findEngineDriver, getConnectionLabel } from 'dbgate-tools';
+  import { extractDbNameFromComposite, findEngineDriver, getConnectionLabel } from 'dbgate-tools';
   import InputTextModal from '../modals/InputTextModal.svelte';
   import { getDatabaseInfo, useUsedApps } from '../utility/metadataLoaders';
   import { openJsonDocument } from '../tabs/JsonTab.svelte';
@@ -390,13 +397,15 @@
   import ConfirmSqlModal, { runOperationOnDatabase, saveScriptToDatabase } from '../modals/ConfirmSqlModal.svelte';
   import { filterAppsForDatabase } from '../utility/appTools';
   import newQuery from '../query/newQuery';
-  import { exportSqlDump } from '../utility/exportFileTools';
   import ImportDatabaseDumpModal from '../modals/ImportDatabaseDumpModal.svelte';
   import ExportDatabaseDumpModal from '../modals/ExportDatabaseDumpModal.svelte';
   import ConfirmModal from '../modals/ConfirmModal.svelte';
   import { closeMultipleTabs } from '../tabpanel/TabsPanel.svelte';
   import NewCollectionModal from '../modals/NewCollectionModal.svelte';
   import hasPermission from '../utility/hasPermission';
+  import { openImportExportTab } from '../utility/importExportTools';
+  import newTable from '../tableeditor/newTable';
+  import { loadSchemaList, switchCurrentDatabase } from '../utility/common';
 
   export let data;
   export let passProps;
@@ -414,6 +423,7 @@
 
   $: isPinned = !!$pinnedDatabases.find(x => x?.name == data.name && x?.connection?._id == data.connection?._id);
   $: apps = useUsedApps();
+  $: isLoadingSchemas = $loadingSchemaLists[`${data?.connection?._id}::${data?.name}`];
 </script>
 
 <AppObjectCore
@@ -425,8 +435,8 @@
   colorMark={passProps?.connectionColorFactory &&
     passProps?.connectionColorFactory({ conid: _.get(data.connection, '_id'), database: data.name }, null, null, false)}
   isBold={_.get($currentDatabase, 'connection._id') == _.get(data.connection, '_id') &&
-    _.get($currentDatabase, 'name') == data.name}
-  on:click={() => ($currentDatabase = data)}
+    extractDbNameFromComposite(_.get($currentDatabase, 'name')) == data.name}
+  on:click={() => switchCurrentDatabase(data)}
   on:dragstart
   on:dragenter
   on:dragend
@@ -436,6 +446,7 @@
       .find(x => x.isNewQuery)
       .onClick();
   }}
+  statusIcon={isLoadingSchemas ? 'icon loading' : ''}
   menu={createMenu}
   showPinnedInsteadOfUnpin={passProps?.showPinnedInsteadOfUnpin}
   onPin={isPinned ? null : () => pinnedDatabases.update(list => [...list, data])}

packages/web/src/appobj/DatabaseObjectAppObject.svelte

@@ -618,15 +618,22 @@
       });
     } else if (menu.isImport) {
       const { conid, database } = data;
-      showModal(ImportExportModal, {
-        initialValues: {
-          sourceStorageType: getDefaultFileFormat(getExtensions()).storageType,
-          targetStorageType: 'database',
-          targetConnectionId: conid,
-          targetDatabaseName: database,
-          fixedTargetPureName: data.pureName,
-        },
+      openImportExportTab({
+        sourceStorageType: getDefaultFileFormat(getExtensions()).storageType,
+        targetStorageType: 'database',
+        targetConnectionId: conid,
+        targetDatabaseName: database,
+        fixedTargetPureName: data.pureName,
       });
+      // showModal(ImportExportModal, {
+      //   initialValues: {
+      //     sourceStorageType: getDefaultFileFormat(getExtensions()).storageType,
+      //     targetStorageType: 'database',
+      //     targetConnectionId: conid,
+      //     targetDatabaseName: database,
+      //     fixedTargetPureName: data.pureName,
+      //   },
+      // });
     } else {
       openDatabaseObjectDetail(
         menu.tab,
@@ -764,31 +771,22 @@
             },
             {
               onClick: () => {
-                // openNewTab(
-                //   {
-                //     tabComponent: 'ImportExportTab',
-                //     title: 'Import/Export',
-                //     icon: 'img export',
-                //   },
-                //   {
-                //     editor: {
-                //       sourceStorageType: 'database',
-                //       sourceConnectionId: data.conid,
-                //       sourceDatabaseName: data.database,
-                //       sourceSchemaName: data.schemaName,
-                //       sourceList: [data.pureName],
-                //     },
-                //   }
-                // );
-                showModal(ImportExportModal, {
-                  initialValues: {
-                    sourceStorageType: 'database',
-                    sourceConnectionId: data.conid,
-                    sourceDatabaseName: data.database,
-                    sourceSchemaName: data.schemaName,
-                    sourceList: [data.pureName],
-                  },
+                openImportExportTab({
+                  sourceStorageType: 'database',
+                  sourceConnectionId: data.conid,
+                  sourceDatabaseName: data.database,
+                  sourceSchemaName: data.schemaName,
+                  sourceList: [data.pureName],
                 });
+                // showModal(ImportExportModal, {
+                //   initialValues: {
+                //     sourceStorageType: 'database',
+                //     sourceConnectionId: data.conid,
+                //     sourceDatabaseName: data.database,
+                //     sourceSchemaName: data.schemaName,
+                //     sourceList: [data.pureName],
+                //   },
+                // });
               },
             }
           );
@@ -832,7 +830,6 @@
   import { filterName, generateDbPairingId, getAlterDatabaseScript, getConnectionLabel } from 'dbgate-tools';
   import { getConnectionInfo, getDatabaseInfo } from '../utility/metadataLoaders';
   import fullDisplayName from '../utility/fullDisplayName';
-  import ImportExportModal from '../modals/ImportExportModal.svelte';
   import { showModal } from '../modals/modalTools';
   import { findEngineDriver } from 'dbgate-tools';
   import uuidv1 from 'uuid/v1';
@@ -848,6 +845,7 @@
   import { format as dateFormat } from 'date-fns';
   import { getDefaultFileFormat } from '../plugins/fileformats';
   import hasPermission from '../utility/hasPermission';
+  import { openImportExportTab } from '../utility/importExportTools';
 
   export let data;
   export let passProps;
@@ -860,6 +858,18 @@
     return createDatabaseObjectMenu(data, passProps?.connection);
   }
 
+  function getExtInfo(data) {
+    const res = [];
+    if (data.tableRowCount != null) {
+      res.push(`${formatRowCount(data.tableRowCount)} rows`);
+    }
+    if (data.tableEngine) {
+      res.push(data.tableEngine);
+    }
+    if (res.length > 0) return res.join(', ');
+    return null;
+  }
+
   $: isPinned = !!$pinnedTables.find(x => testEqual(data, x));
 </script>
 
@@ -867,13 +877,13 @@
   {...$$restProps}
   module={$$props.module}
   {data}
-  title={data.schemaName ? `${data.schemaName}.${data.pureName}` : data.pureName}
+  title={data.schemaName && !passProps?.hideSchemaName ? `${data.schemaName}.${data.pureName}` : data.pureName}
   icon={databaseObjectIcons[data.objectTypeField]}
   menu={createMenu}
   showPinnedInsteadOfUnpin={passProps?.showPinnedInsteadOfUnpin}
   onPin={isPinned ? null : () => pinnedTables.update(list => [...list, data])}
   onUnpin={isPinned ? () => pinnedTables.update(list => list.filter(x => !testEqual(x, data))) : null}
-  extInfo={data.tableRowCount != null ? `${formatRowCount(data.tableRowCount)} rows` : null}
+  extInfo={getExtInfo(data)}
   on:click={() => handleClick()}
   on:middleclick={() => handleClick(true)}
   on:expand

packages/web/src/appobj/SavedFileAppObject.svelte

@@ -65,6 +65,14 @@
     currentConnection: true,
   };
 
+  const jobs: FileTypeHandler = {
+    icon: 'img export',
+    format: 'json',
+    tabComponent: 'ImportExportTab',
+    folder: 'jobs',
+    currentConnection: false,
+  };
+
   const perspectives: FileTypeHandler = {
     icon: 'img perspective',
     format: 'json',
@@ -82,6 +90,7 @@
     sqlite,
     diagrams,
     perspectives,
+    jobs,
   };
 
   export const extractKey = data => data.file;

packages/web/src/clientAuth.ts

@@ -60,7 +60,7 @@ export function handleOauthCallback() {
         internalRedirectTo('/');
       } else {
         console.log('Error when processing OAUTH callback', error || errorMessage);
-        internalRedirectTo(`/?page=not-logged&error=${error || errorMessage}`);
+        internalRedirectTo(`/not-logged.html?error=${error || errorMessage}`);
       }
     });
 
@@ -81,9 +81,9 @@ export function handleOauthCallback() {
       if (authResp.success) {
         window.close();
       } else if (authResp.error) {
-        internalRedirectTo(`/?page=error&error=${encodeURIComponent(authResp.error)}`);
+        internalRedirectTo(`/error.html?error=${encodeURIComponent(authResp.error)}`);
       } else {
-        internalRedirectTo(`/?page=error`);
+        internalRedirectTo(`/error.html`);
       }
     });
 
@@ -105,9 +105,9 @@ export function handleOauthCallback() {
         localStorage.setItem('accessToken', authResp.accessToken);
         internalRedirectTo('/');
       } else if (authResp.error) {
-        internalRedirectTo(`/?page=error&error=${encodeURIComponent(authResp.error)}`);
+        internalRedirectTo(`/error.html?error=${encodeURIComponent(authResp.error)}`);
       } else {
-        internalRedirectTo(`/?page=error`);
+        internalRedirectTo(`/error.html`);
       }
     });
 
@@ -119,15 +119,15 @@ export function handleOauthCallback() {
 
 export async function handleAuthOnStartup(config, isAdminPage = false) {
   if (config.configurationError) {
-    internalRedirectTo(`/?page=error`);
+    internalRedirectTo(`/error.html`);
     return;
   }
 
   if (!config.isLicenseValid) {
     if (config.storageDatabase || getElectron()) {
-      internalRedirectTo(`/?page=license`);
+      internalRedirectTo(`/license.html`);
     } else {
-      internalRedirectTo(`/?page=error`);
+      internalRedirectTo(`/error.html`);
     }
   }
 
@@ -153,7 +153,7 @@ export async function handleAuthOnStartup(config, isAdminPage = false) {
 }
 
 export async function redirectToAdminLogin() {
-  internalRedirectTo('/?page=admin-login');
+  internalRedirectTo('/admin-login.html');
   return;
 }
 
@@ -165,12 +165,12 @@ export async function redirectToLogin(config = null, force = false) {
 
   if (getAuthCategory(config) == 'token') {
     if (!force) {
-      const params = new URLSearchParams(location.search);
-      if (params.get('page') == 'login' || params.get('page') == 'admin-login' || params.get('page') == 'not-logged') {
+      const page = window['dbgate_page']
+      if (page == 'login' || page == 'admin-login' || page == 'not-logged') {
         return;
       }
     }
-    internalRedirectTo('/?page=login');
+    internalRedirectTo('/login.html');
     return;
   }
 
@@ -201,13 +201,13 @@ export async function doLogout() {
 
   if (category == 'admin') {
     localStorage.removeItem('adminAccessToken');
-    internalRedirectTo('/?page=admin-login&is-admin=true');
+    internalRedirectTo('/admin-login.html?is-admin=true');
   } else if (category == 'token') {
     localStorage.removeItem('accessToken');
     if (config.logoutUrl) {
       window.location.href = config.logoutUrl;
     } else {
-      internalRedirectTo('/?page=not-logged');
+      internalRedirectTo('/not-logged.html');
     }
   } else if (category == 'basic') {
     window.location.href = 'config/logout';

packages/web/src/commands/CommandPalette.svelte

@@ -46,7 +46,7 @@
         databaseList.push({
           text: `${db.name} on ${getConnectionLabel(connection)}`,
           icon: 'img database',
-          onClick: () => currentDatabase.set({ connection, name: db.name }),
+          onClick: () => switchCurrentDatabase({ connection, name: db.name }),
         });
       }
     }
@@ -80,7 +80,7 @@
   import { useConnectionList, useDatabaseInfo } from '../utility/metadataLoaders';
   import { getLocalStorage } from '../utility/storageCache';
   import registerCommand from './registerCommand';
-  import { formatKeyText } from '../utility/common';
+  import { formatKeyText, switchCurrentDatabase } from '../utility/common';
 
   let domInput;
   let filter = '';

packages/web/src/commands/changeDatabaseStatusCommand.ts

@@ -3,6 +3,7 @@ import { currentDatabase, getCurrentDatabase } from '../stores';
 import getElectron from '../utility/getElectron';
 import registerCommand from './registerCommand';
 import { apiCall } from '../utility/api';
+import { switchCurrentDatabase } from '../utility/common';
 
 registerCommand({
   id: 'database.changeState',
@@ -40,7 +41,7 @@ registerCommand({
         onClick: () => {
           const electron = getElectron();
           if (electron) apiCall('database-connections/disconnect', dbid);
-          currentDatabase.set(null);
+          switchCurrentDatabase(null);
         },
       },
     ];

packages/web/src/commands/recentDatabaseSwitch.ts

@@ -2,6 +2,7 @@ import _ from 'lodash';
 import { recentDatabases, currentDatabase, getRecentDatabases } from '../stores';
 import registerCommand from './registerCommand';
 import { getConnectionLabel } from 'dbgate-tools';
+import { switchCurrentDatabase } from '../utility/common';
 
 currentDatabase.subscribe(value => {
   if (!value) return;
@@ -17,7 +18,7 @@ currentDatabase.subscribe(value => {
 function switchDatabaseCommand(db) {
   return {
     text: `${db.name} on ${getConnectionLabel(db?.connection, { allowExplicitDatabase: false })}`,
-    onClick: () => currentDatabase.set(db),
+    onClick: () => switchCurrentDatabase(db),
   };
 }
 

packages/web/src/commands/stdCommands.ts

@@ -3,6 +3,7 @@ import {
   currentTheme,
   emptyConnectionGroupNames,
   extensions,
+  getAppUpdaterActive,
   getExtensions,
   getVisibleToolbar,
   visibleToolbar,
@@ -12,7 +13,6 @@ import registerCommand from './registerCommand';
 import { get } from 'svelte/store';
 import AboutModal from '../modals/AboutModal.svelte';
 import SettingsModal from '../settings/SettingsModal.svelte';
-import ImportExportModal from '../modals/ImportExportModal.svelte';
 import SqlGeneratorModal from '../modals/SqlGeneratorModal.svelte';
 import { showModal } from '../modals/modalTools';
 import newQuery, { newDiagram, newPerspective, newQueryDesign } from '../query/newQuery';
@@ -35,7 +35,7 @@ import { apiCall } from '../utility/api';
 import runCommand from './runCommand';
 import { openWebLink } from '../utility/exportFileTools';
 import { getSettings } from '../utility/metadataLoaders';
-import { isMac } from '../utility/common';
+import { isMac, switchCurrentDatabase } from '../utility/common';
 import { doLogout, internalRedirectTo } from '../clientAuth';
 import { disconnectServerConnection } from '../appobj/ConnectionAppObject.svelte';
 import UploadErrorModal from '../modals/UploadErrorModal.svelte';
@@ -43,6 +43,8 @@ import ErrorMessageModal from '../modals/ErrorMessageModal.svelte';
 import NewCollectionModal from '../modals/NewCollectionModal.svelte';
 import ConfirmModal from '../modals/ConfirmModal.svelte';
 import localforage from 'localforage';
+import { openImportExportTab } from '../utility/importExportTools';
+import newTable from '../tableeditor/newTable';
 
 // function themeCommand(theme: ThemeDefinition) {
 //   return {
@@ -252,26 +254,7 @@ registerCommand({
     const $currentDatabase = get(currentDatabase);
     const connection = _.get($currentDatabase, 'connection') || {};
     const database = _.get($currentDatabase, 'name');
-
-    openNewTab(
-      {
-        title: 'Table #',
-        icon: 'img table-structure',
-        tabComponent: 'TableStructureTab',
-        props: {
-          conid: connection._id,
-          database,
-        },
-      },
-      {
-        editor: {
-          columns: [],
-        },
-      },
-      {
-        forceNewTab: true,
-      }
-    );
+    newTable(connection, database);
   },
 });
 
@@ -364,7 +347,7 @@ registerCommand({
       onConfirm: async file => {
         const resp = await apiCall('connections/new-sqlite-database', { file });
         const connection = resp;
-        currentDatabase.set({ connection, name: `${file}.sqlite` });
+        switchCurrentDatabase({ connection, name: `${file}.sqlite` });
       },
     });
   },
@@ -482,10 +465,18 @@ registerCommand({
   toolbar: true,
   icon: 'icon import',
   onClick: () =>
-    showModal(ImportExportModal, {
-      importToCurrentTarget: true,
-      initialValues: { sourceStorageType: getDefaultFileFormat(get(extensions)).storageType },
-    }),
+    openImportExportTab(
+      {
+        sourceStorageType: getDefaultFileFormat(get(extensions)).storageType,
+      },
+      {
+        importToCurrentTarget: true,
+      }
+    ),
+  // showModal(ImportExportModal, {
+  //   importToCurrentTarget: true,
+  //   initialValues: { sourceStorageType: getDefaultFileFormat(get(extensions)).storageType },
+  // }),
 });
 
 registerCommand({
@@ -585,6 +576,15 @@ registerCommand({
   onClick: () => disconnectServerConnection(getCurrentConfig()?.singleConnection?._id),
 });
 
+registerCommand({
+  id: 'file.checkForUpdates',
+  category: 'App',
+  name: 'Check for updates',
+  // testEnabled: () => true,
+  testEnabled: () => getAppUpdaterActive(),
+  onClick: () => getElectron().send('check-for-updates'),
+});
+
 export function registerFileCommands({
   idPrefix,
   category,

packages/web/src/datagrid/CollectionDataGridCore.svelte

@@ -122,8 +122,6 @@
   import { registerQuickExportHandler } from '../buttons/ToolStripExportButton.svelte';
   import registerCommand from '../commands/registerCommand';
   import { extractShellConnection } from '../impexp/createImpExpScript';
-  import ImportExportModal from '../modals/ImportExportModal.svelte';
-  import { showModal } from '../modals/modalTools';
   import { apiCall } from '../utility/api';
 
   import { registerMenu } from '../utility/contextMenu';
@@ -136,6 +134,7 @@
 
   import LoadingDataGridCore from './LoadingDataGridCore.svelte';
   import { mongoFilterBehaviour, standardFilterBehaviours } from 'dbgate-tools';
+  import { openImportExportTab } from '../utility/importExportTools';
 
   export let conid;
   export let display;
@@ -207,7 +206,8 @@
     initialValues.sourceQueryType = coninfo.isReadOnly ? 'json' : 'native';
     initialValues.sourceList = [pureName];
     initialValues[`columns_${pureName}`] = display.getExportColumnMap();
-    showModal(ImportExportModal, { initialValues });
+    openImportExportTab(initialValues);
+    // showModal(ImportExportModal, { initialValues });
   }
 
   export function openQuery() {

packages/web/src/datagrid/ColumnHeaderControl.svelte

@@ -88,9 +88,9 @@
     {/if}
     <ColumnLabel {...column} />
 
-    {#if _.isString(column.dataType) && !order}
+    {#if _.isString(column.displayedDataType || column.dataType) && !order}
       <span class="data-type" title={column.dataType}>
-        {column.dataType.toLowerCase()}
+        {(column.displayedDataType || column.dataType).toLowerCase()}
       </span>
     {/if}
   </div>

packages/web/src/datagrid/JslDataGridCore.svelte

@@ -45,9 +45,6 @@
   import _ from 'lodash';
   import { registerQuickExportHandler } from '../buttons/ToolStripExportButton.svelte';
   import registerCommand from '../commands/registerCommand';
-  import ImportExportModal from '../modals/ImportExportModal.svelte';
-  import { showModal } from '../modals/modalTools';
-  import { extensions } from '../stores';
   import { apiCall, apiOff, apiOn } from '../utility/api';
 
   import { registerMenu } from '../utility/contextMenu';
@@ -58,7 +55,7 @@
   import ChangeSetGrider from './ChangeSetGrider';
 
   import LoadingDataGridCore from './LoadingDataGridCore.svelte';
-  import RowsArrayGrider from './RowsArrayGrider';
+  import { openImportExportTab } from '../utility/importExportTools';
 
   export let jslid;
   export let display;
@@ -152,7 +149,8 @@
       initialValues.sourceList = ['query-data'];
       initialValues[`columns_query-data`] = display.getExportColumnMap();
     }
-    showModal(ImportExportModal, { initialValues });
+    openImportExportTab(initialValues);
+    // showModal(ImportExportModal, { initialValues });
   }
 
   const quickExportHandler = fmt => async () => {

packages/web/src/datagrid/SqlDataGridCore.svelte

@@ -65,13 +65,10 @@
 
 <script lang="ts">
   import _ from 'lodash';
-  import { getContext } from 'svelte';
   import { registerQuickExportHandler } from '../buttons/ToolStripExportButton.svelte';
 
   import registerCommand from '../commands/registerCommand';
   import { extractShellConnection } from '../impexp/createImpExpScript';
-  import ImportExportModal from '../modals/ImportExportModal.svelte';
-  import { showModal } from '../modals/modalTools';
   import { apiCall } from '../utility/api';
 
   import { registerMenu } from '../utility/contextMenu';
@@ -84,6 +81,7 @@
 
   import LoadingDataGridCore from './LoadingDataGridCore.svelte';
   import hasPermission from '../utility/hasPermission';
+  import { openImportExportTab } from '../utility/importExportTools';
 
   export let conid;
   export let display;
@@ -145,7 +143,8 @@
     initialValues.sourceQueryType = coninfo.isReadOnly ? 'json' : 'native';
     initialValues.sourceList = display.baseTableOrSimilar ? [display.baseTableOrSimilar.pureName] : [];
     initialValues[`columns_${pureName}`] = display.getExportColumnMap();
-    showModal(ImportExportModal, { initialValues });
+    openImportExportTab(initialValues);
+    // showModal(ImportExportModal, { initialValues });
   }
 
   export function openQuery() {

packages/web/src/designer/ColumnLine.svelte

@@ -196,7 +196,7 @@
     <div class="space" />
     {#if designer?.style?.showDataType && column?.dataType}
       <div class="ml-2">
-        {column?.dataType.toLowerCase()}
+        {(column?.displayedDataType || column?.dataType).toLowerCase()}
       </div>
     {/if}
     {#if designer?.style?.showNullability}

packages/web/src/elements/ColumnLabel.svelte

@@ -19,6 +19,7 @@
   export let columnName = '';
   export let extInfo = null;
   export let dataType = null;
+  export let displayedDataType = null;
   export let showDataType = false;
   export let foreignKey;
   export let conid = undefined;
@@ -59,7 +60,7 @@
         {/if}
       </span>
     {:else if dataType}
-      <span class="extinfo">{dataType.toLowerCase()}</span>
+      <span class="extinfo">{(displayedDataType || dataType).toLowerCase()}</span>
     {/if}
   {/if}
 </span>

packages/web/src/elements/ColumnMapColumnDropdown.svelte

@@ -0,0 +1,30 @@
+<script alng="ts">
+  import DropDownButton from '../buttons/DropDownButton.svelte';
+  import TextField from '../forms/TextField.svelte';
+
+  export let tableInfo;
+  export let onChange;
+  export let value;
+</script>
+
+{#if tableInfo}
+  <div class="wrapper">
+    <TextField {...$$restProps} {value} on:input={e => onChange(e.target.value)} />
+    <DropDownButton
+      menu={() => {
+        return tableInfo.columns.map(opt => ({
+          text: opt.columnName,
+          onClick: () => onChange(opt.columnName),
+        }));
+      }}
+    />
+  </div>
+{:else}
+  <TextField {value} on:input={e => onChange(e.target.value)} />
+{/if}
+
+<style>
+  .wrapper {
+    display: flex;
+  }
+</style>

packages/web/src/elements/ForeignKeyObjectListControl.svelte

@@ -8,7 +8,7 @@
 
   export let collection;
   export let title;
-  export let clickable;
+  export let clickable = false;
   export let onRemove = null;
   export let onAddNew = null;
   export let emptyMessage = null;