UI fix

default schema refactor
refresh schema list
2024-09-19 13:43:55 +02:00 · 2024-09-19 13:41:49 +02:00 · 2024-09-19 11:15:44 +02:00 · 2024-09-19 10:59:09 +02:00 · 2024-09-19 09:53:55 +02:00 · 2024-09-19 09:52:54 +02:00
379 changed files with 10022 additions and 3492 deletions
@@ -30,6 +30,9 @@ jobs:
      - name: yarn adjustPackageJson
        run: |
          yarn adjustPackageJson
+      - name: setUpdaterChannel beta
+        run: |
+          node setUpdaterChannel beta
      - name: yarn set timeout
        run: |
          yarn config set network-timeout 100000
@@ -99,9 +102,13 @@ jobs:
          mv app/dist/*.deb artifacts/ || true
          mv app/dist/*.snap artifacts/ || true
          mv app/dist/*.dmg artifacts/ || true
+          mv app/dist/*.blockmap artifacts/ || true
+
+          mv app/dist/*.yml artifacts/ || true
+          rm artifacts/builder-debug.yml

      - name: Upload artifacts
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.os }}
          path: artifacts
@@ -0,0 +1,153 @@
+name: Electron app PREMIUM BETA
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+-premium-beta.[0-9]+'
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # os: [windows-2022]
+        # os: [ubuntu-22.04]
+        # os: [windows-2022, ubuntu-22.04]
+        os: [macos-12, windows-2022, ubuntu-22.04]
+        # os: [macOS-10.15]
+
+    steps:
+      - name: Context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 18.x
+          
+      - name: Checkout dbgate/dbgate-pro
+        uses: actions/checkout@v2
+        with:
+          repository: dbgate/dbgate-pro
+          token: ${{ secrets.GH_TOKEN }}
+          path: dbgate-pro
+
+      - name: Merge dbgate/dbgate-pro
+        run: |
+          mkdir ../dbgate-pro
+          mv dbgate-pro/* ../dbgate-pro/
+          cd ..
+          mkdir dbgate-merged
+          cd dbgate-pro
+          cd sync
+          yarn
+          node sync.js --nowatch
+          cd ..
+
+      - name: yarn adjustPackageJson
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn adjustPackageJson
+      - name: adjustPackageJsonPremium
+        run: |
+          cd ..
+          cd dbgate-merged
+          node adjustPackageJsonPremium
+      - name: setUpdaterChannel premium-beta
+        run: |
+          cd ..
+          cd dbgate-merged
+          node setUpdaterChannel premium-beta
+      - name: yarn set timeout
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn config set network-timeout 100000
+      - name: yarn install
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn install
+      - name: setCurrentVersion
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn setCurrentVersion
+      - name: printSecrets
+        run: |
+          cd ..
+          cd dbgate-merged
+            yarn printSecrets
+        env:
+          GIST_UPLOAD_SECRET : ${{secrets.GIST_UPLOAD_SECRET}}
+      - name: fillNativeModulesElectron
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn fillNativeModulesElectron
+      - name: fillPackagedPlugins
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn fillPackagedPlugins
+      - name: Publish
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn run build:app
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
+
+          WIN_CSC_LINK: ${{ secrets.WINCERT_2025 }}
+          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_2025_PASSWORD }}
+          # WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
+          # WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
+
+          CSC_LINK: ${{ secrets.APPLECERT_CERTIFICATE }}
+          CSC_KEY_PASSWORD: ${{ secrets.APPLECERT_PASSWORD }}
+
+          APPLE_ID: ${{ secrets.APPLE_ID }}
+          APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
+          APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
+
+      - name: Copy artifacts
+        run: |
+          mkdir artifacts          
+
+          cp ../dbgate-merged/app/dist/*x86*.AppImage artifacts/dbgate-premium-beta.AppImage || true
+          cp ../dbgate-merged/app/dist/*win*.exe artifacts/dbgate-premium-beta.exe || true
+          cp ../dbgate-merged/app/dist/*-mac_x64.dmg artifacts/dbgate-premium-beta.dmg || true
+
+          mv ../dbgate-merged/app/dist/*.exe artifacts/ || true
+          mv ../dbgate-merged/app/dist/*.zip artifacts/ || true
+          mv ../dbgate-merged/app/dist/*.tar.gz artifacts/ || true
+          mv ../dbgate-merged/app/dist/*.AppImage artifacts/ || true
+          mv ../dbgate-merged/app/dist/*.deb artifacts/ || true
+          mv ../dbgate-merged/app/dist/*.snap artifacts/ || true
+          mv ../dbgate-merged/app/dist/*.dmg artifacts/ || true
+          mv ../dbgate-merged/app/dist/*.blockmap artifacts/ || true
+
+          mv ../dbgate-merged/app/dist/*.yml artifacts/ || true
+          rm artifacts/builder-debug.yml
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.os }}
+          path: artifacts
+
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          files: 'artifacts/**'
+          prerelease: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -0,0 +1,155 @@
+name: Electron app PREMIUM
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+      # - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
+
+    # branches:
+    #   - production
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # os: [ubuntu-22.04, windows-2016]
+        os: [macos-12, windows-2022, ubuntu-22.04]
+
+    steps:
+      - name: Context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 18.x
+
+      - name: Checkout dbgate/dbgate-pro
+        uses: actions/checkout@v2
+        with:
+          repository: dbgate/dbgate-pro
+          token: ${{ secrets.GH_TOKEN }}
+          path: dbgate-pro
+
+      - name: Merge dbgate/dbgate-pro
+        run: |
+          mkdir ../dbgate-pro
+          mv dbgate-pro/* ../dbgate-pro/
+          cd ..
+          mkdir dbgate-merged
+          cd dbgate-pro
+          cd sync
+          yarn
+          node sync.js --nowatch
+          cd ..
+
+      - name: yarn adjustPackageJson
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn adjustPackageJson
+      - name: yarn adjustPackageJsonPremium
+        run: |
+          cd ..
+          cd dbgate-merged
+          node adjustPackageJsonPremium
+      - name: setUpdaterChannel premium
+        run: |
+          cd ..
+          cd dbgate-merged
+          node setUpdaterChannel premium
+      - name: yarn set timeout
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn config set network-timeout 100000
+      - name: yarn install
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn install
+      - name: setCurrentVersion
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn setCurrentVersion
+      - name: printSecrets
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn printSecrets
+        env:
+          GIST_UPLOAD_SECRET : ${{secrets.GIST_UPLOAD_SECRET}}
+      - name: fillNativeModulesElectron
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn fillNativeModulesElectron
+      - name: fillPackagedPlugins
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn fillPackagedPlugins
+      - name: Publish
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn run build:app
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
+
+          WIN_CSC_LINK: ${{ secrets.WINCERT_2025 }}
+          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_2025_PASSWORD }}
+          # WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
+          # WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
+
+          CSC_LINK: ${{ secrets.APPLECERT_CERTIFICATE }}
+          CSC_KEY_PASSWORD: ${{ secrets.APPLECERT_PASSWORD }}
+
+          APPLE_ID: ${{ secrets.APPLE_ID }}
+          APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
+          APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
+
+      - name: Copy artifacts
+        run: |
+          mkdir artifacts          
+
+          cp ../dbgate-merged/app/dist/*x86*.AppImage artifacts/dbgate-premium-latest.AppImage || true
+          cp ../dbgate-merged/app/dist/*.exe artifacts/dbgate-premium-latest.exe || true
+          cp ../dbgate-merged/app/dist/*win_x64.zip artifacts/dbgate-premium-windows-latest.zip || true
+          cp ../dbgate-merged/app/dist/*win_arm64.zip artifacts/dbgate-premium-windows-latest-arm64.zip || true
+          cp ../dbgate-merged/app/dist/*-mac_universal.dmg artifacts/dbgate-premium-latest.dmg || true
+          cp ../dbgate-merged/app/dist/*-mac_x64.dmg artifacts/dbgate-premium-latest-x64.dmg || true
+
+          mv ../dbgate-merged/app/dist/*.exe artifacts/ || true
+          mv ../dbgate-merged/app/dist/*.zip artifacts/ || true
+          mv ../dbgate-merged/app/dist/*.tar.gz artifacts/ || true
+          mv ../dbgate-merged/app/dist/*.AppImage artifacts/ || true
+          mv ../dbgate-merged/app/dist/*.deb artifacts/ || true
+          mv ../dbgate-merged/app/dist/*.dmg artifacts/ || true
+          mv ../dbgate-merged/app/dist/*.blockmap artifacts/ || true
+
+          mv ../dbgate-merged/app/dist/*.yml artifacts/ || true
+          rm artifacts/builder-debug.yml
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.os }}
+          path: artifacts
+
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          files: 'artifacts/**'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -109,6 +109,10 @@ jobs:
          mv app/dist/*.deb artifacts/ || true
          mv app/dist/*.dmg artifacts/ || true
          mv app/dist/*.snap artifacts/dbgate-latest.snap || true
+          mv app/dist/*.blockmap artifacts/ || true
+
+          mv app/dist/*.yml artifacts/ || true
+          rm artifacts/builder-debug.yml

      # - name: Copy artifacts Linux, MacOs
      #   if: matrix.os != 'windows-2016'
@@ -134,24 +138,13 @@ jobs:

      #     mv app/dist/latest.yml artifacts/latest.yml || true

-      - name: Copy latest.yml (windows)
+      - name: Copy PAD file
        if: matrix.os == 'windows-2022'
        run: |
-          mv app/dist/latest.yml artifacts/latest.yml || true
          mv app/dist/dbgate-pad.xml artifacts/ || true

-      - name: Copy latest-linux.yml
-        if: matrix.os == 'ubuntu-22.04'
-        run: |
-          mv app/dist/latest-linux.yml artifacts/latest-linux.yml || true
-
-      - name: Copy latest-mac.yml
-        if: matrix.os == 'macos-12'
-        run: |
-          mv app/dist/latest-mac.yml artifacts/latest-mac.yml || true
-
      - name: Upload artifacts
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.os }}
          path: artifacts
@@ -0,0 +1,104 @@
+name: Docker image PREMIUM
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+-premium-beta.[0-9]+'
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ubuntu-22.04]
+
+    steps:
+      - name: Context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            dbgate/dbgate-premium
+          flavor: |
+            latest=false
+          tags: |
+            type=raw,value=beta,enable=${{ contains(github.ref_name, '-docker.') || contains(github.ref_name, '-beta.') }}
+
+            type=match,pattern=\d+.\d+.\d+,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
+            type=raw,value=latest,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
+
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 18.x
+
+      - name: Checkout dbgate/dbgate-pro
+        uses: actions/checkout@v2
+        with:
+          repository: dbgate/dbgate-pro
+          token: ${{ secrets.GH_TOKEN }}
+          path: dbgate-pro
+
+      - name: Merge dbgate/dbgate-pro
+        run: |
+          mkdir ../dbgate-pro
+          mv dbgate-pro/* ../dbgate-pro/
+          cd ..
+          mkdir dbgate-merged
+          cd dbgate-pro
+          cd sync
+          yarn
+          node sync.js --nowatch
+          cd ..
+
+      - name: yarn install
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn install
+
+      - name: setCurrentVersion
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn setCurrentVersion
+          
+      - name: printSecrets
+        run: |
+          cd ..
+          cd dbgate-merged
+            yarn printSecrets
+        env:
+          GIST_UPLOAD_SECRET : ${{secrets.GIST_UPLOAD_SECRET}}
+      - name: Prepare docker image
+        run: |
+          cd ..
+          cd dbgate-merged
+          yarn run prepare:docker
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          push: true
+          context: ../dbgate-merged/docker
+          tags: ${{ steps.meta.outputs.tags }}
+          platforms: linux/amd64,linux/arm64
@@ -5,7 +5,6 @@ on:
    tags:
      - 'v[0-9]+.[0-9]+.[0-9]+'
      - 'v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+'
-      - 'v[0-9]+.[0-9]+.[0-9]+-docker.[0-9]+'

 jobs:
  build:
@@ -154,3 +154,8 @@ jobs:
        working-directory: plugins/dbgate-plugin-oracle
        run: |
          npm publish
+
+      - name: Publish dbgate-plugin-clickhouse
+        working-directory: plugins/dbgate-plugin-clickhouse
+        run: |
+          npm publish
@@ -4,6 +4,7 @@ on:
    branches:
      - master
      - develop
+      - 'feature/**'

 jobs:
  test-runner:
@@ -77,6 +78,11 @@ jobs:
          ACCEPT_EULA: Y
          SA_PASSWORD: Pwd2020Db
          MSSQL_PID: Express
-          
+
+      clickhouse:
+        image: bitnami/clickhouse:24.8.4
+        env:
+          CLICKHOUSE_ADMIN_PASSWORD: Pwd2020Db
+
      # cockroachdb:
      #   image: cockroachdb/cockroach
@@ -31,4 +31,6 @@ yarn-error.log*
 app/src/nativeModulesContent.js
 packages/api/src/nativeModulesContent.js
 packages/api/src/packagedPluginsContent.js
-.VSCodeCounter
+.VSCodeCounter
+
+packages/web/public/*.html
@@ -8,6 +8,49 @@ Builds:
 - linux - application for linux
 - win - application for Windows

+### 5.4.4
+- CHANGED: Improved autoupdate, notification is now in app
+- CHANGED: Default behaviour of autoupdate, new version is downloaded after click of "Download" button
+- ADDED: Ability to configure autoupdate (check only, check+download, don't check)\
+- ADDED: Option to run check for new version manually
+- FIXED: Fixed autoupgrade channel for premium edition
+- FIXED: Fixes following issues: #886, #865, #782, #375
+
+### 5.4.2
+- FIXED: DbGate now works correctly with Oracle 10g
+- FIXED: Fixed update channel for premium edition
+
+### 5.4.1
+- FIXED: Broken older plugins #881
+- ADDED: Premium edition - "Start trial" button
+
+### 5.4.0
+- ADDED: Support for CosmosDB (Premium only)
+- ADDED: Administration UI (Premium only)
+- ADDED: New application icon
+- ADDED: MongoDB type support in data editing
+- ADDED: MongoDB - posibility to remove field
+- ADDED: Oracle - posibility to connect via SID
+- FIXED: Many improvements in MongoDB filtering
+- FIXED: Switch to form and back to table rows missing #343
+- ADDED: Posibility to deactivate MongoDB Profiler #745
+- ADDED: Ability to use Oracle thick driver - neccessary for connecting older Oracle servers #843
+- FIXED: Connection permissions configuration is broken #860
+- ADDED: ssh key file authentication option missing #876
+- ADDED: Ability to reset layout #878
+- FIXED: Script with escaped backslash causes erro #880
+
+### 5.3.4
+- FIXED: On blank system does not start (window does not appear) #862
+- FIXED: Missing Execute, Export bar #861
+
+### 5.3.3
+- FIXED: The application Window is not visible when openning after changing monitor configuration. #856
+- FIXED: Multi column filter is broken for Postgresql #855
+- ADDED: Do not display internal timescaledb objects in postgres databases #839
+- FIXED: When in splitview mode and Clicking "Refresh" button on the right side, will refresh the left side, and not the right side #810
+- FIXED: Cannot filter by uuid field in psql #538
+
 ### 5.3.1
 - FIXED: Column sorting on query tab not working #819
 - FIXED: Postgres Connection stays in "Loading database structure" until reloading the page #826
@@ -29,6 +29,7 @@ DbGate is licensed under GPL-3.0 license and is free to use for any purpose.
 * Amazon Redshift
 * CockroachDB
 * MariaDB
+* CosmosDB (Premium)

 <!-- Learn more about DbGate features at the [DbGate website](https://dbgate.org/), or try our online [demo application](https://demo.dbgate.org) -->

@@ -6,7 +6,8 @@
  "description": "Opensource database administration tool",
  "dependencies": {
    "electron-log": "^4.4.1",
-    "electron-updater": "^4.6.1",
+    "electron-updater": "^6.3.4",
+    "jsonwebtoken": "^9.0.2",
    "lodash.clonedeepwith": "^4.5.0",
    "patch-package": "^6.4.7"
  },
@@ -27,7 +28,11 @@
      "entitlements": "entitlements.mac.plist",
      "entitlementsInherit": "entitlements.mac.plist",
      "publish": [
-        "github"
+        {
+          "provider": "github",
+          "owner": "dbgate",
+          "repo": "dbgate"
+        }
      ],
      "target": {
        "target": "default",
@@ -55,7 +60,11 @@
      "category": "Development",
      "synopsis": "Database manager for SQL Server, MySQL, PostgreSQL, MongoDB and SQLite",
      "publish": [
-        "github"
+        {
+          "provider": "github",
+          "owner": "dbgate",
+          "repo": "dbgate"
+        }
      ]
    },
    "appImage": {
@@ -90,9 +99,12 @@
      ],
      "icon": "icon.ico",
      "publish": [
-        "github"
-      ],
-      "rfc3161TimeStampServer": "http://sha256timestamp.ws.symantec.com/sha256/timestamp"
+        {
+          "provider": "github",
+          "owner": "dbgate",
+          "repo": "dbgate"
+        }
+      ]
    },
    "files": [
      "packages",
@@ -121,6 +133,7 @@
  },
  "optionalDependencies": {
    "better-sqlite3": "9.6.0",
-    "msnodesqlv8": "^4.2.1"
+    "msnodesqlv8": "^4.2.1",
+    "oracledb": "^6.6.0"
  }
 }
@@ -16,17 +16,20 @@ const BrowserWindow = electron.BrowserWindow;
 const path = require('path');
 const url = require('url');
 const mainMenuDefinition = require('./mainMenuDefinition');
-const { settings } = require('cluster');
-let disableAutoUpgrade = false;
+const { isProApp } = require('./proTools');
+const updaterChannel = require('./updaterChannel');

 // require('@electron/remote/main').initialize();

 const configRootPath = path.join(app.getPath('userData'), 'config-root.json');
+let saveConfigOnExit = true;
 let initialConfig = {};
 let apiLoaded = false;
 let mainModule;
 // let getLogger;
 // let loadLogsContent;
+let appUpdateStatus = '';
+let settingsJson = {};

 process.on('uncaughtException', function (error) {
  console.error('uncaughtException', error);
@@ -50,21 +53,11 @@ const isMac = () => os.platform() == 'darwin';

 try {
  initialConfig = JSON.parse(fs.readFileSync(configRootPath, { encoding: 'utf-8' }));
-  disableAutoUpgrade = initialConfig['disableAutoUpgrade'] || false;
 } catch (err) {
  console.log('Error loading config-root:', err.message);
  initialConfig = {};
 }

-if (process.argv.includes('--disable-auto-upgrade')) {
-  console.log('Disabling auto-upgrade');
-  disableAutoUpgrade = true;
-}
-if (process.argv.includes('--enable-auto-upgrade')) {
-  console.log('Enabling auto-upgrade');
-  disableAutoUpgrade = false;
-}
-
 // Keep a global reference of the window object, if you don't, the window will
 // be closed automatically when the JavaScript object is garbage collected.
 let mainWindow;
@@ -73,6 +66,10 @@ let runCommandOnLoad = null;

 log.transports.file.level = 'debug';
 autoUpdater.logger = log;
+if (updaterChannel) {
+  autoUpdater.channel = updaterChannel;
+  autoUpdater.allowPrerelease = updaterChannel.includes('beta');
+}
 // TODO - create settings for this
 // appUpdater.channel = 'beta';

@@ -173,6 +170,21 @@ ipcMain.on('quit-app', async (event, arg) => {
    mainWindow.close();
  }
 });
+ipcMain.on('reset-settings', async (event, arg) => {
+  try {
+    saveConfigOnExit = false;
+    fs.unlinkSync(configRootPath);
+    console.log('Deleted file:', configRootPath);
+  } catch (err) {
+    console.log('Error deleting config-root:', err.message);
+  }
+
+  if (isMac()) {
+    app.quit();
+  } else {
+    mainWindow.close();
+  }
+});
 ipcMain.on('set-title', async (event, arg) => {
  mainWindow.setTitle(arg);
 });
@@ -191,6 +203,15 @@ ipcMain.on('app-started', async (event, arg) => {
  if (initialConfig['winIsMaximized']) {
    mainWindow.webContents.send('setIsMaximized', true);
  }
+  if (autoUpdater.isUpdaterActive()) {
+    mainWindow.webContents.send('setAppUpdaterActive');
+  }
+  if (!process.env.DEVMODE) {
+    if (settingsJson['app.autoUpdateMode'] != 'skip') {
+      autoUpdater.autoDownload = settingsJson['app.autoUpdateMode'] == 'download';
+      autoUpdater.checkForUpdates();
+    }
+  }
 });
 ipcMain.on('window-action', async (event, arg) => {
  if (!mainWindow) {
@@ -264,6 +285,20 @@ ipcMain.handle('showItemInFolder', async (event, path) => {
 ipcMain.handle('openExternal', async (event, url) => {
  electron.shell.openExternal(url);
 });
+ipcMain.on('downloadUpdate', async (event, url) => {
+  autoUpdater.downloadUpdate();
+  changeAppUpdateStatus({
+    icon: 'icon loading',
+    message: `Downloading update...`,
+  });
+});
+ipcMain.on('applyUpdate', async (event, url) => {
+  autoUpdater.quitAndInstall(false, true);
+});
+ipcMain.on('check-for-updates', async (event, url) => {
+  autoUpdater.autoDownload = false;
+  autoUpdater.checkForUpdates();
+});

 function fillMissingSettings(value) {
  const res = {
@@ -299,9 +334,9 @@ function ensureBoundsVisible(bounds) {
 }

 function createWindow() {
-  let settingsJson = {};
+  const datadir = path.join(os.homedir(), '.dbgate');
+
  try {
-    const datadir = path.join(os.homedir(), '.dbgate');
    settingsJson = fillMissingSettings(
      JSON.parse(fs.readFileSync(path.join(datadir, 'settings.json'), { encoding: 'utf-8' }))
    );
@@ -311,18 +346,20 @@ function createWindow() {
  }

  let bounds = initialConfig['winBounds'];
-  bounds = ensureBoundsVisible(bounds);
+  if (bounds) {
+    bounds = ensureBoundsVisible(bounds);
+  }
  useNativeMenu = settingsJson['app.useNativeMenu'];

  mainWindow = new BrowserWindow({
    width: 1200,
    height: 800,
-    title: 'DbGate',
+    title: isProApp() ? 'DbGate Premium' : 'DbGate',
    frame: useNativeMenu,
    titleBarStyle: useNativeMenu ? undefined : 'hidden',
    ...bounds,
    icon: os.platform() == 'win32' ? 'icon.ico' : path.resolve(__dirname, '../icon.png'),
-    partition: 'persist:dbgate',
+    partition: isProApp() ? 'persist:dbgate-premium' : 'persist:dbgate',
    webPreferences: {
      nodeIntegration: true,
      contextIsolation: false,
@@ -350,24 +387,27 @@ function createWindow() {
      });
    mainWindow.on('close', () => {
      try {
-        fs.writeFileSync(
-          configRootPath,
-          JSON.stringify({
-            winBounds: mainWindow.getBounds(),
-            winIsMaximized: mainWindow.isMaximized(),
-            disableAutoUpgrade,
-          }),
-          'utf-8'
-        );
+        if (saveConfigOnExit) {
+          fs.writeFileSync(
+            configRootPath,
+            JSON.stringify({
+              winBounds: mainWindow.getBounds(),
+              winIsMaximized: mainWindow.isMaximized(),
+            }),
+            'utf-8'
+          );
+        }
      } catch (err) {
        console.log('Error saving config-root:', err.message);
      }
    });
+
+    // mainWindow.webContents.toggleDevTools();
+
    mainWindow.loadURL(startUrl);
    if (os.platform() == 'linux') {
      mainWindow.setIcon(path.resolve(__dirname, '../icon.png'));
    }
-    // mainWindow.webContents.toggleDevTools();

    mainWindow.on('maximize', () => {
      mainWindow.webContents.send('setIsMaximized', true);
@@ -422,13 +462,61 @@ function createWindow() {
  });
 }

+function changeAppUpdateStatus(status) {
+  appUpdateStatus = status;
+  mainWindow.webContents.send('app-update-status', appUpdateStatus);
+}
+
+autoUpdater.on('checking-for-update', () => {
+  console.log('Checking for updates');
+  changeAppUpdateStatus({
+    icon: 'icon loading',
+    message: 'Checking for updates...',
+  });
+});
+
+autoUpdater.on('update-available', info => {
+  console.log('Update available', info);
+  if (autoUpdater.autoDownload) {
+    changeAppUpdateStatus({
+      icon: 'icon loading',
+      message: `Downloading update...`,
+    });
+  } else {
+    mainWindow.webContents.send('update-available', info.version);
+    changeAppUpdateStatus({
+      icon: 'icon download',
+      message: `Update available`,
+    });
+  }
+});
+
+autoUpdater.on('update-not-available', info => {
+  console.log('Update not available', info);
+  changeAppUpdateStatus({
+    icon: 'icon check',
+    message: `No new updates`,
+  });
+});
+
+autoUpdater.on('update-downloaded', info => {
+  console.log('Update downloaded from', info);
+  changeAppUpdateStatus({
+    icon: 'icon download',
+    message: `Downloaded ${info.version}`,
+  });
+  mainWindow.webContents.send('downloaded-new-version', info.version);
+});
+
+autoUpdater.on('error', error => {
+  changeAppUpdateStatus({
+    icon: 'icon error',
+    message: `Autoupdate error`,
+  });
+  console.error('Update error', error);
+});
+
 function onAppReady() {
-  if (disableAutoUpgrade) {
-    console.log('Auto-upgrade is disabled, run dbgate --enable-auto-upgrade to enable');
-  }
-  if (!process.env.DEVMODE && !disableAutoUpgrade) {
-    autoUpdater.checkForUpdatesAndNotify();
-  }
  createWindow();
 }

@@ -91,6 +91,7 @@ module.exports = ({ editMenu }) => [
      { command: 'folder.showLogs', hideDisabled: true },
      { command: 'folder.showData', hideDisabled: true },
      { command: 'new.gist', hideDisabled: true },
+      { command: 'app.resetSettings', hideDisabled: true },
    ],
  },
  {
@@ -104,6 +105,8 @@ module.exports = ({ editMenu }) => [
      { command: 'settings.commands', hideDisabled: true },
      { command: 'tabs.changelog', hideDisabled: true },
      { command: 'about.show', hideDisabled: true },
+      { divider: true },
+      { command: 'file.checkForUpdates', hideDisabled: true },
    ],
  },
 ];
@@ -0,0 +1,12 @@
+function isProApp() {
+  return false;
+}
+
+function checkLicense(license) {
+  return null;
+}
+
+module.exports = {
+  isProApp,
+  checkLicense,
+};
@@ -0,0 +1 @@
+module.exports = null;
@@ -193,11 +193,6 @@
  dependencies:
    "@types/node" "*"

-"@types/semver@^7.3.6":
-  version "7.5.8"
-  resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.8.tgz#8268a8c57a3e4abd25c165ecd36237db7948a55e"
-  integrity sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==
-
 "@types/verror@^1.10.3":
  version "1.10.10"
  resolved "https://registry.yarnpkg.com/@types/verror/-/verror-1.10.10.tgz#d5a4b56abac169bfbc8b23d291363a682e6fa087"
@@ -476,6 +471,11 @@ buffer-crc32@~0.2.3:
  resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242"
  integrity sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==

+buffer-equal-constant-time@1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819"
+  integrity sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==
+
 buffer-equal@1.0.0:
  version "1.0.0"
  resolved "https://registry.yarnpkg.com/buffer-equal/-/buffer-equal-1.0.0.tgz#59616b498304d556abd466966b22eeda3eca5fbe"
@@ -499,14 +499,6 @@ buffer@^5.1.0, buffer@^5.5.0:
    base64-js "^1.3.1"
    ieee754 "^1.1.13"

-builder-util-runtime@8.9.2:
-  version "8.9.2"
-  resolved "https://registry.yarnpkg.com/builder-util-runtime/-/builder-util-runtime-8.9.2.tgz#a9669ae5b5dcabfe411ded26678e7ae997246c28"
-  integrity sha512-rhuKm5vh7E0aAmT6i8aoSfEjxzdYEFX7zDApK+eNgOhjofnWb74d9SRJv0H/8nsgOkos0TZ4zxW0P8J4N7xQ2A==
-  dependencies:
-    debug "^4.3.2"
-    sax "^1.2.4"
-
 builder-util-runtime@9.0.2:
  version "9.0.2"
  resolved "https://registry.yarnpkg.com/builder-util-runtime/-/builder-util-runtime-9.0.2.tgz#dc54f8581bbcf1e0428da4483fa46d09524be857"
@@ -515,6 +507,14 @@ builder-util-runtime@9.0.2:
    debug "^4.3.4"
    sax "^1.2.4"

+builder-util-runtime@9.2.5:
+  version "9.2.5"
+  resolved "https://registry.yarnpkg.com/builder-util-runtime/-/builder-util-runtime-9.2.5.tgz#0afdffa0adb5c84c14926c7dd2cf3c6e96e9be83"
+  integrity sha512-HjIDfhvqx/8B3TDN4GbABQcgpewTU4LMRTQPkVpKYV3lsuxEJoIfvg09GyWTNmfVNSUAYf+fbTN//JX4TH20pg==
+  dependencies:
+    debug "^4.3.4"
+    sax "^1.2.4"
+
 builder-util@23.0.9:
  version "23.0.9"
  resolved "https://registry.yarnpkg.com/builder-util/-/builder-util-23.0.9.tgz#8b1aeeeee679060e39ad2bd0f50f5b3f3cb53a59"
@@ -786,7 +786,7 @@ crypto-random-string@^2.0.0:
  resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-2.0.0.tgz#ef2a7a966ec11083388369baa02ebead229b30d5"
  integrity sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==

-debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4:
+debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.4:
  version "4.3.4"
  resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865"
  integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==
@@ -927,6 +927,13 @@ duplexer3@^0.1.4:
  resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.5.tgz#0b5e4d7bad5de8901ea4440624c8e1d20099217e"
  integrity sha512-1A8za6ws41LQgv9HrE/66jyC5yuSjQ3L/KOpFtoBilsAK2iA2wuS5rTt1OCzIvtS2V7nVmedsUU+DGRcjBmOYA==

+ecdsa-sig-formatter@1.0.11:
+  version "1.0.11"
+  resolved "https://registry.yarnpkg.com/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz#ae0f0fa2d85045ef14a817daa3ce9acd0489e5bf"
+  integrity sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==
+  dependencies:
+    safe-buffer "^5.0.1"
+
 ejs@^3.1.7:
  version "3.1.10"
  resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.10.tgz#69ab8358b14e896f80cc39e62087b88500c3ac3b"
@@ -1000,19 +1007,19 @@ electron-publish@23.0.9:
    lazy-val "^1.0.5"
    mime "^2.5.2"

-electron-updater@^4.6.1:
-  version "4.6.5"
-  resolved "https://registry.yarnpkg.com/electron-updater/-/electron-updater-4.6.5.tgz#e9a75458bbfd6bb41a58a829839e150ad2eb2d3d"
-  integrity sha512-kdTly8O9mSZfm9fslc1mnCY+mYOeaYRy7ERa2Fed240u01BKll3aiupzkd07qKw69KvhBSzuHroIW3mF0D8DWA==
+electron-updater@^6.3.4:
+  version "6.3.4"
+  resolved "https://registry.yarnpkg.com/electron-updater/-/electron-updater-6.3.4.tgz#3934bc89875bb524c2cbbd11041114e97c0c2496"
+  integrity sha512-uZUo7p1Y53G4tl6Cgw07X1yF8Jlz6zhaL7CQJDZ1fVVkOaBfE2cWtx80avwDVi8jHp+I/FWawrMgTAeCCNIfAg==
  dependencies:
-    "@types/semver" "^7.3.6"
-    builder-util-runtime "8.9.2"
-    fs-extra "^10.0.0"
+    builder-util-runtime "9.2.5"
+    fs-extra "^10.1.0"
    js-yaml "^4.1.0"
    lazy-val "^1.0.5"
    lodash.escaperegexp "^4.1.2"
    lodash.isequal "^4.5.0"
-    semver "^7.3.5"
+    semver "^7.6.3"
+    tiny-typed-emitter "^2.1.0"

 electron@30.0.2:
  version "30.0.2"
@@ -1663,6 +1670,39 @@ jsonfile@^6.0.1:
  optionalDependencies:
    graceful-fs "^4.1.6"

+jsonwebtoken@^9.0.2:
+  version "9.0.2"
+  resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz#65ff91f4abef1784697d40952bb1998c504caaf3"
+  integrity sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ==
+  dependencies:
+    jws "^3.2.2"
+    lodash.includes "^4.3.0"
+    lodash.isboolean "^3.0.3"
+    lodash.isinteger "^4.0.4"
+    lodash.isnumber "^3.0.3"
+    lodash.isplainobject "^4.0.6"
+    lodash.isstring "^4.0.1"
+    lodash.once "^4.0.0"
+    ms "^2.1.1"
+    semver "^7.5.4"
+
+jwa@^1.4.1:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/jwa/-/jwa-1.4.1.tgz#743c32985cb9e98655530d53641b66c8645b039a"
+  integrity sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==
+  dependencies:
+    buffer-equal-constant-time "1.0.1"
+    ecdsa-sig-formatter "1.0.11"
+    safe-buffer "^5.0.1"
+
+jws@^3.2.2:
+  version "3.2.2"
+  resolved "https://registry.yarnpkg.com/jws/-/jws-3.2.2.tgz#001099f3639468c9414000e99995fa52fb478304"
+  integrity sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==
+  dependencies:
+    jwa "^1.4.1"
+    safe-buffer "^5.0.1"
+
 keyv@^3.0.0:
  version "3.1.0"
  resolved "https://registry.yarnpkg.com/keyv/-/keyv-3.1.0.tgz#ecc228486f69991e49e9476485a5be1e8fc5c4d9"
@@ -1718,11 +1758,46 @@ lodash.escaperegexp@^4.1.2:
  resolved "https://registry.yarnpkg.com/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz#64762c48618082518ac3df4ccf5d5886dae20347"
  integrity sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==

+lodash.includes@^4.3.0:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/lodash.includes/-/lodash.includes-4.3.0.tgz#60bb98a87cb923c68ca1e51325483314849f553f"
+  integrity sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==
+
+lodash.isboolean@^3.0.3:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz#6c2e171db2a257cd96802fd43b01b20d5f5870f6"
+  integrity sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==
+
 lodash.isequal@^4.5.0:
  version "4.5.0"
  resolved "https://registry.yarnpkg.com/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0"
  integrity sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==

+lodash.isinteger@^4.0.4:
+  version "4.0.4"
+  resolved "https://registry.yarnpkg.com/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz#619c0af3d03f8b04c31f5882840b77b11cd68343"
+  integrity sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==
+
+lodash.isnumber@^3.0.3:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz#3ce76810c5928d03352301ac287317f11c0b1ffc"
+  integrity sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==
+
+lodash.isplainobject@^4.0.6:
+  version "4.0.6"
+  resolved "https://registry.yarnpkg.com/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz#7c526a52d89b45c45cc690b88163be0497f550cb"
+  integrity sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==
+
+lodash.isstring@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/lodash.isstring/-/lodash.isstring-4.0.1.tgz#d527dfb5456eca7cc9bb95d5daeaf88ba54a5451"
+  integrity sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==
+
+lodash.once@^4.0.0:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac"
+  integrity sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==
+
 lodash@^4.17.15:
  version "4.17.21"
  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
@@ -1860,6 +1935,11 @@ ms@2.1.2:
  resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"
  integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==

+ms@^2.1.1:
+  version "2.1.3"
+  resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
+  integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
+
 msnodesqlv8@^4.2.1:
  version "4.2.1"
  resolved "https://registry.yarnpkg.com/msnodesqlv8/-/msnodesqlv8-4.2.1.tgz#59f2930e7f3b9b201d7288425a6ffa923ea1a573"
@@ -1944,6 +2024,11 @@ open@^7.4.2:
    is-docker "^2.0.0"
    is-wsl "^2.1.1"

+oracledb@^6.6.0:
+  version "6.6.0"
+  resolved "https://registry.yarnpkg.com/oracledb/-/oracledb-6.6.0.tgz#bb40adbe81a84a1e544c48af9f120c61f030e936"
+  integrity sha512-T3dx+o3j+tVN53wQyr4yGTmoPHLy+a2V8yb1T2PmWrsj3ZlSt2Yu1BgV2yTDqnmBZYpRi/I3yJXRCOHHD7PiyA==
+
 os-tmpdir@~1.0.2:
  version "1.0.2"
  resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274"
@@ -2317,6 +2402,11 @@ semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7:
  resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.1.tgz#60bfe090bf907a25aa8119a72b9f90ef7ca281b2"
  integrity sha512-f/vbBsu+fOiYt+lmwZV0rVwJScl46HppnOA1ZvIuBWKOTlllpyJ3bfVax76/OrhCH38dyxoDIA8K7uB963IYgA==

+semver@^7.5.4, semver@^7.6.3:
+  version "7.6.3"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143"
+  integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==
+
 serialize-error@^7.0.1:
  version "7.0.1"
  resolved "https://registry.yarnpkg.com/serialize-error/-/serialize-error-7.0.1.tgz#f1360b0447f61ffb483ec4157c737fab7d778e18"
@@ -2555,6 +2645,11 @@ through2@^2.0.1:
    readable-stream "~2.3.6"
    xtend "~4.0.1"

+tiny-typed-emitter@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/tiny-typed-emitter/-/tiny-typed-emitter-2.1.0.tgz#b3b027fdd389ff81a152c8e847ee2f5be9fad7b5"
+  integrity sha512-qVtvMxeXbVej0cQWKqVSSAHmKZEHAvxdF8HEUBFWts8h+xEo5m/lEiPakuyZ3BnCBjOD8i24kzNOiOLLgsSxhA==
+
 tmp-promise@^3.0.2:
  version "3.0.3"
  resolved "https://registry.yarnpkg.com/tmp-promise/-/tmp-promise-3.0.3.tgz#60a1a1cc98c988674fcbfd23b6e3367bdeac4ce7"
@@ -3,9 +3,10 @@ const fs = require('fs');
 let fillContent = '';

 if (process.platform == 'win32') {
-  fillContent += `content.msnodesqlv8 = () => require('msnodesqlv8');`;
+  fillContent += `content.msnodesqlv8 = () => require('msnodesqlv8');\n`;
 }
-fillContent += `content['better-sqlite3'] = () => require('better-sqlite3');`;
+fillContent += `content['better-sqlite3'] = () => require('better-sqlite3');\n`;
+fillContent += `content['oracledb'] = () => require('oracledb');\n`;

 const getContent = empty => `
 // this file is generated automatically by script fillNativeModules.js, do not edit it manually
@@ -7,7 +7,9 @@ const { getAlterDatabaseScript, extendDatabaseInfo, generateDbPairingId } = requ

 function flatSource() {
  return _.flatten(
-    engines.map(engine => (engine.objects || []).map(object => [engine.label, object.type, object, engine]))
+    engines
+      .filter(x => !x.skipReferences)
+      .map(engine => (engine.objects || []).map(object => [engine.label, object.type, object, engine]))
  );
 }

@@ -41,7 +43,7 @@ async function testDatabaseDiff(conn, driver, mangle, createObject = null) {
 }

 describe('Alter database', () => {
-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipReferences).map(engine => [engine.label, engine]))(
    'Drop referenced table - %s',
    testWrapper(async (conn, driver, engine) => {
      await testDatabaseDiff(conn, driver, db => {
@@ -65,3 +67,4 @@ describe('Alter database', () => {
    })
  );
 });
+
@@ -6,39 +6,44 @@ const engines = require('../engines');
 const crypto = require('crypto');
 const { getAlterTableScript, extendDatabaseInfo, generateDbPairingId } = require('dbgate-tools');

-function pickImportantTableInfo(table) {
+function pickImportantTableInfo(engine, table) {
+  const props = ['columnName'];
+  if (!engine.skipNullability) props.push('notNull');
+  if (!engine.skipAutoIncrement) props.push('autoIncrement');
  return {
    pureName: table.pureName,
-    columns: table.columns
-      .filter(x => x.columnName != 'rowid')
-      .map(fp.pick(['columnName', 'notNull', 'autoIncrement'])),
+    columns: table.columns.filter(x => x.columnName != 'rowid').map(fp.pick(props)),
  };
 }

-function checkTableStructure(t1, t2) {
+function checkTableStructure(engine, t1, t2) {
  // expect(t1.pureName).toEqual(t2.pureName)
-  expect(pickImportantTableInfo(t1)).toEqual(pickImportantTableInfo(t2));
+  expect(pickImportantTableInfo(engine, t1)).toEqual(pickImportantTableInfo(engine, t2));
 }

-async function testTableDiff(conn, driver, mangle) {
+async function testTableDiff(engine, conn, driver, mangle) {
  await driver.query(conn, `create table t0 (id int not null primary key)`);

  await driver.query(
    conn,
    `create table t1 (
    col_pk int not null primary key, 
-    col_std int null, 
-    col_def int null default 12,
-    col_fk int null references t0(id),
-    col_idx int null,
-    col_uq int null unique,
-    col_ref int null unique
+    col_std int, 
+    col_def int default 12,
+    ${engine.skipReferences ? '' : 'col_fk int references t0(id),'}
+    col_idx int,
+    col_uq int ${engine.skipUnique ? '' : 'unique'} ,
+    col_ref int ${engine.skipUnique ? '' : 'unique'}
  )`
  );

-  await driver.query(conn, `create index idx1 on t1(col_idx)`);
+  if (!engine.skipIndexes) {
+    await driver.query(conn, `create index idx1 on t1(col_idx)`);
+  }

-  await driver.query(conn, `create table t2 (id int not null primary key, fkval int null references t1(col_ref))`);
+  if (!engine.skipReferences) {
+    await driver.query(conn, `create table t2 (id int not null primary key, fkval int null references t1(col_ref))`);
+  }

  const tget = x => x.tables.find(y => y.pureName == 't1');
  const structure1 = generateDbPairingId(extendDatabaseInfo(await driver.analyseFull(conn)));
@@ -53,7 +58,7 @@ async function testTableDiff(conn, driver, mangle) {

  const structure2Real = extendDatabaseInfo(await driver.analyseFull(conn));

-  checkTableStructure(tget(structure2Real), tget(structure2));
+  checkTableStructure(engine, tget(structure2Real), tget(structure2));
  // expect(stableStringify(structure2)).toEqual(stableStringify(structure2Real));
 }

@@ -65,14 +70,22 @@ const TESTED_COLUMNS = ['col_pk', 'col_std', 'col_def', 'col_fk', 'col_ref', 'co
 // const TESTED_COLUMNS = ['col_ref'];

 function engines_columns_source() {
-  return _.flatten(engines.map(engine => TESTED_COLUMNS.map(column => [engine.label, column, engine])));
+  return _.flatten(
+    engines.map(engine =>
+      TESTED_COLUMNS.filter(col => !col.endsWith('_pk') || !engine.skipPkColumnTesting).map(column => [
+        engine.label,
+        column,
+        engine,
+      ])
+    )
+  );
 }

 describe('Alter table', () => {
  test.each(engines.map(engine => [engine.label, engine]))(
    'Add column - %s',
    testWrapper(async (conn, driver, engine) => {
-      await testTableDiff(conn, driver, tbl => {
+      await testTableDiff(engine, conn, driver, tbl => {
        tbl.columns.push({
          columnName: 'added',
          dataType: 'int',
@@ -87,7 +100,7 @@ describe('Alter table', () => {
  test.each(engines_columns_source())(
    'Drop column - %s - %s',
    testWrapper(async (conn, driver, column, engine) => {
-      await testTableDiff(conn, driver, tbl => (tbl.columns = tbl.columns.filter(x => x.columnName != column)));
+      await testTableDiff(engine, conn, driver, tbl => (tbl.columns = tbl.columns.filter(x => x.columnName != column)));
    })
  );

@@ -95,6 +108,7 @@ describe('Alter table', () => {
    'Change nullability - %s - %s',
    testWrapper(async (conn, driver, column, engine) => {
      await testTableDiff(
+        engine,
        conn,
        driver,
        tbl => (tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, notNull: true } : x)))
@@ -106,6 +120,7 @@ describe('Alter table', () => {
    'Rename column - %s - %s',
    testWrapper(async (conn, driver, column, engine) => {
      await testTableDiff(
+        engine,
        conn,
        driver,
        tbl => (tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, columnName: 'col_renamed' } : x)))
@@ -116,7 +131,7 @@ describe('Alter table', () => {
  test.each(engines.map(engine => [engine.label, engine]))(
    'Drop index - %s',
    testWrapper(async (conn, driver, engine) => {
-      await testTableDiff(conn, driver, tbl => {
+      await testTableDiff(engine, conn, driver, tbl => {
        tbl.indexes = [];
      });
    })
@@ -5,7 +5,7 @@ const dataDuplicator = require('dbgate-api/src/shell/dataDuplicator');
 const { runCommandOnDriver } = require('dbgate-tools');

 describe('Data duplicator', () => {
-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipDataDuplicator).map(engine => [engine.label, engine]))(
    'Insert simple data - %s',
    testWrapper(async (conn, driver, engine) => {
      runCommandOnDriver(conn, driver, dmp =>
@@ -167,7 +167,7 @@ describe('Deploy database', () => {
    })
  );

-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipReferences).map(engine => [engine.label, engine]))(
    'Foreign keys - %s',
    testWrapper(async (conn, driver, engine) => {
      await testDatabaseDeploy(
@@ -222,7 +222,7 @@ describe('Deploy database', () => {
    })
  );

-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipDataModifications).map(engine => [engine.label, engine]))(
    'Deploy preloaded data - %s',
    testWrapper(async (conn, driver, engine) => {
      await testDatabaseDeploy(conn, driver, [
@@ -251,7 +251,7 @@ describe('Deploy database', () => {
    })
  );

-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipDataModifications).map(engine => [engine.label, engine]))(
    'Deploy preloaded data - update - %s',
    testWrapper(async (conn, driver, engine) => {
      await testDatabaseDeploy(conn, driver, [
@@ -0,0 +1,151 @@
+const dbgateApi = require('dbgate-api/src/shell');
+// const jsonLinesWriter = require('dbgate-api/src/shell/jsonLinesWriter');
+const tmp = require('tmp');
+// const dbgatePluginCsv = require('dbgate-plugin-csv/src/backend');
+const fs = require('fs');
+const requirePlugin = require('dbgate-api/src/shell/requirePlugin');
+
+const CSV_DATA = `Issue Number; Title; Github URL; Labels; State; Created At; Updated At; Reporter; Assignee
+801; "Does it 'burst' the database on startup or first lUI load ? "; https://github.com/dbgate/dbgate/issues/801; ""; open; 05/23/2024; 05/23/2024; rgarrigue; 
+799; "BUG: latest AppImage crashes on opening in Fedora 39"; https://github.com/dbgate/dbgate/issues/799; ""; open; 05/21/2024; 05/24/2024; BenGraham-Git; 
+798; "MongoDB write operations fail"; https://github.com/dbgate/dbgate/issues/798; "bug,solved"; open; 05/21/2024; 05/24/2024; mahmed0715; 
+797; "BUG: Unable to open SQL files"; https://github.com/dbgate/dbgate/issues/797; "bug"; open; 05/20/2024; 05/21/2024; cesarValdivia; 
+795; "BUG: MS SQL Server connection error (KEY_USAGE_BIT_INCORRECT)"; https://github.com/dbgate/dbgate/issues/795; ""; open; 05/20/2024; 05/20/2024; keskinonur; 
+794; "GLIBC_2.29' not found and i have 2.31"; https://github.com/dbgate/dbgate/issues/794; ""; closed; 05/20/2024; 05/21/2024; MFdanGM; 
+793; "BUG: PostgresSQL doesn't show tables when connected"; https://github.com/dbgate/dbgate/issues/793; ""; open; 05/20/2024; 05/22/2024; stomper013; 
+792; "FEAT: Wayland support"; https://github.com/dbgate/dbgate/issues/792; ""; closed; 05/19/2024; 05/21/2024; VosaXalo; 
+`;
+
+async function getReaderRows(reader) {
+  const jsonLinesFileName = tmp.tmpNameSync();
+
+  const writer = await dbgateApi.jsonLinesWriter({
+    fileName: jsonLinesFileName,
+  });
+  await dbgateApi.copyStream(reader, writer);
+
+  const jsonData = fs.readFileSync(jsonLinesFileName, 'utf-8');
+  const rows = jsonData
+    .split('\n')
+    .filter(x => x.trim() !== '')
+    .map(x => JSON.parse(x));
+
+  return rows;
+}
+
+test('csv import test', async () => {
+  const dbgatePluginCsv = requirePlugin('dbgate-plugin-csv');
+
+  const csvFileName = tmp.tmpNameSync();
+
+  fs.writeFileSync(csvFileName, CSV_DATA);
+
+  const reader = await dbgatePluginCsv.shellApi.reader({
+    fileName: csvFileName,
+  });
+
+  const rows = await getReaderRows(reader);
+
+  expect(rows[0].columns).toEqual([
+    { columnName: 'Issue Number' },
+    { columnName: 'Title' },
+    { columnName: 'Github URL' },
+    { columnName: 'Labels' },
+    { columnName: 'State' },
+    { columnName: 'Created At' },
+    { columnName: 'Updated At' },
+    { columnName: 'Reporter' },
+    { columnName: 'Assignee' },
+  ]);
+  expect(rows.length).toEqual(9);
+  expect(rows[1]).toEqual({
+    'Issue Number': '801',
+    Title: "Does it 'burst' the database on startup or first lUI load ? ",
+    'Github URL': 'https://github.com/dbgate/dbgate/issues/801',
+    Labels: '',
+    State: 'open',
+    'Created At': '05/23/2024',
+    'Updated At': '05/23/2024',
+    Reporter: 'rgarrigue',
+    Assignee: '',
+  });
+});
+
+test('JSON array import test', async () => {
+  const jsonFileName = tmp.tmpNameSync();
+
+  fs.writeFileSync(
+    jsonFileName,
+    JSON.stringify([
+      { id: 1, val: 'v1' },
+      { id: 2, val: 'v2' },
+    ])
+  );
+
+  const reader = await dbgateApi.jsonReader({
+    fileName: jsonFileName,
+  });
+
+  const rows = await getReaderRows(reader);
+
+  expect(rows.length).toEqual(2);
+  expect(rows).toEqual([
+    { id: 1, val: 'v1' },
+    { id: 2, val: 'v2' },
+  ]);
+});
+
+test('JSON object import test', async () => {
+  const jsonFileName = tmp.tmpNameSync();
+
+  fs.writeFileSync(
+    jsonFileName,
+    JSON.stringify({
+      k1: { id: 1, val: 'v1' },
+      k2: { id: 2, val: 'v2' },
+    })
+  );
+
+  const reader = await dbgateApi.jsonReader({
+    fileName: jsonFileName,
+    jsonStyle: 'object',
+    keyField: 'mykey',
+  });
+
+  const rows = await getReaderRows(reader);
+
+  expect(rows.length).toEqual(2);
+  expect(rows).toEqual([
+    { mykey: 'k1', id: 1, val: 'v1' },
+    { mykey: 'k2', id: 2, val: 'v2' },
+  ]);
+});
+
+test('JSON filtered object import test', async () => {
+  const jsonFileName = tmp.tmpNameSync();
+
+  fs.writeFileSync(
+    jsonFileName,
+    JSON.stringify({
+      filtered: {
+        k1: { id: 1, val: 'v1' },
+        k2: { id: 2, val: 'v2' },
+      },
+    })
+  );
+
+  const reader = await dbgateApi.jsonReader({
+    fileName: jsonFileName,
+    jsonStyle: 'object',
+    keyField: 'mykey',
+    rootField: 'filtered',
+  });
+
+  const rows = await getReaderRows(reader);
+
+  expect(rows.length).toEqual(2);
+  expect(rows).toEqual([
+    { mykey: 'k1', id: 1, val: 'v1' },
+    { mykey: 'k2', id: 2, val: 'v2' },
+  ]);
+});
@@ -2,7 +2,7 @@ const { testWrapper } = require('../tools');
 const engines = require('../engines');
 const _ = require('lodash');

-const initSql = ['CREATE TABLE t1 (id int)', 'CREATE TABLE t2 (id int)'];
+const initSql = ['CREATE TABLE t1 (id int primary key)', 'CREATE TABLE t2 (id int primary key)'];

 function flatSource() {
  return _.flatten(
@@ -26,9 +26,9 @@ describe('Object analyse', () => {
  test.each(flatSource())(
    'Full analysis - %s - %s',
    testWrapper(async (conn, driver, type, object, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql);
+      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });

-      await driver.query(conn, object.create1);
+      await driver.query(conn, object.create1, { discardResult: true });
      const structure = await driver.analyseFull(conn);

      expect(structure[type].length).toEqual(1);
@@ -39,11 +39,11 @@ describe('Object analyse', () => {
  test.each(flatSource())(
    'Incremental analysis - add - %s - %s',
    testWrapper(async (conn, driver, type, object, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql);
+      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });

-      await driver.query(conn, object.create2);
+      await driver.query(conn, object.create2, { discardResult: true });
      const structure1 = await driver.analyseFull(conn);
-      await driver.query(conn, object.create1);
+      await driver.query(conn, object.create1, { discardResult: true });
      const structure2 = await driver.analyseIncremental(conn, structure1);

      expect(structure2[type].length).toEqual(2);
@@ -54,12 +54,12 @@ describe('Object analyse', () => {
  test.each(flatSource())(
    'Incremental analysis - drop - %s - %s',
    testWrapper(async (conn, driver, type, object, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql);
+      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });

-      await driver.query(conn, object.create1);
-      await driver.query(conn, object.create2);
+      await driver.query(conn, object.create1, { discardResult: true });
+      await driver.query(conn, object.create2, { discardResult: true });
      const structure1 = await driver.analyseFull(conn);
-      await driver.query(conn, object.drop2);
+      await driver.query(conn, object.drop2, { discardResult: true });
      const structure2 = await driver.analyseIncremental(conn, structure1);

      expect(structure2[type].length).toEqual(1);
@@ -70,15 +70,15 @@ describe('Object analyse', () => {
  test.each(flatSource())(
    'Create SQL - add - %s - %s',
    testWrapper(async (conn, driver, type, object, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql);
+      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });

-      await driver.query(conn, object.create1);
+      await driver.query(conn, object.create1, { discardResult: true });
      const structure1 = await driver.analyseFull(conn);
-      await driver.query(conn, object.drop1);
+      await driver.query(conn, object.drop1, { discardResult: true });
      const structure2 = await driver.analyseIncremental(conn, structure1);
      expect(structure2[type].length).toEqual(0);

-      await driver.query(conn, structure1[type][0].createSql);
+      await driver.query(conn, structure1[type][0].createSql, { discardResult: true });

      const structure3 = await driver.analyseIncremental(conn, structure2);

@@ -2,7 +2,11 @@ const engines = require('../engines');
 const { splitQuery } = require('dbgate-query-splitter');
 const { testWrapper } = require('../tools');

-const initSql = ['CREATE TABLE t1 (id int)', 'INSERT INTO t1 (id) VALUES (1)', 'INSERT INTO t1 (id) VALUES (2)'];
+const initSql = [
+  'CREATE TABLE t1 (id int primary key)',
+  'INSERT INTO t1 (id) VALUES (1)',
+  'INSERT INTO t1 (id) VALUES (2)',
+];

 expect.extend({
  dataRow(row, expected) {
@@ -64,7 +68,7 @@ describe('Query', () => {
  test.each(engines.map(engine => [engine.label, engine]))(
    'Simple query - %s',
    testWrapper(async (conn, driver, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql);
+      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });

      const res = await driver.query(conn, 'SELECT id FROM t1 ORDER BY id');
      expect(res.columns).toEqual([
@@ -87,7 +91,7 @@ describe('Query', () => {
  test.each(engines.map(engine => [engine.label, engine]))(
    'Simple stream query - %s',
    testWrapper(async (conn, driver, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql);
+      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
      const results = await executeStream(driver, conn, 'SELECT id FROM t1 ORDER BY id');
      expect(results.length).toEqual(1);
      const res = results[0];
@@ -100,7 +104,7 @@ describe('Query', () => {
  test.each(engines.map(engine => [engine.label, engine]))(
    'More queries - %s',
    testWrapper(async (conn, driver, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql);
+      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
      const results = await executeStream(
        driver,
        conn,
@@ -124,7 +128,7 @@ describe('Query', () => {
      const results = await executeStream(
        driver,
        conn,
-        'CREATE TABLE t1 (id int); INSERT INTO t1 (id) VALUES (1); INSERT INTO t1 (id) VALUES (2); SELECT id FROM t1 ORDER BY id; '
+        'CREATE TABLE t1 (id int primary key); INSERT INTO t1 (id) VALUES (1); INSERT INTO t1 (id) VALUES (2); SELECT id FROM t1 ORDER BY id; '
      );
      expect(results.length).toEqual(1);

@@ -146,14 +150,15 @@ describe('Query', () => {
    })
  );

-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipDataModifications).map(engine => [engine.label, engine]))(
    'Save data query - %s',
    testWrapper(async (conn, driver, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql);
+      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });

      await driver.script(
        conn,
-        'INSERT INTO t1 (id) VALUES (3);INSERT INTO t1 (id) VALUES (4);UPDATE t1 SET id=10 WHERE id=1;DELETE FROM t1 WHERE id=2;'
+        'INSERT INTO t1 (id) VALUES (3);INSERT INTO t1 (id) VALUES (4);UPDATE t1 SET id=10 WHERE id=1;DELETE FROM t1 WHERE id=2;',
+        { discardResult: true }
      );
      const res = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM t1');
      // console.log(res);
@@ -0,0 +1,71 @@
+const stableStringify = require('json-stable-stringify');
+const _ = require('lodash');
+const fp = require('lodash/fp');
+const { testWrapper } = require('../tools');
+const engines = require('../engines');
+const { runCommandOnDriver } = require('dbgate-tools');
+
+async function baseStructure(conn, driver) {
+  await driver.query(conn, `create table t1 (id int not null primary key)`);
+
+  await driver.query(
+    conn,
+    `create table t2 (
+        id int not null primary key, 
+        t1_id int
+      )`
+  );
+}
+
+describe('Schema tests', () => {
+  test.each(engines.filter(x => x.supportSchemas).map(engine => [engine.label, engine]))(
+    'Create schema - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await baseStructure(conn, driver);
+      const structure1 = await driver.analyseFull(conn);
+      const schemas1 = await driver.listSchemas(conn);
+      expect(schemas1.find(x => x.schemaName == 'myschema')).toBeFalsy();
+      const count = schemas1.length;
+      expect(structure1.tables.length).toEqual(2);
+      await runCommandOnDriver(conn, driver, dmp => dmp.createSchema('myschema'));
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+      const schemas2 = await driver.listSchemas(conn);
+      expect(schemas2.find(x => x.schemaName == 'myschema')).toBeTruthy();
+      expect(schemas2.length).toEqual(count + 1);
+      expect(schemas2.find(x => x.isDefault).schemaName).toEqual(engine.defaultSchemaName);
+      expect(structure2).toBeNull();
+    })
+  );
+
+  test.each(engines.filter(x => x.supportSchemas).map(engine => [engine.label, engine]))(
+    'Drop schema - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await baseStructure(conn, driver);
+      await runCommandOnDriver(conn, driver, dmp => dmp.createSchema('myschema'));
+
+      const structure1 = await driver.analyseFull(conn);
+      const schemas1 = await driver.listSchemas(conn);
+      expect(schemas1.find(x => x.schemaName == 'myschema')).toBeTruthy();
+      expect(structure1.tables.length).toEqual(2);
+      await runCommandOnDriver(conn, driver, dmp => dmp.dropSchema('myschema'));
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+      const schemas2 = await driver.listSchemas(conn);
+      expect(schemas2.find(x => x.schemaName == 'myschema')).toBeFalsy();
+      expect(structure2).toBeNull();
+    })
+  );
+});
+
+describe('Base analyser test', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Structure without change - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await baseStructure(conn, driver);
+
+      const structure1 = await driver.analyseFull(conn);
+      expect(structure1.tables.length).toEqual(2);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+      expect(structure2).toBeNull();
+    })
+  );
+});
@@ -1,32 +1,37 @@
 const engines = require('../engines');
 const { testWrapper } = require('../tools');

-const t1Sql = 'CREATE TABLE t1 (id int not null primary key, val1 varchar(50) null)';
+const t1Sql = 'CREATE TABLE t1 (id int not null primary key, val1 varchar(50))';
 const ix1Sql = 'CREATE index ix1 ON t1(val1, id)';
-const t2Sql = 'CREATE TABLE t2 (id int not null primary key, val2 varchar(50) null unique)';
+const t2Sql = engine =>
+  `CREATE TABLE t2 (id int not null primary key, val2 varchar(50) ${engine.skipUnique ? '' : 'unique'})`;
 const t3Sql = 'CREATE TABLE t3 (id int not null primary key, valfk int, foreign key (valfk) references t2(id))';
 // const fkSql = 'ALTER TABLE t3 ADD FOREIGN KEY (valfk) REFERENCES t2(id)'

-const txMatch = (tname, vcolname, nextcol) =>
+const txMatch = (engine, tname, vcolname, nextcol) =>
  expect.objectContaining({
    pureName: tname,
    columns: [
      expect.objectContaining({
        columnName: 'id',
-        notNull: true,
-        dataType: expect.stringMatching(/int/i),
+        dataType: expect.stringMatching(/int.*/i),
+        ...(engine.skipNullability ? {} : { notNull: true }),
      }),
      expect.objectContaining({
        columnName: vcolname,
-        notNull: false,
-        dataType: expect.stringMatching(/.*char.*\(50\)/),
+        ...(engine.skipNullability ? {} : { notNull: false }),
+        dataType: engine.skipStringLength
+          ? expect.stringMatching(/.*string|char.*/i)
+          : expect.stringMatching(/.*char.*\(50\)/i),
      }),
      ...(nextcol
        ? [
            expect.objectContaining({
              columnName: 'nextcol',
-              notNull: false,
-              dataType: expect.stringMatching(/.*char.*\(50\)/),
+              ...(engine.skipNullability ? {} : { notNull: false }),
+              dataType: engine.skipStringLength
+                ? expect.stringMatching(/.*string.*|char.*/i)
+                : expect.stringMatching(/.*char.*\(50\).*/i),
            }),
          ]
        : []),
@@ -40,9 +45,9 @@ const txMatch = (tname, vcolname, nextcol) =>
    }),
  });

-const t1Match = txMatch('t1', 'val1');
-const t2Match = txMatch('t2', 'val2');
-const t2NextColMatch = txMatch('t2', 'val2', true);
+const t1Match = engine => txMatch(engine, 't1', 'val1');
+const t2Match = engine => txMatch(engine, 't2', 'val2');
+const t2NextColMatch = engine => txMatch(engine, 't2', 'val2', true);

 describe('Table analyse', () => {
  test.each(engines.map(engine => [engine.label, engine]))(
@@ -53,25 +58,25 @@ describe('Table analyse', () => {
      const structure = await driver.analyseFull(conn);

      expect(structure.tables.length).toEqual(1);
-      expect(structure.tables[0]).toEqual(t1Match);
+      expect(structure.tables[0]).toEqual(t1Match(engine));
    })
  );

  test.each(engines.map(engine => [engine.label, engine]))(
    'Table add - incremental analysis - %s',
    testWrapper(async (conn, driver, engine) => {
-      await driver.query(conn, t2Sql);
+      await driver.query(conn, t2Sql(engine));

      const structure1 = await driver.analyseFull(conn);
      expect(structure1.tables.length).toEqual(1);
-      expect(structure1.tables[0]).toEqual(t2Match);
+      expect(structure1.tables[0]).toEqual(t2Match(engine));

      await driver.query(conn, t1Sql);
      const structure2 = await driver.analyseIncremental(conn, structure1);

      expect(structure2.tables.length).toEqual(2);
-      expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
-      expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2Match);
+      expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match(engine));
+      expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2Match(engine));
    })
  );

@@ -79,17 +84,17 @@ describe('Table analyse', () => {
    'Table remove - incremental analysis - %s',
    testWrapper(async (conn, driver, engine) => {
      await driver.query(conn, t1Sql);
-      await driver.query(conn, t2Sql);
+      await driver.query(conn, t2Sql(engine));
      const structure1 = await driver.analyseFull(conn);
      expect(structure1.tables.length).toEqual(2);
-      expect(structure1.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
-      expect(structure1.tables.find(x => x.pureName == 't2')).toEqual(t2Match);
+      expect(structure1.tables.find(x => x.pureName == 't1')).toEqual(t1Match(engine));
+      expect(structure1.tables.find(x => x.pureName == 't2')).toEqual(t2Match(engine));

      await driver.query(conn, 'DROP TABLE t2');
      const structure2 = await driver.analyseIncremental(conn, structure1);

      expect(structure2.tables.length).toEqual(1);
-      expect(structure2.tables[0]).toEqual(t1Match);
+      expect(structure2.tables[0]).toEqual(t1Match(engine));
    })
  );

@@ -97,23 +102,26 @@ describe('Table analyse', () => {
    'Table change - incremental analysis - %s',
    testWrapper(async (conn, driver, engine) => {
      await driver.query(conn, t1Sql);
-      await driver.query(conn, t2Sql);
+      await driver.query(conn, t2Sql(engine));
      const structure1 = await driver.analyseFull(conn);

      if (engine.dbSnapshotBySeconds) await new Promise(resolve => setTimeout(resolve, 1100));

-      await driver.query(conn, 'ALTER TABLE t2 ADD nextcol varchar(50)');
+      await driver.query(
+        conn,
+        `ALTER TABLE t2 ADD ${engine.alterTableAddColumnSyntax ? 'COLUMN' : ''} nextcol varchar(50)`
+      );
      const structure2 = await driver.analyseIncremental(conn, structure1);

      expect(structure2).toBeTruthy(); // if falsy, no modification is detected

      expect(structure2.tables.length).toEqual(2);
-      expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
-      expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2NextColMatch);
+      expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match(engine));
+      expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2NextColMatch(engine));
    })
  );

-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipIndexes).map(engine => [engine.label, engine]))(
    'Index - full analysis - %s',
    testWrapper(async (conn, driver, engine) => {
      await driver.query(conn, t1Sql);
@@ -128,10 +136,10 @@ describe('Table analyse', () => {
    })
  );

-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipUnique).map(engine => [engine.label, engine]))(
    'Unique - full analysis - %s',
    testWrapper(async (conn, driver, engine) => {
-      await driver.query(conn, t2Sql);
+      await driver.query(conn, t2Sql(engine));
      const structure = await driver.analyseFull(conn);

      const t2 = structure.tables.find(x => x.pureName == 't2');
@@ -142,10 +150,10 @@ describe('Table analyse', () => {
    })
  );

-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipReferences).map(engine => [engine.label, engine]))(
    'Foreign key - full analysis - %s',
    testWrapper(async (conn, driver, engine) => {
-      await driver.query(conn, t2Sql);
+      await driver.query(conn, t2Sql(engine));
      await driver.query(conn, t3Sql);
      // await driver.query(conn, fkSql);

@@ -62,7 +62,7 @@ describe('Table create', () => {
    })
  );

-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipIndexes).map(engine => [engine.label, engine]))(
    'Table with index - %s',
    testWrapper(async (conn, driver, engine) => {
      await testTableCreate(conn, driver, {
@@ -92,7 +92,7 @@ describe('Table create', () => {
    })
  );

-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipReferences).map(engine => [engine.label, engine]))(
    'Table with foreign key - %s',
    testWrapper(async (conn, driver, engine) => {
      await testTableCreate(conn, driver, {
@@ -122,7 +122,7 @@ describe('Table create', () => {
    })
  );

-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipUnique).map(engine => [engine.label, engine]))(
    'Table with unique - %s',
    testWrapper(async (conn, driver, engine) => {
      await testTableCreate(conn, driver, {
@@ -1,12 +1,12 @@
 version: '3'
 services:
-  # postgres:
-  #   image: postgres
-  #   restart: always
-  #   environment:
-  #     POSTGRES_PASSWORD: Pwd2020Db
-  #   ports: 
-  #     - 15000:5432
+  postgres:
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_PASSWORD: Pwd2020Db
+    ports: 
+      - 15000:5432

  # mariadb:
  #   image: mariadb
@@ -26,15 +26,23 @@ services:
  #   environment:
  #     - MYSQL_ROOT_PASSWORD=Pwd2020Db

-  mssql:
-    image: mcr.microsoft.com/mssql/server
-    restart: always
-    ports: 
-      - 15002:1433
-    environment:
-      - ACCEPT_EULA=Y
-      - SA_PASSWORD=Pwd2020Db
-      - MSSQL_PID=Express
+  # clickhouse:
+  #   image: bitnami/clickhouse:24.8.4
+  #   restart: always
+  #   ports: 
+  #     - 15005:8123
+  #   environment:
+  #     - CLICKHOUSE_ADMIN_PASSWORD=Pwd2020Db
+
+  # mssql:
+  #   image: mcr.microsoft.com/mssql/server
+  #   restart: always
+  #   ports: 
+  #     - 15002:1433
+  #   environment:
+  #     - ACCEPT_EULA=Y
+  #     - SA_PASSWORD=Pwd2020Db
+  #     - MSSQL_PID=Express
      
  # cockroachdb:
  #   image: cockroachdb/cockroach
@@ -81,6 +81,8 @@ const engines = [
        drop2: 'DROP FUNCTION obj2',
      },
    ],
+    supportSchemas: true,
+    defaultSchemaName: 'public',
  },
  {
    label: 'SQL Server',
@@ -105,6 +107,8 @@ const engines = [
        drop2: 'DROP PROCEDURE obj2',
      },
    ],
+    supportSchemas: true,
+    defaultSchemaName: 'dbo',
  },
  {
    label: 'SQLite',
@@ -129,16 +133,41 @@ const engines = [
    skipOnCI: true,
    objects: [views, matviews],
  },
+  {
+    label: 'ClickHouse',
+    connection: {
+      engine: 'clickhouse@dbgate-plugin-clickhouse',
+      databaseUrl: 'http://clickhouse:8123',
+      password: 'Pwd2020Db',
+    },
+    local: {
+      databaseUrl: 'http://localhost:15005',
+    },
+    skipOnCI: false,
+    objects: [views],
+    skipDataModifications: true,
+    skipReferences: true,
+    skipIndexes: true,
+    skipNullability: true,
+    skipUnique: true,
+    skipAutoIncrement: true,
+    skipPkColumnTesting: true,
+    skipDataDuplicator: true,
+    skipStringLength: true,
+    alterTableAddColumnSyntax: true,
+    dbSnapshotBySeconds: true,
+  },
 ];

 const filterLocal = [
  // filter local testing
  '-MySQL',
  '-MariaDB',
-  '-PostgreSQL',
+  'PostgreSQL',
  '-SQL Server',
-  'SQLite',
+  '-SQLite',
  '-CockroachDB',
+  '-ClickHouse',
 ];

 const enginesPostgre = engines.filter(x => x.label == 'PostgreSQL');
@@ -0,0 +1,3 @@
+module.exports = {
+  setupFilesAfterEnv: ['<rootDir>/setupTests.js'],
+};
@@ -11,12 +11,9 @@
  "scripts": {
    "wait:local": "cross-env DEVMODE=1 LOCALTEST=1 node wait.js",
    "wait:ci": "cross-env DEVMODE=1 CITEST=1 node wait.js",
-
    "test:local": "cross-env DEVMODE=1 LOCALTEST=1 jest",
    "test:local:path": "cross-env DEVMODE=1 LOCALTEST=1 jest --runTestsByPath __tests__/data-duplicator.spec.js",
-    
    "test:ci": "cross-env DEVMODE=1 CITEST=1 jest --runInBand --json --outputFile=result.json --testLocationInResults",
-    
    "run:local": "docker-compose down && docker-compose up -d && yarn wait:local && yarn test:local"
  },
  "jest": {
@@ -24,7 +21,8 @@
  },
  "devDependencies": {
    "cross-env": "^7.0.3",
-    "jest": "^27.0.1"
-  },
-  "dependencies": {}
+    "jest": "^27.0.1",
+    "pino-pretty": "^11.2.2",
+    "tmp": "^0.2.3"
+  }
 }
@@ -0,0 +1,30 @@
+global.DBGATE_PACKAGES = {
+  'dbgate-tools': require('dbgate-tools'),
+  'dbgate-sqltree': require('dbgate-sqltree'),
+};
+
+const { prettyFactory } = require('pino-pretty');
+const tmp = require('tmp');
+
+const pretty = prettyFactory({
+  colorize: true,
+  translateTime: 'SYS:standard',
+  ignore: 'pid,hostname',
+});
+
+global.console = {
+  ...console,
+  log: (...messages) => {
+    try {
+      const parsedMessage = JSON.parse(messages[0]);
+      process.stdout.write(pretty(parsedMessage));
+    } catch (error) {
+      process.stdout.write(messages.join(' ') + '\n');
+    }
+  },
+  debug: (...messages) => {
+    process.stdout.write(messages.join(' ') + '\n');
+  },
+};
+
+tmp.setGracefulCleanup();
@@ -1,4 +1,3 @@
-global.DBGATE_TOOLS = require('dbgate-tools');
 const requireEngineDriver = require('dbgate-api/src/utility/requireEngineDriver');
 const crypto = require('crypto');

@@ -44,16 +43,18 @@ async function connect(engine, database) {
  }
 }

-const testWrapper = body => async (label, ...other) => {
-  const engine = other[other.length - 1];
-  const driver = requireEngineDriver(engine.connection);
-  const conn = await connect(engine, randomDbName());
-  try {
-    await body(conn, driver, ...other);
-  } finally {
-    await driver.close(conn);
-  }
-};
+const testWrapper =
+  body =>
+  async (label, ...other) => {
+    const engine = other[other.length - 1];
+    const driver = requireEngineDriver(engine.connection);
+    const conn = await connect(engine, randomDbName());
+    try {
+      await body(conn, driver, ...other);
+    } finally {
+      await driver.close(conn);
+    }
+  };

 module.exports = {
  randomDbName,
@@ -1,7 +1,10 @@
 const requireEngineDriver = require('dbgate-api/src/utility/requireEngineDriver');
 const engines = require('./engines');
 const { extractConnection } = require('./tools');
-global.DBGATE_TOOLS = require('dbgate-tools');
+global.DBGATE_PACKAGES = {
+  'dbgate-tools': require('dbgate-tools'),
+  'dbgate-sqltree': require('dbgate-sqltree'),
+};

 async function connectEngine(engine) {
  const connection = extractConnection(engine);
@@ -35,38 +35,40 @@ RIGHT=850



-magick  \
-    \( \
-        -size 1000x1000 -define gradient:direction=east 'gradient:#0050b3-#1890ff' \
-        \( +clone -fill Black -colorize 100 \
-        -fill White -stroke White -draw "arc $LEFT,750 $RIGHT,950 0,360" -draw "rectangle $LEFT,150 $RIGHT,850" \
-        \) \
-        -alpha off \
-        -compose CopyOpacity -composite \
-    \) \
-    \( \
-        -size 1000x1000 -define gradient:direction=east 'gradient:#096dd9-#40a9ff' \
-        \( +clone -fill Black -colorize 100 \
-        -fill White -draw "arc $LEFT,50 $RIGHT,250 0,360" \
-        \) \
-        -alpha off \
-        -compose CopyOpacity -composite \
-    \) \
-    -compose Over -composite \
-    -strokewidth $STROKE_WIDTH -stroke '#0050b3' -fill transparent \
-    -draw "arc $LEFT,225 $RIGHT,425  0,180" \
-    -draw "arc $LEFT,400 $RIGHT,600  0,180" \
-    -draw "arc $LEFT,575 $RIGHT,775  0,180" \
-    -draw "arc $LEFT,750 $RIGHT,950  0,180" \
-    -draw "arc $LEFT,50 $RIGHT,250  0,360" \
-    -draw "line $LEFT,150 $LEFT,850" \
-    -draw "line $RIGHT,150 $RIGHT,850" \
-    -fill '#fafafa' -stroke '#8c8c8c' -strokewidth 3 \
-    -pointsize 800 -font './Mcbungus-Regular.ttf' \
-    -gravity center \
-    -draw 'text 0,100 "G"' \
-    icon.png
+# magick  \
+#     \( \
+#         -size 1000x1000 -define gradient:direction=east 'gradient:#0050b3-#1890ff' \
+#         \( +clone -fill Black -colorize 100 \
+#         -fill White -stroke White -draw "arc $LEFT,750 $RIGHT,950 0,360" -draw "rectangle $LEFT,150 $RIGHT,850" \
+#         \) \
+#         -alpha off \
+#         -compose CopyOpacity -composite \
+#     \) \
+#     \( \
+#         -size 1000x1000 -define gradient:direction=east 'gradient:#096dd9-#40a9ff' \
+#         \( +clone -fill Black -colorize 100 \
+#         -fill White -draw "arc $LEFT,50 $RIGHT,250 0,360" \
+#         \) \
+#         -alpha off \
+#         -compose CopyOpacity -composite \
+#     \) \
+#     -compose Over -composite \
+#     -strokewidth $STROKE_WIDTH -stroke '#0050b3' -fill transparent \
+#     -draw "arc $LEFT,225 $RIGHT,425  0,180" \
+#     -draw "arc $LEFT,400 $RIGHT,600  0,180" \
+#     -draw "arc $LEFT,575 $RIGHT,775  0,180" \
+#     -draw "arc $LEFT,750 $RIGHT,950  0,180" \
+#     -draw "arc $LEFT,50 $RIGHT,250  0,360" \
+#     -draw "line $LEFT,150 $LEFT,850" \
+#     -draw "line $RIGHT,150 $RIGHT,850" \
+#     -fill '#fafafa' -stroke '#8c8c8c' -strokewidth 3 \
+#     -pointsize 800 -font './Mcbungus-Regular.ttf' \
+#     -gravity center \
+#     -draw 'text 0,100 "G"' \
+#     icon.png

+convert icon-input.png -background white -alpha remove -alpha off icon.png 
+convert -size 1000x1000 xc:none -fill white -draw "circle 500,500 500,0" icon.png -compose SrcIn -composite icon.png

 # magick \
 #   \( \
@@ -106,4 +108,6 @@ magick icon.png -resize 512x512! ../packages/web/public/logo512.png
 magick icon.png -define icon:auto-resize="256,128,96,64,48,32,16" ../packages/web/public/favicon.ico

 convert icon.png -resize 800x800 -background transparent -gravity center -extent 1000x1000 iconmac.png
-magick composite  iconmac.png macbg.png -resize 600x600! ../app/icon512-mac.png
+
+convert macbg.png icon.png -compose SrcIn -composite -resize 600x600! ../app/icon512-mac.png
+# magick composite  iconmac.png macbg.png -resize 600x600! ../app/icon512-mac.png
@@ -1,6 +1,6 @@
 {
  "private": true,
-  "version": "5.3.3",
+  "version": "5.4.5-beta.7",
  "name": "dbgate-all",
  "workspaces": [
    "packages/*",
@@ -20,6 +20,8 @@
    "start:api:singledb": "yarn workspace dbgate-api start:singledb | pino-pretty",
    "start:api:auth": "yarn workspace dbgate-api start:auth | pino-pretty",
    "start:api:dblogin": "yarn workspace dbgate-api start:dblogin | pino-pretty",
+    "start:api:storage": "yarn workspace dbgate-api start:storage | pino-pretty",
+    "sync:pro": "cd sync && yarn start",
    "start:web": "yarn workspace dbgate-web dev",
    "start:sqltree": "yarn workspace dbgate-sqltree start",
    "start:tools": "yarn workspace dbgate-tools start",
@@ -34,7 +36,9 @@
    "build:api": "yarn workspace dbgate-api build",
    "build:web:docker": "yarn workspace dbgate-web build",
    "build:plugins:frontend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:frontend",
+    "build:plugins:backend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:backend",
    "build:plugins:frontend:watch": "workspaces-run --parallel --only=\"dbgate-plugin-*\" -- yarn build:frontend:watch",
+    "storage-json": "dbmodel model-to-json storage-db packages/api/src/storageModel.js --commonjs",
    "plugins:copydist": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn copydist",
    "build:app:local": "yarn plugins:copydist && cd app && yarn build:local",
    "start:app:local": "cd app && yarn start:local",
@@ -48,8 +52,8 @@
    "resetPackagedPlugins": "node resetPackagedPlugins",
    "prettier": "prettier --write packages/api/src && prettier --write packages/datalib/src && prettier --write packages/filterparser/src && prettier --write packages/sqltree/src && prettier --write packages/tools/src && prettier --write packages/types && prettier --write packages/web/src && prettier --write app/src",
    "copy:docker:build": "copyfiles packages/api/dist/* docker -f && copyfiles packages/web/public/* docker -u 2 && copyfiles \"packages/web/public/**/*\" docker -u 2 && copyfiles \"plugins/dist/**/*\" docker/plugins -u 2",
-    "install:sqlite:docker": "cd docker && yarn init --yes && yarn add better-sqlite3 && cd ..",
-    "prepare:docker": "yarn plugins:copydist && yarn build:web:docker && yarn build:api && yarn copy:docker:build && yarn install:sqlite:docker",
+    "install:drivers:docker": "cd docker && yarn init --yes && yarn add better-sqlite3 && yarn add oracledb && cd ..",
+    "prepare:docker": "yarn plugins:copydist && yarn build:web:docker && yarn build:api && yarn copy:docker:build && yarn install:drivers:docker",
    "start": "concurrently --kill-others-on-fail \"yarn start:api\" \"yarn start:web\"",
    "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\" \"yarn build:plugins:frontend:watch\"",
    "ts:api": "yarn workspace dbgate-api ts",
@@ -27,7 +27,7 @@
    "cors": "^2.8.5",
    "cross-env": "^6.0.3",
    "dbgate-datalib": "^5.0.0-alpha.1",
-    "dbgate-query-splitter": "^4.10.1",
+    "dbgate-query-splitter": "^4.10.3",
    "dbgate-sqltree": "^5.0.0-alpha.1",
    "dbgate-tools": "^5.0.0-alpha.1",
    "debug": "^4.3.4",
@@ -57,6 +57,7 @@
    "rimraf": "^3.0.0",
    "simple-encryptor": "^4.0.0",
    "ssh2": "^1.11.0",
+    "stream-json": "^1.8.0",
    "tar": "^6.0.5"
  },
  "scripts": {
@@ -66,6 +67,7 @@
    "start:auth": "env-cmd -f env/auth/.env node src/index.js --listen-api",
    "start:dblogin": "env-cmd -f env/dblogin/.env node src/index.js --listen-api",
    "start:filedb": "env-cmd node src/index.js /home/jena/test/chinook/Chinook.db --listen-api",
+    "start:storage": "env-cmd -f env/storage/.env node src/index.js --listen-api",
    "start:singleconn": "env-cmd node src/index.js --server localhost --user root --port 3307 --engine mysql@dbgate-plugin-mysql --password test --listen-api",
    "ts": "tsc",
    "build": "webpack"
@@ -83,6 +85,7 @@
  },
  "optionalDependencies": {
    "better-sqlite3": "9.6.0",
-    "msnodesqlv8": "^4.2.1"
+    "msnodesqlv8": "^4.2.1",
+    "oracledb": "^6.6.0"
  }
 }
@@ -0,0 +1,16 @@
+const crypto = require('crypto');
+
+const tokenSecret = crypto.randomUUID();
+
+function getTokenLifetime() {
+  return process.env.TOKEN_LIFETIME || '1d';
+}
+
+function getTokenSecret() {
+  return tokenSecret;
+}
+
+module.exports = {
+  getTokenLifetime,
+  getTokenSecret,
+};
@@ -0,0 +1,322 @@
+const { getTokenSecret, getTokenLifetime } = require('./authCommon');
+const _ = require('lodash');
+const axios = require('axios');
+const { getLogger, getPredefinedPermissions } = require('dbgate-tools');
+
+const AD = require('activedirectory2').promiseWrapper;
+const jwt = require('jsonwebtoken');
+
+const logger = getLogger('authProvider');
+
+class AuthProviderBase {
+  amoid = 'none';
+
+  async login(login, password, options = undefined) {
+    return {
+      accessToken: jwt.sign(
+        {
+          amoid: this.amoid,
+        },
+        getTokenSecret(),
+        { expiresIn: getTokenLifetime() }
+      ),
+    };
+  }
+
+  oauthToken(params) {
+    return {};
+  }
+
+  getCurrentLogin(req) {
+    const login = req?.user?.login ?? req?.auth?.user ?? null;
+    return login;
+  }
+
+  isUserLoggedIn(req) {
+    return !!req?.user || !!req?.auth;
+  }
+
+  getCurrentPermissions(req) {
+    const login = this.getCurrentLogin(req);
+    const permissions = process.env[`LOGIN_PERMISSIONS_${login}`];
+    return permissions || process.env.PERMISSIONS;
+  }
+
+  getLoginPageConnections() {
+    return null;
+  }
+
+  getSingleConnectionId(req) {
+    return null;
+  }
+
+  toJson() {
+    return {
+      amoid: this.amoid,
+      workflowType: 'anonymous',
+      name: 'Anonymous',
+    };
+  }
+
+  async redirect({ state }) {
+    return {
+      status: 'error',
+    };
+  }
+
+  async getLogoutUrl() {
+    return null;
+  }
+}
+
+class OAuthProvider extends AuthProviderBase {
+  amoid = 'oauth';
+
+  async oauthToken(params) {
+    const { redirectUri, code } = params;
+
+    const scopeParam = process.env.OAUTH_SCOPE ? `&scope=${process.env.OAUTH_SCOPE}` : '';
+    const resp = await axios.default.post(
+      `${process.env.OAUTH_TOKEN}`,
+      `grant_type=authorization_code&code=${encodeURIComponent(code)}&redirect_uri=${encodeURIComponent(
+        redirectUri
+      )}&client_id=${process.env.OAUTH_CLIENT_ID}&client_secret=${process.env.OAUTH_CLIENT_SECRET}${scopeParam}`
+    );
+
+    const { access_token, refresh_token } = resp.data;
+
+    const payload = jwt.decode(access_token);
+
+    logger.info({ payload }, 'User payload returned from OAUTH');
+
+    const login =
+      process.env.OAUTH_LOGIN_FIELD && payload && payload[process.env.OAUTH_LOGIN_FIELD]
+        ? payload[process.env.OAUTH_LOGIN_FIELD]
+        : 'oauth';
+
+    if (
+      process.env.OAUTH_ALLOWED_LOGINS &&
+      !process.env.OAUTH_ALLOWED_LOGINS.split(',').find(x => x.toLowerCase().trim() == login.toLowerCase().trim())
+    ) {
+      return { error: `Username ${login} not allowed to log in` };
+    }
+
+    const groups =
+      process.env.OAUTH_GROUP_FIELD && payload && payload[process.env.OAUTH_GROUP_FIELD]
+        ? payload[process.env.OAUTH_GROUP_FIELD]
+        : [];
+
+    const allowedGroups = process.env.OAUTH_ALLOWED_GROUPS
+      ? process.env.OAUTH_ALLOWED_GROUPS.split(',').map(group => group.toLowerCase().trim())
+      : [];
+
+    if (process.env.OAUTH_ALLOWED_GROUPS && !groups.some(group => allowedGroups.includes(group.toLowerCase().trim()))) {
+      return { error: `Username ${login} does not belong to an allowed group` };
+    }
+
+    if (access_token) {
+      return {
+        accessToken: jwt.sign({ login }, getTokenSecret(), { expiresIn: getTokenLifetime() }),
+      };
+    }
+
+    return { error: 'Token not found' };
+  }
+
+  async getLogoutUrl() {
+    return process.env.OAUTH_LOGOUT;
+  }
+
+  toJson() {
+    return {
+      ...super.toJson(),
+      workflowType: 'redirect',
+      name: 'OAuth 2.0',
+    };
+  }
+
+  redirect({ state, redirectUri }) {
+    const scopeParam = process.env.OAUTH_SCOPE ? `&scope=${process.env.OAUTH_SCOPE}` : '';
+    return {
+      status: 'ok',
+      uri: `${process.env.OAUTH_AUTH}?client_id=${
+        process.env.OAUTH_CLIENT_ID
+      }&response_type=code&redirect_uri=${encodeURIComponent(redirectUri)}&state=${encodeURIComponent(
+        state
+      )}${scopeParam}`,
+    };
+  }
+}
+
+class ADProvider extends AuthProviderBase {
+  amoid = 'ad';
+
+  async login(login, password, options = undefined) {
+    const adConfig = {
+      url: process.env.AD_URL,
+      baseDN: process.env.AD_BASEDN,
+      username: process.env.AD_USERNAME,
+      password: process.env.AD_PASSWORD,
+    };
+    const ad = new AD(adConfig);
+    try {
+      const res = await ad.authenticate(login, password);
+      if (!res) {
+        return { error: 'Login failed' };
+      }
+      if (
+        process.env.AD_ALLOWED_LOGINS &&
+        !process.env.AD_ALLOWED_LOGINS.split(',').find(x => x.toLowerCase().trim() == login.toLowerCase().trim())
+      ) {
+        return { error: `Username ${login} not allowed to log in` };
+      }
+      return {
+        accessToken: jwt.sign(
+          {
+            amoid: this.amoid,
+            login,
+          },
+          getTokenSecret(),
+          { expiresIn: getTokenLifetime() }
+        ),
+      };
+    } catch (e) {
+      return { error: 'Login failed' };
+    }
+  }
+
+  toJson() {
+    return {
+      ...super.toJson(),
+      workflowType: 'credentials',
+      name: 'Active Directory',
+    };
+  }
+}
+
+class LoginsProvider extends AuthProviderBase {
+  amoid = 'logins';
+
+  async login(login, password, options = undefined) {
+    if (password == process.env[`LOGIN_PASSWORD_${login}`]) {
+      return {
+        accessToken: jwt.sign(
+          {
+            amoid: this.amoid,
+            login,
+          },
+          getTokenSecret(),
+          { expiresIn: getTokenLifetime() }
+        ),
+      };
+    }
+    return { error: 'Invalid credentials' };
+  }
+
+  toJson() {
+    return {
+      ...super.toJson(),
+      workflowType: 'credentials',
+      name: 'Login & Password',
+    };
+  }
+}
+
+class DenyAllProvider extends AuthProviderBase {
+  amoid = 'deny';
+
+  async login(login, password, options = undefined) {
+    return { error: 'Login not allowed' };
+  }
+
+  toJson() {
+    return {
+      ...super.toJson(),
+      workflowType: 'credentials',
+      name: 'Deny all',
+    };
+  }
+}
+
+function hasEnvLogins() {
+  if (process.env.LOGIN && process.env.PASSWORD) {
+    return true;
+  }
+  for (const key in process.env) {
+    if (key.startsWith('LOGIN_PASSWORD_')) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function detectEnvAuthProvider() {
+  if (process.env.AUTH_PROVIDER) {
+    return process.env.AUTH_PROVIDER;
+  }
+
+  if (process.env.STORAGE_DATABASE) {
+    return 'denyall';
+  }
+  if (process.env.OAUTH_AUTH) {
+    return 'oauth';
+  }
+  if (process.env.AD_URL) {
+    return 'ad';
+  }
+  if (hasEnvLogins()) {
+    return 'logins';
+  }
+  return 'none';
+}
+
+function createEnvAuthProvider() {
+  const authProvider = detectEnvAuthProvider();
+  switch (authProvider) {
+    case 'oauth':
+      return new OAuthProvider();
+    case 'ad':
+      return new ADProvider();
+    case 'logins':
+      return new LoginsProvider();
+    case 'denyall':
+      return new DenyAllProvider();
+    default:
+      return new AuthProviderBase();
+  }
+}
+
+let defaultAuthProvider = createEnvAuthProvider();
+let authProviders = [defaultAuthProvider];
+
+function getAuthProviders() {
+  return authProviders;
+}
+
+function getAuthProviderById(amoid) {
+  return authProviders.find(x => x.amoid == amoid);
+}
+
+function getDefaultAuthProvider() {
+  return defaultAuthProvider;
+}
+
+function getAuthProviderFromReq(req) {
+  const authProviderId = req?.auth?.amoid || req?.user?.amoid;
+  return getAuthProviderById(authProviderId) ?? getDefaultAuthProvider();
+}
+
+function setAuthProviders(value, defaultProvider = null) {
+  authProviders = value;
+  defaultAuthProvider = defaultProvider || value[0];
+}
+
+module.exports = {
+  AuthProviderBase,
+  detectEnvAuthProvider,
+  getAuthProviders,
+  getDefaultAuthProvider,
+  setAuthProviders,
+  getAuthProviderById,
+  getAuthProviderFromReq,
+};
@@ -1,24 +1,20 @@
 const axios = require('axios');
 const jwt = require('jsonwebtoken');
 const getExpressPath = require('../utility/getExpressPath');
-const { getLogins } = require('../utility/hasPermission');
 const { getLogger } = require('dbgate-tools');
 const AD = require('activedirectory2').promiseWrapper;
 const crypto = require('crypto');
+const { getTokenSecret, getTokenLifetime } = require('../auth/authCommon');
+const {
+  getAuthProviderFromReq,
+  getAuthProviders,
+  getDefaultAuthProvider,
+  getAuthProviderById,
+} = require('../auth/authProvider');
+const storage = require('./storage');

 const logger = getLogger('auth');

-const tokenSecret = crypto.randomUUID();
-
-function shouldAuthorizeApi() {
-  const logins = getLogins();
-  return !!process.env.OAUTH_AUTH || !!process.env.AD_URL || (!!logins && !process.env.BASIC_AUTH);
-}
-
-function getTokenLifetime() {
-  return process.env.TOKEN_LIFETIME || '1d';
-}
-
 function unauthorizedResponse(req, res, text) {
  // if (req.path == getExpressPath('/config/get-settings')) {
  //   return res.json({});
@@ -30,11 +26,32 @@ function unauthorizedResponse(req, res, text) {
 }

 function authMiddleware(req, res, next) {
-  const SKIP_AUTH_PATHS = ['/config/get', '/auth/oauth-token', '/auth/login', '/stream'];
+  const SKIP_AUTH_PATHS = [
+    '/config/get',
+    '/config/logout',
+    '/config/get-settings',
+    '/config/save-license-key',
+    '/auth/oauth-token',
+    '/auth/login',
+    '/auth/redirect',
+    '/stream',
+    'storage/get-connections-for-login-page',
+    'auth/get-providers',
+    '/connections/dblogin-web',
+    '/connections/dblogin-app',
+    '/connections/dblogin-auth',
+    '/connections/dblogin-auth-token',
+  ];

-  if (!shouldAuthorizeApi()) {
+  // console.log('********************* getAuthProvider()', getAuthProvider());
+
+  // const isAdminPage = req.headers['x-is-admin-page'] == 'true';
+
+  if (process.env.BASIC_AUTH) {
+    // API is not authorized for basic auth
    return next();
  }
+
  let skipAuth = !!SKIP_AUTH_PATHS.find(x => req.path == getExpressPath(x));

  const authHeader = req.headers.authorization;
@@ -46,7 +63,7 @@ function authMiddleware(req, res, next) {
  }
  const token = authHeader.split(' ')[1];
  try {
-    const decoded = jwt.verify(token, tokenSecret);
+    const decoded = jwt.verify(token, getTokenSecret());
    req.user = decoded;
    return next();
  } catch (err) {
@@ -63,106 +80,49 @@ function authMiddleware(req, res, next) {
 module.exports = {
  oauthToken_meta: true,
  async oauthToken(params) {
-    const { redirectUri, code } = params;
-
-    const scopeParam = process.env.OAUTH_SCOPE ? `&scope=${process.env.OAUTH_SCOPE}` : '';
-    const resp = await axios.default.post(
-      `${process.env.OAUTH_TOKEN}`,
-      `grant_type=authorization_code&code=${encodeURIComponent(code)}&redirect_uri=${encodeURIComponent(
-        redirectUri
-      )}&client_id=${process.env.OAUTH_CLIENT_ID}&client_secret=${process.env.OAUTH_CLIENT_SECRET}${scopeParam}`
-    );
-
-    const { access_token, refresh_token } = resp.data;
-
-    const payload = jwt.decode(access_token);
-
-    logger.info({ payload }, 'User payload returned from OAUTH');
-
-    const login =
-      process.env.OAUTH_LOGIN_FIELD && payload && payload[process.env.OAUTH_LOGIN_FIELD]
-        ? payload[process.env.OAUTH_LOGIN_FIELD]
-        : 'oauth';
-
-    if (
-      process.env.OAUTH_ALLOWED_LOGINS &&
-      !process.env.OAUTH_ALLOWED_LOGINS.split(',').find(x => x.toLowerCase().trim() == login.toLowerCase().trim())
-    ) {
-      return { error: `Username ${login} not allowed to log in` };
-    }
-
-    const groups = 
-      process.env.OAUTH_GROUP_FIELD && payload && payload[process.env.OAUTH_GROUP_FIELD]
-        ? payload[process.env.OAUTH_GROUP_FIELD]
-        : [];
-
-    const allowedGroups = 
-      process.env.OAUTH_ALLOWED_GROUPS
-        ? process.env.OAUTH_ALLOWED_GROUPS.split(',').map(group => group.toLowerCase().trim())
-        : [];
-    
-    if (
-      process.env.OAUTH_ALLOWED_GROUPS && 
-      !groups.some(group => allowedGroups.includes(group.toLowerCase().trim()))
-    ) {
-      return { error: `Username ${login} does not belong to an allowed group` };
-    }
-
-    if (access_token) {
-      return {
-        accessToken: jwt.sign({ login }, tokenSecret, { expiresIn: getTokenLifetime() }),
-      };
-    }
-
-    return { error: 'Token not found' };
+    const { amoid } = params;
+    return getAuthProviderById(amoid).oauthToken(params);
  },
  login_meta: true,
  async login(params) {
-    const { login, password } = params;
+    const { amoid, login, password, isAdminPage } = params;

-    if (process.env.AD_URL) {
-      const adConfig = {
-        url: process.env.AD_URL,
-        baseDN: process.env.AD_BASEDN,
-        username: process.env.AD_USERNAME,
-        password: process.env.AD_PASSOWRD,
-      };
-      const ad = new AD(adConfig);
-      try {
-        const res = await ad.authenticate(login, password);
-        if (!res) {
-          return { error: 'Login failed' };
-        }
-        if (
-          process.env.AD_ALLOWED_LOGINS &&
-          !process.env.AD_ALLOWED_LOGINS.split(',').find(x => x.toLowerCase().trim() == login.toLowerCase().trim())
-        ) {
-          return { error: `Username ${login} not allowed to log in` };
-        }
+    if (isAdminPage) {
+      if (process.env.ADMIN_PASSWORD && process.env.ADMIN_PASSWORD == password) {
        return {
-          accessToken: jwt.sign({ login }, tokenSecret, { expiresIn: getTokenLifetime() }),
-        };
-      } catch (err) {
-        logger.error({ err }, 'Failed active directory authentization');
-        return {
-          error: err.message,
+          accessToken: jwt.sign(
+            {
+              login: 'superadmin',
+              permissions: await storage.loadSuperadminPermissions(),
+              roleId: -3,
+            },
+            getTokenSecret(),
+            {
+              expiresIn: getTokenLifetime(),
+            }
+          ),
        };
      }
+
+      return { error: 'Login failed' };
    }

-    const logins = getLogins();
-    if (!logins) {
-      return { error: 'Logins not configured' };
-    }
-    const foundLogin = logins.find(x => x.login == login);
-    if (foundLogin && foundLogin.password && foundLogin.password == password) {
-      return {
-        accessToken: jwt.sign({ login }, tokenSecret, { expiresIn: getTokenLifetime() }),
-      };
-    }
-    return { error: 'Invalid credentials' };
+    return getAuthProviderById(amoid).login(login, password);
+  },
+
+  getProviders_meta: true,
+  getProviders() {
+    return {
+      providers: getAuthProviders().map(x => x.toJson()),
+      default: getDefaultAuthProvider()?.amoid,
+    };
+  },
+
+  redirect_meta: true,
+  async redirect(params) {
+    const { amoid } = params;
+    return getAuthProviderById(amoid).redirect(params);
  },

  authMiddleware,
-  shouldAuthorizeApi,
 };
@@ -3,14 +3,20 @@ const os = require('os');
 const path = require('path');
 const axios = require('axios');
 const { datadir, getLogsFilePath } = require('../utility/directories');
-const { hasPermission, getLogins } = require('../utility/hasPermission');
+const { hasPermission } = require('../utility/hasPermission');
 const socket = require('../utility/socket');
 const _ = require('lodash');
 const AsyncLock = require('async-lock');
+const jwt = require('jsonwebtoken');

 const currentVersion = require('../currentVersion');
 const platformInfo = require('../utility/platformInfo');
 const connections = require('../controllers/connections');
+const { getAuthProviderFromReq } = require('../auth/authProvider');
+const { checkLicense, checkLicenseKey } = require('../utility/checkLicense');
+const storage = require('./storage');
+const { getAuthProxyUrl } = require('../utility/authProxy');
+const { getPublicHardwareFingerprint } = require('../utility/hardwareFingerprint');

 const lock = new AsyncLock();

@@ -27,27 +33,51 @@ module.exports = {

  get_meta: true,
  async get(_params, req) {
-    const logins = getLogins();
-    const loginName =
-      req && req.user && req.user.login ? req.user.login : req && req.auth && req.auth.user ? req.auth.user : null;
-    const login = logins && loginName ? logins.find(x => x.login == loginName) : null;
-    const permissions = login ? login.permissions : process.env.PERMISSIONS;
+    const authProvider = getAuthProviderFromReq(req);
+    const login = authProvider.getCurrentLogin(req);
+    const permissions = authProvider.getCurrentPermissions(req);
+    const isUserLoggedIn = authProvider.isUserLoggedIn(req);
+
+    const singleConid = authProvider.getSingleConnectionId(req);
+
+    const singleConnection = singleConid
+      ? await connections.getCore({ conid: singleConid })
+      : connections.singleConnection;
+
+    let configurationError = null;
+    if (process.env.STORAGE_DATABASE && process.env.BASIC_AUTH) {
+      configurationError =
+        'Basic authentization is not allowed, when using storage. Cannot use both STORAGE_DATABASE and BASIC_AUTH';
+    }
+
+    const checkedLicense = await checkLicense();
+    const isLicenseValid = checkedLicense?.status == 'ok';

    return {
      runAsPortal: !!connections.portalConnections,
      singleDbConnection: connections.singleDbConnection,
-      singleConnection: connections.singleConnection,
+      singleConnection: singleConnection,
+      isUserLoggedIn,
      // hideAppEditor: !!process.env.HIDE_APP_EDITOR,
      allowShellConnection: platformInfo.allowShellConnection,
      allowShellScripting: platformInfo.allowShellScripting,
      isDocker: platformInfo.isDocker,
+      isElectron: platformInfo.isElectron,
+      isLicenseValid,
+      checkedLicense,
+      configurationError,
+      logoutUrl: await authProvider.getLogoutUrl(),
      permissions,
      login,
-      oauth: process.env.OAUTH_AUTH,
-      oauthClient: process.env.OAUTH_CLIENT_ID,
-      oauthScope: process.env.OAUTH_SCOPE,
-      oauthLogout: process.env.OAUTH_LOGOUT,
-      isLoginForm: !!process.env.AD_URL || (!!logins && !process.env.BASIC_AUTH),
+      // ...additionalConfigProps,
+      isBasicAuth: !!process.env.BASIC_AUTH,
+      isAdminLoginForm: !!(
+        process.env.STORAGE_DATABASE &&
+        process.env.ADMIN_PASSWORD &&
+        !process.env.BASIC_AUTH &&
+        checkedLicense?.type == 'premium'
+      ),
+      storageDatabase: process.env.STORAGE_DATABASE,
      logsFilePath: getLogsFilePath(),
      connectionsFilePath: path.join(datadir(), 'connections.jsonl'),
      ...currentVersion,
@@ -75,6 +105,12 @@ module.exports = {
    return res;
  },

+  deleteSettings_meta: true,
+  async deleteSettings() {
+    await fs.unlink(path.join(datadir(), 'settings.json'));
+    return true;
+  },
+
  fillMissingSettings(value) {
    const res = {
      ...value,
@@ -97,12 +133,80 @@ module.exports = {
  async loadSettings() {
    try {
      const settingsText = await fs.readFile(path.join(datadir(), 'settings.json'), { encoding: 'utf-8' });
-      return this.fillMissingSettings(JSON.parse(settingsText));
+      return {
+        ...this.fillMissingSettings(JSON.parse(settingsText)),
+        'other.licenseKey': platformInfo.isElectron ? await this.loadLicenseKey() : undefined,
+      };
    } catch (err) {
      return this.fillMissingSettings({});
    }
  },

+  async loadLicenseKey() {
+    try {
+      const licenseKey = await fs.readFile(path.join(datadir(), 'license.key'), { encoding: 'utf-8' });
+      return licenseKey;
+    } catch (err) {
+      return null;
+    }
+  },
+
+  saveLicenseKey_meta: true,
+  async saveLicenseKey({ licenseKey }) {
+    const decoded = jwt.decode(licenseKey);
+    if (!decoded) {
+      return {
+        status: 'error',
+        errorMessage: 'Invalid license key',
+      };
+    }
+
+    const { exp } = decoded;
+    if (exp * 1000 < Date.now()) {
+      return {
+        status: 'error',
+        errorMessage: 'License key is expired',
+      };
+    }
+
+    try {
+      if (process.env.STORAGE_DATABASE) {
+        await storage.writeConfig({ group: 'license', config: { licenseKey } });
+        // await storageWriteConfig('license', { licenseKey });
+      } else {
+        await fs.writeFile(path.join(datadir(), 'license.key'), licenseKey);
+      }
+      socket.emitChanged(`config-changed`);
+      return { status: 'ok' };
+    } catch (err) {
+      return {
+        status: 'error',
+        errorMessage: err.message,
+      };
+    }
+  },
+
+  startTrial_meta: true,
+  async startTrial() {
+    try {
+      const fingerprint = await getPublicHardwareFingerprint();
+
+      const resp = await axios.default.post(`${getAuthProxyUrl()}/trial-license`, {
+        type: 'premium-trial',
+        days: 30,
+        fingerprint,
+      });
+      const { token } = resp.data;
+
+      return await this.saveLicenseKey({ licenseKey: token });
+    } catch (err) {
+      return {
+        status: 'error',
+        errorMessage: err.message,
+      };
+    }
+  },
+
  updateSettings_meta: true,
  async updateSettings(values, req) {
    if (!hasPermission(`settings/change`, req)) return false;
@@ -112,10 +216,16 @@ module.exports = {
      try {
        const updated = {
          ...currentValue,
-          ...values,
+          ..._.omit(values, ['other.licenseKey']),
        };
        await fs.writeFile(path.join(datadir(), 'settings.json'), JSON.stringify(updated, undefined, 2));
        // this.settingsValue = updated;
+
+        if (currentValue['other.licenseKey'] != values['other.licenseKey']) {
+          await this.saveLicenseKey({ licenseKey: values['other.licenseKey'] });
+          socket.emitChanged(`config-changed`);
+        }
+
        socket.emitChanged(`settings-changed`);
        return updated;
      } catch (err) {
@@ -130,4 +240,10 @@ module.exports = {
    const resp = await axios.default.get('https://raw.githubusercontent.com/dbgate/dbgate/master/CHANGELOG.md');
    return resp.data;
  },
+
+  checkLicense_meta: true,
+  async checkLicense({ licenseKey }) {
+    const resp = await checkLicenseKey(licenseKey);
+    return resp;
+  },
 };
@@ -16,6 +16,9 @@ const { safeJsonParse, getLogger } = require('dbgate-tools');
 const platformInfo = require('../utility/platformInfo');
 const { connectionHasPermission, testConnectionPermission } = require('../utility/hasPermission');
 const pipeForkLogs = require('../utility/pipeForkLogs');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const { getAuthProviderById } = require('../auth/authProvider');
+const { startTokenChecking } = require('../utility/authProxy');

 const logger = getLogger('connections');

@@ -70,6 +73,8 @@ function getPortalCollections() {
      displayName: process.env[`LABEL_${id}`],
      isReadOnly: process.env[`READONLY_${id}`],
      databases: process.env[`DBCONFIG_${id}`] ? safeJsonParse(process.env[`DBCONFIG_${id}`]) : null,
+      allowedDatabases: process.env[`ALLOWED_DATABASES_${id}`]?.replace(/\|/g, '\n'),
+      allowedDatabasesRegex: process.env[`ALLOWED_DATABASES_REGEX_${id}`],
      parent: process.env[`PARENT_${id}`] || undefined,

      // SSH tunnel
@@ -199,6 +204,12 @@ module.exports = {

  list_meta: true,
  async list(_params, req) {
+    const storage = require('./storage');
+
+    const storageConnections = await storage.connections(req);
+    if (storageConnections) {
+      return storageConnections;
+    }
    if (portalConnections) {
      if (platformInfo.allowShellConnection) return portalConnections;
      return portalConnections.map(maskConnection).filter(x => connectionHasPermission(x, req));
@@ -236,14 +247,16 @@ module.exports = {
  },

  saveVolatile_meta: true,
-  async saveVolatile({ conid, user, password, test }) {
+  async saveVolatile({ conid, user = undefined, password = undefined, accessToken = undefined, test = false }) {
    const old = await this.getCore({ conid });
    const res = {
      ...old,
      _id: crypto.randomUUID(),
      password,
+      accessToken,
      passwordMode: undefined,
      unsaved: true,
+      useRedirectDbLogin: false,
    };
    if (old.passwordMode == 'askUser') {
      res.user = user;
@@ -336,6 +349,14 @@ module.exports = {
    if (volatile) {
      return volatile;
    }
+
+    const storage = require('./storage');
+
+    const storageConnection = await storage.getConnection({ conid });
+    if (storageConnection) {
+      return storageConnection;
+    }
+
    if (portalConnections) {
      const res = portalConnections.find(x => x._id == conid) || null;
      return mask && !platformInfo.allowShellConnection ? maskConnection(res) : res;
@@ -365,4 +386,95 @@ module.exports = {
    });
    return res;
  },
+
+  dbloginWeb_meta: {
+    raw: true,
+    method: 'get',
+  },
+  async dbloginWeb(req, res) {
+    const { conid, state, redirectUri } = req.query;
+    const connection = await this.getCore({ conid });
+    const driver = requireEngineDriver(connection);
+    const authResp = await driver.getRedirectAuthUrl(connection, {
+      redirectUri,
+      state,
+      client: 'web',
+    });
+    res.redirect(authResp.url);
+  },
+
+  dbloginApp_meta: true,
+  async dbloginApp({ conid, state }) {
+    const connection = await this.getCore({ conid });
+    const driver = requireEngineDriver(connection);
+    const resp = await driver.getRedirectAuthUrl(connection, {
+      state,
+      client: 'app',
+    });
+    startTokenChecking(resp.sid, async token => {
+      const volatile = await this.saveVolatile({ conid, accessToken: token });
+      socket.emit('got-volatile-token', { savedConId: conid, volatileConId: volatile._id });
+    });
+    return resp;
+  },
+
+  dbloginToken_meta: true,
+  async dbloginToken({ code, conid, strmid, redirectUri, sid }) {
+    try {
+      const connection = await this.getCore({ conid });
+      const driver = requireEngineDriver(connection);
+      const accessToken = await driver.getAuthTokenFromCode(connection, { sid, code, redirectUri });
+      const volatile = await this.saveVolatile({ conid, accessToken });
+      // console.log('******************************** WE HAVE ACCESS TOKEN', accessToken);
+      socket.emit('got-volatile-token', { strmid, savedConId: conid, volatileConId: volatile._id });
+      return { success: true };
+    } catch (err) {
+      logger.error({ err }, 'Error getting DB token');
+      return { error: err.message };
+    }
+  },
+
+  dbloginAuthToken_meta: true,
+  async dbloginAuthToken({ amoid, code, conid, redirectUri, sid }) {
+    try {
+      const connection = await this.getCore({ conid });
+      const driver = requireEngineDriver(connection);
+      const accessToken = await driver.getAuthTokenFromCode(connection, { code, redirectUri, sid });
+      const volatile = await this.saveVolatile({ conid, accessToken });
+      const authProvider = getAuthProviderById(amoid);
+      const resp = await authProvider.login(null, null, { conid: volatile._id });
+      return resp;
+    } catch (err) {
+      logger.error({ err }, 'Error getting DB token');
+      return { error: err.message };
+    }
+  },
+
+  dbloginAuth_meta: true,
+  async dbloginAuth({ amoid, conid, user, password }) {
+    if (user || password) {
+      const saveResp = await this.saveVolatile({ conid, user, password, test: true });
+      if (saveResp.msgtype == 'connected') {
+        const loginResp = await getAuthProviderById(amoid).login(user, password, { conid: saveResp._id });
+        return loginResp;
+      }
+      return saveResp;
+    }
+
+    // user and password is stored in connection, volatile connection is not needed
+    const loginResp = await getAuthProviderById(amoid).login(null, null, { conid });
+    return loginResp;
+  },
+
+  volatileDbloginFromAuth_meta: true,
+  async volatileDbloginFromAuth({ conid }, req) {
+    const connection = await this.getCore({ conid });
+    const driver = requireEngineDriver(connection);
+    const accessToken = await driver.getAccessTokenFromAuth(connection, req);
+    if (accessToken) {
+      const volatile = await this.saveVolatile({ conid, accessToken });
+      return volatile;
+    }
+    return null;
+  },
 };
@@ -89,6 +89,9 @@ module.exports = {
    if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') {
      throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode });
    }
+    if (connection.useRedirectDbLogin) {
+      throw new MissingCredentialsError({ conid, redirectToDbLogin: true });
+    }
    const subprocess = fork(
      global['API_PACKAGE'] || process.argv[1],
      [
@@ -179,6 +182,15 @@ module.exports = {
    return res;
  },

+  runOperation_meta: true,
+  async runOperation({ conid, database, operation, useTransaction }, req) {
+    testConnectionPermission(conid, req);
+    logger.info({ conid, database, operation }, 'Processing operation');
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'runOperation', operation, useTransaction });
+    return res;
+  },
+
  collectionData_meta: true,
  async collectionData({ conid, database, options }, req) {
    testConnectionPermission(conid, req);
@@ -201,6 +213,17 @@ module.exports = {
    return res.result || null;
  },

+  schemaList_meta: true,
+  async schemaList({ conid, database }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('schemaList', { conid, database });
+  },
+
+  dispatchDatabaseChangedEvent_meta: true,
+  dispatchDatabaseChangedEvent({ event, conid, database }) {
+    socket.emitChanged(event, { conid, database });
+  },
+
  loadKeys_meta: true,
  async loadKeys({ conid, database, root, filter }, req) {
    testConnectionPermission(conid, req);
@@ -18,11 +18,14 @@ function readFirstLine(file) {
      }
      if (reader.hasNextLine()) {
        reader.nextLine((err, line) => {
-          if (err) reject(err);
-          resolve(line);
+          if (err) {
+            reader.close(() => reject(err)); // Ensure reader is closed on error
+            return;
+          }
+          reader.close(() => resolve(line)); // Ensure reader is closed after reading
        });
      } else {
-        resolve(null);
+        reader.close(() => resolve(null)); // Properly close if no lines are present
      }
    });
  });
@@ -42,13 +42,14 @@ module.exports = {

  info_meta: true,
  async info({ packageName }) {
+    // @ts-ignore
+    const isPackaged = await fs.exists(path.join(packagedPluginsDir(), packageName));
+
    try {
      const infoResp = await axios.default.get(`https://registry.npmjs.org/${packageName}`);
      const { latest } = infoResp.data['dist-tags'];
      const manifest = infoResp.data.versions[latest];
      const { readme } = infoResp.data;
-      // @ts-ignore
-      const isPackaged = await fs.exists(path.join(packagedPluginsDir(), packageName));

      return {
        readme,
@@ -57,6 +58,7 @@ module.exports = {
      };
    } catch (err) {
      return {
+        isPackaged,
        state: 'error',
        error: err.message,
      };
@@ -1,3 +1,4 @@
+const crypto = require('crypto');
 const connections = require('./connections');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
@@ -56,6 +57,9 @@ module.exports = {
      if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') {
        throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode });
      }
+      if (connection.useRedirectDbLogin) {
+        throw new MissingCredentialsError({ conid, redirectToDbLogin: true });
+      }
      const subprocess = fork(
        global['API_PACKAGE'] || process.argv[1],
        [
@@ -181,22 +185,29 @@ module.exports = {
    return { status: 'ok' };
  },

-  createDatabase_meta: true,
-  async createDatabase({ conid, name }, req) {
+  async sendDatabaseOp({ conid, msgtype, name }, req) {
    testConnectionPermission(conid, req);
    const opened = await this.ensureOpened(conid);
    if (opened.connection.isReadOnly) return false;
-    opened.subprocess.send({ msgtype: 'createDatabase', name });
-    return { status: 'ok' };
+    const res = await this.sendRequest(opened, { msgtype, name });
+    if (res.errorMessage) {
+      console.error(res.errorMessage);
+
+      return {
+        apiErrorMessage: res.errorMessage,
+      };
+    }
+    return res.result || null;
+  },
+
+  createDatabase_meta: true,
+  async createDatabase({ conid, name }, req) {
+    return this.sendDatabaseOp({ conid, msgtype: 'createDatabase', name }, req);
  },

  dropDatabase_meta: true,
  async dropDatabase({ conid, name }, req) {
-    testConnectionPermission(conid, req);
-    const opened = await this.ensureOpened(conid);
-    if (opened.connection.isReadOnly) return false;
-    opened.subprocess.send({ msgtype: 'dropDatabase', name });
-    return { status: 'ok' };
+    return this.sendDatabaseOp({ conid, msgtype: 'dropDatabase', name }, req);
  },

  sendRequest(conn, message) {
@@ -0,0 +1,20 @@
+module.exports = {
+  connections_meta: true,
+  async connections(req) {
+    return null;
+  },
+
+  getConnection_meta: true,
+  async getConnection({ conid }) {
+    return null;
+  },
+
+  async loadSuperadminPermissions() {
+    return [];
+  },
+
+  getConnectionsForLoginPage_meta: true,
+  async getConnectionsForLoginPage() {
+    return null;
+  },
+};
@@ -97,9 +97,12 @@ if (processArgs.listenApi) {
 }

 const shell = require('./shell/index');
-const dbgateTools = require('dbgate-tools');
+const currentVersion = require('./currentVersion');

-global['DBGATE_TOOLS'] = dbgateTools;
+global.DBGATE_PACKAGES = {
+  'dbgate-tools': require('dbgate-tools'),
+  'dbgate-sqltree': require('dbgate-sqltree'),
+};

 if (processArgs.startProcess) {
  const proc = require('./proc');
@@ -116,6 +119,7 @@ module.exports = {
  ...shell,
  getLogger,
  configureLogger,
+  currentVersion,
  // loadLogsContent,
  getMainModule: () => require('./main'),
 };
@@ -18,6 +18,7 @@ const sessions = require('./controllers/sessions');
 const runners = require('./controllers/runners');
 const jsldata = require('./controllers/jsldata');
 const config = require('./controllers/config');
+const storage = require('./controllers/storage');
 const archive = require('./controllers/archive');
 const apps = require('./controllers/apps');
 const auth = require('./controllers/auth');
@@ -31,9 +32,9 @@ const onFinished = require('on-finished');
 const { rundir } = require('./utility/directories');
 const platformInfo = require('./utility/platformInfo');
 const getExpressPath = require('./utility/getExpressPath');
-const { getLogins } = require('./utility/hasPermission');
 const _ = require('lodash');
 const { getLogger } = require('dbgate-tools');
+const { getDefaultAuthProvider } = require('./auth/authProvider');

 const logger = getLogger('main');

@@ -44,11 +45,23 @@ function start() {

  const server = http.createServer(app);

-  const logins = getLogins();
-  if (logins && process.env.BASIC_AUTH) {
+  if (process.env.BASIC_AUTH && !process.env.STORAGE_DATABASE) {
+    async function authorizer(username, password, cb) {
+      try {
+        const resp = await getDefaultAuthProvider().login(username, password);
+        if (resp.accessToken) {
+          cb(null, true);
+        } else {
+          cb(null, false);
+        }
+      } catch (err) {
+        cb(err, false);
+      }
+    }
    app.use(
      basicAuth({
-        users: _.fromPairs(logins.filter(x => x.password).map(x => [x.login, x.password])),
+        authorizer,
+        authorizeAsync: true,
        challenge: true,
        realm: 'DbGate Web App',
      })
@@ -72,9 +85,7 @@ function start() {
    });
  }

-  if (auth.shouldAuthorizeApi()) {
-    app.use(auth.authMiddleware);
-  }
+  app.use(auth.authMiddleware);

  app.get(getExpressPath('/stream'), async function (req, res) {
    const strmid = req.query.strmid;
@@ -162,6 +173,7 @@ function useAllControllers(app, electron) {
  useController(app, electron, '/runners', runners);
  useController(app, electron, '/jsldata', jsldata);
  useController(app, electron, '/config', config);
+  useController(app, electron, '/storage', storage);
  useController(app, electron, '/archive', archive);
  useController(app, electron, '/uploads', uploads);
  useController(app, electron, '/plugins', plugins);
@@ -170,6 +170,18 @@ async function handleRunScript({ msgid, sql, useTransaction }, skipReadonlyCheck
  }
 }

+async function handleRunOperation({ msgid, operation, useTransaction }, skipReadonlyCheck = false) {
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+  try {
+    if (!skipReadonlyCheck) ensureExecuteCustomScript(driver);
+    await driver.operation(systemConnection, operation, { useTransaction });
+    process.send({ msgtype: 'response', msgid });
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+  }
+}
+
 async function handleQueryData({ msgid, sql }, skipReadonlyCheck = false) {
  await waitConnected();
  const driver = requireEngineDriver(storedConnection);
@@ -201,6 +213,10 @@ async function handleDriverDataCore(msgid, callMethod) {
  }
 }

+async function handleSchemaList({ msgid }) {
+  return handleDriverDataCore(msgid, driver => driver.listSchemas(systemConnection));
+}
+
 async function handleCollectionData({ msgid, options }) {
  return handleDriverDataCore(msgid, driver => driver.readCollection(systemConnection, options));
 }
@@ -311,6 +327,7 @@ const messageHandlers = {
  connect: handleConnect,
  queryData: handleQueryData,
  runScript: handleRunScript,
+  runOperation: handleRunOperation,
  updateCollection: handleUpdateCollection,
  collectionData: handleCollectionData,
  loadKeys: handleLoadKeys,
@@ -324,6 +341,7 @@ const messageHandlers = {
  loadFieldValues: handleLoadFieldValues,
  sqlSelect: handleSqlSelect,
  exportKeys: handleExportKeys,
+  schemaList: handleSchemaList,
  // runCommand: handleRunCommand,
 };

@@ -16,7 +16,18 @@ let afterConnectCallbacks = [];
 async function handleRefresh() {
  const driver = requireEngineDriver(storedConnection);
  try {
-    const databases = await driver.listDatabases(systemConnection);
+    let databases = await driver.listDatabases(systemConnection);
+    if (storedConnection?.allowedDatabases?.trim()) {
+      const allowedDatabaseList = storedConnection.allowedDatabases
+        .split('\n')
+        .map(x => x.trim().toLowerCase())
+        .filter(x => x);
+      databases = databases.filter(x => allowedDatabaseList.includes(x.name.toLocaleLowerCase()));
+    }
+    if (storedConnection?.allowedDatabasesRegex?.trim()) {
+      const regex = new RegExp(storedConnection.allowedDatabasesRegex, 'i');
+      databases = databases.filter(x => regex.test(x.name));
+    }
    setStatusName('ok');
    const databasesString = stableStringify(databases);
    if (lastDatabases != databasesString) {
@@ -94,18 +105,24 @@ function handlePing() {
  lastPing = new Date().getTime();
 }

-async function handleDatabaseOp(op, { name }) {
-  const driver = requireEngineDriver(storedConnection);
-  systemConnection = await connectUtility(driver, storedConnection, 'app');
-  if (driver[op]) {
-    await driver[op](systemConnection, name);
-  } else {
-    const dmp = driver.createDumper();
-    dmp[op](name);
-    logger.info({ sql: dmp.s }, 'Running script');
-    await driver.query(systemConnection, dmp.s);
+async function handleDatabaseOp(op, { msgid, name }) {
+  try {
+    const driver = requireEngineDriver(storedConnection);
+    systemConnection = await connectUtility(driver, storedConnection, 'app');
+    if (driver[op]) {
+      await driver[op](systemConnection, name);
+    } else {
+      const dmp = driver.createDumper();
+      dmp[op](name);
+      logger.info({ sql: dmp.s }, 'Running script');
+      await driver.query(systemConnection, dmp.s, { discardResult: true });
+    }
+    await handleRefresh();
+
+    process.send({ msgtype: 'response', msgid, status: 'ok' });
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
  }
-  await handleRefresh();
 }

 async function handleDriverDataCore(msgid, callMethod) {
@@ -15,10 +15,12 @@ async function getSshConnection(connection) {
    agentForward: connection.sshMode == 'agent',
    passphrase: connection.sshMode == 'keyFile' ? connection.sshKeyfilePassword : undefined,
    username: connection.sshLogin,
-    password: connection.sshMode == 'userPassword' ? connection.sshPassword : undefined,
+    password: (connection.sshMode || 'userPassword') == 'userPassword' ? connection.sshPassword : undefined,
    agentSocket: connection.sshMode == 'agent' ? platformInfo.sshAuthSock : undefined,
    privateKey:
-      connection.sshMode == 'keyFile' && connection.sshKeyfile ? await fs.readFile(connection.sshKeyfile) : undefined,
+      connection.sshMode == 'keyFile' && (connection.sshKeyfile || platformInfo?.defaultKeyfile)
+        ? await fs.readFile(connection.sshKeyfile || platformInfo?.defaultKeyfile)
+        : undefined,
    skipAutoPrivateKey: true,
    noReadline: true,
  };
@@ -0,0 +1,16 @@
+const importDbModel = require('../utility/importDbModel');
+const fs = require('fs');
+
+async function dbModelToJson({ modelFolder, outputFile, commonjs }) {
+  const dbInfo = await importDbModel(modelFolder);
+
+  const json = JSON.stringify(dbInfo, null, 2);
+  if (commonjs) {
+    fs.writeFileSync(outputFile, `module.exports = ${json};`);
+    return;
+  } else {
+    fs.writeFileSync(outputFile, json);
+  }
+}
+
+module.exports = dbModelToJson;
@@ -6,7 +6,7 @@ const copyStream = require('./copyStream');
 const fakeObjectReader = require('./fakeObjectReader');
 const consoleObjectWriter = require('./consoleObjectWriter');
 const jsonLinesWriter = require('./jsonLinesWriter');
-const jsonArrayWriter = require('./jsonArrayWriter');
+const jsonWriter = require('./jsonWriter');
 const jsonLinesReader = require('./jsonLinesReader');
 const sqlDataWriter = require('./sqlDataWriter');
 const jslDataReader = require('./jslDataReader');
@@ -27,6 +27,9 @@ const loadDatabase = require('./loadDatabase');
 const generateModelSql = require('./generateModelSql');
 const modifyJsonLinesReader = require('./modifyJsonLinesReader');
 const dataDuplicator = require('./dataDuplicator');
+const dbModelToJson = require('./dbModelToJson');
+const jsonToDbModel = require('./jsonToDbModel');
+const jsonReader = require('./jsonReader');

 const dbgateApi = {
  queryReader,
@@ -35,8 +38,9 @@ const dbgateApi = {
  tableReader,
  copyStream,
  jsonLinesWriter,
-  jsonArrayWriter,
  jsonLinesReader,
+  jsonReader,
+  jsonWriter,
  sqlDataWriter,
  fakeObjectReader,
  consoleObjectWriter,
@@ -57,6 +61,8 @@ const dbgateApi = {
  generateModelSql,
  modifyJsonLinesReader,
  dataDuplicator,
+  dbModelToJson,
+  jsonToDbModel,
 };

 requirePlugin.initializeDbgateApi(dbgateApi);
@@ -1,52 +0,0 @@
-const { getLogger } = require('dbgate-tools');
-const fs = require('fs');
-const stream = require('stream');
-
-const logger = getLogger('jsonArrayWriter');
-
-class StringifyStream extends stream.Transform {
-  constructor() {
-    super({ objectMode: true });
-    this.wasHeader = false;
-    this.wasRecord = false;
-  }
-  _transform(chunk, encoding, done) {
-    let skip = false;
-
-    if (!this.wasHeader) {
-      skip = chunk.__isStreamHeader;
-      this.wasHeader = true;
-    }
-    if (!skip) {
-      if (!this.wasRecord) {
-        this.push('[\n');
-      } else {
-        this.push(',\n');
-      }
-      this.wasRecord = true;
-
-      this.push(JSON.stringify(chunk));
-    }
-    done();
-  }
-
-  _flush(done) {
-    if (!this.wasRecord) {
-      this.push('[]\n');
-    } else {
-      this.push('\n]\n');
-    }
-    done();
-  }
-}
-
-async function jsonArrayWriter({ fileName, encoding = 'utf-8' }) {
-  logger.info(`Writing file ${fileName}`);
-  const stringify = new StringifyStream();
-  const fileStream = fs.createWriteStream(fileName, encoding);
-  stringify.pipe(fileStream);
-  stringify['finisher'] = fileStream;
-  return stringify;
-}
-
-module.exports = jsonArrayWriter;
@@ -2,6 +2,7 @@ const fs = require('fs');
 const stream = require('stream');
 const byline = require('byline');
 const { getLogger } = require('dbgate-tools');
+const download = require('./download');
 const logger = getLogger('jsonLinesReader');

 class ParseStream extends stream.Transform {
@@ -35,8 +36,10 @@ class ParseStream extends stream.Transform {
 async function jsonLinesReader({ fileName, encoding = 'utf-8', limitRows = undefined }) {
  logger.info(`Reading file ${fileName}`);

+  const downloadedFile = await download(fileName);
+
  const fileStream = fs.createReadStream(
-    fileName,
+    downloadedFile,
    // @ts-ignore
    encoding
  );
@@ -0,0 +1,84 @@
+const fs = require('fs');
+const stream = require('stream');
+const byline = require('byline');
+const { getLogger } = require('dbgate-tools');
+const { parser } = require('stream-json');
+const { pick } = require('stream-json/filters/Pick');
+const { streamArray } = require('stream-json/streamers/StreamArray');
+const { streamObject } = require('stream-json/streamers/StreamObject');
+const download = require('./download');
+
+const logger = getLogger('jsonReader');
+
+
+class ParseStream extends stream.Transform {
+  constructor({ limitRows, jsonStyle, keyField }) {
+    super({ objectMode: true });
+    this.wasHeader = false;
+    this.limitRows = limitRows;
+    this.jsonStyle = jsonStyle;
+    this.keyField = keyField || '_key';
+    this.rowsWritten = 0;
+  }
+  _transform(chunk, encoding, done) {
+    if (!this.wasHeader) {
+      this.push({
+        __isStreamHeader: true,
+        __isDynamicStructure: true,
+      });
+
+      this.wasHeader = true;
+    }
+    if (!this.limitRows || this.rowsWritten < this.limitRows) {
+      if (this.jsonStyle === 'object') {
+        this.push({
+          ...chunk.value,
+          [this.keyField]: chunk.key,
+        });
+      } else {
+        this.push(chunk.value);
+      }
+
+      this.rowsWritten += 1;
+    }
+    done();
+  }
+}
+
+async function jsonReader({
+  fileName,
+  jsonStyle,
+  keyField = '_key',
+  rootField = null,
+  encoding = 'utf-8',
+  limitRows = undefined,
+}) {
+  logger.info(`Reading file ${fileName}`);
+
+  const downloadedFile = await download(fileName);
+  const fileStream = fs.createReadStream(
+    downloadedFile,
+    // @ts-ignore
+    encoding
+  );
+  const parseJsonStream = parser();
+  fileStream.pipe(parseJsonStream);
+
+  const parseStream = new ParseStream({ limitRows, jsonStyle, keyField });
+
+  const tramsformer = jsonStyle === 'object' ? streamObject() : streamArray();
+
+  if (rootField) {
+    const filterStream = pick({ filter: rootField });
+    parseJsonStream.pipe(filterStream);
+    filterStream.pipe(tramsformer);
+  } else {
+    parseJsonStream.pipe(tramsformer);
+  }
+
+  tramsformer.pipe(parseStream);
+
+  return parseStream;
+}
+
+module.exports = jsonReader;
@@ -0,0 +1,9 @@
+const exportDbModel = require('../utility/exportDbModel');
+const fs = require('fs');
+
+async function jsonToDbModel({ modelFile, outputDir }) {
+  const dbInfo = JSON.parse(fs.readFileSync(modelFile, 'utf-8'));
+  await exportDbModel(dbInfo, outputDir);
+}
+
+module.exports = jsonToDbModel;
@@ -0,0 +1,97 @@
+const { getLogger } = require('dbgate-tools');
+const fs = require('fs');
+const stream = require('stream');
+const _ = require('lodash');
+
+const logger = getLogger('jsonArrayWriter');
+
+class StringifyStream extends stream.Transform {
+  constructor({ jsonStyle, keyField, rootField }) {
+    super({ objectMode: true });
+    this.wasHeader = false;
+    this.wasRecord = false;
+    this.jsonStyle = jsonStyle;
+    this.keyField = keyField || '_key';
+    this.rootField = rootField;
+  }
+  _transform(chunk, encoding, done) {
+    let skip = false;
+
+    if (!this.wasHeader) {
+      skip = chunk.__isStreamHeader;
+      this.wasHeader = true;
+    }
+    if (!skip) {
+      if (!this.wasRecord) {
+        if (this.rootField) {
+          if (this.jsonStyle === 'object') {
+            this.push(`{"${this.rootField}": {\n`);
+          } else {
+            this.push(`{"${this.rootField}": [\n`);
+          }
+        } else {
+          if (this.jsonStyle === 'object') {
+            this.push('{\n');
+          } else {
+            this.push('[\n');
+          }
+        }
+      } else {
+        this.push(',\n');
+      }
+      this.wasRecord = true;
+
+      if (this.jsonStyle === 'object') {
+        const key = chunk[this.keyField] ?? chunk[Object.keys(chunk)[0]];
+        this.push(`"${key}": ${JSON.stringify(_.omit(chunk, [this.keyField]))}`);
+      } else {
+        this.push(JSON.stringify(chunk));
+      }
+    }
+    done();
+  }
+
+  _flush(done) {
+    if (!this.wasRecord) {
+      if (this.rootField) {
+        if (this.jsonStyle === 'object') {
+          this.push(`{"${this.rootField}": {}}\n`);
+        } else {
+          this.push(`{"${this.rootField}": []}\n`);
+        }
+      } else {
+        if (this.jsonStyle === 'object') {
+          this.push('{}\n');
+        } else {
+          this.push('[]\n');
+        }
+      }
+    } else {
+      if (this.rootField) {
+        if (this.jsonStyle === 'object') {
+          this.push('\n}}\n');
+        } else {
+          this.push('\n]}\n');
+        }
+      } else {
+        if (this.jsonStyle === 'object') {
+          this.push('\n}\n');
+        } else {
+          this.push('\n]\n');
+        }
+      }
+    }
+    done();
+  }
+}
+
+async function jsonWriter({ fileName, jsonStyle, keyField = '_key', rootField, encoding = 'utf-8' }) {
+  logger.info(`Writing file ${fileName}`);
+  const stringify = new StringifyStream({ jsonStyle, keyField, rootField });
+  const fileStream = fs.createWriteStream(fileName, encoding);
+  stringify.pipe(fileStream);
+  stringify['finisher'] = fileStream;
+  return stringify;
+}
+
+module.exports = jsonWriter;
@@ -61,10 +61,13 @@ class ParseStream extends stream.Transform {
          if (update.document) {
            obj = update.document;
          } else {
-            obj = {
-              ...obj,
-              ...update.fields,
-            };
+            obj = _.omitBy(
+              {
+                ...obj,
+                ...update.fields,
+              },
+              (v, k) => v?.$$undefined$$
+            );
          }
        }

@@ -3,6 +3,7 @@ const fs = require('fs');
 const { pluginsdir, packagedPluginsDir, getPluginBackendPath } = require('../utility/directories');
 const nativeModules = require('../nativeModules');
 const platformInfo = require('../utility/platformInfo');
+const authProxy = require('../utility/authProxy');
 const { getLogger } = require('dbgate-tools');
 const logger = getLogger('requirePlugin');

@@ -11,6 +12,8 @@ const loadedPlugins = {};
 const dbgateEnv = {
  dbgateApi: null,
  nativeModules,
+  platformInfo,
+  authProxy,
 };
 function requirePlugin(packageName, requiredPlugin = null) {
  if (!packageName) throw new Error('Missing packageName in plugin');
@@ -0,0 +1,25 @@
+function isAuthProxySupported() {
+  return false;
+}
+
+async function authProxyGetRedirectUrl(options) {
+  return null;
+}
+
+async function authProxyGetTokenFromCode(options) {
+  return null;
+}
+
+function startTokenChecking(sid, callback) {}
+
+function getAuthProxyUrl() {
+  return 'https://auth.dbgate.eu';
+}
+
+module.exports = {
+  isAuthProxySupported,
+  authProxyGetRedirectUrl,
+  authProxyGetTokenFromCode,
+  startTokenChecking,
+  getAuthProxyUrl,
+};
@@ -0,0 +1,18 @@
+function checkLicense() {
+  return {
+    status: 'ok',
+    type: 'community',
+  };
+}
+
+function checkLicenseKey(key) {
+  return {
+    status: 'ok',
+    type: 'community',
+  };
+}
+
+module.exports = {
+  checkLicense,
+  checkLicenseKey,
+};
@@ -0,0 +1,88 @@
+const axios = require('axios');
+const os = require('os');
+const crypto = require('crypto');
+const platformInfo = require('./platformInfo');
+
+async function getPublicIpInfo() {
+  try {
+    const resp = await axios.default.get('https://ipinfo.io/json');
+    if (!resp.data?.ip) {
+      return { ip: 'unknown-ip' };
+    }
+    return resp.data;
+  } catch (err) {
+    return { ip: 'unknown-ip' };
+  }
+}
+
+function getMacAddress() {
+  try {
+    const interfaces = os.networkInterfaces();
+    for (let iface of Object.values(interfaces)) {
+      for (let config of iface) {
+        if (config.mac && config.mac !== '00:00:00:00:00:00') {
+          return config.mac;
+        }
+      }
+    }
+    return '00:00:00:00:00:00';
+  } catch (err) {
+    return '00:00:00:00:00:00';
+  }
+}
+
+async function getHardwareFingerprint() {
+  const publicIpInfo = await getPublicIpInfo();
+  const macAddress = getMacAddress();
+  const platform = os.platform();
+  const release = os.release();
+  const hostname = os.hostname();
+  const totalMemory = os.totalmem();
+
+  return {
+    publicIp: publicIpInfo.ip,
+    country: publicIpInfo.country,
+    region: publicIpInfo.region,
+    city: publicIpInfo.city,
+    macAddress,
+    platform,
+    release,
+    hostname,
+    totalMemory,
+  };
+}
+
+async function getHardwareFingerprintHash(data = undefined) {
+  if (!data) {
+    data = await getHardwareFingerprint();
+  }
+  const fingerprintData = JSON.stringify(data);
+  const hash = crypto.createHash('sha256').update(fingerprintData).digest('hex');
+  return hash;
+}
+
+async function getPublicHardwareFingerprint() {
+  const fingerprint = await getHardwareFingerprint();
+  const hash = await getHardwareFingerprintHash(fingerprint);
+  return {
+    hash,
+    payload: {
+      platform: fingerprint.platform,
+      city: fingerprint.city,
+      country: fingerprint.country,
+      region: fingerprint.region,
+      isDocker: platformInfo.isDocker,
+      isElectron: platformInfo.isElectron,
+    },
+  };
+}
+
+// getHardwareFingerprint().then(console.log);
+// getHardwareFingerprintHash().then(console.log);
+// getPublicHardwareFingerprint().then(console.log);
+
+module.exports = {
+  getHardwareFingerprint,
+  getHardwareFingerprintHash,
+  getPublicHardwareFingerprint,
+};
@@ -1,72 +1,81 @@
 const { compilePermissions, testPermission } = require('dbgate-tools');
 const _ = require('lodash');
+const { getAuthProviderFromReq } = require('../auth/authProvider');

-const userPermissions = {};
+const cachedPermissions = {};

 function hasPermission(tested, req) {
  if (!req) {
    // request object not available, allow all
    return true;
  }
-  const { user } = (req && req.auth) || {};
-  const { login } = (process.env.OAUTH_PERMISSIONS && req && req.user) || {};
-  const key = user || login || '';
-  const logins = getLogins();

-  if (!userPermissions[key]) {
-    if (logins) {
-      const login = logins.find(x => x.login == user);
-      userPermissions[key] = compilePermissions(login ? login.permissions : null);
-    } else {
-      userPermissions[key] = compilePermissions(process.env.PERMISSIONS);
-    }
+  const permissions = getAuthProviderFromReq(req).getCurrentPermissions(req);
+
+  if (!cachedPermissions[permissions]) {
+    cachedPermissions[permissions] = compilePermissions(permissions);
  }
-  return testPermission(tested, userPermissions[key]);
+
+  return testPermission(tested, cachedPermissions[permissions]);
+
+  // const { user } = (req && req.auth) || {};
+  // const { login } = (process.env.OAUTH_PERMISSIONS && req && req.user) || {};
+  // const key = user || login || '';
+  // const logins = getLogins();
+
+  // if (!userPermissions[key]) {
+  //   if (logins) {
+  //     const login = logins.find(x => x.login == user);
+  //     userPermissions[key] = compilePermissions(login ? login.permissions : null);
+  //   } else {
+  //     userPermissions[key] = compilePermissions(process.env.PERMISSIONS);
+  //   }
+  // }
+  // return testPermission(tested, userPermissions[key]);
 }

-let loginsCache = null;
-let loginsLoaded = false;
+// let loginsCache = null;
+// let loginsLoaded = false;

-function getLogins() {
-  if (loginsLoaded) {
-    return loginsCache;
-  }
+// function getLogins() {
+//   if (loginsLoaded) {
+//     return loginsCache;
+//   }

-  const res = [];
-  if (process.env.LOGIN && process.env.PASSWORD) {
-    res.push({
-      login: process.env.LOGIN,
-      password: process.env.PASSWORD,
-      permissions: process.env.PERMISSIONS,
-    });
-  }
-  if (process.env.LOGINS) {
-    const logins = _.compact(process.env.LOGINS.split(',').map(x => x.trim()));
-    for (const login of logins) {
-      const password = process.env[`LOGIN_PASSWORD_${login}`];
-      const permissions = process.env[`LOGIN_PERMISSIONS_${login}`];
-      if (password) {
-        res.push({
-          login,
-          password,
-          permissions,
-        });
-      }
-    }
-  }
-  else if (process.env.OAUTH_PERMISSIONS) {
-    const login_permission_keys = Object.keys(process.env).filter((key) => _.startsWith(key, 'LOGIN_PERMISSIONS_'))
-    for (const permissions_key of login_permission_keys) {
-      const login = permissions_key.replace('LOGIN_PERMISSIONS_', '');
-      const permissions = process.env[permissions_key];
-      userPermissions[login] = compilePermissions(permissions);
-    }
-  }
+//   const res = [];
+//   if (process.env.LOGIN && process.env.PASSWORD) {
+//     res.push({
+//       login: process.env.LOGIN,
+//       password: process.env.PASSWORD,
+//       permissions: process.env.PERMISSIONS,
+//     });
+//   }
+//   if (process.env.LOGINS) {
+//     const logins = _.compact(process.env.LOGINS.split(',').map(x => x.trim()));
+//     for (const login of logins) {
+//       const password = process.env[`LOGIN_PASSWORD_${login}`];
+//       const permissions = process.env[`LOGIN_PERMISSIONS_${login}`];
+//       if (password) {
+//         res.push({
+//           login,
+//           password,
+//           permissions,
+//         });
+//       }
+//     }
+//   } else if (process.env.OAUTH_PERMISSIONS) {
+//     const login_permission_keys = Object.keys(process.env).filter(key => _.startsWith(key, 'LOGIN_PERMISSIONS_'));
+//     for (const permissions_key of login_permission_keys) {
+//       const login = permissions_key.replace('LOGIN_PERMISSIONS_', '');
+//       const permissions = process.env[permissions_key];
+//       userPermissions[login] = compilePermissions(permissions);
+//     }
+//   }

-  loginsCache = res.length > 0 ? res : null;
-  loginsLoaded = true;
-  return loginsCache;
-}
+//   loginsCache = res.length > 0 ? res : null;
+//   loginsLoaded = true;
+//   return loginsCache;
+// }

 function connectionHasPermission(connection, req) {
  if (!connection) {
@@ -87,7 +96,6 @@ function testConnectionPermission(connection, req) {

 module.exports = {
  hasPermission,
-  getLogins,
  connectionHasPermission,
  testConnectionPermission,
 };
@@ -31,6 +31,9 @@ module.exports = {
      electronSender.send(message, data == null ? null : data);
    }
    for (const strmid in sseResponses) {
+      if (data?.strmid && data?.strmid != strmid) {
+        continue;
+      }
      let skipThisStream = false;
      if (sseResponses[strmid].filter) {
        for (const key in sseResponses[strmid].filter) {
@@ -47,7 +50,7 @@ module.exports = {
      }

      sseResponses[strmid].response?.write(
-        `event: ${message}\ndata: ${stableStringify(data == null ? null : data)}\n\n`
+        `event: ${message}\ndata: ${stableStringify(data == null ? null : _.omit(data, ['strmid']))}\n\n`
      );
    }
  },
@@ -67,7 +67,7 @@ module.exports = function useController(app, electron, route, controller) {
    }

    if (raw) {
-      router[method](routeAction, controller[key]);
+      router[method](routeAction, (req, res) => controller[key](req, res));
    } else {
      router[method](routeAction, async (req, res) => {
        // if (controller._init && !controller._init_called) {
@@ -47,6 +47,8 @@ var config = {
  ],
  externals: {
    'better-sqlite3': 'commonjs better-sqlite3',
+    'oracledb': 'commonjs oracledb',
+    'msnodesqlv8': 'commonjs msnodesqlv8',
  },
 };

@@ -20,6 +20,7 @@ export interface ChangeSetItem {
  document?: any;
  condition?: { [column: string]: string };
  fields?: { [column: string]: string };
+  insertIfNotExistsFields?: { [column: string]: string };
 }

 export interface ChangeSetItemFields {
@@ -229,13 +230,23 @@ export function batchUpdateChangeSet(
  return changeSet;
 }

-function extractFields(item: ChangeSetItem, allowNulls = true): UpdateField[] {
-  return _.keys(item.fields)
-    .filter(targetColumn => allowNulls || item.fields[targetColumn] != null)
+function extractFields(item: ChangeSetItem, allowNulls = true, allowedDocumentColumns: string[] = []): UpdateField[] {
+  const allFields = {
+    ...item.fields,
+  };
+
+  for (const docField in item.document || {}) {
+    if (allowedDocumentColumns.includes(docField)) {
+      allFields[docField] = item.document[docField];
+    }
+  }
+
+  return _.keys(allFields)
+    .filter(targetColumn => allowNulls || allFields[targetColumn] != null)
    .map(targetColumn => ({
      targetColumn,
      exprType: 'value',
-      value: item.fields[targetColumn],
+      value: allFields[targetColumn],
    }));
 }

@@ -243,17 +254,19 @@ function changeSetInsertToSql(
  item: ChangeSetItem,
  dbinfo: DatabaseInfo = null
 ): [AllowIdentityInsert, Insert, AllowIdentityInsert] {
-  const fields = extractFields(item, false);
+  const table = dbinfo?.tables?.find(x => x.schemaName == item.schemaName && x.pureName == item.pureName);
+  const fields = extractFields(
+    item,
+    false,
+    table?.columns?.map(x => x.columnName)
+  );
  if (fields.length == 0) return null;
  let autoInc = false;
-  if (dbinfo) {
-    const table = dbinfo.tables.find(x => x.schemaName == item.schemaName && x.pureName == item.pureName);
-    if (table) {
-      const autoIncCol = table.columns.find(x => x.autoIncrement);
-      // console.log('autoIncCol', autoIncCol);
-      if (autoIncCol && fields.find(x => x.targetColumn == autoIncCol.columnName)) {
-        autoInc = true;
-      }
+  if (table) {
+    const autoIncCol = table.columns.find(x => x.autoIncrement);
+    // console.log('autoIncCol', autoIncCol);
+    if (autoIncCol && fields.find(x => x.targetColumn == autoIncCol.columnName)) {
+      autoInc = true;
    }
  }
  const targetTable = {
@@ -272,6 +285,9 @@ function changeSetInsertToSql(
      targetTable,
      commandType: 'insert',
      fields,
+      insertWhereNotExistsCondition: item.insertIfNotExistsFields
+        ? compileSimpleChangeSetCondition(item.insertIfNotExistsFields)
+        : null,
    },
    autoInc
      ? {
@@ -320,7 +336,41 @@ export function extractChangeSetCondition(item: ChangeSetItem, alias?: string):
  };
 }

-function changeSetUpdateToSql(item: ChangeSetItem): Update {
+function compileSimpleChangeSetCondition(fields: { [column: string]: string }): Condition {
+  function getColumnCondition(columnName: string): Condition {
+    const value = fields[columnName];
+    const expr: Expression = {
+      exprType: 'column',
+      columnName,
+    };
+    if (value == null) {
+      return {
+        conditionType: 'isNull',
+        expr,
+      };
+    } else {
+      return {
+        conditionType: 'binary',
+        operator: '=',
+        left: expr,
+        right: {
+          exprType: 'value',
+          value,
+        },
+      };
+    }
+  }
+  return {
+    conditionType: 'and',
+    conditions: _.keys(fields).map(columnName => getColumnCondition(columnName)),
+  };
+}
+
+function changeSetUpdateToSql(item: ChangeSetItem, dbinfo: DatabaseInfo = null): Update {
+  const table = dbinfo?.tables?.find(x => x.schemaName == item.schemaName && x.pureName == item.pureName);
+
+  const autoIncCol = table?.columns?.find(x => x.autoIncrement);
+
  return {
    from: {
      name: {
@@ -329,7 +379,11 @@ function changeSetUpdateToSql(item: ChangeSetItem): Update {
      },
    },
    commandType: 'update',
-    fields: extractFields(item),
+    fields: extractFields(
+      item,
+      true,
+      table?.columns?.map(x => x.columnName).filter(x => x != autoIncCol?.columnName)
+    ),
    where: extractChangeSetCondition(item),
  };
 }
@@ -351,7 +405,7 @@ export function changeSetToSql(changeSet: ChangeSet, dbinfo: DatabaseInfo): Comm
  return _.compact(
    _.flatten([
      ...(changeSet.inserts.map(item => changeSetInsertToSql(item, dbinfo)) as any),
-      ...changeSet.updates.map(changeSetUpdateToSql),
+      ...changeSet.updates.map(item => changeSetUpdateToSql(item, dbinfo)),
      ...changeSet.deletes.map(changeSetDeleteToSql),
    ])
  );
@@ -446,7 +500,12 @@ export function changeSetInsertNewRow(changeSet: ChangeSet, name?: NamedObjectIn
  };
 }

-export function changeSetInsertDocuments(changeSet: ChangeSet, documents: any[], name?: NamedObjectInfo): ChangeSet {
+export function changeSetInsertDocuments(
+  changeSet: ChangeSet,
+  documents: any[],
+  name?: NamedObjectInfo,
+  insertIfNotExistsFieldNames?: string[]
+): ChangeSet {
  const insertedRows = getChangeSetInsertedRows(changeSet, name);
  return {
    ...changeSet,
@@ -456,6 +515,7 @@ export function changeSetInsertDocuments(changeSet: ChangeSet, documents: any[],
        ...name,
        insertedRowIndex: insertedRows.length + index,
        fields: doc,
+        insertIfNotExistsFields: insertIfNotExistsFieldNames ? _.pick(doc, insertIfNotExistsFieldNames) : null,
      })),
    ],
  };
@@ -472,3 +532,7 @@ export function changeSetContainsChanges(changeSet: ChangeSet) {
    changeSet.dataUpdateCommands?.length > 0
  );
 }
+
+export function changeSetChangedCount(changeSet: ChangeSet) {
+  return changeSet.deletes.length + changeSet.updates.length + changeSet.inserts.length;
+}
@@ -2,6 +2,7 @@ import _ from 'lodash';
 import { GridDisplay, ChangeCacheFunc, ChangeConfigFunc, DisplayColumn } from './GridDisplay';
 import type { EngineDriver, ViewInfo, ColumnInfo, CollectionInfo } from 'dbgate-types';
 import { GridConfig, GridCache } from './GridConfig';
+import { mongoFilterBehaviour, standardFilterBehaviours } from 'dbgate-tools';

 function getObjectKeys(obj) {
  if (_.isArray(obj)) {
@@ -52,7 +53,7 @@ function getColumnsForObject(basePath, obj, res: any[], display) {
  }
 }

-function getDisplayColumn(basePath, columnName, display) {
+function getDisplayColumn(basePath, columnName, display: CollectionGridDisplay) {
  const uniquePath = [...basePath, columnName];
  const uniqueName = uniquePath.join('.');
  return {
@@ -62,9 +63,13 @@ function getDisplayColumn(basePath, columnName, display) {
    uniquePath,
    isStructured: true,
    parentHeaderText: createHeaderText(basePath),
-    filterType: 'mongo',
+    filterBehaviour: display?.driver?.getFilterBehaviour(null, standardFilterBehaviours) ?? mongoFilterBehaviour,
    pureName: display.collection?.pureName,
    schemaName: display.collection?.schemaName,
+
+    isPartitionKey: !!display?.collection?.partitionKey?.find(x => x.columnName == uniqueName),
+    isClusterKey: !!display?.collection?.clusterKey?.find(x => x.columnName == uniqueName),
+    isUniqueKey: !!display?.collection?.uniqueKey?.find(x => x.columnName == uniqueName),
  };
 }

@@ -95,7 +100,7 @@ export class CollectionGridDisplay extends GridDisplay {
    cache: GridCache,
    setCache: ChangeCacheFunc,
    loadedRows,
-    changeSet, 
+    changeSet,
    readOnly = false
  ) {
    super(config, setConfig, cache, setCache, driver);
@@ -104,10 +109,10 @@ export class CollectionGridDisplay extends GridDisplay {
    this.columns = analyseCollectionDisplayColumns([...(loadedRows || []), ...changedDocs, ...insertedDocs], this);
    this.filterable = true;
    this.sortable = true;
-    this.editable = !readOnly;
+    this.editable = !readOnly && collection?.uniqueKey?.length > 0;
    this.supportsReload = true;
    this.isDynamicStructure = true;
-    this.changeSetKeyFields = ['_id'];
+    this.changeSetKeyFields = collection?.uniqueKey?.map(x => x.columnName);
    this.baseCollection = collection;
  }
 }
@@ -0,0 +1,91 @@
+import _ from 'lodash';
+import { filterName, isTableColumnUnique } from 'dbgate-tools';
+import { GridDisplay, ChangeCacheFunc, DisplayColumn, DisplayedColumnInfo, ChangeConfigFunc } from './GridDisplay';
+import type {
+  TableInfo,
+  EngineDriver,
+  ViewInfo,
+  ColumnInfo,
+  NamedObjectInfo,
+  DatabaseInfo,
+  ForeignKeyInfo,
+} from 'dbgate-types';
+import { GridConfig, GridCache, createGridCache } from './GridConfig';
+import { Expression, Select, treeToSql, dumpSqlSelect, ColumnRefExpression, Condition } from 'dbgate-sqltree';
+
+export interface CustomGridColumn {
+  columnName: string;
+  columnLabel: string;
+  isPrimaryKey?: boolean;
+}
+
+export class CustomGridDisplay extends GridDisplay {
+  customColumns: CustomGridColumn[];
+
+  constructor(
+    public tableName: NamedObjectInfo,
+    columns: CustomGridColumn[],
+    driver: EngineDriver,
+    config: GridConfig,
+    setConfig: ChangeConfigFunc,
+    cache: GridCache,
+    setCache: ChangeCacheFunc,
+    dbinfo: DatabaseInfo,
+    serverVersion,
+    isReadOnly = false,
+    public additionalcondition: Condition = null
+  ) {
+    super(config, setConfig, cache, setCache, driver, dbinfo, serverVersion);
+
+    this.customColumns = columns;
+
+    this.columns = columns.map(col => ({
+      columnName: col.columnName,
+      headerText: col.columnLabel,
+      uniqueName: col.columnName,
+      uniquePath: [col.columnName],
+      isPrimaryKey: col.isPrimaryKey,
+      isForeignKeyUnique: false,
+      schemaName: tableName.schemaName,
+      pureName: tableName.pureName,
+    }));
+
+    this.changeSetKeyFields = columns.filter(x => x.isPrimaryKey).map(x => x.columnName);
+    this.baseTable = {
+      ...tableName,
+      columns: this.columns.map(x => ({ ...tableName, columnName: x.columnName, dataType: 'string' })),
+      foreignKeys: [],
+    };
+
+    this.filterable = true;
+    this.sortable = true;
+    this.groupable = false;
+    this.editable = !isReadOnly;
+    this.supportsReload = true;
+  }
+
+  createSelect(options = {}) {
+    const select = this.createSelectBase(
+      this.tableName,
+      [],
+      // @ts-ignore
+      // this.columns.map(col => ({
+      //   columnName: col.columnName,
+      // })),
+      options,
+      this.customColumns.find(x => x.isPrimaryKey)?.columnName
+    );
+    select.selectAll = true;
+    if (this.additionalcondition) {
+      if (select.where) {
+        select.where = {
+          conditionType: 'and',
+          conditions: [select.where, this.additionalcondition],
+        };
+      } else {
+        select.where = this.additionalcondition;
+      }
+    }
+    return select;
+  }
+}
@@ -10,12 +10,13 @@ import type {
  CollectionInfo,
  SqlDialect,
  ViewInfo,
+  FilterBehaviour,
 } from 'dbgate-types';
-import { parseFilter, getFilterType } from 'dbgate-filterparser';
+import { parseFilter } from 'dbgate-filterparser';
 import { filterName } from 'dbgate-tools';
 import { ChangeSetFieldDefinition, ChangeSetRowDefinition } from './ChangeSet';
 import { Expression, Select, treeToSql, dumpSqlSelect, Condition, CompoudCondition } from 'dbgate-sqltree';
-import { isTypeLogical } from 'dbgate-tools';
+import { isTypeLogical, standardFilterBehaviours, detectSqlFilterBehaviour, stringFilterBehaviour } from 'dbgate-tools';

 export interface DisplayColumn {
  schemaName: string;
@@ -27,13 +28,19 @@ export interface DisplayColumn {
  notNull?: boolean;
  autoIncrement?: boolean;
  isPrimaryKey?: boolean;
+
+  // NoSQL specific
+  isPartitionKey?: boolean;
+  isClusterKey?: boolean;
+  isUniqueKey?: boolean;
+
  foreignKey?: ForeignKeyInfo;
  isForeignKeyUnique?: boolean;
  isExpandable?: boolean;
  isChecked?: boolean;
  hintColumnNames?: string[];
  dataType?: string;
-  filterType?: boolean;
+  filterBehaviour?: FilterBehaviour;
  isStructured?: boolean;
 }

@@ -92,20 +99,26 @@ export abstract class GridDisplay {
  isLoadedCorrectly = true;
  supportsReload = false;
  isDynamicStructure = false;
-  filterTypeOverride = null;
+  filterBehaviourOverride = null;

  setColumnVisibility(uniquePath: string[], isVisible: boolean) {
    const uniqueName = uniquePath.join('.');
    if (uniquePath.length == 1) {
-      this.includeInColumnSet('hiddenColumns', uniqueName, !isVisible);
+      this.includeInColumnSet([
+        { field: 'hiddenColumns', uniqueName, isIncluded: !isVisible },
+        isVisible == false && this.isDynamicStructure && { field: 'addedColumns', uniqueName, isIncluded: false },
+      ]);
    } else {
-      this.includeInColumnSet('addedColumns', uniqueName, isVisible);
+      this.includeInColumnSet([{ field: 'addedColumns', uniqueName, isIncluded: isVisible }]);
      if (!this.isDynamicStructure) this.reload();
    }
  }

  addDynamicColumn(name: string) {
-    this.includeInColumnSet('addedColumns', name, true);
+    this.includeInColumnSet([
+      { field: 'addedColumns', uniqueName: name, isIncluded: true },
+      { field: 'hiddenColumns', uniqueName: name, isIncluded: false },
+    ]);
  }

  focusColumns(uniqueNames: string[]) {
@@ -143,19 +156,30 @@ export abstract class GridDisplay {
    this.setCache(reloadDataCacheFunc);
  }

-  includeInColumnSet(field: keyof GridConfigColumns, uniqueName: string, isIncluded: boolean) {
-    // console.log('includeInColumnSet', field, uniqueName, isIncluded);
-    if (isIncluded) {
-      this.setConfig(cfg => ({
-        ...cfg,
-        [field]: [...(cfg[field] || []), uniqueName],
-      }));
-    } else {
-      this.setConfig(cfg => ({
-        ...cfg,
-        [field]: (cfg[field] || []).filter(x => x != uniqueName),
-      }));
-    }
+  includeInColumnSet(
+    modifications: ({ field: keyof GridConfigColumns; uniqueName: string; isIncluded: boolean } | null)[]
+  ) {
+    this.setConfig(cfg => {
+      let res = cfg;
+      for (const modification of modifications) {
+        if (!modification) {
+          continue;
+        }
+        const { field, uniqueName, isIncluded } = modification;
+        if (isIncluded) {
+          res = {
+            ...res,
+            [field]: [...(cfg[field] || []), uniqueName],
+          };
+        } else {
+          res = {
+            ...res,
+            [field]: (cfg[field] || []).filter(x => x != uniqueName),
+          };
+        }
+      }
+      return res;
+    });
  }

  showAllColumns() {
@@ -192,12 +216,16 @@ export abstract class GridDisplay {
      const column = displayedColumnInfo[uniqueName];
      if (!column) continue;
      try {
-        const condition = parseFilter(filter, getFilterType(column.dataType));
+        const condition = parseFilter(
+          filter,
+          this.driver?.getFilterBehaviour(column.dataType, standardFilterBehaviours) ??
+            detectSqlFilterBehaviour(column.dataType)
+        );
        if (condition) {
          conditions.push(
            _.cloneDeepWith(condition, (expr: Expression) => {
              if (expr.exprType == 'placeholder') {
-                return this.createColumnExpression(column, { alias: column.sourceAlias });
+                return this.createColumnExpression(column, { alias: column.sourceAlias }, undefined, 'filter');
              }
              // return {
              //   exprType: 'column',
@@ -220,12 +248,12 @@ export abstract class GridDisplay {
      };
      for (const column of this.baseTableOrView.columns) {
        try {
-          const condition = parseFilter(this.config.multiColumnFilter, getFilterType(column.dataType));
+          const condition = parseFilter(this.config.multiColumnFilter, detectSqlFilterBehaviour(column.dataType));
          if (condition) {
            orCondition.conditions.push(
              _.cloneDeepWith(condition, (expr: Expression) => {
                if (expr.exprType == 'placeholder') {
-                  return this.createColumnExpression(column, { alias: 'basetbl' });
+                  return this.createColumnExpression(column, { alias: 'basetbl' }, undefined, 'filter');
                }
              })
            );
@@ -344,7 +372,9 @@ export abstract class GridDisplay {
  }

  toggleExpandedColumn(uniqueName: string, value?: boolean) {
-    this.includeInColumnSet('expandedColumns', uniqueName, value == null ? !this.isExpandedColumn(uniqueName) : value);
+    this.includeInColumnSet([
+      { field: 'expandedColumns', uniqueName, isIncluded: value == null ? !this.isExpandedColumn(uniqueName) : value },
+    ]);
  }

  getFilter(uniqueName: string) {
@@ -540,10 +570,10 @@ export abstract class GridDisplay {

  processReferences(select: Select, displayedColumnInfo: DisplayedColumnInfo, options) {}

-  createColumnExpression(col, source, alias?) {
+  createColumnExpression(col, source, alias?, purpose: 'view' | 'filter' = 'view') {
    let expr = null;
    if (this.dialect.createColumnViewExpression) {
-      expr = this.dialect.createColumnViewExpression(col.columnName, col.dataType, source, alias);
+      expr = this.dialect.createColumnViewExpression(col.columnName, col.dataType, source, alias, purpose);
      if (expr) {
        return expr;
      }
@@ -556,16 +586,16 @@ export abstract class GridDisplay {
    };
  }

-  createSelectBase(name: NamedObjectInfo, columns: ColumnInfo[], options) {
+  createSelectBase(name: NamedObjectInfo, columns: ColumnInfo[], options, defaultOrderColumnName?: string) {
    if (!columns) return null;
-    const orderColumnName = columns[0].columnName;
+    const orderColumnName = defaultOrderColumnName ?? columns[0]?.columnName;
    const select: Select = {
      commandType: 'select',
      from: {
        name: _.pick(name, ['schemaName', 'pureName']),
        alias: 'basetbl',
      },
-      columns: columns.map(col => this.createColumnExpression(col, { alias: 'basetbl' })),
+      columns: columns.map(col => this.createColumnExpression(col, { alias: 'basetbl' }, undefined, 'view')),
      orderBy: this.driver?.requiresDefaultSortCriteria
        ? [
            {
@@ -595,7 +625,7 @@ export abstract class GridDisplay {
      columns: [
        ...select.columns,
        {
-          alias: '_rowNumber',
+          alias: '_RowNumber',
          exprType: 'rowNumber',
          orderBy: select.orderBy
            ? select.orderBy.map(x =>
@@ -653,7 +683,7 @@ export abstract class GridDisplay {
    let select = this.createSelect();
    if (!select) return null;
    if (this.dialect.rangeSelect) select.range = { offset: offset, limit: count };
-    else if (this.dialect.rowNumberOverPaging && offset > 0)
+    else if (this.dialect.rowNumberOverPaging && (offset > 0 || !this.dialect.topRecords))
      select = this.getRowNumberOverSelect(select, offset, count);
    else if (this.dialect.limitSelect) select.topRecords = count;
    return select;
@@ -734,6 +764,7 @@ export abstract class GridDisplay {
          alias: 'count',
        },
      ];
+      select.selectAll = false;
    }
    return select;
    // const sql = treeToSql(this.driver, select, dumpSqlSelect);
@@ -747,10 +778,12 @@ export abstract class GridDisplay {
    for (const name in filters) {
      const column = this.isDynamicStructure ? null : this.columns.find(x => x.columnName == name);
      if (!this.isDynamicStructure && !column) continue;
-      const filterType =
-        this.filterTypeOverride ?? (this.isDynamicStructure ? 'mongo' : getFilterType(column.dataType));
+      const filterBehaviour =
+        this.filterBehaviourOverride ??
+        this.driver?.getFilterBehaviour(column.dataType, standardFilterBehaviours) ??
+        detectSqlFilterBehaviour(column.dataType);
      try {
-        const condition = parseFilter(filters[name], filterType);
+        const condition = parseFilter(filters[name], filterBehaviour);
        const replaced = _.cloneDeepWith(condition, (expr: Expression) => {
          if (expr.exprType == 'placeholder')
            return {
@@ -765,7 +798,7 @@ export abstract class GridDisplay {
    }

    if (this.config.multiColumnFilter) {
-      const placeholderCondition = parseFilter(this.config.multiColumnFilter, 'string');
+      const placeholderCondition = parseFilter(this.config.multiColumnFilter, stringFilterBehaviour);
      if (placeholderCondition) {
        conditions.push({
          conditionType: 'anyColumnPass',
@@ -2,6 +2,7 @@ import _ from 'lodash';
 import { GridDisplay, ChangeCacheFunc, ChangeConfigFunc } from './GridDisplay';
 import { GridConfig, GridCache } from './GridConfig';
 import { analyseCollectionDisplayColumns } from './CollectionGridDisplay';
+import { evalFilterBehaviour } from 'dbgate-tools';

 export class JslGridDisplay extends GridDisplay {
  constructor(
@@ -22,7 +23,7 @@ export class JslGridDisplay extends GridDisplay {
    this.sortable = true;
    this.supportsReload = supportsReload;
    this.isDynamicStructure = isDynamicStructure;
-    this.filterTypeOverride = 'eval';
+    this.filterBehaviourOverride = evalFilterBehaviour;
    this.editable = editable;
    this.editableStructure = editable ? structure : null;

@@ -105,7 +105,6 @@ export class PerspectiveCache {
        'databaseConfig',
        'orderBy',
        'sqlCondition',
-        'mongoCondition',
      ])
    );
    let res = this.tables[tableKey];
@@ -1,4 +1,4 @@
-import type { DatabaseInfo, ForeignKeyInfo, NamedObjectInfo, TableInfo } from 'dbgate-types';
+import type { DatabaseInfo, FilterBehaviour, ForeignKeyInfo, NamedObjectInfo, TableInfo } from 'dbgate-types';
 import uuidv1 from 'uuid/v1';

 // export interface PerspectiveConfigColumns {
@@ -31,7 +31,7 @@ export interface PerspectiveCustomJoinConfig {

 export interface PerspectiveFilterColumnInfo {
  columnName: string;
-  filterType: string;
+  filterBehaviour: FilterBehaviour;
  pureName: string;
  schemaName: string;
  foreignKey: ForeignKeyInfo;
@@ -5,6 +5,7 @@ import _zipObject from 'lodash/zipObject';
 import _mapValues from 'lodash/mapValues';
 import _isArray from 'lodash/isArray';
 import { safeJsonParse } from 'dbgate-tools';
+import { CollectionAggregateDefinition } from 'dbgate-types';

 function normalizeLoadedRow(row) {
  return _mapValues(row, v => safeJsonParse(v) || v);
@@ -59,24 +60,6 @@ export class PerspectiveDataLoader {
      : null;
  }

-  buildMongoCondition(props: PerspectiveDataLoadProps): {} {
-    const { schemaName, pureName, bindingColumns, bindingValues, dataColumns, orderBy, mongoCondition } = props;
-
-    const conditions = [];
-
-    if (mongoCondition) {
-      conditions.push(mongoCondition);
-    }
-
-    if (bindingColumns?.length == 1) {
-      conditions.push({
-        [bindingColumns[0]]: { $in: bindingValues.map(x => x[0]) },
-      });
-    }
-
-    return conditions.length == 1 ? conditions[0] : conditions.length > 0 ? { $and: conditions } : null;
-  }
-
  async loadGroupingSqlDb(props: PerspectiveDataLoadProps) {
    const { schemaName, pureName, bindingColumns } = props;

@@ -135,18 +118,28 @@ export class PerspectiveDataLoader {
  async loadGroupingDocDb(props: PerspectiveDataLoadProps) {
    const { schemaName, pureName, bindingColumns } = props;

-    const aggregate = [
-      { $match: this.buildMongoCondition(props) },
-      {
-        $group: {
-          _id: _zipObject(
-            bindingColumns,
-            bindingColumns.map(col => '$' + col)
-          ),
-          count: { $sum: 1 },
+    const aggregate: CollectionAggregateDefinition = {
+      condition: this.buildSqlCondition(props),
+      groupByColumns: bindingColumns,
+      aggregateColumns: [
+        {
+          alias: 'pergrpsize',
+          aggregateFunction: 'count',
        },
-      },
-    ];
+      ],
+    };
+    // const aggregate = [
+    //   { $match: this.buildMongoCondition(props) },
+    //   {
+    //     $group: {
+    //       _id: _zipObject(
+    //         bindingColumns,
+    //         bindingColumns.map(col => '$' + col)
+    //       ),
+    //       count: { $sum: 1 },
+    //     },
+    //   },
+    // ];

    if (dbg?.enabled) {
      dbg(`LOAD COUNTS, table=${props.pureName}, columns=${bindingColumns?.join(',')}`);
@@ -163,8 +156,8 @@ export class PerspectiveDataLoader {

    if (response.errorMessage) return response;
    return response.rows.map(row => ({
-      ...row._id,
-      _perspective_group_size_: parseInt(row.count),
+      ...row,
+      _perspective_group_size_: parseInt(row.pergrpsize),
    }));
  }

@@ -244,16 +237,23 @@ export class PerspectiveDataLoader {
    const { pureName } = props;
    const res: any = {
      pureName,
-      condition: this.buildMongoCondition(props),
+      condition: this.buildSqlCondition(props),
+      sort:
+        useSort && props.orderBy?.length > 0
+          ? props.orderBy.map(x => ({
+              ...x,
+              direction: x.order,
+            }))
+          : undefined,
      skip: props.range?.offset,
      limit: props.range?.limit,
    };
-    if (useSort && props.orderBy?.length > 0) {
-      res.sort = _zipObject(
-        props.orderBy.map(col => col.columnName),
-        props.orderBy.map(col => (col.order == 'DESC' ? -1 : 1))
-      );
-    }
+    // if (useSort && props.orderBy?.length > 0) {
+    //   res.sort = _zipObject(
+    //     props.orderBy.map(col => col.columnName),
+    //     props.orderBy.map(col => (col.order == 'DESC' ? -1 : 1))
+    //   );
+    // }

    return res;
  }
@@ -25,7 +25,6 @@ export interface PerspectiveDataLoadProps {
  range?: RangeDefinition;
  topCount?: number;
  sqlCondition?: Condition;
-  mongoCondition?: any;
  engineType: PerspectiveDatabaseEngineType;
 }

@@ -2,13 +2,22 @@ import type {
  CollectionInfo,
  ColumnInfo,
  DatabaseInfo,
+  FilterBehaviour,
  ForeignKeyInfo,
  NamedObjectInfo,
  RangeDefinition,
  TableInfo,
  ViewInfo,
 } from 'dbgate-types';
-import { equalFullName, isCollectionInfo, isTableInfo, isViewInfo } from 'dbgate-tools';
+import {
+  detectSqlFilterBehaviour,
+  equalFullName,
+  isCollectionInfo,
+  isTableInfo,
+  isViewInfo,
+  mongoFilterBehaviour,
+  stringFilterBehaviour,
+} from 'dbgate-tools';
 import {
  ChangePerspectiveConfigFunc,
  createPerspectiveNodeConfig,
@@ -33,8 +42,7 @@ import _cloneDeepWith from 'lodash/cloneDeepWith';
 import _findIndex from 'lodash/findIndex';
 import { PerspectiveDataLoadProps, PerspectiveDataProvider } from './PerspectiveDataProvider';
 import stableStringify from 'json-stable-stringify';
-import { getFilterType, parseFilter } from 'dbgate-filterparser';
-import { FilterType } from 'dbgate-filterparser/lib/types';
+import { parseFilter } from 'dbgate-filterparser';
 import { CompoudCondition, Condition, Expression, Select } from 'dbgate-sqltree';
 // import { getPerspectiveDefaultColumns } from './getPerspectiveDefaultColumns';
 import uuidv1 from 'uuid/v1';
@@ -197,8 +205,8 @@ export abstract class PerspectiveTreeNode {
  get columnTitle() {
    return this.title;
  }
-  get filterType(): FilterType {
-    return 'string';
+  get filterBehaviour(): FilterBehaviour {
+    return stringFilterBehaviour;
  }
  get columnName() {
    return null;
@@ -341,12 +349,27 @@ export abstract class PerspectiveTreeNode {
    );
  }

+  getMutliColumnCondition(source): Condition {
+    if (!this.nodeConfig?.multiColumnFilter) return null;
+
+    const base = this.getBaseTableFromThis() as TableInfo | ViewInfo | CollectionInfo;
+    if (!base) return null;
+
+    const isDocDb = isCollectionInfo(base);
+    if (isDocDb) {
+      return this.getMutliColumnNoSqlCondition();
+    } else {
+      return this.getMutliColumnSqlCondition(source);
+    }
+  }
+
  getMutliColumnSqlCondition(source): Condition {
    if (!this.nodeConfig?.multiColumnFilter) return null;
+
    const base = this.getBaseTableFromThis() as TableInfo | ViewInfo;
    if (!base) return null;
    try {
-      const condition = parseFilter(this.nodeConfig?.multiColumnFilter, 'string');
+      const condition = parseFilter(this.nodeConfig?.multiColumnFilter, stringFilterBehaviour);
      if (condition) {
        const orCondition: CompoudCondition = {
          conditionType: 'or',
@@ -375,32 +398,40 @@ export abstract class PerspectiveTreeNode {
    return null;
  }

-  getMutliColumnMongoCondition(): {} {
+  getMutliColumnNoSqlCondition(): Condition {
    if (!this.nodeConfig?.multiColumnFilter) return null;
    const pattern = this.dataProvider?.dataPatterns?.[this.designerId];
    if (!pattern) return null;

-    const condition = parseFilter(this.nodeConfig?.multiColumnFilter, 'mongo');
+    const condition = parseFilter(this.nodeConfig?.multiColumnFilter, mongoFilterBehaviour);
    if (!condition) return null;
-    const res = pattern.columns.map(col => {
-      return _cloneDeepWith(condition, expr => {
-        if (expr.__placeholder__) {
-          return {
-            [col.name]: expr.__placeholder__,
-          };
-        }
-      });
-    });
-    return {
-      $or: res,
+
+    const orCondition: CompoudCondition = {
+      conditionType: 'or',
+      conditions: [],
    };
+    for (const column of pattern.columns || []) {
+      orCondition.conditions.push(
+        _cloneDeepWith(condition, (expr: Expression) => {
+          if (expr.exprType == 'placeholder') {
+            return {
+              exprType: 'column',
+              columnName: column.name,
+            };
+          }
+        })
+      );
+    }
+    if (orCondition.conditions.length > 0) {
+      return orCondition;
+    }
  }

  getChildrenSqlCondition(source = null): Condition {
    const conditions = _compact([
      ...this.childNodes.map(x => x.parseFilterCondition(source)),
      ...this.buildParentFilterConditions(),
-      this.getMutliColumnSqlCondition(source),
+      this.getMutliColumnCondition(source),
    ]);
    if (conditions.length == 0) {
      return null;
@@ -414,20 +445,6 @@ export abstract class PerspectiveTreeNode {
    };
  }

-  getChildrenMongoCondition(source = null): {} {
-    const conditions = _compact([
-      ...this.childNodes.map(x => x.parseFilterCondition(source)),
-      this.getMutliColumnMongoCondition(),
-    ]);
-    if (conditions.length == 0) {
-      return null;
-    }
-    if (conditions.length == 1) {
-      return conditions[0];
-    }
-    return { $and: conditions };
-  }
-
  getOrderBy(table: TableInfo | ViewInfo | CollectionInfo): PerspectiveDataLoadProps['orderBy'] {
    const res = _compact(
      this.childNodes.map(node => {
@@ -446,6 +463,8 @@ export abstract class PerspectiveTreeNode {
      order: 'ASC' as 'ASC',
    }));
    if (pkColumns) return pkColumns;
+    const uqColumns = (table as CollectionInfo)?.uniqueKey;
+    if (uqColumns?.length >= 1) return uqColumns.map(x => ({ columnName: x.columnName, order: 'ASC' }));
    const columns = (table as TableInfo | ViewInfo)?.columns;
    if (columns) return [{ columnName: columns[0].columnName, order: 'ASC' }];
    return [{ columnName: '_id', order: 'ASC' }];
@@ -714,8 +733,8 @@ export class PerspectiveTableColumnNode extends PerspectiveTreeNode {
    return true;
  }

-  get filterType(): FilterType {
-    return getFilterType(this.column.dataType);
+  get filterBehaviour(): FilterBehaviour {
+    return detectSqlFilterBehaviour(this.column.dataType);
  }

  get isCircular() {
@@ -767,7 +786,7 @@ export class PerspectiveTableColumnNode extends PerspectiveTreeNode {
  get filterInfo(): PerspectiveFilterColumnInfo {
    return {
      columnName: this.columnName,
-      filterType: this.filterType,
+      filterBehaviour: this.filterBehaviour,
      pureName: this.column.pureName,
      schemaName: this.column.schemaName,
      foreignKey: this.foreignKey,
@@ -777,7 +796,7 @@ export class PerspectiveTableColumnNode extends PerspectiveTreeNode {
  parseFilterCondition(source = null): Condition {
    const filter = this.getFilter();
    if (!filter) return null;
-    const condition = parseFilter(filter, this.filterType);
+    const condition = parseFilter(filter, this.filterBehaviour);
    if (!condition) return null;
    return _cloneDeepWith(condition, (expr: Expression) => {
      if (expr.exprType == 'placeholder') {
@@ -949,9 +968,9 @@ export class PerspectivePatternColumnNode extends PerspectiveTreeNode {
    return !this.isChildColumn;
  }

-  get filterType(): FilterType {
-    if (this.tableColumn) return getFilterType(this.tableColumn.dataType);
-    return 'mongo';
+  get filterBehaviour(): FilterBehaviour {
+    if (this.tableColumn) return detectSqlFilterBehaviour(this.tableColumn.dataType);
+    return mongoFilterBehaviour;
  }

  get preloadedLevelData() {
@@ -1083,7 +1102,7 @@ export class PerspectivePatternColumnNode extends PerspectiveTreeNode {

    return {
      columnName: this.columnName,
-      filterType: this.filterType,
+      filterBehaviour: this.filterBehaviour,
      pureName: this.table.pureName,
      schemaName: this.table.schemaName,
      foreignKey: this.foreignKey,
@@ -1093,7 +1112,7 @@ export class PerspectivePatternColumnNode extends PerspectiveTreeNode {
  parseFilterCondition(source = null): {} {
    const filter = this.getFilter();
    if (!filter) return null;
-    const condition = parseFilter(filter, 'mongo');
+    const condition = parseFilter(filter, mongoFilterBehaviour);
    if (!condition) return null;
    return _cloneDeepWith(condition, expr => {
      if (expr.__placeholder__) {
@@ -1150,17 +1169,16 @@ export class PerspectiveTableNode extends PerspectiveTreeNode {
  }

  getNodeLoadProps(parentRows: any[]): PerspectiveDataLoadProps {
-    const isMongo = isCollectionInfo(this.table);
+    const isDocDb = isCollectionInfo(this.table);
    return {
      schemaName: this.table.schemaName,
      pureName: this.table.pureName,
      dataColumns: this.getDataLoadColumns(),
-      allColumns: isMongo,
+      allColumns: isDocDb,
      databaseConfig: this.databaseConfig,
      orderBy: this.getOrderBy(this.table),
-      sqlCondition: isMongo ? null : this.getChildrenSqlCondition(),
-      mongoCondition: isMongo ? this.getChildrenMongoCondition() : null,
-      engineType: isMongo ? 'docdb' : 'sqldb',
+      sqlCondition: this.getChildrenSqlCondition(),
+      engineType: isDocDb ? 'docdb' : 'sqldb',
    };
  }

@@ -1364,7 +1382,7 @@ export class PerspectiveCustomJoinTreeNode extends PerspectiveTableNode {
    // console.log('PARENT ROWS', parentRows);

    // console.log('this.getDataLoadColumns()', this.getDataLoadColumns());
-    const isMongo = isCollectionInfo(this.table);
+    const isDocDb = isCollectionInfo(this.table);

    // const bindingValues = [];

@@ -1424,12 +1442,12 @@ export class PerspectiveCustomJoinTreeNode extends PerspectiveTableNode {
      bindingColumns: this.getParentMatchColumns(),
      bindingValues: _uniqBy(bindingValues, x => JSON.stringify(x)),
      dataColumns: this.getDataLoadColumns(),
-      allColumns: isMongo,
+      allColumns: isDocDb,
      databaseConfig: this.databaseConfig,
      orderBy: this.getOrderBy(this.table),
-      sqlCondition: isMongo ? null : this.getChildrenSqlCondition(),
-      mongoCondition: isMongo ? this.getChildrenMongoCondition() : null,
-      engineType: isMongo ? 'docdb' : 'sqldb',
+      sqlCondition: this.getChildrenSqlCondition(),
+      // mongoCondition: isMongo ? this.getChildrenMongoCondition() : null,
+      engineType: isDocDb ? 'docdb' : 'sqldb',
    };
  }

@@ -63,7 +63,7 @@ export class TableGridDisplay extends GridDisplay {
        ? this.table.primaryKey.columns.map(x => x.columnName)
        : this.table.columns.map(x => x.columnName);
    }
-    
+
    if (this.config.isFormView) {
      this.addAllExpandedColumnsToSelected = true;
      this.hintBaseColumns = this.formColumns;
@@ -287,7 +287,7 @@ export class TableGridDisplay extends GridDisplay {
    for (const column of columns) {
      if (this.addAllExpandedColumnsToSelected || this.config.addedColumns.includes(column.uniqueName)) {
        select.columns.push(
-          this.createColumnExpression(column, { name: column, alias: parentAlias }, column.uniqueName)
+          this.createColumnExpression(column, { name: column, alias: parentAlias }, column.uniqueName, 'view')
        );
        displayedColumnInfo[column.uniqueName] = {
          ...column,
@@ -1,6 +1,6 @@
 import _ from 'lodash';
 import { GridDisplay, ChangeCacheFunc, ChangeConfigFunc } from './GridDisplay';
-import type { EngineDriver, ViewInfo, ColumnInfo } from 'dbgate-types';
+import type { EngineDriver, ViewInfo, ColumnInfo, DatabaseInfo } from 'dbgate-types';
 import { GridConfig, GridCache } from './GridConfig';

 export class ViewGridDisplay extends GridDisplay {
@@ -11,9 +11,10 @@ export class ViewGridDisplay extends GridDisplay {
    setConfig: ChangeConfigFunc,
    cache: GridCache,
    setCache: ChangeCacheFunc,
+    dbinfo: DatabaseInfo,
    serverVersion
  ) {
-    super(config, setConfig, cache, setCache, driver, serverVersion);
+    super(config, setConfig, cache, setCache, driver, dbinfo, serverVersion);
    this.columns = this.getDisplayColumns(view);
    this.formColumns = this.columns;
    this.filterable = true;
@@ -21,3 +21,4 @@ export * from './perspectiveTools';
 export * from './DataDuplicator';
 export * from './FreeTableGridDisplay';
 export * from './FreeTableModel';
+export * from './CustomGridDisplay';
@@ -26,6 +26,8 @@ async function runAndExit(promise) {
  }
 }

+program.version(dbgateApi.currentVersion.version);
+
 program
  .option('-s, --server <server>', 'server host')
  .option('-u, --user <user>', 'user name')
@@ -36,7 +38,8 @@ program
    '--load-data-condition <condition>',
    'regex, which table data will be loaded and stored in model (in load command)'
  )
-  .requiredOption('-e, --engine <engine>', 'engine name, eg. mysql@dbgate-plugin-mysql');
+  .option('-e, --engine <engine>', 'engine name, eg. mysql@dbgate-plugin-mysql')
+  .option('--commonjs', 'Creates CommonJS module');

 program
  .command('deploy <modelFolder>')
@@ -115,4 +118,30 @@ program
    );
  });

+program
+  .command('json-to-model <jsonFile> <modelFolder>')
+  .description('Converts JSON file to model')
+  .action((jsonFile, modelFolder) => {
+    runAndExit(
+      dbgateApi.jsonToDbModel({
+        modelFile: jsonFile,
+        outputDir: modelFolder,
+      })
+    );
+  });
+
+program
+  .command('model-to-json <modelFolder> <jsonFile>')
+  .description('Converts model to JSON file')
+  .action((modelFolder, jsonFile) => {
+    const { commonjs } = program.opts();
+    runAndExit(
+      dbgateApi.dbModelToJson({
+        modelFolder,
+        outputFile: jsonFile,
+        commonjs,
+      })
+    );
+  });
+
 program.parse(process.argv);
@@ -1,310 +0,0 @@
-import P from 'parsimmon';
-import moment from 'moment';
-import { FilterType } from './types';
-import { Condition } from 'dbgate-sqltree';
-import type { TransformType } from 'dbgate-types';
-import { interpretEscapes, token, word, whitespace } from './common';
-
-const compoudCondition = conditionType => conditions => {
-  if (conditions.length == 1) return conditions[0];
-  return {
-    [conditionType]: conditions,
-  };
-};
-
-function getTransformCondition(transform: TransformType, value) {
-  return {
-    conditionType: 'binary',
-    operator: '=',
-    left: {
-      exprType: 'transform',
-      transform,
-      expr: {
-        exprType: 'placeholder',
-      },
-    },
-    right: {
-      exprType: 'value',
-      value,
-    },
-  };
-}
-
-const yearCondition = () => value => {
-  return getTransformCondition('YEAR', value);
-};
-
-const yearMonthCondition = () => value => {
-  const m = value.match(/(\d\d\d\d)-(\d\d?)/);
-
-  return {
-    conditionType: 'and',
-    conditions: [getTransformCondition('YEAR', m[1]), getTransformCondition('MONTH', m[2])],
-  };
-};
-
-const yearMonthDayCondition = () => value => {
-  const m = value.match(/(\d\d\d\d)-(\d\d?)-(\d\d?)/);
-
-  return {
-    conditionType: 'and',
-    conditions: [
-      getTransformCondition('YEAR', m[1]),
-      getTransformCondition('MONTH', m[2]),
-      getTransformCondition('DAY', m[3]),
-    ],
-  };
-};
-
-const yearEdge = edgeFunction => value => {
-  return moment(new Date(parseInt(value), 0, 1))
-    [edgeFunction]('year')
-    .format('YYYY-MM-DDTHH:mm:ss.SSS');
-};
-
-const yearMonthEdge = edgeFunction => value => {
-  const m = value.match(/(\d\d\d\d)-(\d\d?)/);
-
-  return moment(new Date(parseInt(m[1]), parseInt(m[2]) - 1, 1))
-    [edgeFunction]('month')
-    .format('YYYY-MM-DDTHH:mm:ss.SSS');
-};
-
-const yearMonthDayEdge = edgeFunction => value => {
-  const m = value.match(/(\d\d\d\d)-(\d\d?)-(\d\d?)/);
-
-  return moment(new Date(parseInt(m[1]), parseInt(m[2]) - 1, parseInt(m[3])))
-    [edgeFunction]('day')
-    .format('YYYY-MM-DDTHH:mm:ss.SSS');
-};
-
-const yearMonthDayMinuteEdge = edgeFunction => value => {
-  const m = value.match(/(\d\d\d\d)-(\d\d?)-(\d\d?)\s+(\d\d?):(\d\d?)/);
-  const year = m[1];
-  const month = m[2];
-  const day = m[3];
-  const hour = m[4];
-  const minute = m[5];
-  const dateObject = new Date(year, month - 1, day, hour, minute);
-
-  return moment(dateObject)[edgeFunction]('minute').format('YYYY-MM-DDTHH:mm:ss.SSS');
-};
-
-const yearMonthDayMinuteSecondEdge = edgeFunction => value => {
-  const m = value.match(/(\d\d\d\d)-(\d\d?)-(\d\d?)(T|\s+)(\d\d?):(\d\d?):(\d\d?)/);
-  const year = m[1];
-  const month = m[2];
-  const day = m[3];
-  const hour = m[5];
-  const minute = m[6];
-  const second = m[7];
-  const dateObject = new Date(year, month - 1, day, hour, minute, second);
-
-  return moment(dateObject)[edgeFunction]('second').format('YYYY-MM-DDTHH:mm:ss.SSS');
-};
-
-const createIntervalCondition = (start, end) => {
-  return {
-    conditionType: 'and',
-    conditions: [
-      {
-        conditionType: 'binary',
-        operator: '>=',
-        left: {
-          exprType: 'placeholder',
-        },
-        right: {
-          exprType: 'value',
-          value: start,
-        },
-      },
-      {
-        conditionType: 'binary',
-        operator: '<=',
-        left: {
-          exprType: 'placeholder',
-        },
-        right: {
-          exprType: 'value',
-          value: end,
-        },
-      },
-    ],
-  };
-};
-
-const createDateIntervalCondition = (start, end) => {
-  return createIntervalCondition(start.format('YYYY-MM-DDTHH:mm:ss.SSS'), end.format('YYYY-MM-DDTHH:mm:ss.SSS'));
-};
-
-const fixedMomentIntervalCondition = (intervalType, diff) => () => {
-  return createDateIntervalCondition(
-    moment().add(intervalType, diff).startOf(intervalType),
-    moment().add(intervalType, diff).endOf(intervalType)
-  );
-};
-
-const yearMonthDayMinuteCondition = () => value => {
-  const m = value.match(/(\d\d\d\d)-(\d\d?)-(\d\d?)\s+(\d\d?):(\d\d?)/);
-  const year = m[1];
-  const month = m[2];
-  const day = m[3];
-  const hour = m[4];
-  const minute = m[5];
-  const dateObject = new Date(year, month - 1, day, hour, minute);
-
-  return createDateIntervalCondition(moment(dateObject).startOf('minute'), moment(dateObject).endOf('minute'));
-};
-
-const yearMonthDaySecondCondition = () => value => {
-  const m = value.match(/(\d\d\d\d)-(\d\d?)-(\d\d?)(T|\s+)(\d\d?):(\d\d?):(\d\d?)/);
-  const year = m[1];
-  const month = m[2];
-  const day = m[3];
-  const hour = m[5];
-  const minute = m[6];
-  const second = m[7];
-  const dateObject = new Date(year, month - 1, day, hour, minute, second);
-
-  return createDateIntervalCondition(moment(dateObject).startOf('second'), moment(dateObject).endOf('second'));
-};
-
-const binaryCondition = operator => value => ({
-  conditionType: 'binary',
-  operator,
-  left: {
-    exprType: 'placeholder',
-  },
-  right: {
-    exprType: 'value',
-    value,
-  },
-});
-
-const unaryCondition = conditionType => () => {
-  return {
-    conditionType,
-    expr: {
-      exprType: 'placeholder',
-    },
-  };
-};
-
-const sqlTemplate = templateSql => {
-  return {
-    conditionType: 'rawTemplate',
-    templateSql,
-    expr: {
-      exprType: 'placeholder',
-    },
-  };
-};
-
-const createParser = () => {
-  const langDef = {
-    comma: () => word(','),
-
-    not: () => word('NOT'),
-    notNull: r => r.not.then(r.null).map(unaryCondition('isNotNull')),
-    null: () => word('NULL').map(unaryCondition('isNull')),
-
-    sql: () =>
-      token(P.regexp(/\{(.*?)\}/, 1))
-        .map(sqlTemplate)
-        .desc('sql literal'),
-
-    yearNum: () => P.regexp(/\d\d\d\d/).map(yearCondition()),
-    yearMonthNum: () => P.regexp(/\d\d\d\d-\d\d?/).map(yearMonthCondition()),
-    yearMonthDayNum: () => P.regexp(/\d\d\d\d-\d\d?-\d\d?/).map(yearMonthDayCondition()),
-    yearMonthDayMinute: () => P.regexp(/\d\d\d\d-\d\d?-\d\d?\s+\d\d?:\d\d?/).map(yearMonthDayMinuteCondition()),
-    yearMonthDaySecond: () =>
-      P.regexp(/\d\d\d\d-\d\d?-\d\d?(\s+|T)\d\d?:\d\d?:\d\d?/).map(yearMonthDaySecondCondition()),
-
-    yearNumStart: () => P.regexp(/\d\d\d\d/).map(yearEdge('startOf')),
-    yearNumEnd: () => P.regexp(/\d\d\d\d/).map(yearEdge('endOf')),
-    yearMonthStart: () => P.regexp(/\d\d\d\d-\d\d?/).map(yearMonthEdge('startOf')),
-    yearMonthEnd: () => P.regexp(/\d\d\d\d-\d\d?/).map(yearMonthEdge('endOf')),
-    yearMonthDayStart: () => P.regexp(/\d\d\d\d-\d\d?-\d\d?/).map(yearMonthDayEdge('startOf')),
-    yearMonthDayEnd: () => P.regexp(/\d\d\d\d-\d\d?-\d\d?/).map(yearMonthDayEdge('endOf')),
-    yearMonthDayMinuteStart: () =>
-      P.regexp(/\d\d\d\d-\d\d?-\d\d?\s+\d\d?:\d\d?/).map(yearMonthDayMinuteEdge('startOf')),
-    yearMonthDayMinuteEnd: () => P.regexp(/\d\d\d\d-\d\d?-\d\d?\s+\d\d?:\d\d?/).map(yearMonthDayMinuteEdge('endOf')),
-    yearMonthDayMinuteSecondStart: () =>
-      P.regexp(/\d\d\d\d-\d\d?-\d\d?(\s+|T)\d\d?:\d\d?:\d\d?/).map(yearMonthDayMinuteSecondEdge('startOf')),
-    yearMonthDayMinuteSecondEnd: () =>
-      P.regexp(/\d\d\d\d-\d\d?-\d\d?(\s+|T)\d\d?:\d\d?:\d\d?/).map(yearMonthDayMinuteSecondEdge('endOf')),
-
-    this: () => word('THIS'),
-    last: () => word('LAST'),
-    next: () => word('NEXT'),
-    week: () => word('WEEK'),
-    month: () => word('MONTH'),
-    year: () => word('YEAR'),
-
-    yesterday: () => word('YESTERDAY').map(fixedMomentIntervalCondition('day', -1)),
-    today: () => word('TODAY').map(fixedMomentIntervalCondition('day', 0)),
-    tomorrow: () => word('TOMORROW').map(fixedMomentIntervalCondition('day', 1)),
-
-    lastWeek: r => r.last.then(r.week).map(fixedMomentIntervalCondition('week', -1)),
-    thisWeek: r => r.this.then(r.week).map(fixedMomentIntervalCondition('week', 0)),
-    nextWeek: r => r.next.then(r.week).map(fixedMomentIntervalCondition('week', 1)),
-
-    lastMonth: r => r.last.then(r.month).map(fixedMomentIntervalCondition('month', -1)),
-    thisMonth: r => r.this.then(r.month).map(fixedMomentIntervalCondition('month', 0)),
-    nextMonth: r => r.next.then(r.month).map(fixedMomentIntervalCondition('month', 1)),
-
-    lastYear: r => r.last.then(r.year).map(fixedMomentIntervalCondition('year', -1)),
-    thisYear: r => r.this.then(r.year).map(fixedMomentIntervalCondition('year', 0)),
-    nextYear: r => r.next.then(r.year).map(fixedMomentIntervalCondition('year', 1)),
-
-    valueStart: r =>
-      P.alt(
-        r.yearMonthDayMinuteSecondStart,
-        r.yearMonthDayMinuteStart,
-        r.yearMonthDayStart,
-        r.yearMonthStart,
-        r.yearNumStart
-      ),
-    valueEnd: r =>
-      P.alt(r.yearMonthDayMinuteSecondEnd, r.yearMonthDayMinuteEnd, r.yearMonthDayEnd, r.yearMonthEnd, r.yearNumEnd),
-
-    le: r => word('<=').then(r.valueEnd).map(binaryCondition('<=')),
-    ge: r => word('>=').then(r.valueStart).map(binaryCondition('>=')),
-    lt: r => word('<').then(r.valueStart).map(binaryCondition('<')),
-    gt: r => word('>').then(r.valueEnd).map(binaryCondition('>')),
-
-    element: r =>
-      P.alt(
-        r.yearMonthDaySecond,
-        r.yearMonthDayMinute,
-        r.yearMonthDayNum,
-        r.yearMonthNum,
-        r.yearNum,
-        r.yesterday,
-        r.today,
-        r.tomorrow,
-        r.lastWeek,
-        r.thisWeek,
-        r.nextWeek,
-        r.lastMonth,
-        r.thisMonth,
-        r.nextMonth,
-        r.lastYear,
-        r.thisYear,
-        r.nextYear,
-        r.null,
-        r.notNull,
-        r.le,
-        r.lt,
-        r.ge,
-        r.gt,
-        r.sql
-      ).trim(whitespace),
-    factor: r => r.element.sepBy(whitespace).map(compoudCondition('$and')),
-    list: r => r.factor.sepBy(r.comma).map(compoudCondition('$or')),
-  };
-
-  return P.createLanguage(langDef);
-};
-
-export const datetimeParser = createParser();
@@ -1,11 +0,0 @@
-import { isTypeNumber, isTypeString, isTypeLogical, isTypeDateTime } from 'dbgate-tools';
-import { FilterType } from './types';
-
-export function getFilterType(dataType: string): FilterType {
-  if (!dataType) return 'string';
-  if (isTypeNumber(dataType)) return 'number';
-  if (isTypeString(dataType)) return 'string';
-  if (isTypeLogical(dataType)) return 'logical';
-  if (isTypeDateTime(dataType)) return 'datetime';
-  return 'string';
-}
--- a/Show More
+++ b/Show More