AD_ALLOWED_LOGINS support

FEAT: Folders support

.github/workflows/build-app-beta.yaml

@@ -27,6 +27,9 @@ jobs:
         uses: actions/setup-node@v1
         with:
           node-version: 14.x
+      - name: yarn adjustPackageJson
+        run: |
+          yarn adjustPackageJson
       - name: yarn install
         run: |
           yarn install
@@ -48,8 +51,10 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
 
-          WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
-          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
+          WIN_CSC_LINK: ${{ secrets.WINCERT_2025 }}
+          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_2025_PASSWORD }}
+          # WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
+          # WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
 
           CSC_LINK: ${{ secrets.APPLECERT_CERTIFICATE }}
           CSC_KEY_PASSWORD: ${{ secrets.APPLECERT_PASSWORD }}

.github/workflows/build-app.yaml

@@ -31,6 +31,9 @@ jobs:
         uses: actions/setup-node@v1
         with:
           node-version: 14.x
+      - name: yarn adjustPackageJson
+        run: |
+          yarn adjustPackageJson
       - name: yarn install
         run: |
           # yarn --version
@@ -54,8 +57,10 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
 
-          WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
-          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
+          WIN_CSC_LINK: ${{ secrets.WINCERT_2025 }}
+          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_2025_PASSWORD }}
+          # WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
+          # WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
 
           CSC_LINK: ${{ secrets.APPLECERT_CERTIFICATE }}
           CSC_KEY_PASSWORD: ${{ secrets.APPLECERT_PASSWORD }}

.github/workflows/build-docker-beta.yaml

@@ -1,58 +0,0 @@
-name: Docker image BETA
-
-# on: [push]
-
-on:
-  push:
-    tags:
-      - 'v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+'
-      - 'v[0-9]+.[0-9]+.[0-9]+-docker.[0-9]+'
-
-jobs:
-  build:
-
-    runs-on: ${{ matrix.os }}
-
-    strategy:
-      matrix:
-        os: [ubuntu-18.04]
-
-    steps:
-      - name: Context
-        env:
-          GITHUB_CONTEXT: ${{ toJson(github) }}
-        run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Use Node.js 14.x
-        uses: actions/setup-node@v1
-        with:
-          node-version: 14.x
-      - name: yarn install
-        run: |
-          # yarn --version
-          # yarn config set network-timeout 300000
-          yarn install
-      - name: setCurrentVersion
-        run: |
-          yarn setCurrentVersion
-      - name: Prepare docker image
-        run: |
-          yarn run prepare:docker
-      - name: Build docker image
-        run: |
-          docker build ./docker -t dbgate
-      - name: Push docker image
-        run: |
-          docker tag dbgate dbgate/dbgate:beta
-          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
-          docker push dbgate/dbgate:beta
-      - name: Build alpine docker image
-        run: |
-          docker build ./docker -t dbgate -f docker/Dockerfile-alpine
-      - name: Push alpine docker image
-        run: |
-          docker tag dbgate dbgate/dbgate:beta-alpine
-          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
-          docker push dbgate/dbgate:beta-alpine

.github/workflows/build-docker.yaml

@@ -1,17 +1,11 @@
 name: Docker image
 
-# on: [push]
-
 on:
   push:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+'
-    # - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
-
-# on:
-#   push:
-#     branches:
-#       - production
+      - 'v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+-docker.[0-9]+'
 
 jobs:
   build:
@@ -30,12 +24,43 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 1
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            dbgate/dbgate
+          flavor: |
+            latest=false
+          tags: |
+            type=raw,value=beta,enable=${{ contains(github.ref_name, '-docker.') || contains(github.ref_name, '-beta.') }}
+
+            type=match,pattern=\d+.\d+.\d+,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
+            type=raw,value=latest,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
+
+      - name: Docker alpine meta
+        id: alpmeta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            dbgate/dbgate
+          flavor: |
+            latest=false
+          tags: |
+            type=raw,value=beta-alpine,enable=${{ contains(github.ref_name, '-docker.') || contains(github.ref_name, '-beta.') }}
+
+            type=match,pattern=\d+.\d+.\d+,suffix=-alpine,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
+            type=raw,value=alpine,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
+
       - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
           node-version: 14.x
       - name: yarn install
         run: |
+          # yarn --version
+          # yarn config set network-timeout 300000
           yarn install
       - name: setCurrentVersion
         run: |
@@ -43,19 +68,28 @@ jobs:
       - name: Prepare docker image
         run: |
           yarn run prepare:docker
-      - name: Build docker image
-        run: |
-          docker build ./docker -t dbgate
-      - name: Push docker image
-        run: |
-          docker tag dbgate dbgate/dbgate
-          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
-          docker push dbgate/dbgate
-      - name: Build alpine docker image
-        run: |
-          docker build ./docker -t dbgate -f docker/Dockerfile-alpine
-      - name: Push alpine docker image
-        run: |
-          docker tag dbgate dbgate/dbgate:alpine
-          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
-          docker push dbgate/dbgate:alpine
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          push: true
+          context: ./docker
+          tags: ${{ steps.meta.outputs.tags }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
+
+      - name: Build and push alpine
+        uses: docker/build-push-action@v3
+        with:
+          push: true
+          context: ./docker
+          file: ./docker/Dockerfile-alpine
+          tags: ${{ steps.alpmeta.outputs.tags }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v7

.github/workflows/run-tests.yaml

@@ -31,6 +31,11 @@ jobs:
         run: |
           cd packages/filterparser
           yarn test:ci
+      - name: Datalib (perspective) tests
+        if: always()
+        run: |
+          cd packages/datalib
+          yarn test:ci
       - uses: tanmen/jest-reporter@v1
         if: always()
         with:
@@ -43,6 +48,12 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           result-file: packages/filterparser/result.json
           action-name: Filter parser test results
+      - uses: tanmen/jest-reporter@v1
+        if: always()
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          result-file: packages/datalib/result.json
+          action-name: Datalib (perspectives) test results
 
     services:
       postgres:

.node-version

@@ -0,0 +1 @@
+16.14.2

.vscode/restore-terminals.json

@@ -0,0 +1,20 @@
+{
+  "terminals": [
+    {
+      "splitTerminals": [
+        {
+          "name": "lib",
+          "commands": ["yarn lib"]
+        },
+        {
+          "name": "web",
+          "commands": ["yarn start:web"]
+        },
+        {
+          "name": "api",
+          "commands": ["yarn start:api"]
+        }
+      ]
+    }
+  ]
+}

.vscode/settings.json

@@ -1,3 +1,6 @@
 {
-    "jestrunner.jestCommand": "node_modules/.bin/cross-env DEVMODE=1 LOCALTEST=1 node_modules/.bin/jest"
+    "jestrunner.jestCommand": "node_modules/.bin/cross-env DEVMODE=1 LOCALTEST=1 node_modules/.bin/jest",
+    "cSpell.words": [
+        "dbgate"
+    ]
 }

CHANGELOG.md

@@ -8,6 +8,80 @@ Builds:
  - linux - application for linux
  - win - application for Windows
 
+### 5.1.5
+- ADDED: Support perspectives for MongoDB - MongoDB query designer
+- ADDED: Show JSON content directly in the overview #395
+- CHANGED: OSX Command H shortcut for hiding window #390
+- ADDED: Uppercase Autocomplete Suggestions #389
+- FIXED: Record view left/right arrows cause start record number to be treated as string #388
+- FIXED: MongoDb ObjectId behaviour not consistent in nested objects #387
+- FIXED: demo.dbgate.org - beta version crash 5.1.5-beta.3 #386
+- ADDED: connect via socket - configurable via environment variables #358
+
+### 5.1.4
+- ADDED: Drop database commands #384
+- ADDED: Customizable Redis key separator #379
+- ADDED: ARM support for docker images
+- ADDED: Version tags for docker images
+- ADDED: Better SQL command splitting and highlighting
+- ADDED: Unsaved marker for SQL files
+
+### 5.1.3
+- ADDED: Editing multiline cell values #378 #371 #359
+- ADDED: Truncate table #333
+- ADDED: Perspectives - show row count
+- ADDED: Query - error markers in gutter area
+- ADDED: Query - ability to execute query elements from gutter
+- FIXED: Correct error line numbers returned from queries
+
+### 5.1.2
+- FIXED: MongoDb any export function does not work. #373
+- ADDED: Query Designer short order more flexibility #372
+- ADDED: Form View move between records #370
+- ADDED: Custom SQL conditions in query designer and table filtering #369
+- ADDED: Query Designer filter eq to X or IS NULL #368
+- FIXED: Query designer, open a saved query lost sort order #363
+- ADDED: Query designer reorder columns #362
+- ADDED: connect via socket #358
+- FIXED: Show affected rows after UPDATE/DELETE/INSERT #361
+- ADDED: Perspective cell formatters - JSON, image
+- ADDED: Perspectives - cells without joined data are gray
+
+### 5.1.1
+- ADDED: Perspective designer
+- FIXED: NULL,NOT NULL filter datatime columns #356
+- FIXED: Recognize computed columns on SQL server #354
+- ADDED: Hotkey for clear filter #352
+- FIXED: Change column type on Postgres #350
+- ADDED: Ability to open qdesign file #349
+- ADDED: Custom editor font size #345
+- ADDED: Ability to open perspective files
+
+
+### 5.1.0
+- ADDED: Perspectives (docs: https://dbgate.org/docs/perspectives.html )
+- CHANGED: Upgraded SQLite engine version (driver better-sqlite3: 7.6.2)
+- CHANGED: Upgraded ElectronJS version (from version 13 to version 17)
+- CHANGED: Upgraded all dependencies with current available minor version updates
+- CHANGED: By default, connect on click #332˝
+- CHANGED: Improved keyboard navigation, when editing table data #331
+- ADDED:  Option to skip Save changes dialog #329
+- FIXED: Unsigned column doesn't work correctly. #324
+- FIXED: Connect to MS SQL with domain user now works also under Linux and Mac #305
+
+### 5.0.9
+- FIXED: Fixed problem with SSE events on web version
+- ADDED: Added menu command "New query designer"
+- ADDED: Added menu command "New ER diagram"
+
+### 5.0.8
+- ADDED: SQL Server - support using domain logins under Linux and Mac #305
+- ADDED: Permissions for connections #318
+- ADDED: Ability to change editor front #308
+- ADDED: Custom expression in query designer #306
+- ADDED: OR conditions in query designer #321
+- ADDED: Ability to configure settings view environment variables #304
+ 
 ### 5.0.7
 - FIXED: Fixed some problems with SSH tunnel (upgraded SSH client) #315
 - FIXED: Fixed MognoDB executing find query #312

README.md

@@ -72,6 +72,7 @@ DbGate is licensed under MIT license and is completely free.
 * Charts, export chart to HTML page
 * For detailed info, how to run DbGate in docker container, visit [docker hub](https://hub.docker.com/r/dbgate/dbgate)
 * Extensible plugin architecture
+* Perspectives - nested table view over complex relational data
 
 ## How to contribute
 Any contributions are welcome. If you want to contribute without coding, consider following:
@@ -79,7 +80,8 @@ Any contributions are welcome. If you want to contribute without coding, conside
 * Tell your friends about DbGate or share on social networks - when more people will use DbGate, it will grow to be better
 * Write review on [Slant.co](https://www.slant.co/improve/options/41086/~dbgate-review) or [G2](https://www.g2.com/products/dbgate/reviews) 
 * Create issue, if you find problem in app, or you have idea to new feature. If issue already exists, you could leave comment on it, to prioritise most wanted issues.
-* Become a backer on [Open collective](https://opencollective.com/dbgate)
+* Create some tutorial video on [youtube](https://www.youtube.com/playlist?list=PLCo7KjCVXhr0RfUSjM9wJMsp_ShL1q61A)
+* Become a backer on [GitHub sponsors](https://github.com/sponsors/dbgate) or [Open collective](https://opencollective.com/dbgate)
 * Where a small coding is acceptable for you, you could [create plugin](https://dbgate.org/docs/plugin-development.html). Plugins for new themes can be created actually without JS coding.
 
 Thank you!

adjustPackageJson.js

@@ -0,0 +1,12 @@
+const fs = require('fs');
+
+function adjustFile(file) {
+  const json = JSON.parse(fs.readFileSync(file, { encoding: 'utf-8' }));
+  if (process.platform != 'win32') {
+    delete json.optionalDependencies.msnodesqlv8;
+  }
+  fs.writeFileSync(file, JSON.stringify(json, null, 2), 'utf-8');
+}
+
+adjustFile('packages/api/package.json');
+adjustFile('app/package.json');

app/package.json

@@ -107,12 +107,12 @@
   "devDependencies": {
     "copyfiles": "^2.2.0",
     "cross-env": "^6.0.3",
-    "electron": "13.6.3",
-    "electron-builder": "22.14.5",
-    "electron-builder-notarize": "^1.4.0"
+    "electron": "17.4.10",
+    "electron-builder": "23.1.0",
+    "electron-builder-notarize": "^1.5.0"
   },
   "optionalDependencies": {
-    "better-sqlite3": "7.5.0",
-    "msnodesqlv8": "^2.4.4"
+    "better-sqlite3": "7.6.2",
+    "msnodesqlv8": "^2.6.0"
   }
 }

app/src/mainMenuDefinition.js

@@ -6,6 +6,9 @@ module.exports = ({ editMenu }) => [
       { command: 'new.sqliteDatabase', hideDisabled: true },
       { divider: true },
       { command: 'new.query', hideDisabled: true },
+      { command: 'new.queryDesign', hideDisabled: true },
+      { command: 'new.diagram', hideDisabled: true },
+      { command: 'new.perspective', hideDisabled: true },
       { command: 'new.freetable', hideDisabled: true },
       { command: 'new.shell', hideDisabled: true },
       { command: 'new.jsonl', hideDisabled: true },
@@ -66,6 +69,7 @@ module.exports = ({ editMenu }) => [
       { command: 'app.toggleDevTools', hideDisabled: true },
       { command: 'app.toggleFullScreen', hideDisabled: true },
       { command: 'app.minimize', hideDisabled: true },
+      { command: 'toggle.sidebar' },
       { divider: true },
       { command: 'theme.changeTheme', hideDisabled: true },
       { command: 'settings.show' },

misc/play-dark-mode.svg

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Generator: Adobe Illustrator 18.1.1, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 17.804 17.804" style="enable-background:new 0 0 17.804 17.804;" xml:space="preserve">
+<g>
+	<g id="c98_play">
+		<path fill='#ccc' d="M2.067,0.043C2.21-0.028,2.372-0.008,2.493,0.085l13.312,8.503c0.094,0.078,0.154,0.191,0.154,0.313
+			c0,0.12-0.061,0.237-0.154,0.314L2.492,17.717c-0.07,0.057-0.162,0.087-0.25,0.087l-0.176-0.04
+			c-0.136-0.065-0.222-0.207-0.222-0.361V0.402C1.844,0.25,1.93,0.107,2.067,0.043z"/>
+	</g>
+	<g id="Capa_1_78_">
+	</g>
+</g>
+</svg>

misc/play-light-mode.svg

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Generator: Adobe Illustrator 18.1.1, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 17.804 17.804" style="enable-background:new 0 0 17.804 17.804;" xml:space="preserve">
+<g>
+	<g id="c98_play">
+		<path fill='#444' d="M2.067,0.043C2.21-0.028,2.372-0.008,2.493,0.085l13.312,8.503c0.094,0.078,0.154,0.191,0.154,0.313
+			c0,0.12-0.061,0.237-0.154,0.314L2.492,17.717c-0.07,0.057-0.162,0.087-0.25,0.087l-0.176-0.04
+			c-0.136-0.065-0.222-0.207-0.222-0.361V0.402C1.844,0.25,1.93,0.107,2.067,0.043z"/>
+	</g>
+	<g id="Capa_1_78_">
+	</g>
+</g>
+</svg>

package.json

@@ -1,6 +1,6 @@
 {
   "private": true,
-  "version": "5.0.8",
+  "version": "5.1.6-beta.7",
   "name": "dbgate-all",
   "workspaces": [
     "packages/*",
@@ -16,6 +16,7 @@
     "start:app:debug:ssh": "cd app && cross-env DEBUG=ssh yarn start",
     "start:api:portal": "yarn workspace dbgate-api start:portal",
     "start:api:singledb": "yarn workspace dbgate-api start:singledb",
+    "start:api:auth": "yarn workspace dbgate-api start:auth",
     "start:web": "yarn workspace dbgate-web dev",
     "start:sqltree": "yarn workspace dbgate-sqltree start",
     "start:tools": "yarn workspace dbgate-tools start",
@@ -36,6 +37,7 @@
     "start:app:local": "cd app && yarn start:local",
     "setCurrentVersion": "node setCurrentVersion",
     "generatePadFile": "node generatePadFile",
+    "adjustPackageJson": "node adjustPackageJson",
     "fillNativeModules": "node fillNativeModules",
     "fillNativeModulesElectron": "node fillNativeModules --electron",
     "fillPackagedPlugins": "node fillPackagedPlugins",

packages/api/env/auth/.gitignore

@@ -0,0 +1 @@
+.env

packages/api/package.json

@@ -17,6 +17,7 @@
     "dbgate"
   ],
   "dependencies": {
+    "activedirectory2": "^2.1.0",
     "async-lock": "^1.2.4",
     "axios": "^0.21.1",
     "body-parser": "^1.19.0",
@@ -25,7 +26,7 @@
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-query-splitter": "^4.9.0",
+    "dbgate-query-splitter": "^4.9.2",
     "dbgate-sqltree": "^5.0.0-alpha.1",
     "dbgate-tools": "^5.0.0-alpha.1",
     "debug": "^4.3.4",
@@ -42,10 +43,12 @@
     "is-electron": "^2.2.1",
     "js-yaml": "^4.1.0",
     "json-stable-stringify": "^1.0.1",
+    "jsonwebtoken": "^8.5.1",
     "line-reader": "^0.4.0",
     "lodash": "^4.17.21",
     "ncp": "^2.0.0",
     "node-cron": "^2.0.3",
+    "on-finished": "^2.4.1",
     "portfinder": "^1.0.28",
     "simple-encryptor": "^4.0.0",
     "ssh2": "^1.11.0",
@@ -56,6 +59,7 @@
     "start": "env-cmd node src/index.js --listen-api",
     "start:portal": "env-cmd -f env/portal/.env node src/index.js --listen-api",
     "start:singledb": "env-cmd -f env/singledb/.env node src/index.js --listen-api",
+    "start:auth": "env-cmd -f env/auth/.env node src/index.js --listen-api",
     "start:filedb": "env-cmd node src/index.js /home/jena/test/chinook/Chinook.db --listen-api",
     "start:singleconn": "env-cmd node src/index.js --server localhost --user root --port 3307 --engine mysql@dbgate-plugin-mysql --password test --listen-api",
     "ts": "tsc",
@@ -73,7 +77,7 @@
     "webpack-cli": "^3.3.11"
   },
   "optionalDependencies": {
-    "better-sqlite3": "7.5.0",
-    "msnodesqlv8": "^2.4.4"
+    "better-sqlite3": "7.6.2",
+    "msnodesqlv8": "^2.6.0"
   }
 }

packages/api/src/controllers/auth.js

@@ -0,0 +1,142 @@
+const axios = require('axios');
+const jwt = require('jsonwebtoken');
+const getExpressPath = require('../utility/getExpressPath');
+const uuidv1 = require('uuid/v1');
+const { getLogins } = require('../utility/hasPermission');
+const AD = require('activedirectory2').promiseWrapper;
+
+const tokenSecret = uuidv1();
+
+function shouldAuthorizeApi() {
+  const logins = getLogins();
+  return !!process.env.OAUTH_AUTH || !!process.env.AD_URL || (!!logins && !process.env.BASIC_AUTH);
+}
+
+function getTokenLifetime() {
+  return process.env.TOKEN_LIFETIME || '1d';
+}
+
+function unauthorizedResponse(req, res, text) {
+  // if (req.path == getExpressPath('/config/get-settings')) {
+  //   return res.json({});
+  // }
+  // if (req.path == getExpressPath('/connections/list')) {
+  //   return res.json([]);
+  // }
+  return res.sendStatus(401).send(text);
+}
+
+function authMiddleware(req, res, next) {
+  const SKIP_AUTH_PATHS = ['/config/get', '/auth/oauth-token', '/auth/login', '/stream'];
+
+  if (!shouldAuthorizeApi()) {
+    return next();
+  }
+  let skipAuth = !!SKIP_AUTH_PATHS.find(x => req.path == getExpressPath(x));
+
+  const authHeader = req.headers.authorization;
+  if (!authHeader) {
+    if (skipAuth) {
+      return next();
+    }
+    return unauthorizedResponse(req, res, 'missing authorization header');
+  }
+  const token = authHeader.split(' ')[1];
+  try {
+    const decoded = jwt.verify(token, tokenSecret);
+    req.user = decoded;
+    return next();
+  } catch (err) {
+    if (skipAuth) {
+      return next();
+    }
+
+    console.log('Sending invalid token error', err.message);
+
+    return unauthorizedResponse(req, res, 'invalid token');
+  }
+}
+
+module.exports = {
+  oauthToken_meta: true,
+  async oauthToken(params) {
+    const { redirectUri, code } = params;
+
+    const resp = await axios.default.post(
+      `${process.env.OAUTH_TOKEN}`,
+      `grant_type=authorization_code&code=${encodeURIComponent(code)}&redirect_uri=${encodeURIComponent(
+        redirectUri
+      )}&client_id=${process.env.OAUTH_CLIENT_ID}&client_secret=${process.env.OAUTH_CLIENT_SECRET}`
+    );
+
+    const { access_token, refresh_token } = resp.data;
+
+    const payload = jwt.decode(access_token);
+
+    console.log('User payload returned from OAUTH:', payload);
+
+    const login = process.env.OAUTH_LOGIN_FIELD ? payload[process.env.OAUTH_LOGIN_FIELD] : 'oauth';
+
+    if (
+      process.env.OAUTH_ALLOWED_LOGINS &&
+      !process.env.OAUTH_ALLOWED_LOGINS.split(',').find(x => x.toLowerCase().trim() == login.toLowerCase().trim())
+    ) {
+      return { error: `Username ${login} not allowed to log in` };
+    }
+    if (access_token) {
+      return {
+        accessToken: jwt.sign({ login }, tokenSecret, { expiresIn: getTokenLifetime() }),
+      };
+    }
+
+    return { error: 'Token not found' };
+  },
+  login_meta: true,
+  async login(params) {
+    const { login, password } = params;
+
+    if (process.env.AD_URL) {
+      const adConfig = {
+        url: process.env.AD_URL,
+        baseDN: process.env.AD_BASEDN,
+        username: process.env.AD_USERNAME,
+        password: process.env.AD_PASSOWRD,
+      };
+      const ad = new AD(adConfig);
+      try {
+        const res = await ad.authenticate(login, password);
+        if (!res) {
+          return { error: 'Login failed' };
+        }
+        if (
+          process.env.AD_ALLOWED_LOGINS &&
+          !process.env.AD_ALLOWED_LOGINS.split(',').find(x => x.toLowerCase().trim() == login.toLowerCase().trim())
+        ) {
+          return { error: `Username ${login} not allowed to log in` };
+        }        
+        return {
+          accessToken: jwt.sign({ login }, tokenSecret, { expiresIn: getTokenLifetime() }),
+        };
+      } catch (err) {
+        console.log('Failed active directory authentization', err.message);
+        return {
+          error: err.message,
+        };
+      }
+    }
+
+    const logins = getLogins();
+    if (!logins) {
+      return { error: 'Logins not configured' };
+    }
+    if (logins.find(x => x.login == login)?.password == password) {
+      return {
+        accessToken: jwt.sign({ login }, tokenSecret, { expiresIn: getTokenLifetime() }),
+      };
+    }
+    return { error: 'Invalid credentials' };
+  },
+
+  authMiddleware,
+  shouldAuthorizeApi,
+};

packages/api/src/controllers/config.js

@@ -28,7 +28,7 @@ module.exports = {
   get_meta: true,
   async get(_params, req) {
     const logins = getLogins();
-    const login = logins ? logins.find(x => x.login == (req.auth && req.auth.user)) : null;
+    const login = req.user ? req.user.login : logins ? logins.find(x => x.login == (req.auth && req.auth.user)) : null;
     const permissions = login ? login.permissions : process.env.PERMISSIONS;
 
     return {
@@ -40,6 +40,9 @@ module.exports = {
       isDocker: platformInfo.isDocker,
       permissions,
       login,
+      oauth: process.env.OAUTH_AUTH,
+      oauthLogout: process.env.OAUTH_LOGOUT,
+      isLoginForm: !!process.env.AD_URL || (!!logins && !process.env.BASIC_AUTH),
       ...currentVersion,
     };
   },

packages/api/src/controllers/connections.js

@@ -53,6 +53,8 @@ function getPortalCollections() {
       databaseUrl: process.env[`URL_${id}`],
       useDatabaseUrl: !!process.env[`URL_${id}`],
       databaseFile: process.env[`FILE_${id}`],
+      socketPath: process.env[`SOCKET_PATH_${id}`],
+      authType: process.env[`AUTH_TYPE_${id}`] || (process.env[`SOCKET_PATH_${id}`] ? 'socket' : undefined),
       defaultDatabase:
         process.env[`DATABASE_${id}`] ||
         (process.env[`FILE_${id}`] ? getDatabaseFileLabel(process.env[`FILE_${id}`]) : null),
@@ -60,6 +62,7 @@ function getPortalCollections() {
       displayName: process.env[`LABEL_${id}`],
       isReadOnly: process.env[`READONLY_${id}`],
       databases: process.env[`DBCONFIG_${id}`] ? safeJsonParse(process.env[`DBCONFIG_${id}`]) : null,
+      parent: process.env[`PARENT_${id}`] || undefined,
 
       // SSH tunnel
       useSshTunnel: process.env[`USE_SSH_${id}`],

packages/api/src/controllers/serverConnections.js

@@ -152,4 +152,13 @@ module.exports = {
     opened.subprocess.send({ msgtype: 'createDatabase', name });
     return { status: 'ok' };
   },
+
+  dropDatabase_meta: true,
+  async dropDatabase({ conid, name }, req) {
+    testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid);
+    if (opened.connection.isReadOnly) return false;
+    opened.subprocess.send({ msgtype: 'dropDatabase', name });
+    return { status: 'ok' };
+  },
 };

packages/api/src/controllers/sessions.js

@@ -103,6 +103,12 @@ module.exports = {
       if (handleProcessCommunication(message, subprocess)) return;
       this[`handle_${msgtype}`](sesid, message);
     });
+    subprocess.on('exit', () => {
+      this.opened = this.opened.filter(x => x.sesid != sesid);
+      this.dispatchMessage(sesid, 'Query session closed');
+      socket.emit(`session-closed-${sesid}`);
+    });
+
     subprocess.send({ msgtype: 'connect', ...connection, database });
     return _.pick(newOpened, ['conid', 'database', 'sesid']);
   },
@@ -165,6 +171,17 @@ module.exports = {
     return { state: 'ok' };
   },
 
+  ping_meta: true,
+  async ping({ sesid }) {
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (!session) {
+      throw new Error('Invalid session');
+    }
+    session.subprocess.send({ msgtype: 'ping' });
+
+    return { state: 'ok' };
+  },
+
   // runCommand_meta: true,
   // async runCommand({ conid, database, sql }) {
   //   console.log(`Running SQL command , conid=${conid}, database=${database}, sql=${sql}`);

packages/api/src/main.js

@@ -20,11 +20,13 @@ const jsldata = require('./controllers/jsldata');
 const config = require('./controllers/config');
 const archive = require('./controllers/archive');
 const apps = require('./controllers/apps');
+const auth = require('./controllers/auth');
 const uploads = require('./controllers/uploads');
 const plugins = require('./controllers/plugins');
 const files = require('./controllers/files');
 const scheduler = require('./controllers/scheduler');
 const queryHistory = require('./controllers/queryHistory');
+const onFinished = require('on-finished');
 
 const { rundir } = require('./utility/directories');
 const platformInfo = require('./utility/platformInfo');
@@ -40,7 +42,7 @@ function start() {
   const server = http.createServer(app);
 
   const logins = getLogins();
-  if (logins) {
+  if (logins && process.env.BASIC_AUTH) {
     app.use(
       basicAuth({
         users: _.fromPairs(logins.map(x => [x.login, x.password])),
@@ -52,6 +54,10 @@ function start() {
 
   app.use(cors());
 
+  if (auth.shouldAuthorizeApi()) {
+    app.use(auth.authMiddleware);
+  }
+
   app.get(getExpressPath('/stream'), async function (req, res) {
     res.set({
       'Cache-Control': 'no-cache',
@@ -63,7 +69,10 @@ function start() {
 
     // Tell the client to retry every 10 seconds if connectivity is lost
     res.write('retry: 10000\n\n');
-    socket.setSseResponse(res);
+    socket.addSseResponse(res);
+    onFinished(req, () => {
+      socket.removeSseResponse(res);
+    });
   });
 
   app.use(bodyParser.json({ limit: '50mb' }));
@@ -153,6 +162,7 @@ function useAllControllers(app, electron) {
   useController(app, electron, '/scheduler', scheduler);
   useController(app, electron, '/query-history', queryHistory);
   useController(app, electron, '/apps', apps);
+  useController(app, electron, '/auth', auth);
 }
 
 function setElectronSender(electronSender) {

packages/api/src/proc/databaseConnectionProcess.js

@@ -173,10 +173,11 @@ async function handleQueryData({ msgid, sql }, skipReadonlyCheck = false) {
   const driver = requireEngineDriver(storedConnection);
   try {
     if (!skipReadonlyCheck) ensureExecuteCustomScript(driver);
+    // console.log(sql);
     const res = await driver.query(systemConnection, sql);
     process.send({ msgtype: 'response', msgid, ...res });
   } catch (err) {
-    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+    process.send({ msgtype: 'response', msgid, errorMessage: err.message || 'Error executing SQL script' });
   }
 }
 
@@ -334,11 +335,11 @@ function start() {
 
   setInterval(() => {
     const time = new Date().getTime();
-    if (time - lastPing > 120 * 1000) {
+    if (time - lastPing > 40 * 1000) {
       console.log('Database connection not alive, exiting');
       process.exit(0);
     }
-  }, 60 * 1000);
+  }, 10 * 1000);
 
   process.on('message', async message => {
     if (handleProcessCommunication(message)) return;

packages/api/src/proc/serverConnectionProcess.js

@@ -2,7 +2,6 @@ const stableStringify = require('json-stable-stringify');
 const { extractBoolSettingsValue, extractIntSettingsValue } = require('dbgate-tools');
 const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const { decryptConnection } = require('../utility/crypting');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
 
@@ -81,14 +80,16 @@ function handlePing() {
   lastPing = new Date().getTime();
 }
 
-async function handleCreateDatabase({ name }) {
+async function handleDatabaseOp(op, { name }) {
   const driver = requireEngineDriver(storedConnection);
   systemConnection = await connectUtility(driver, storedConnection, 'app');
-  console.log(`RUNNING SCRIPT: CREATE DATABASE ${driver.dialect.quoteIdentifier(name)}`);
-  if (driver.createDatabase) {
-    await driver.createDatabase(systemConnection, name);
+  if (driver[op]) {
+    await driver[op](systemConnection, name);
   } else {
-    await driver.query(systemConnection, `CREATE DATABASE ${driver.dialect.quoteIdentifier(name)}`);
+    const dmp = driver.createDumper();
+    dmp[op](name);
+    console.log(`RUNNING SCRIPT: ${dmp.s}`);
+    await driver.query(systemConnection, dmp.s);
   }
   await handleRefresh();
 }
@@ -96,7 +97,8 @@ async function handleCreateDatabase({ name }) {
 const messageHandlers = {
   connect: handleConnect,
   ping: handlePing,
-  createDatabase: handleCreateDatabase,
+  createDatabase: props => handleDatabaseOp('createDatabase', props),
+  dropDatabase: props => handleDatabaseOp('dropDatabase', props),
 };
 
 async function handleMessage({ msgtype, ...other }) {
@@ -109,11 +111,11 @@ function start() {
 
   setInterval(() => {
     const time = new Date().getTime();
-    if (time - lastPing > 120 * 1000) {
+    if (time - lastPing > 40 * 1000) {
       console.log('Server connection not alive, exiting');
       process.exit(0);
     }
-  }, 60 * 1000);
+  }, 10 * 1000);
 
   process.on('message', async message => {
     if (handleProcessCommunication(message)) return;

packages/api/src/proc/sessionProcess.js

@@ -15,6 +15,7 @@ let systemConnection;
 let storedConnection;
 let afterConnectCallbacks = [];
 // let currentHandlers = [];
+let lastPing = null;
 
 class TableWriter {
   constructor() {
@@ -101,8 +102,9 @@ class TableWriter {
 }
 
 class StreamHandler {
-  constructor(resultIndexHolder, resolve) {
+  constructor(resultIndexHolder, resolve, startLine) {
     this.recordset = this.recordset.bind(this);
+    this.startLine = startLine;
     this.row = this.row.bind(this);
     // this.error = this.error.bind(this);
     this.done = this.done.bind(this);
@@ -155,14 +157,21 @@ class StreamHandler {
     this.resolve();
   }
   info(info) {
+    if (info && info.line != null) {
+      info = {
+        ...info,
+        line: this.startLine + info.line,
+      };
+    }
     process.send({ msgtype: 'info', info });
   }
 }
 
-function handleStream(driver, resultIndexHolder, sql) {
+function handleStream(driver, resultIndexHolder, sqlItem) {
   return new Promise((resolve, reject) => {
-    const handler = new StreamHandler(resultIndexHolder, resolve);
-    driver.stream(systemConnection, sql, handler);
+    const start = sqlItem.trimStart || sqlItem.start;
+    const handler = new StreamHandler(resultIndexHolder, resolve, start && start.line);
+    driver.stream(systemConnection, sqlItem.text, handler);
   });
 }
 
@@ -221,7 +230,10 @@ async function handleExecuteQuery({ sql }) {
   const resultIndexHolder = {
     value: 0,
   };
-  for (const sqlItem of splitQuery(sql, driver.getQuerySplitterOptions('stream'))) {
+  for (const sqlItem of splitQuery(sql, {
+    ...driver.getQuerySplitterOptions('stream'),
+    returnRichInfo: true,
+  })) {
     await handleStream(driver, resultIndexHolder, sqlItem);
     // const handler = new StreamHandler(resultIndex);
     // const stream = await driver.stream(systemConnection, sqlItem, handler);
@@ -260,10 +272,15 @@ async function handleExecuteReader({ jslid, sql, fileName }) {
   });
 }
 
+function handlePing() {
+  lastPing = new Date().getTime();
+}
+
 const messageHandlers = {
   connect: handleConnect,
   executeQuery: handleExecuteQuery,
   executeReader: handleExecuteReader,
+  ping: handlePing,
   // cancel: handleCancel,
 };
 
@@ -274,6 +291,17 @@ async function handleMessage({ msgtype, ...other }) {
 
 function start() {
   childProcessChecker();
+
+  lastPing = new Date().getTime();
+
+  setInterval(() => {
+    const time = new Date().getTime();
+    if (time - lastPing > 25 * 1000) {
+      console.log('Session not alive, exiting');
+      process.exit(0);
+    }
+  }, 10 * 1000);
+
   process.on('message', async message => {
     if (handleProcessCommunication(message)) return;
     try {

packages/api/src/shell/importDatabase.js

@@ -47,7 +47,7 @@ async function importDatabase({ connection = undefined, systemConnection = undef
   const downloadedFile = await download(inputFile);
 
   const fileStream = fs.createReadStream(downloadedFile, 'utf-8');
-  const splittedStream = splitQueryStream(fileStream, driver.getQuerySplitterOptions());
+  const splittedStream = splitQueryStream(fileStream, driver.getQuerySplitterOptions('script'));
   const importStream = new ImportStream(pool, driver);
   // @ts-ignore
   splittedStream.pipe(importStream);

packages/api/src/shell/queryReader.js

@@ -20,7 +20,9 @@ async function queryReader({
   const driver = requireEngineDriver(connection);
   const pool = await connectUtility(driver, connection, queryType == 'json' ? 'read' : 'script');
   console.log(`Connected.`);
-  return queryType == 'json' ? await driver.readJsonQuery(pool, query) : await driver.readQuery(pool, query || sql);
+  const reader =
+    queryType == 'json' ? await driver.readJsonQuery(pool, query) : await driver.readQuery(pool, query || sql);
+  return reader;
 }
 
 module.exports = queryReader;

packages/api/src/utility/socket.js

@@ -1,36 +1,33 @@
-let sseResponse = null;
+const _ = require('lodash');
+
+const sseResponses = [];
 let electronSender = null;
-let init = [];
+let pingConfigured = false;
 
 module.exports = {
-  setSseResponse(value) {
-    sseResponse = value;
-    setInterval(() => this.emit('ping'), 29 * 1000);
+  ensurePing() {
+    if (!pingConfigured) {
+      setInterval(() => this.emit('ping'), 29 * 1000);
+      pingConfigured = true;
+    }
+  },
+  addSseResponse(value) {
+    sseResponses.push(value);
+    this.ensurePing();
+  },
+  removeSseResponse(value) {
+    _.remove(sseResponses, x => x == value);
   },
   setElectronSender(value) {
     electronSender = value;
+    this.ensurePing();
   },
   emit(message, data) {
     if (electronSender) {
-      if (init.length > 0) {
-        for (const item of init) {
-          electronSender.send(item.message, item.data == null ? null : item.data);
-        }
-        init = [];
-      }
       electronSender.send(message, data == null ? null : data);
-    } else if (sseResponse) {
-      if (init.length > 0) {
-        for (const item of init) {
-          sseResponse.write(
-            `event: ${item.message}\ndata: ${JSON.stringify(item.data == null ? null : item.data)}\n\n`
-          );
-        }
-        init = [];
-      }
-      sseResponse.write(`event: ${message}\ndata: ${JSON.stringify(data == null ? null : data)}\n\n`);
-    } else {
-      init.push([{ message, data }]);
+    }
+    for (const res of sseResponses) {
+      res.write(`event: ${message}\ndata: ${JSON.stringify(data == null ? null : data)}\n\n`);
     }
   },
   emitChanged(key) {

packages/datalib/jest.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  moduleFileExtensions: ['js'],
+};

packages/datalib/package.json

@@ -5,6 +5,8 @@
   "typings": "lib/index.d.ts",
   "scripts": {
     "build": "tsc",
+    "test": "jest",
+    "test:ci": "jest --json --outputFile=result.json --testLocationInResults",
     "start": "tsc --watch"
   },
   "files": [
@@ -12,11 +14,14 @@
   ],
   "dependencies": {
     "dbgate-sqltree": "^5.0.0-alpha.1",
+    "dbgate-tools": "^5.0.0-alpha.1",
     "dbgate-filterparser": "^5.0.0-alpha.1"
   },
   "devDependencies": {
     "dbgate-types": "^5.0.0-alpha.1",
     "@types/node": "^13.7.0",
+    "jest": "^28.1.3",
+    "ts-jest": "^28.0.7",
     "typescript": "^4.4.3"
   }
 }

packages/datalib/src/ChangeSet.ts

@@ -9,7 +9,7 @@ import {
   AllowIdentityInsert,
   Expression,
 } from 'dbgate-sqltree';
-import { NamedObjectInfo, DatabaseInfo } from 'dbgate-types';
+import type { NamedObjectInfo, DatabaseInfo } from 'dbgate-types';
 
 export interface ChangeSetItem {
   pureName: string;

packages/datalib/src/CollectionGridDisplay.ts

@@ -1,6 +1,6 @@
 import _ from 'lodash';
 import { GridDisplay, ChangeCacheFunc, ChangeConfigFunc, DisplayColumn } from './GridDisplay';
-import { EngineDriver, ViewInfo, ColumnInfo, CollectionInfo } from 'dbgate-types';
+import type { EngineDriver, ViewInfo, ColumnInfo, CollectionInfo } from 'dbgate-types';
 import { GridConfig, GridCache } from './GridConfig';
 
 function getObjectKeys(obj) {

packages/datalib/src/FormViewDisplay.ts

@@ -1,6 +1,6 @@
 import _ from 'lodash';
 import { GridConfig, GridCache, GridConfigColumns, createGridCache, GroupFunc } from './GridConfig';
-import { TableInfo, EngineDriver, DatabaseInfo, SqlDialect } from 'dbgate-types';
+import type { TableInfo, EngineDriver, DatabaseInfo, SqlDialect } from 'dbgate-types';
 import { getFilterValueExpression } from 'dbgate-filterparser';
 import { ChangeCacheFunc, ChangeConfigFunc, DisplayColumn } from './GridDisplay';
 

packages/datalib/src/FreeTableGridDisplay.ts

@@ -1,5 +1,5 @@
 import _ from 'lodash';
-import { EngineDriver, ViewInfo, ColumnInfo } from 'dbgate-types';
+import type { EngineDriver, ViewInfo, ColumnInfo } from 'dbgate-types';
 import { GridDisplay, ChangeCacheFunc, ChangeConfigFunc } from './GridDisplay';
 import { GridConfig, GridCache } from './GridConfig';
 import { FreeTableModel } from './FreeTableModel';

packages/datalib/src/FreeTableModel.ts

@@ -1,4 +1,4 @@
-import { TableInfo } from 'dbgate-types';
+import type { TableInfo } from 'dbgate-types';
 
 export interface FreeTableModel {
   structure: TableInfo;

packages/datalib/src/GridConfig.ts

@@ -1,6 +1,3 @@
-import { DisplayColumn } from './GridDisplay';
-import { TableInfo } from 'dbgate-types';
-
 export interface GridConfigColumns {
   hiddenColumns: string[];
   expandedColumns: string[];

packages/datalib/src/GridDisplay.ts

@@ -1,6 +1,6 @@
 import _ from 'lodash';
 import { GridConfig, GridCache, GridConfigColumns, createGridCache, GroupFunc, createGridConfig } from './GridConfig';
-import {
+import type {
   ForeignKeyInfo,
   TableInfo,
   ColumnInfo,
@@ -24,7 +24,7 @@ export interface DisplayColumn {
   headerText: string;
   uniqueName: string;
   uniquePath: string[];
-  notNull: boolean;
+  notNull?: boolean;
   autoIncrement?: boolean;
   isPrimaryKey?: boolean;
   foreignKey?: ForeignKeyInfo;

packages/datalib/src/JslGridDisplay.ts

@@ -1,6 +1,5 @@
 import _ from 'lodash';
 import { GridDisplay, ChangeCacheFunc, ChangeConfigFunc } from './GridDisplay';
-import { QueryResultColumn } from 'dbgate-types';
 import { GridConfig, GridCache } from './GridConfig';
 import { analyseCollectionDisplayColumns } from './CollectionGridDisplay';
 

packages/datalib/src/PerspectiveCache.ts

@@ -0,0 +1,128 @@
+import { PerspectiveDataLoadProps } from './PerspectiveDataProvider';
+import _pick from 'lodash/pick';
+import _zip from 'lodash/zip';
+import _difference from 'lodash/difference';
+import debug from 'debug';
+import stableStringify from 'json-stable-stringify';
+import { PerspectiveDataPattern } from './PerspectiveDataPattern';
+
+const dbg = debug('dbgate:PerspectiveCache');
+
+export class PerspectiveBindingGroup {
+  constructor(public table: PerspectiveCacheTable) {}
+
+  groupSize?: number;
+  loadedAll: boolean;
+  loadedRows: any[] = [];
+  bindingValues: any[];
+
+  matchRow(row) {
+    return this.table.bindingColumns.every((column, index) => row[column] == this.bindingValues[index]);
+  }
+}
+
+export class PerspectiveCacheTable {
+  constructor(props: PerspectiveDataLoadProps, public cache: PerspectiveCache) {
+    this.schemaName = props.schemaName;
+    this.pureName = props.pureName;
+    this.bindingColumns = props.bindingColumns;
+    this.dataColumns = props.dataColumns;
+    this.loadedAll = false;
+  }
+
+  schemaName: string;
+  pureName: string;
+  bindingColumns?: string[];
+  dataColumns: string[];
+  allColumns?: boolean;
+  loadedAll: boolean;
+  loadedRows: any[] = [];
+  bindingGroups: { [bindingKey: string]: PerspectiveBindingGroup } = {};
+  allRowCount: number = null;
+
+  get loadedCount() {
+    return this.loadedRows.length;
+  }
+
+  getRowsResult(props: PerspectiveDataLoadProps): { rows: any[]; incomplete: boolean } {
+    return {
+      rows: this.loadedRows.slice(0, props.topCount),
+      incomplete: props.topCount < this.loadedCount || !this.loadedAll,
+    };
+  }
+
+  getBindingGroup(groupValues: any[]) {
+    const key = stableStringify(groupValues);
+    return this.bindingGroups[key];
+  }
+
+  getUncachedBindingGroups(props: PerspectiveDataLoadProps): any[][] {
+    const uncached = [];
+    for (const group of props.bindingValues) {
+      const key = stableStringify(group);
+      const item = this.bindingGroups[key];
+      if (!item) {
+        uncached.push(group);
+      }
+    }
+    return uncached;
+  }
+
+  storeGroupSize(props: PerspectiveDataLoadProps, bindingValues: any[], count: number) {
+    const originalBindingValue = props.bindingValues.find(v => _zip(v, bindingValues).every(([x, y]) => x == y));
+    if (originalBindingValue) {
+      const key = stableStringify(originalBindingValue);
+      // console.log('SET SIZE', originalBindingValue, bindingValues, key, count);
+      const group = new PerspectiveBindingGroup(this);
+      group.bindingValues = bindingValues;
+      group.groupSize = count;
+      this.bindingGroups[key] = group;
+    } else {
+      dbg('Group not found', bindingValues);
+    }
+  }
+}
+
+export class PerspectiveCache {
+  constructor() {}
+
+  tables: { [tableKey: string]: PerspectiveCacheTable } = {};
+  dataPatterns: PerspectiveDataPattern[] = [];
+
+  getTableCache(props: PerspectiveDataLoadProps) {
+    const tableKey = stableStringify(
+      _pick(props, [
+        'schemaName',
+        'pureName',
+        'bindingColumns',
+        'databaseConfig',
+        'orderBy',
+        'sqlCondition',
+        'mongoCondition',
+      ])
+    );
+    let res = this.tables[tableKey];
+
+    if (res && _difference(props.dataColumns, res.dataColumns).length > 0 && !res.allColumns) {
+      dbg('Delete cache because incomplete columns', props.pureName, res.dataColumns);
+
+      // we have incomplete cache
+      delete this.tables[tableKey];
+      res = null;
+    }
+
+    if (!res) {
+      res = new PerspectiveCacheTable(props, this);
+      this.tables[tableKey] = res;
+      return res;
+    }
+
+    // cache could be used
+    return res;
+  }
+
+  clear() {
+    this.tables = {};
+    this.dataPatterns = [];
+  }
+}

packages/datalib/src/PerspectiveConfig.ts

@@ -0,0 +1,185 @@
+import type { DatabaseInfo, ForeignKeyInfo, NamedObjectInfo, TableInfo } from 'dbgate-types';
+import uuidv1 from 'uuid/v1';
+
+// export interface PerspectiveConfigColumns {
+//   expandedColumns: string[];
+//   checkedColumns: string[];
+//   uncheckedColumns: string[];
+// }
+
+export type PerspectiveDatabaseEngineType = 'sqldb' | 'docdb';
+
+export interface PerspectiveDatabaseConfig {
+  conid: string;
+  database: string;
+}
+
+export interface PerspectiveCustomJoinConfig {
+  refNodeDesignerId: string;
+  referenceDesignerId: string;
+  joinName: string;
+  baseDesignerId: string;
+  conid?: string;
+  database?: string;
+  refSchemaName?: string;
+  refTableName: string;
+  columns: {
+    baseColumnName: string;
+    refColumnName: string;
+  }[];
+}
+
+export interface PerspectiveFilterColumnInfo {
+  columnName: string;
+  filterType: string;
+  pureName: string;
+  schemaName: string;
+  foreignKey: ForeignKeyInfo;
+}
+
+// export interface PerspectiveParentFilterConfig {
+//   uniqueName: string;
+// }
+// export interface PerspectiveConfig extends PerspectiveConfigColumns {
+//   rootObject: { schemaName?: string; pureName: string };
+//   filters: { [uniqueName: string]: string };
+//   sort: {
+//     [parentUniqueName: string]: {
+//       uniqueName: string;
+//       order: 'ASC' | 'DESC';
+//     }[];
+//   };
+//   customJoins: PerspectiveCustomJoinConfig[];
+//   parentFilters: PerspectiveParentFilterConfig[];
+// }
+
+export interface PerspectiveNodeConfig {
+  designerId: string;
+  schemaName?: string;
+  pureName: string;
+  defaultColumnsProcessed?: boolean;
+
+  alias?: string;
+
+  conid?: string;
+  database?: string;
+
+  isParentFilter?: boolean;
+
+  expandedColumns: string[];
+  checkedColumns: string[];
+  columnDisplays: {};
+  // uncheckedColumns: string[];
+
+  sort: {
+    columnName: string;
+    order: 'ASC' | 'DESC';
+  }[];
+
+  filters: { [uniqueName: string]: string };
+  isAutoGenerated?: true | undefined;
+  isNodeChecked?: boolean;
+
+  position?: {
+    x: number;
+    y: number;
+  };
+}
+
+export interface PerspectiveReferenceConfig {
+  designerId: string;
+
+  sourceId: string;
+  targetId: string;
+
+  columns: {
+    source: string;
+    target: string;
+  }[];
+
+  isAutoGenerated?: true | undefined;
+}
+
+export interface PerspectiveConfig {
+  rootDesignerId: string;
+  isArranged: boolean;
+  nodes: PerspectiveNodeConfig[];
+  references: PerspectiveReferenceConfig[];
+}
+
+export function createPerspectiveNodeConfig(name: { schemaName?: string; pureName: string }) {
+  const node: PerspectiveNodeConfig = {
+    pureName: name.pureName,
+    schemaName: name.schemaName,
+    designerId: uuidv1(),
+
+    expandedColumns: [],
+    checkedColumns: [],
+    columnDisplays: {},
+
+    sort: [],
+    filters: {},
+
+    isNodeChecked: true,
+  };
+
+  return node;
+}
+
+export function createPerspectiveConfig(rootObject?: { schemaName?: string; pureName: string }): PerspectiveConfig {
+  if (!rootObject) {
+    return {
+      nodes: [],
+      references: [],
+      isArranged: true,
+      rootDesignerId: null,
+    };
+  }
+
+  const rootNode = createPerspectiveNodeConfig(rootObject);
+  return {
+    nodes: [rootNode],
+    references: [],
+    rootDesignerId: rootNode.designerId,
+    isArranged: true,
+  };
+}
+
+export type ChangePerspectiveConfigFunc = (
+  changeFunc: (config: PerspectiveConfig) => PerspectiveConfig,
+  reload?: boolean
+) => void;
+
+export function extractPerspectiveDatabases(
+  { conid, database },
+  cfg: PerspectiveConfig
+): { conid: string; database: string }[] {
+  const res: { conid: string; database: string }[] = [];
+  res.push({ conid, database });
+
+  function add(conid, database) {
+    if (res.find(x => x.conid == conid && x.database == database)) return;
+    res.push({ conid, database });
+  }
+
+  for (const node of cfg.nodes) {
+    add(node.conid || conid, node.database || database);
+  }
+  return res;
+}
+
+export interface MultipleDatabaseInfo {
+  [conid: string]: {
+    [database: string]: DatabaseInfo;
+  };
+}
+
+export function switchPerspectiveReferenceDirection(ref: PerspectiveReferenceConfig): PerspectiveReferenceConfig {
+  return {
+    designerId: ref.designerId,
+    sourceId: ref.targetId,
+    targetId: ref.sourceId,
+    isAutoGenerated: ref.isAutoGenerated,
+    columns: ref.columns.map(x => ({ source: x.target, target: x.source })),
+  };
+}

packages/datalib/src/PerspectiveDataLoader.ts

@@ -0,0 +1,373 @@
+import { Condition, Expression, Select } from 'dbgate-sqltree';
+import { PerspectiveDataLoadProps } from './PerspectiveDataProvider';
+import debug from 'debug';
+import _zipObject from 'lodash/zipObject';
+import _mapValues from 'lodash/mapValues';
+import _isArray from 'lodash/isArray';
+import { safeJsonParse } from 'dbgate-tools';
+
+function normalizeLoadedRow(row) {
+  return _mapValues(row, v => safeJsonParse(v) || v);
+}
+
+function normalizeResult(result) {
+  if (_isArray(result)) {
+    return result.map(normalizeLoadedRow);
+  }
+  if (result.errorMessage) {
+    return result;
+  }
+  return {
+    ...result,
+    errorMessage: 'Unspecified error',
+  };
+}
+
+const dbg = debug('dbgate:PerspectiveDataLoader');
+
+export class PerspectiveDataLoader {
+  constructor(public apiCall) {}
+
+  buildSqlCondition(props: PerspectiveDataLoadProps): Condition {
+    const { schemaName, pureName, bindingColumns, bindingValues, dataColumns, orderBy, sqlCondition } = props;
+
+    const conditions = [];
+
+    if (sqlCondition) {
+      conditions.push(sqlCondition);
+    }
+
+    if (bindingColumns?.length == 1) {
+      conditions.push({
+        conditionType: 'in',
+        expr: {
+          exprType: 'column',
+          columnName: bindingColumns[0],
+          source: {
+            name: { schemaName, pureName },
+          },
+        },
+        values: bindingValues.map(x => x[0]),
+      });
+    }
+
+    return conditions.length > 0
+      ? {
+          conditionType: 'and',
+          conditions,
+        }
+      : null;
+  }
+
+  buildMongoCondition(props: PerspectiveDataLoadProps): {} {
+    const { schemaName, pureName, bindingColumns, bindingValues, dataColumns, orderBy, mongoCondition } = props;
+
+    const conditions = [];
+
+    if (mongoCondition) {
+      conditions.push(mongoCondition);
+    }
+
+    if (bindingColumns?.length == 1) {
+      conditions.push({
+        [bindingColumns[0]]: { $in: bindingValues.map(x => x[0]) },
+      });
+    }
+
+    return conditions.length == 1 ? conditions[0] : conditions.length > 0 ? { $and: conditions } : null;
+  }
+
+  async loadGroupingSqlDb(props: PerspectiveDataLoadProps) {
+    const { schemaName, pureName, bindingColumns } = props;
+
+    const bindingColumnExpressions = bindingColumns.map(
+      columnName =>
+        ({
+          exprType: 'column',
+          columnName,
+          source: {
+            name: { schemaName, pureName },
+          },
+        } as Expression)
+    );
+
+    const select: Select = {
+      commandType: 'select',
+      from: {
+        name: { schemaName, pureName },
+      },
+      columns: [
+        {
+          exprType: 'call',
+          func: 'COUNT',
+          args: [
+            {
+              exprType: 'raw',
+              sql: '*',
+            },
+          ],
+          alias: '_perspective_group_size_',
+        },
+        ...bindingColumnExpressions,
+      ],
+      where: this.buildSqlCondition(props),
+    };
+
+    select.groupBy = bindingColumnExpressions;
+
+    if (dbg?.enabled) {
+      dbg(`LOAD COUNTS, table=${props.pureName}, columns=${bindingColumns?.join(',')}`);
+    }
+
+    const response = await this.apiCall('database-connections/sql-select', {
+      conid: props.databaseConfig.conid,
+      database: props.databaseConfig.database,
+      select,
+    });
+
+    if (response.errorMessage) return response;
+    return response.rows.map(row => ({
+      ...row,
+      _perspective_group_size_: parseInt(row._perspective_group_size_),
+    }));
+  }
+
+  async loadGroupingDocDb(props: PerspectiveDataLoadProps) {
+    const { schemaName, pureName, bindingColumns } = props;
+
+    const aggregate = [
+      { $match: this.buildMongoCondition(props) },
+      {
+        $group: {
+          _id: _zipObject(
+            bindingColumns,
+            bindingColumns.map(col => '$' + col)
+          ),
+          count: { $sum: 1 },
+        },
+      },
+    ];
+
+    if (dbg?.enabled) {
+      dbg(`LOAD COUNTS, table=${props.pureName}, columns=${bindingColumns?.join(',')}`);
+    }
+
+    const response = await this.apiCall('database-connections/collection-data', {
+      conid: props.databaseConfig.conid,
+      database: props.databaseConfig.database,
+      options: {
+        pureName,
+        aggregate,
+      },
+    });
+
+    if (response.errorMessage) return response;
+    return response.rows.map(row => ({
+      ...row._id,
+      _perspective_group_size_: parseInt(row.count),
+    }));
+  }
+
+  async loadGrouping(props: PerspectiveDataLoadProps) {
+    const { engineType } = props;
+    switch (engineType) {
+      case 'sqldb':
+        return this.loadGroupingSqlDb(props);
+      case 'docdb':
+        return this.loadGroupingDocDb(props);
+    }
+  }
+
+  async loadDataSqlDb(props: PerspectiveDataLoadProps) {
+    const {
+      schemaName,
+      pureName,
+      bindingColumns,
+      bindingValues,
+      dataColumns,
+      orderBy,
+      sqlCondition: condition,
+      engineType,
+    } = props;
+
+    if (dataColumns?.length == 0) {
+      return [];
+    }
+
+    const select: Select = {
+      commandType: 'select',
+      from: {
+        name: { schemaName, pureName },
+      },
+      columns: dataColumns?.map(columnName => ({
+        exprType: 'column',
+        columnName,
+        source: {
+          name: { schemaName, pureName },
+        },
+      })),
+      selectAll: !dataColumns,
+      orderBy:
+        orderBy?.length > 0
+          ? orderBy?.map(({ columnName, order }) => ({
+              exprType: 'column',
+              columnName,
+              direction: order,
+              source: {
+                name: { schemaName, pureName },
+              },
+            }))
+          : null,
+      range: props.range,
+      where: this.buildSqlCondition(props),
+    };
+
+    if (dbg?.enabled) {
+      dbg(
+        `LOAD DATA, table=${props.pureName}, columns=${props.dataColumns?.join(',')}, range=${props.range?.offset},${
+          props.range?.limit
+        }`
+      );
+    }
+
+    const response = await this.apiCall('database-connections/sql-select', {
+      conid: props.databaseConfig.conid,
+      database: props.databaseConfig.database,
+      select,
+    });
+
+    if (response.errorMessage) return response;
+    return response.rows;
+  }
+
+  getDocDbLoadOptions(props: PerspectiveDataLoadProps, useSort: boolean) {
+    const { pureName } = props;
+    const res: any = {
+      pureName,
+      condition: this.buildMongoCondition(props),
+      skip: props.range?.offset,
+      limit: props.range?.limit,
+    };
+    if (useSort && props.orderBy?.length > 0) {
+      res.sort = _zipObject(
+        props.orderBy.map(col => col.columnName),
+        props.orderBy.map(col => (col.order == 'DESC' ? -1 : 1))
+      );
+    }
+
+    return res;
+  }
+
+  async loadDataDocDb(props: PerspectiveDataLoadProps) {
+    const {
+      schemaName,
+      pureName,
+      bindingColumns,
+      bindingValues,
+      dataColumns,
+      orderBy,
+      sqlCondition: condition,
+      engineType,
+    } = props;
+
+    if (dbg?.enabled) {
+      dbg(
+        `LOAD DATA, collection=${props.pureName}, columns=${props.dataColumns?.join(',')}, range=${
+          props.range?.offset
+        },${props.range?.limit}`
+      );
+    }
+
+    const options = this.getDocDbLoadOptions(props, true);
+
+    const response = await this.apiCall('database-connections/collection-data', {
+      conid: props.databaseConfig.conid,
+      database: props.databaseConfig.database,
+      options,
+    });
+
+    if (response.errorMessage) return response;
+    return response.rows;
+  }
+
+  async loadData(props: PerspectiveDataLoadProps) {
+    const { engineType } = props;
+    switch (engineType) {
+      case 'sqldb':
+        return normalizeResult(await this.loadDataSqlDb(props));
+      case 'docdb':
+        return normalizeResult(await this.loadDataDocDb(props));
+    }
+  }
+
+  async loadRowCountSqlDb(props: PerspectiveDataLoadProps) {
+    const {
+      schemaName,
+      pureName,
+      bindingColumns,
+      bindingValues,
+      dataColumns,
+      orderBy,
+      sqlCondition: condition,
+    } = props;
+
+    const select: Select = {
+      commandType: 'select',
+      from: {
+        name: { schemaName, pureName },
+      },
+      columns: [
+        {
+          exprType: 'raw',
+          sql: 'COUNT(*)',
+          alias: 'count',
+        },
+      ],
+      where: this.buildSqlCondition(props),
+    };
+
+    const response = await this.apiCall('database-connections/sql-select', {
+      conid: props.databaseConfig.conid,
+      database: props.databaseConfig.database,
+      select,
+    });
+
+    if (response.errorMessage) return response;
+    return response.rows[0];
+  }
+
+  async loadRowCountDocDb(props: PerspectiveDataLoadProps) {
+    const {
+      schemaName,
+      pureName,
+      bindingColumns,
+      bindingValues,
+      dataColumns,
+      orderBy,
+      sqlCondition: condition,
+    } = props;
+
+    const options = {
+      ...this.getDocDbLoadOptions(props, false),
+      countDocuments: true,
+    };
+
+    const response = await this.apiCall('database-connections/collection-data', {
+      conid: props.databaseConfig.conid,
+      database: props.databaseConfig.database,
+      options,
+    });
+
+    return response;
+  }
+
+  async loadRowCount(props: PerspectiveDataLoadProps) {
+    const { engineType } = props;
+    switch (engineType) {
+      case 'sqldb':
+        return this.loadRowCountSqlDb(props);
+      case 'docdb':
+        return this.loadRowCountDocDb(props);
+    }
+  }
+}

packages/datalib/src/PerspectiveDataPattern.ts

@@ -0,0 +1,95 @@
+import { PerspectiveDataLoader } from './PerspectiveDataLoader';
+import { PerspectiveDataLoadProps } from './PerspectiveDataProvider';
+import _isString from 'lodash/isString';
+import _isPlainObject from 'lodash/isPlainObject';
+import _isNumber from 'lodash/isNumber';
+import _isBoolean from 'lodash/isBoolean';
+import _isArray from 'lodash/isArray';
+import { safeJsonParse } from 'dbgate-tools';
+
+export type PerspectiveDataPatternColumnType = 'null' | 'oid' | 'string' | 'number' | 'boolean' | 'json';
+
+export interface PerspectiveDataPatternColumn {
+  name: string;
+  types: PerspectiveDataPatternColumnType[];
+  columns: PerspectiveDataPatternColumn[];
+}
+
+export interface PerspectiveDataPattern {
+  conid: string;
+  database: string;
+  schemaName?: string;
+  pureName: string;
+  columns: PerspectiveDataPatternColumn[];
+}
+
+export type PerspectiveDataPatternDict = { [designerId: string]: PerspectiveDataPattern };
+
+function detectValueType(value): PerspectiveDataPatternColumnType {
+  if (_isString(value)) return 'string';
+  if (_isNumber(value)) return 'number';
+  if (_isBoolean(value)) return 'boolean';
+  if (value?.$oid) return 'oid';
+  if (_isPlainObject(value) || _isArray(value)) return 'json';
+  if (value == null) return 'null';
+}
+
+function addObjectToColumns(columns: PerspectiveDataPatternColumn[], row) {
+  if (_isPlainObject(row)) {
+    for (const key of Object.keys(row)) {
+      let column: PerspectiveDataPatternColumn = columns.find(x => x.name == key);
+      if (!column) {
+        column = {
+          name: key,
+          types: [],
+          columns: [],
+        };
+        columns.push(column);
+      }
+      const value = row[key];
+      const type = detectValueType(value);
+      if (!column.types.includes(type)) {
+        column.types.push(type);
+      }
+      if (_isPlainObject(value)) {
+        addObjectToColumns(column.columns, value);
+      }
+      if (_isArray(value)) {
+        for (const item of value) {
+          addObjectToColumns(column.columns, item);
+        }
+      }
+      if (_isString(value)) {
+        const json = safeJsonParse(value);
+        if (json && (_isPlainObject(json) || _isArray(json))) {
+          if (!column.types.includes('json')) {
+            column.types.push('json');
+          }
+          if (_isPlainObject(json)) {
+            addObjectToColumns(column.columns, json);
+          }
+          if (_isArray(json)) {
+            for (const item of json) {
+              addObjectToColumns(column.columns, item);
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+export function analyseDataPattern(
+  patternBase: Omit<PerspectiveDataPattern, 'columns'>,
+  rows: any[]
+): PerspectiveDataPattern {
+  const res: PerspectiveDataPattern = {
+    ...patternBase,
+    columns: [],
+  };
+  // console.log('ROWS', rows);
+  for (const row of rows) {
+    addObjectToColumns(res.columns, row);
+  }
+  return res;
+}

packages/datalib/src/PerspectiveDataProvider.ts

@@ -0,0 +1,229 @@
+import debug from 'debug';
+import { Condition } from 'dbgate-sqltree';
+import type { RangeDefinition } from 'dbgate-types';
+import { PerspectiveBindingGroup, PerspectiveCache } from './PerspectiveCache';
+import { PerspectiveDataLoader } from './PerspectiveDataLoader';
+import { PerspectiveDataPatternDict } from './PerspectiveDataPattern';
+import { PerspectiveDatabaseConfig, PerspectiveDatabaseEngineType } from './PerspectiveConfig';
+
+export const PERSPECTIVE_PAGE_SIZE = 100;
+
+const dbg = debug('dbgate:PerspectiveDataProvider');
+
+export interface PerspectiveDataLoadProps {
+  databaseConfig: PerspectiveDatabaseConfig;
+  schemaName?: string;
+  pureName: string;
+  dataColumns?: string[];
+  allColumns?: boolean;
+  orderBy: {
+    columnName: string;
+    order: 'ASC' | 'DESC';
+  }[];
+  bindingColumns?: string[];
+  bindingValues?: any[][];
+  range?: RangeDefinition;
+  topCount?: number;
+  sqlCondition?: Condition;
+  mongoCondition?: any;
+  engineType: PerspectiveDatabaseEngineType;
+}
+
+export class PerspectiveDataProvider {
+  constructor(
+    public cache: PerspectiveCache,
+    public loader: PerspectiveDataLoader,
+    public dataPatterns: PerspectiveDataPatternDict
+  ) {}
+  async loadData(props: PerspectiveDataLoadProps): Promise<{ rows: any[]; incomplete: boolean }> {
+    dbg('load data', props);
+    // console.log('LOAD DATA', props);
+    if (props.bindingColumns) {
+      return this.loadDataNested(props);
+    } else {
+      return this.loadDataFlat(props);
+    }
+  }
+
+  async loadDataNested(props: PerspectiveDataLoadProps): Promise<{ rows: any[]; incomplete: boolean }> {
+    const tableCache = this.cache.getTableCache(props);
+
+    const uncached = tableCache.getUncachedBindingGroups(props);
+    if (uncached.length > 0) {
+      const counts = await this.loader.loadGrouping({
+        ...props,
+        bindingValues: uncached,
+      });
+      // console.log('COUNTS', counts);
+      for (const resetItem of uncached) {
+        tableCache.storeGroupSize(props, resetItem, 0);
+      }
+      for (const countItem of counts) {
+        const { _perspective_group_size_, ...fields } = countItem;
+        tableCache.storeGroupSize(
+          props,
+          props.bindingColumns.map(col => fields[col]),
+          _perspective_group_size_
+        );
+      }
+    }
+
+    const rows = [];
+
+    // console.log('CACHE', tableCache.bindingGroups);
+
+    let groupIndex = 0;
+    let loadCalled = false;
+    let shouldReturn = false;
+    for (; groupIndex < props.bindingValues.length; groupIndex++) {
+      const groupValues = props.bindingValues[groupIndex];
+      const group = tableCache.getBindingGroup(groupValues);
+
+      if (!group.loadedAll) {
+        if (loadCalled) {
+          shouldReturn = true;
+        } else {
+          // we need to load next data
+          await this.loadNextGroup(props, groupIndex);
+          loadCalled = true;
+        }
+      }
+
+      // console.log('GRP', groupValues, group);
+      rows.push(...group.loadedRows);
+      if (rows.length >= props.topCount || shouldReturn) {
+        return {
+          rows: rows.slice(0, props.topCount),
+          incomplete: props.topCount < rows.length || !group.loadedAll || groupIndex < props.bindingValues.length - 1,
+        };
+      }
+    }
+    if (groupIndex >= props.bindingValues.length) {
+      // all groups are fully loaded
+      return { rows, incomplete: false };
+    }
+  }
+
+  async loadNextGroup(props: PerspectiveDataLoadProps, groupIndex: number) {
+    const tableCache = this.cache.getTableCache(props);
+
+    const planLoadingGroupIndexes: number[] = [];
+    const planLoadingGroups: PerspectiveBindingGroup[] = [];
+    let planLoadRowCount = 0;
+
+    const loadPlanned = async () => {
+      // console.log(
+      //   'LOAD PLANNED',
+      //   planLoadingGroupIndexes,
+      //   planLoadingGroupIndexes.map(idx => props.bindingValues[idx])
+      // );
+      const rows = await this.loader.loadData({
+        ...props,
+        bindingValues: planLoadingGroupIndexes.map(idx => props.bindingValues[idx]),
+      });
+      // console.log('LOADED PLANNED', rows);
+      // distribute rows into groups
+      for (const row of rows) {
+        const group = planLoadingGroups.find(x => x.matchRow(row));
+        if (group) {
+          group.loadedRows.push(row);
+        }
+      }
+      for (const group of planLoadingGroups) {
+        group.loadedAll = true;
+      }
+    };
+
+    for (; groupIndex < props.bindingValues.length; groupIndex++) {
+      const groupValues = props.bindingValues[groupIndex];
+      const group = tableCache.getBindingGroup(groupValues);
+      if (!group) continue;
+      if (group.loadedAll) continue;
+      if (group.groupSize == 0) {
+        group.loadedAll = true;
+        continue;
+      }
+      if (group.groupSize >= PERSPECTIVE_PAGE_SIZE) {
+        if (planLoadingGroupIndexes.length > 0) {
+          await loadPlanned();
+          return;
+        }
+        const nextRows = await this.loader.loadData({
+          ...props,
+          topCount: null,
+          range: {
+            offset: group.loadedRows.length,
+            limit: PERSPECTIVE_PAGE_SIZE,
+          },
+          bindingValues: [group.bindingValues],
+        });
+        group.loadedRows = [...group.loadedRows, ...nextRows];
+        group.loadedAll = nextRows.length < PERSPECTIVE_PAGE_SIZE;
+        return;
+      } else {
+        if (planLoadRowCount + group.groupSize > PERSPECTIVE_PAGE_SIZE) {
+          await loadPlanned();
+          return;
+        }
+        planLoadingGroupIndexes.push(groupIndex);
+        planLoadingGroups.push(group);
+        planLoadRowCount += group.groupSize;
+      }
+    }
+
+    if (planLoadingGroupIndexes.length > 0) {
+      await loadPlanned();
+    }
+  }
+
+  async loadDataFlat(props: PerspectiveDataLoadProps): Promise<{ rows: any[]; incomplete: boolean }> {
+    const tableCache = this.cache.getTableCache(props);
+
+    if (props.topCount <= tableCache.loadedCount) {
+      return tableCache.getRowsResult(props);
+    }
+
+    // load missing rows
+    tableCache.dataColumns = props.dataColumns;
+    tableCache.allColumns = props.allColumns;
+
+    const nextRows = await this.loader.loadData({
+      ...props,
+      topCount: null,
+      range: {
+        offset: tableCache.loadedCount,
+        limit: props.topCount - tableCache.loadedCount,
+      },
+    });
+
+    if (nextRows.errorMessage) {
+      throw new Error(nextRows.errorMessage);
+    }
+
+    tableCache.loadedRows = [...tableCache.loadedRows, ...nextRows];
+    tableCache.loadedAll = nextRows.length < props.topCount - tableCache.loadedCount;
+
+    // const rows=tableCache.getRows(props);
+
+    return tableCache.getRowsResult(props);
+  }
+
+  async loadRowCount(props: PerspectiveDataLoadProps): Promise<number> {
+    const tableCache = this.cache.getTableCache(props);
+
+    if (tableCache.allRowCount != null) {
+      return tableCache.allRowCount;
+    }
+
+    const result = await this.loader.loadRowCount({
+      ...props,
+    });
+
+    if (result.errorMessage) {
+      throw new Error(result.errorMessage);
+    }
+
+    tableCache.allRowCount = parseInt(result.count);
+    return tableCache.allRowCount;
+  }
+}

packages/datalib/src/PerspectiveDisplay.ts

@@ -0,0 +1,300 @@
+import { getTableChildPerspectiveNodes, PerspectiveTableNode, PerspectiveTreeNode } from './PerspectiveTreeNode';
+import _max from 'lodash/max';
+import _range from 'lodash/max';
+import _fill from 'lodash/fill';
+import _findIndex from 'lodash/findIndex';
+import _isPlainObject from 'lodash/isPlainObject';
+import _isArray from 'lodash/isArray';
+import debug from 'debug';
+
+const dbg = debug('dbgate:PerspectiveDisplay');
+
+let lastJoinId = 0;
+function getJoinId(): number {
+  lastJoinId += 1;
+  return lastJoinId;
+}
+
+export class PerspectiveDisplayColumn {
+  title: string;
+  dataField: string;
+  displayType: string;
+  parentNodes: PerspectiveTreeNode[] = [];
+  colSpanAtLevel = {};
+  columnIndex = 0;
+  dataNode: PerspectiveTreeNode = null;
+
+  constructor(public display: PerspectiveDisplay) {}
+
+  get rowSpan() {
+    return this.display.columnLevelCount - this.parentNodes.length;
+  }
+
+  showParent(level: number) {
+    return !!this.colSpanAtLevel[level];
+  }
+
+  getColSpan(level: number) {
+    return this.colSpanAtLevel[level];
+  }
+
+  isVisible(level: number) {
+    return level == this.columnLevel;
+  }
+
+  get columnLevel() {
+    return this.parentNodes.length;
+  }
+
+  getParentName(level) {
+    return this.parentNodes[level]?.title;
+  }
+
+  getParentNode(level) {
+    return this.parentNodes[level];
+  }
+
+  getParentTableDesignerId(level) {
+    return this.parentNodes[level]?.headerTableAttributes ? this.parentNodes[level]?.designerId : '';
+  }
+
+  // hasParentNode(node: PerspectiveTreeNode) {
+  //   return this.parentNodes.includes(node);
+  // }
+}
+
+interface PerspectiveSubRowCollection {
+  rows: CollectedPerspectiveDisplayRow[];
+}
+
+interface CollectedPerspectiveDisplayRow {
+  columnIndexes: number[];
+  rowData: any[];
+  subRowCollections: PerspectiveSubRowCollection[];
+  incompleteRowsIndicator?: string[];
+}
+
+export class PerspectiveDisplayRow {
+  constructor(public display: PerspectiveDisplay) {
+    this.rowData = _fill(Array(display.columns.length), undefined);
+    this.rowSpans = _fill(Array(display.columns.length), 1);
+    this.rowJoinIds = _fill(Array(display.columns.length), 0);
+    this.rowCellSkips = _fill(Array(display.columns.length), false);
+  }
+
+  rowData: any[] = [];
+  rowSpans: number[] = null;
+  rowCellSkips: boolean[] = null;
+
+  rowJoinIds: number[] = [];
+}
+
+export class PerspectiveDisplay {
+  columns: PerspectiveDisplayColumn[] = [];
+  rows: PerspectiveDisplayRow[] = [];
+  readonly columnLevelCount: number;
+  loadIndicatorsCounts: { [designerId: string]: number } = {};
+
+  constructor(public root: PerspectiveTreeNode, rows: any[]) {
+    // dbg('source rows', rows);
+    this.fillColumns(root.childNodes, [root]);
+    if (this.columns.length > 0) {
+      this.columns[0].colSpanAtLevel[0] = this.columns.length;
+    }
+    this.columnLevelCount = _max(this.columns.map(x => x.parentNodes.length)) + 1;
+    const collectedRows = this.collectRows(rows, root.childNodes);
+    dbg('collected rows', collectedRows);
+    // console.log('COLLECTED', JSON.stringify(collectedRows, null, 2));
+    // this.mergeRows(collectedRows);
+    this.mergeRows(collectedRows);
+    // dbg('merged rows', this.rows);
+
+    // console.log(
+    //   'MERGED',
+    //   this.rows.map(r =>
+    //     r.incompleteRowsIndicator
+    //       ? `************************************ ${r.incompleteRowsIndicator.join('|')}`
+    //       : r.rowData.join('|')
+    //   )
+    // );
+  }
+
+  private getRowAt(rowIndex) {
+    while (this.rows.length <= rowIndex) {
+      this.rows.push(new PerspectiveDisplayRow(this));
+    }
+    return this.rows[rowIndex];
+  }
+
+  fillColumns(children: PerspectiveTreeNode[], parentNodes: PerspectiveTreeNode[]) {
+    for (const child of children) {
+      if (child.generatesHiearchicGridColumn || child.generatesDataGridColumn) {
+        this.processColumn(child, parentNodes);
+      }
+    }
+  }
+
+  processColumn(node: PerspectiveTreeNode, parentNodes: PerspectiveTreeNode[]) {
+    if (node.generatesDataGridColumn) {
+      const column = new PerspectiveDisplayColumn(this);
+      column.title = node.columnTitle;
+      column.dataField = node.dataField;
+      column.parentNodes = parentNodes;
+      column.display = this;
+      column.columnIndex = this.columns.length;
+      column.dataNode = node;
+      column.displayType = node.parentNodeConfig?.columnDisplays?.[node.columnName];
+      this.columns.push(column);
+    }
+
+    if (node.generatesHiearchicGridColumn) {
+      const countBefore = this.columns.length;
+      this.fillColumns(node.childNodes, [...parentNodes, node]);
+
+      if (this.columns.length > countBefore) {
+        this.columns[countBefore].colSpanAtLevel[parentNodes.length] = this.columns.length - countBefore;
+      }
+    }
+  }
+
+  findColumnIndexFromNode(node: PerspectiveTreeNode) {
+    return _findIndex(
+      this.columns,
+      x =>
+        x.dataNode.columnName == node.columnName && x.dataNode?.parentNode?.designerId == node?.parentNode?.designerId
+    );
+  }
+
+  // findColumnIndexFromNode(node: PerspectiveTreeNode) {
+  //   return _findIndex(this.columns, x => x.dataNode.designerId == node.designerId);
+  // }
+
+  extractArray(value) {
+    if (_isArray(value)) return value;
+    if (_isPlainObject(value)) return [value];
+    return [];
+  }
+
+  collectRows(sourceRows: any[], nodes: PerspectiveTreeNode[]): CollectedPerspectiveDisplayRow[] {
+    // console.log('********** COLLECT ROWS', sourceRows);
+    const columnNodes = nodes.filter(x => x.generatesDataGridColumn);
+    const treeNodes = nodes.filter(x => x.generatesHiearchicGridColumn);
+
+    // console.log(
+    //   'columnNodes',
+    //   columnNodes.map(x => x.title)
+    // );
+    // console.log(
+    //   'treeNodes',
+    //   treeNodes.map(x => x.title)
+    // );
+
+    // console.log(
+    //   'nodes',
+    //   nodes.map(x => x.title)
+    // );
+
+    const columnIndexes = columnNodes.map(node => this.findColumnIndexFromNode(node));
+
+    const res: CollectedPerspectiveDisplayRow[] = [];
+    for (const sourceRow of sourceRows) {
+      // console.log('PROCESS SOURCE', sourceRow);
+      // row.startIndex = startIndex;
+      const rowData = columnNodes.map(node => sourceRow[node.columnName]);
+      const subRowCollections = [];
+
+      for (const node of treeNodes) {
+        // console.log('sourceRow[node.fieldName]', node.fieldName, sourceRow[node.fieldName]);
+        if (sourceRow[node.fieldName]) {
+          const subrows = {
+            rows: this.collectRows(this.extractArray(sourceRow[node.fieldName]), node.childNodes),
+          };
+          subRowCollections.push(subrows);
+        }
+      }
+
+      res.push({
+        rowData,
+        columnIndexes,
+        subRowCollections,
+        incompleteRowsIndicator: sourceRow.incompleteRowsIndicator,
+      });
+    }
+
+    return res;
+  }
+
+  fillRowSpans() {
+    for (let col = 0; col < this.columns.length; col++) {
+      // let lastFilledJoinId = null;
+      let lastFilledRow = 0;
+      let rowIndex = 0;
+
+      for (const row of this.rows) {
+        if (
+          row.rowData[col] === undefined &&
+          row.rowJoinIds[col] == this.rows[lastFilledRow].rowJoinIds[col] &&
+          row.rowJoinIds[col]
+        ) {
+          row.rowCellSkips[col] = true;
+          this.rows[lastFilledRow].rowSpans[col] = rowIndex - lastFilledRow + 1;
+        } else {
+          lastFilledRow = rowIndex;
+        }
+        rowIndex++;
+      }
+    }
+  }
+
+  mergeRows(collectedRows: CollectedPerspectiveDisplayRow[]) {
+    let rowIndex = 0;
+    for (const collectedRow of collectedRows) {
+      const count = this.mergeRow(collectedRow, rowIndex);
+      rowIndex += count;
+    }
+    this.fillRowSpans();
+  }
+
+  mergeRow(collectedRow: CollectedPerspectiveDisplayRow, rowIndex: number): number {
+    if (collectedRow.incompleteRowsIndicator?.length > 0) {
+      for (const indicator of collectedRow.incompleteRowsIndicator) {
+        if (!this.loadIndicatorsCounts[indicator]) {
+          this.loadIndicatorsCounts[indicator] = rowIndex;
+        }
+        if (rowIndex < this.loadIndicatorsCounts[indicator]) {
+          this.loadIndicatorsCounts[indicator] = rowIndex;
+        }
+      }
+      return 0;
+    }
+
+    const mainRow = this.getRowAt(rowIndex);
+    for (let i = 0; i < collectedRow.columnIndexes.length; i++) {
+      mainRow.rowData[collectedRow.columnIndexes[i]] = collectedRow.rowData[i];
+    }
+
+    let rowCount = 1;
+    for (const subrows of collectedRow.subRowCollections) {
+      let additionalRowCount = 0;
+      let currentRowIndex = rowIndex;
+      for (const subrow of subrows.rows) {
+        const count = this.mergeRow(subrow, currentRowIndex);
+        additionalRowCount += count;
+        currentRowIndex += count;
+      }
+      if (additionalRowCount > rowCount) {
+        rowCount = additionalRowCount;
+      }
+    }
+
+    const joinId = getJoinId();
+    for (let radd = 0; radd < rowCount; radd++) {
+      const row = this.getRowAt(rowIndex + radd);
+      for (let i = 0; i < collectedRow.columnIndexes.length; i++) {
+        row.rowJoinIds[collectedRow.columnIndexes[i]] = joinId;
+      }
+    }
+
+    return rowCount;
+  }
+}

packages/datalib/src/TableFormViewDisplay.ts

@@ -1,7 +1,7 @@
 import { FormViewDisplay } from './FormViewDisplay';
 import _ from 'lodash';
 import { ChangeCacheFunc, DisplayColumn, ChangeConfigFunc } from './GridDisplay';
-import { EngineDriver, NamedObjectInfo, DatabaseInfo } from 'dbgate-types';
+import type { EngineDriver, NamedObjectInfo, DatabaseInfo } from 'dbgate-types';
 import { GridConfig, GridCache } from './GridConfig';
 import { mergeConditions, Condition, OrderByExpression } from 'dbgate-sqltree';
 import { TableGridDisplay } from './TableGridDisplay';

packages/datalib/src/TableGridDisplay.ts

@@ -1,7 +1,7 @@
 import _ from 'lodash';
 import { filterName, isTableColumnUnique } from 'dbgate-tools';
 import { GridDisplay, ChangeCacheFunc, DisplayColumn, DisplayedColumnInfo, ChangeConfigFunc } from './GridDisplay';
-import {
+import type {
   TableInfo,
   EngineDriver,
   ViewInfo,

packages/datalib/src/ViewGridDisplay.ts

@@ -1,6 +1,6 @@
 import _ from 'lodash';
 import { GridDisplay, ChangeCacheFunc, ChangeConfigFunc } from './GridDisplay';
-import { EngineDriver, ViewInfo, ColumnInfo } from 'dbgate-types';
+import type { EngineDriver, ViewInfo, ColumnInfo } from 'dbgate-types';
 import { GridConfig, GridCache } from './GridConfig';
 
 export class ViewGridDisplay extends GridDisplay {

packages/datalib/src/deleteCascade.ts

@@ -1,6 +1,6 @@
 import _ from 'lodash';
 import { Command, Insert, Update, Delete, UpdateField, Condition, AllowIdentityInsert } from 'dbgate-sqltree';
-import { NamedObjectInfo, DatabaseInfo, ForeignKeyInfo, TableInfo } from 'dbgate-types';
+import type { NamedObjectInfo, DatabaseInfo, ForeignKeyInfo, TableInfo } from 'dbgate-types';
 import { ChangeSet, ChangeSetItem, extractChangeSetCondition } from './ChangeSet';
 
 export interface ChangeSetDeleteCascade {

packages/datalib/src/index.ts

@@ -1,5 +1,7 @@
 export * from './GridDisplay';
 export * from './GridConfig';
+export * from './PerspectiveConfig';
+export * from './PerspectiveTreeNode';
 export * from './TableGridDisplay';
 export * from './ViewGridDisplay';
 export * from './JslGridDisplay';
@@ -12,3 +14,10 @@ export * from './FormViewDisplay';
 export * from './TableFormViewDisplay';
 export * from './CollectionGridDisplay';
 export * from './deleteCascade';
+export * from './PerspectiveDisplay';
+export * from './PerspectiveDataProvider';
+export * from './PerspectiveCache';
+export * from './PerspectiveConfig';
+export * from './processPerspectiveDefaultColunns';
+export * from './PerspectiveDataPattern';
+export * from './PerspectiveDataLoader';

packages/datalib/src/processPerspectiveDefaultColunns.ts

@@ -0,0 +1,255 @@
+import { findForeignKeyForColumn } from 'dbgate-tools';
+import type { DatabaseInfo, TableInfo, ViewInfo } from 'dbgate-types';
+import { createPerspectiveNodeConfig, MultipleDatabaseInfo, PerspectiveConfig } from './PerspectiveConfig';
+import { PerspectiveDataPattern, PerspectiveDataPatternDict } from './PerspectiveDataPattern';
+import { PerspectiveTableNode } from './PerspectiveTreeNode';
+
+const namePredicates = [
+  x => x.toLowerCase() == 'name',
+  x => x.toLowerCase() == 'title',
+  x => x.toLowerCase().includes('name'),
+  x => x.toLowerCase().includes('title'),
+  x => x.toLowerCase().includes('subject'),
+];
+
+function getPerspectiveDefaultColumns(
+  table: TableInfo | ViewInfo,
+  db: DatabaseInfo,
+  circularColumns?: string[]
+): [string[], string[]] {
+  const columns = table.columns.map(x => x.columnName);
+  const predicates = [
+    ...namePredicates,
+    x =>
+      table.columns
+        .find(y => y.columnName == x)
+        ?.dataType?.toLowerCase()
+        ?.includes('char'),
+  ];
+
+  for (const predicate of predicates) {
+    const col = columns.find(predicate);
+    if (col) return [[col], null];
+  }
+
+  if (circularColumns) {
+    const keyPredicates = [
+      x => findForeignKeyForColumn(table as TableInfo, x)?.columns?.length == 1 && !circularColumns.includes(x),
+      x => findForeignKeyForColumn(table as TableInfo, x)?.columns?.length == 1,
+    ];
+
+    for (const predicate of keyPredicates) {
+      const col = columns.find(predicate);
+      if (col) return [null, [col]];
+    }
+  }
+
+  return [[columns[0]], null];
+}
+
+function getPerspectiveDefaultCollectionColumns(pattern: PerspectiveDataPattern): string[] {
+  const columns = pattern.columns.map(x => x.name);
+  const predicates = [...namePredicates, x => pattern.columns.find(y => y.name == x)?.types?.includes('string')];
+
+  for (const predicate of predicates) {
+    const col = columns.find(predicate);
+    if (col) return [col];
+  }
+}
+
+export function perspectiveNodesHaveStructure(
+  config: PerspectiveConfig,
+  dbInfos: MultipleDatabaseInfo,
+  dataPatterns: PerspectiveDataPatternDict,
+  conid: string,
+  database: string
+) {
+  for (const node of config.nodes) {
+    const db = dbInfos?.[node.conid || conid]?.[node.database || database];
+    if (!db) return false;
+
+    const table = db.tables.find(x => x.pureName == node.pureName && x.schemaName == node.schemaName);
+    const view = db.views.find(x => x.pureName == node.pureName && x.schemaName == node.schemaName);
+    const collection = db.collections.find(x => x.pureName == node.pureName && x.schemaName == node.schemaName);
+
+    if (!table && !view && !collection) return false;
+    if (collection && !dataPatterns?.[node.designerId]) return false;
+  }
+
+  return true;
+}
+
+export function shouldProcessPerspectiveDefaultColunns(
+  config: PerspectiveConfig,
+  dbInfos: MultipleDatabaseInfo,
+  dataPatterns: PerspectiveDataPatternDict,
+  conid: string,
+  database: string
+) {
+  const nodesNotProcessed = config.nodes.filter(x => !x.defaultColumnsProcessed);
+  if (nodesNotProcessed.length == 0) return false;
+
+  return perspectiveNodesHaveStructure(config, dbInfos, dataPatterns, conid, database);
+}
+
+function processPerspectiveDefaultColunnsStep(
+  config: PerspectiveConfig,
+  dbInfos: MultipleDatabaseInfo,
+  dataPatterns: PerspectiveDataPatternDict,
+  conid: string,
+  database: string
+) {
+  const rootNode = config.nodes.find(x => x.designerId == config.rootDesignerId);
+  if (!rootNode) return null;
+  const rootDb = dbInfos?.[rootNode.conid || conid]?.[rootNode.database || database];
+  if (!rootDb) return null;
+  const rootTable = rootDb.tables.find(x => x.pureName == rootNode.pureName && x.schemaName == rootNode.schemaName);
+  const rootView = rootDb.views.find(x => x.pureName == rootNode.pureName && x.schemaName == rootNode.schemaName);
+
+  const root = new PerspectiveTableNode(
+    rootTable || rootView,
+    dbInfos,
+    config,
+    null,
+    null,
+    { conid, database },
+    null,
+    config.rootDesignerId
+  );
+
+  for (const node of config.nodes) {
+    if (node.defaultColumnsProcessed) continue;
+
+    const db = dbInfos?.[node.conid || conid]?.[node.database || database];
+    if (!db) continue;
+
+    const table = db.tables.find(x => x.pureName == node.pureName && x.schemaName == node.schemaName);
+    const view = db.views.find(x => x.pureName == node.pureName && x.schemaName == node.schemaName);
+    const collection = db.collections.find(x => x.pureName == node.pureName && x.schemaName == node.schemaName);
+
+    if (table || view) {
+      const treeNode = root.findNodeByDesignerId(node.designerId);
+
+      if (!treeNode) {
+        const [defaultColumns] = getPerspectiveDefaultColumns(table || view, db, null);
+
+        return {
+          ...config,
+          nodes: config.nodes.map(n =>
+            n.designerId == node.designerId
+              ? {
+                  ...n,
+                  defaultColumnsProcessed: true,
+                  checkedColumns: defaultColumns,
+                }
+              : n
+          ),
+        };
+      }
+
+      const circularColumns = treeNode.childNodes.filter(x => x.isCircular).map(x => x.columnName);
+      const [defaultColumns, defaultRefs] = getPerspectiveDefaultColumns(table || view, db, circularColumns);
+
+      if (defaultRefs) {
+        const childNode = treeNode.childNodes.find(x => x.columnName == defaultRefs[0]);
+        if (childNode?.designerId) {
+          return {
+            ...config,
+            nodes: config.nodes.map(n =>
+              n.designerId == childNode.designerId
+                ? {
+                    ...n,
+                    isNodeChecked: true,
+                  }
+                : n.designerId == node.designerId
+                ? {
+                    ...n,
+                    defaultColumnsProcessed: true,
+                  }
+                : n
+            ),
+          };
+        } else if (childNode) {
+          const [newConfig, nodeConfig] = childNode.ensureNodeConfig(config);
+          nodeConfig.isNodeChecked = true;
+
+          return {
+            ...newConfig,
+            nodes: newConfig.nodes.map(n =>
+              n.designerId == node.designerId
+                ? {
+                    ...n,
+                    defaultColumnsProcessed: true,
+                  }
+                : n
+            ),
+          };
+        }
+      } else {
+        return {
+          ...config,
+          nodes: config.nodes.map(n =>
+            n.designerId == node.designerId
+              ? {
+                  ...n,
+                  defaultColumnsProcessed: true,
+                  checkedColumns: defaultColumns,
+                }
+              : n
+          ),
+        };
+      }
+    }
+
+    if (collection) {
+      const defaultColumns = getPerspectiveDefaultCollectionColumns(dataPatterns?.[node.designerId]);
+      return {
+        ...config,
+        nodes: config.nodes.map(n =>
+          n.designerId == node.designerId
+            ? {
+                ...n,
+                defaultColumnsProcessed: true,
+                checkedColumns: defaultColumns,
+              }
+            : n
+        ),
+      };
+    }
+  }
+
+  return null;
+}
+
+function markAllProcessed(config: PerspectiveConfig): PerspectiveConfig {
+  return {
+    ...config,
+    nodes: config.nodes.map(x => ({
+      ...x,
+      defaultColumnsProcessed: true,
+    })),
+  };
+}
+
+export function processPerspectiveDefaultColunns(
+  config: PerspectiveConfig,
+  dbInfos: MultipleDatabaseInfo,
+  dataPatterns: PerspectiveDataPatternDict,
+  conid: string,
+  database: string
+): PerspectiveConfig {
+  while (config.nodes.filter(x => !x.defaultColumnsProcessed).length > 0) {
+    const newConfig = processPerspectiveDefaultColunnsStep(config, dbInfos, dataPatterns, conid, database);
+    if (!newConfig) {
+      return markAllProcessed(config);
+    }
+    if (
+      newConfig.nodes.filter(x => x.defaultColumnsProcessed).length <=
+      config.nodes.filter(x => x.defaultColumnsProcessed).length
+    ) {
+      return markAllProcessed(config);
+    }
+    config = newConfig;
+  }
+  return markAllProcessed(config);
+}

packages/datalib/src/tests/PerspectiveDisplay.test.ts

@@ -0,0 +1,142 @@
+import { PerspectiveDisplay } from '../PerspectiveDisplay';
+import { PerspectiveTableNode } from '../PerspectiveTreeNode';
+import { chinookDbInfo } from './chinookDbInfo';
+import { createPerspectiveConfig, createPerspectiveNodeConfig } from '../PerspectiveConfig';
+import artistDataFlat from './artistDataFlat';
+import artistDataAlbum from './artistDataAlbum';
+import artistDataAlbumTrack from './artistDataAlbumTrack';
+import { processPerspectiveDefaultColunns } from '../processPerspectiveDefaultColunns';
+
+test('test flat view', () => {
+  const artistTable = chinookDbInfo.tables.find(x => x.pureName == 'Artist');
+  const configColumns = processPerspectiveDefaultColunns(
+    createPerspectiveConfig({ pureName: 'Artist' }),
+    { conid: { db: chinookDbInfo } },
+    null,
+    'conid',
+    'db'
+  );
+  const root = new PerspectiveTableNode(
+    artistTable,
+    { conid: { db: chinookDbInfo } },
+    configColumns,
+    null,
+    null,
+    { conid: 'conid', database: 'db' },
+    null,
+    configColumns.rootDesignerId
+  );
+  const display = new PerspectiveDisplay(root, artistDataFlat);
+
+  expect(display.rows.length).toEqual(4);
+  expect(display.rows[0].rowData).toEqual(['AC/DC']);
+  expect(display.loadIndicatorsCounts).toEqual({
+    Artist: 4,
+  });
+});
+
+test('test one level nesting', () => {
+  const artistTable = chinookDbInfo.tables.find(x => x.pureName == 'Artist');
+
+  const config = createPerspectiveConfig({ pureName: 'Artist' });
+  config.nodes.push(createPerspectiveNodeConfig({ pureName: 'Album' }));
+  config.references.push({
+    sourceId: config.nodes[0].designerId,
+    targetId: config.nodes[1].designerId,
+    designerId: '1',
+    columns: [{ source: 'ArtistId', target: 'ArtistId' }],
+  });
+
+  const configColumns = processPerspectiveDefaultColunns(config, { conid: { db: chinookDbInfo } }, null, 'conid', 'db');
+
+  // const config = createPerspectiveConfig({ pureName: 'Artist' });
+  // config.nodes[0].checkedColumns = ['Album'];
+
+  const root = new PerspectiveTableNode(
+    artistTable,
+    { conid: { db: chinookDbInfo } },
+    configColumns,
+    null,
+    null,
+    { conid: 'conid', database: 'db' },
+    null,
+    configColumns.nodes[0].designerId
+  );
+  const display = new PerspectiveDisplay(root, artistDataAlbum);
+
+  console.log(display.loadIndicatorsCounts);
+  // console.log(display.rows);
+
+  expect(display.rows.length).toEqual(6);
+
+  expect(display.rows[0].rowData).toEqual(['AC/DC', 'For Those About To Rock We Salute You']);
+  expect(display.rows[0].rowSpans).toEqual([2, 1]);
+  expect(display.rows[0].rowCellSkips).toEqual([false, false]);
+
+  expect(display.rows[1].rowData).toEqual([undefined, 'Let There Be Rock']);
+  expect(display.rows[1].rowSpans).toEqual([1, 1]);
+  expect(display.rows[1].rowCellSkips).toEqual([true, false]);
+
+  expect(display.rows[2].rowData).toEqual(['Accept', 'Balls to the Wall']);
+  expect(display.rows[2].rowSpans).toEqual([2, 1]);
+  expect(display.rows[2].rowCellSkips).toEqual([false, false]);
+
+  expect(display.rows[5].rowData).toEqual(['Alanis Morissette', 'Jagged Little Pill']);
+  expect(display.rows[5].rowSpans).toEqual([1, 1]);
+
+  expect(display.loadIndicatorsCounts).toEqual({
+    Artist: 6,
+    'Artist.Album': 6,
+  });
+});
+
+test('test two level nesting', () => {
+  const artistTable = chinookDbInfo.tables.find(x => x.pureName == 'Artist');
+  const config = createPerspectiveConfig({ pureName: 'Artist' });
+  config.nodes.push(createPerspectiveNodeConfig({ pureName: 'Album' }));
+  config.nodes.push(createPerspectiveNodeConfig({ pureName: 'Track' }));
+  config.references.push({
+    sourceId: config.nodes[0].designerId,
+    targetId: config.nodes[1].designerId,
+    designerId: '1',
+    columns: [{ source: 'ArtistId', target: 'ArtistId' }],
+  });
+  config.references.push({
+    sourceId: config.nodes[1].designerId,
+    targetId: config.nodes[2].designerId,
+    designerId: '2',
+    columns: [{ source: 'AlbumId', target: 'AlbumId' }],
+  });
+  const configColumns = processPerspectiveDefaultColunns(config, { conid: { db: chinookDbInfo } }, null, 'conid', 'db');
+
+  const root = new PerspectiveTableNode(
+    artistTable,
+    { conid: { db: chinookDbInfo } },
+    configColumns,
+    null,
+    null,
+    { conid: 'conid', database: 'db' },
+    null,
+    configColumns.nodes[0].designerId
+  );
+  const display = new PerspectiveDisplay(root, artistDataAlbumTrack);
+
+  console.log(display.rows);
+  expect(display.rows.length).toEqual(8);
+
+  expect(display.rows[0].rowData).toEqual([
+    'AC/DC',
+    'For Those About To Rock We Salute You',
+    'For Those About To Rock (We Salute You)',
+  ]);
+  expect(display.rows[0].rowSpans).toEqual([4, 2, 1]);
+  expect(display.rows[0].rowCellSkips).toEqual([false, false, false]);
+
+  expect(display.rows[1].rowData).toEqual([undefined, undefined, 'Put The Finger On You']);
+  expect(display.rows[1].rowSpans).toEqual([1, 1, 1]);
+  expect(display.rows[1].rowCellSkips).toEqual([true, true, false]);
+
+  expect(display.rows[2].rowData).toEqual([undefined, 'Let There Be Rock', 'Go Down']);
+  expect(display.rows[2].rowSpans).toEqual([1, 2, 1]);
+  expect(display.rows[2].rowCellSkips).toEqual([true, false, false]);
+});

packages/datalib/src/tests/PerspectiveDisplayNoSql.test.ts

@@ -0,0 +1,98 @@
+import { PerspectiveDisplay } from '../PerspectiveDisplay';
+import { PerspectiveTableNode } from '../PerspectiveTreeNode';
+import { createPerspectiveConfig, PerspectiveNodeConfig } from '../PerspectiveConfig';
+import { processPerspectiveDefaultColunns } from '../processPerspectiveDefaultColunns';
+import { DatabaseAnalyser } from 'dbgate-tools';
+import { analyseDataPattern } from '../PerspectiveDataPattern';
+import { PerspectiveDataProvider } from '../PerspectiveDataProvider';
+
+const accountData = [
+  {
+    name: 'jan',
+    email: 'jan@foo.co',
+    follows: [{ name: 'lucie' }, { name: 'petr' }],
+    nested: { email: 'jan@nest.cz' },
+  },
+  {
+    name: 'romeo',
+    email: 'romeo@foo.co',
+    follows: [{ name: 'julie' }, { name: 'wiliam' }],
+    nested: { email: 'romeo@nest.cz' },
+  },
+];
+
+function createDisplay(cfgFunc?: (cfg: PerspectiveNodeConfig) => void) {
+  const collectionInfo = {
+    objectTypeField: 'collections',
+    pureName: 'Account',
+  };
+  const dbInfo = {
+    ...DatabaseAnalyser.createEmptyStructure(),
+    collections: [collectionInfo],
+  };
+  const config = createPerspectiveConfig({ pureName: 'Account' });
+  const dataPatterns = {
+    [config.rootDesignerId]: analyseDataPattern(
+      {
+        conid: 'conid',
+        database: 'db',
+        pureName: 'Account',
+      },
+      accountData
+    ),
+  };
+  const configColumns = processPerspectiveDefaultColunns(
+    config,
+    { conid: { db: dbInfo } },
+    dataPatterns,
+    'conid',
+    'db'
+  );
+  if (cfgFunc) {
+    cfgFunc(configColumns.nodes[0]);
+  }
+  const root = new PerspectiveTableNode(
+    collectionInfo,
+    { conid: { db: dbInfo } },
+    configColumns,
+    null,
+    new PerspectiveDataProvider(null, null, dataPatterns),
+    { conid: 'conid', database: 'db' },
+    null,
+    configColumns.rootDesignerId
+  );
+
+  const display = new PerspectiveDisplay(root, accountData);
+
+  return display;
+}
+
+test('test nosql display', () => {
+  const display = createDisplay();
+
+  expect(display.rows.length).toEqual(2);
+  expect(display.rows[0].rowData).toEqual(['jan']);
+  expect(display.rows[1].rowData).toEqual(['romeo']);
+});
+
+test('test nosql nested array display', () => {
+  const display = createDisplay(cfg => {
+    cfg.checkedColumns = ['name', 'follows::name'];
+  });
+
+  expect(display.rows.length).toEqual(4);
+  expect(display.rows[0].rowData).toEqual(['jan', 'lucie']);
+  expect(display.rows[1].rowData).toEqual([undefined, 'petr']);
+  expect(display.rows[2].rowData).toEqual(['romeo', 'julie']);
+  expect(display.rows[3].rowData).toEqual([undefined, 'wiliam']);
+});
+
+test('test nosql nested object', () => {
+  const display = createDisplay(cfg => {
+    cfg.checkedColumns = ['name', 'nested::email'];
+  });
+
+  expect(display.rows.length).toEqual(2);
+  expect(display.rows[0].rowData).toEqual(['jan', 'jan@nest.cz']);
+  expect(display.rows[1].rowData).toEqual(['romeo', 'romeo@nest.cz']);
+});

packages/datalib/src/tests/artistDataAlbum.ts

@@ -0,0 +1,56 @@
+export default [
+  {
+    ArtistId: 1,
+    Name: 'AC/DC',
+    Album: [
+      {
+        Title: 'For Those About To Rock We Salute You',
+        ArtistId: 1,
+      },
+      {
+        Title: 'Let There Be Rock',
+        ArtistId: 1,
+      },
+    ],
+  },
+  {
+    ArtistId: 2,
+    Name: 'Accept',
+    Album: [
+      {
+        Title: 'Balls to the Wall',
+        ArtistId: 2,
+      },
+      {
+        Title: 'Restless and Wild',
+        ArtistId: 2,
+      },
+    ],
+  },
+  {
+    ArtistId: 3,
+    Name: 'Aerosmith',
+    Album: [
+      {
+        Title: 'Big Ones',
+        ArtistId: 3,
+      },
+    ],
+  },
+  {
+    ArtistId: 4,
+    Name: 'Alanis Morissette',
+    Album: [
+      {
+        Title: 'Jagged Little Pill',
+        ArtistId: 4,
+      },
+      {
+        incompleteRowsIndicator: ['Artist.Album'],
+      },
+    ],
+  },
+  {
+    incompleteRowsIndicator: ['Artist'],
+  },
+];

packages/datalib/src/tests/artistDataAlbumTrack.ts

@@ -0,0 +1,78 @@
+export default [
+  {
+    ArtistId: 1,
+    Name: 'AC/DC',
+    Album: [
+      {
+        Title: 'For Those About To Rock We Salute You',
+        AlbumId: 1,
+        ArtistId: 1,
+        Track: [
+          {
+            Name: 'For Those About To Rock (We Salute You)',
+            AlbumId: 1,
+          },
+          {
+            Name: 'Put The Finger On You',
+            AlbumId: 1,
+          },
+        ],
+      },
+      {
+        Title: 'Let There Be Rock',
+        AlbumId: 4,
+        ArtistId: 1,
+        Track: [
+          {
+            Name: 'Go Down',
+            AlbumId: 4,
+          },
+          {
+            Name: 'Dog Eat Dog',
+            AlbumId: 4,
+          },
+        ],
+      },
+    ],
+  },
+  {
+    ArtistId: 2,
+    Name: 'Accept',
+    Album: [
+      {
+        Title: 'Balls to the Wall',
+        AlbumId: 2,
+        ArtistId: 2,
+        Track: [
+          {
+            Name: 'Balls to the Wall',
+            AlbumId: 2,
+          },
+        ],
+      },
+      {
+        Title: 'Restless and Wild',
+        AlbumId: 3,
+        ArtistId: 2,
+        Track: [
+          {
+            Name: 'Fast As a Shark',
+            AlbumId: 3,
+          },
+          {
+            Name: 'Restless and Wild',
+            AlbumId: 3,
+          },
+          {
+            Name: 'Princess of the Dawn',
+            AlbumId: 3,
+          },
+        ],
+      },
+    ],
+  },
+
+  {
+    incompleteRowsIndicator: ['Artist'],
+  },
+];

packages/datalib/src/tests/artistDataFlat.ts

@@ -0,0 +1,21 @@
+export default [
+  {
+    ArtistId: 1,
+    Name: 'AC/DC',
+  },
+  {
+    ArtistId: 2,
+    Name: 'Accept',
+  },
+  {
+    ArtistId: 3,
+    Name: 'Aerosmith',
+  },
+  {
+    ArtistId: 4,
+    Name: 'Alanis Morissette',
+  },
+  {
+    incompleteRowsIndicator: ['Artist'],
+  },
+];

packages/filterparser/package.json

@@ -16,8 +16,8 @@
     "dbgate-types": "^5.0.0-alpha.1",
     "@types/jest": "^25.1.4",
     "@types/node": "^13.7.0",
-    "jest": "^24.9.0",
-    "ts-jest": "^25.2.1",
+    "jest": "^28.1.3",
+    "ts-jest": "^28.0.7",
     "typescript": "^4.4.3"
   },
   "dependencies": {

packages/filterparser/src/datetimeParser.ts

@@ -2,7 +2,7 @@ import P from 'parsimmon';
 import moment from 'moment';
 import { FilterType } from './types';
 import { Condition } from 'dbgate-sqltree';
-import { TransformType } from 'dbgate-types';
+import type { TransformType } from 'dbgate-types';
 import { interpretEscapes, token, word, whitespace } from './common';
 
 const compoudCondition = conditionType => conditions => {
@@ -181,10 +181,38 @@ const binaryCondition = operator => value => ({
   },
 });
 
+const unaryCondition = conditionType => () => {
+  return {
+    conditionType,
+    expr: {
+      exprType: 'placeholder',
+    },
+  };
+};
+
+const sqlTemplate = templateSql => {
+  return {
+    conditionType: 'rawTemplate',
+    templateSql,
+    expr: {
+      exprType: 'placeholder',
+    },
+  };
+};
+
 const createParser = () => {
   const langDef = {
     comma: () => word(','),
 
+    not: () => word('NOT'),
+    notNull: r => r.not.then(r.null).map(unaryCondition('isNotNull')),
+    null: () => word('NULL').map(unaryCondition('isNull')),
+
+    sql: () =>
+      token(P.regexp(/\{(.*?)\}/, 1))
+        .map(sqlTemplate)
+        .desc('sql literal'),
+
     yearNum: () => P.regexp(/\d\d\d\d/).map(yearCondition()),
     yearMonthNum: () => P.regexp(/\d\d\d\d-\d\d?/).map(yearMonthCondition()),
     yearMonthDayNum: () => P.regexp(/\d\d\d\d-\d\d?-\d\d?/).map(yearMonthDayCondition()),
@@ -264,10 +292,13 @@ const createParser = () => {
         r.lastYear,
         r.thisYear,
         r.nextYear,
+        r.null,
+        r.notNull,
         r.le,
         r.lt,
         r.ge,
-        r.gt
+        r.gt,
+        r.sql
       ).trim(whitespace),
     factor: r => r.element.sepBy(whitespace).map(compoudCondition('$and')),
     list: r => r.factor.sepBy(r.comma).map(compoudCondition('$or')),

packages/filterparser/src/filterTool.ts

@@ -3,7 +3,7 @@ import moment from 'moment';
 
 export type FilterMultipleValuesMode = 'is' | 'is_not' | 'contains' | 'begins' | 'ends';
 
-export function getFilterValueExpression(value, dataType) {
+export function getFilterValueExpression(value, dataType?) {
   if (value == null) return 'NULL';
   if (isTypeDateTime(dataType)) return moment(value).format('YYYY-MM-DD HH:mm:ss');
   if (value === true) return 'TRUE';

packages/filterparser/src/parseFilter.ts

@@ -2,7 +2,6 @@ import P from 'parsimmon';
 import moment from 'moment';
 import { FilterType } from './types';
 import { Condition } from 'dbgate-sqltree';
-import { TransformType } from 'dbgate-types';
 import { interpretEscapes, token, word, whitespace } from './common';
 import { mongoParser } from './mongoParser';
 import { datetimeParser } from './datetimeParser';
@@ -68,6 +67,16 @@ const negateCondition = condition => {
   };
 };
 
+const sqlTemplate = templateSql => {
+  return {
+    conditionType: 'rawTemplate',
+    templateSql,
+    expr: {
+      exprType: 'placeholder',
+    },
+  };
+};
+
 const createParser = (filterType: FilterType) => {
   const langDef = {
     string1: () =>
@@ -97,6 +106,11 @@ const createParser = (filterType: FilterType) => {
 
     noQuotedString: () => P.regexp(/[^\s^,^'^"]+/).desc('string unquoted'),
 
+    sql: () =>
+      token(P.regexp(/\{(.*?)\}/, 1))
+        .map(sqlTemplate)
+        .desc('sql literal'),
+
     value: r => P.alt(...allowedValues.map(x => r[x])),
     valueTestEq: r => r.value.map(binaryCondition('=')),
     valueTestStr: r => r.value.map(likeCondition('like', '%#VALUE#%')),
@@ -139,7 +153,7 @@ const createParser = (filterType: FilterType) => {
     allowedValues.push('string1Num', 'string2Num', 'number');
   }
 
-  const allowedElements = ['null', 'notNull', 'eq', 'ne', 'ne2'];
+  const allowedElements = ['null', 'notNull', 'eq', 'ne', 'ne2', 'sql'];
   if (filterType == 'number' || filterType == 'datetime' || filterType == 'eval') {
     allowedElements.push('le', 'ge', 'lt', 'gt');
   }

packages/filterparser/src/parserFilter.test.ts

@@ -1,4 +1,4 @@
-import { parseFilter } from './parseFilter';
+const { parseFilter } = require('./parseFilter');
 
 test('parse string', () => {
   const ast = parseFilter('"123"', 'string');

packages/sqltree/src/dumpSqlCommand.ts

@@ -1,4 +1,4 @@
-import { SqlDumper } from 'dbgate-types';
+import type { SqlDumper } from 'dbgate-types';
 import { Command, Select, Update, Delete, Insert } from './types';
 import { dumpSqlExpression } from './dumpSqlExpression';
 import { dumpSqlFromDefinition, dumpSqlSourceRef } from './dumpSqlSource';

packages/sqltree/src/dumpSqlCondition.ts

@@ -1,4 +1,4 @@
-import { SqlDumper } from 'dbgate-types';
+import type { SqlDumper } from 'dbgate-types';
 import { Condition, BinaryCondition } from './types';
 import { dumpSqlExpression } from './dumpSqlExpression';
 import { dumpSqlSelect } from './dumpSqlCommand';
@@ -68,5 +68,19 @@ export function dumpSqlCondition(dmp: SqlDumper, condition: Condition) {
       dmp.put(' ^and ');
       dumpSqlExpression(dmp, condition.right);
       break;
+    case 'in':
+      dumpSqlExpression(dmp, condition.expr);
+      dmp.put(' ^in (%,v)', condition.values);
+      break;
+    case 'rawTemplate':
+      let was = false;
+      for (const item of condition.templateSql.split('$$')) {
+        if (was) {
+          dumpSqlExpression(dmp, condition.expr);
+        }
+        dmp.putRaw(item);
+        was = true;
+      }
+      break;
   }
 }

packages/sqltree/src/dumpSqlExpression.ts

@@ -1,5 +1,5 @@
 import _ from 'lodash';
-import { SqlDumper } from 'dbgate-types';
+import type { SqlDumper } from 'dbgate-types';
 import { Expression, ColumnRefExpression } from './types';
 import { dumpSqlSourceRef } from './dumpSqlSource';
 

packages/sqltree/src/dumpSqlSource.ts

@@ -1,5 +1,5 @@
 import { Source, FromDefinition, Relation } from './types';
-import { SqlDumper } from 'dbgate-types';
+import type { SqlDumper } from 'dbgate-types';
 import { dumpSqlSelect } from './dumpSqlCommand';
 import { dumpSqlCondition } from './dumpSqlCondition';
 

packages/sqltree/src/evaluateCondition.ts

@@ -1,10 +1,6 @@
-import { SqlDumper } from 'dbgate-types';
 import _ from 'lodash';
 import { Condition, BinaryCondition } from './types';
-import { dumpSqlExpression } from './dumpSqlExpression';
-import { link } from 'fs';
 import { evaluateExpression } from './evaluateExpression';
-import { cond } from 'lodash';
 
 function isEmpty(value) {
   if (value == null) return true;

packages/sqltree/src/types.ts

@@ -1,4 +1,4 @@
-import { NamedObjectInfo, RangeDefinition, TransformType } from 'dbgate-types';
+import type { NamedObjectInfo, RangeDefinition, TransformType } from 'dbgate-types';
 
 // export interface Command {
 // }
@@ -99,6 +99,18 @@ export interface BetweenCondition {
   right: Expression;
 }
 
+export interface InCondition {
+  conditionType: 'in';
+  expr: Expression;
+  values: any[];
+}
+
+export interface RawTemplateCondition {
+  conditionType: 'rawTemplate';
+  templateSql: string;
+  expr: Expression;
+}
+
 export type Condition =
   | BinaryCondition
   | NotCondition
@@ -107,7 +119,9 @@ export type Condition =
   | LikeCondition
   | ExistsCondition
   | NotExistsCondition
-  | BetweenCondition;
+  | BetweenCondition
+  | InCondition
+  | RawTemplateCondition;
 
 export interface Source {
   name?: NamedObjectInfo;

packages/sqltree/src/utility.ts

@@ -1,4 +1,4 @@
-import { EngineDriver, SqlDumper } from 'dbgate-types';
+import type { EngineDriver, SqlDumper } from 'dbgate-types';
 import { Command, Condition } from './types';
 import { dumpSqlCommand } from './dumpSqlCommand';
 

packages/tools/package.json

@@ -31,9 +31,11 @@
     "typescript": "^4.4.3"
   },
   "dependencies": {
-    "lodash": "^4.17.21",
-    "dbgate-query-splitter": "^4.9.0",
+    "dbgate-query-splitter": "^4.9.2",
     "dbgate-sqltree": "^5.0.0-alpha.1",
+    "debug": "^4.3.4",
+    "json-stable-stringify": "^1.0.1",
+    "lodash": "^4.17.21",
     "uuid": "^3.4.0"
   }
 }

packages/tools/src/SqlDumper.ts

@@ -1,5 +1,5 @@
 import _ from 'lodash';
-import {
+import type {
   ColumnInfo,
   ConstraintInfo,
   EngineDriver,
@@ -32,6 +32,12 @@ export class SqlDumper implements AlterProcessor {
   dialect: SqlDialect;
   indentLevel = 0;
 
+  static keywordsCase = 'upperCase';
+  static convertKeywordCase(keyword: any): string {
+    if (this.keywordsCase == 'lowerCase') return keyword?.toString()?.toLowerCase();
+    return keyword?.toString()?.toUpperCase();
+  }
+
   constructor(driver: EngineDriver) {
     this.driver = driver;
     this.dialect = driver.dialect;
@@ -60,10 +66,10 @@ export class SqlDumper implements AlterProcessor {
     this.putRaw("'");
   }
   putByteArrayValue(value) {
-    this.putRaw('NULL');
+    this.put('^null');
   }
   putValue(value) {
-    if (value === null) this.putRaw('NULL');
+    if (value === null) this.put('^null');
     else if (value === true) this.putRaw('1');
     else if (value === false) this.putRaw('0');
     else if (_isString(value)) this.putStringValue(value);
@@ -71,7 +77,7 @@ export class SqlDumper implements AlterProcessor {
     else if (_isDate(value)) this.putStringValue(new Date(value).toISOString());
     else if (value?.type == 'Buffer' && _isArray(value?.data)) this.putByteArrayValue(value?.data);
     else if (_isPlainObject(value) || _isArray(value)) this.putStringValue(JSON.stringify(value));
-    else this.putRaw('NULL');
+    else this.put('^null');
   }
   putCmd(format, ...args) {
     this.put(format, ...args);
@@ -92,7 +98,7 @@ export class SqlDumper implements AlterProcessor {
       case 'k':
         {
           if (value) {
-            this.putRaw(value.toUpperCase());
+            this.putRaw(SqlDumper.convertKeywordCase(value));
           }
         }
         break;
@@ -128,7 +134,7 @@ export class SqlDumper implements AlterProcessor {
       switch (c) {
         case '^':
           while (i < length && format[i].match(/[a-z0-9_]/i)) {
-            this.putRaw(format[i].toUpperCase());
+            this.putRaw(SqlDumper.convertKeywordCase(format[i]));
             i++;
           }
           break;
@@ -181,6 +187,14 @@ export class SqlDumper implements AlterProcessor {
     this.put(' ^auto_increment');
   }
 
+  createDatabase(name: string) {
+    this.putCmd('^create ^database %i', name);
+  }
+
+  dropDatabase(name: string) {
+    this.putCmd('^drop ^database %i', name);
+  }
+
   specialColumnOptions(column) {}
 
   columnDefinition(column: ColumnInfo, { includeDefault = true, includeNullable = true, includeCollate = true } = {}) {
@@ -527,7 +541,7 @@ export class SqlDumper implements AlterProcessor {
   }
 
   truncateTable(name: NamedObjectInfo) {
-    this.putCmd('^delete ^from %f', name);
+    this.putCmd('^truncate ^table %f', name);
   }
 
   dropConstraints(table: TableInfo, dropReferences = false) {

packages/tools/src/SqlGenerator.ts

@@ -1,4 +1,4 @@
-import {
+import type {
   DatabaseInfo,
   EngineDriver,
   FunctionInfo,

packages/tools/src/computeDiffRows.ts

@@ -1,5 +1,5 @@
 import { DbDiffOptions, testEqualColumns, testEqualTables, testEqualSqlObjects } from './diffTools';
-import { DatabaseInfo, EngineDriver, SqlObjectInfo, TableInfo } from 'dbgate-types';
+import type { DatabaseInfo, EngineDriver, SqlObjectInfo, TableInfo } from 'dbgate-types';
 import _ from 'lodash';
 
 export function computeDiffRowsCore(sourceList, targetList, testEqual) {

packages/tools/src/createBulkInsertStreamBase.ts

@@ -1,4 +1,3 @@
-import { EngineDriver } from 'dbgate-types';
 import _intersection from 'lodash/intersection';
 import { prepareTableForImport } from './tableTransforms';
 

packages/tools/src/diffTools.ts

@@ -1,4 +1,4 @@
-import {
+import type {
   ColumnInfo,
   ConstraintInfo,
   DatabaseInfo,

packages/tools/src/driverBase.ts

@@ -16,6 +16,7 @@ const dialect = {
     isSparse: false,
     isPersisted: false,
   },
+  defaultSchemaName: null,
 };
 
 export const driverBase = {

packages/tools/src/nameTools.ts

@@ -1,5 +1,14 @@
-import _ from 'lodash';
-import { ColumnInfo, ColumnReference, DatabaseInfo, DatabaseInfoObjects, SqlDialect, TableInfo } from 'dbgate-types';
+import _cloneDeep from 'lodash/cloneDeep';
+import _isString from 'lodash/isString';
+import type {
+  ColumnInfo,
+  ColumnReference,
+  DatabaseInfo,
+  DatabaseInfoObjects,
+  NamedObjectInfo,
+  SqlDialect,
+  TableInfo,
+} from 'dbgate-types';
 
 export function fullNameFromString(name) {
   const m = name.match(/\[([^\]]+)\]\.\[([^\]]+)\]/);
@@ -38,6 +47,11 @@ export function equalStringLike(s1, s2) {
   return (s1 || '').toLowerCase().trim() == (s2 || '').toLowerCase().trim();
 }
 
+export function equalFullName(name1: NamedObjectInfo, name2: NamedObjectInfo) {
+  if (!name1 || !name2) return name1 == name2;
+  return name1.pureName == name2.pureName && name1.schemaName == name2.schemaName;
+}
+
 export function findObjectLike(
   { pureName, schemaName },
   dbinfo: DatabaseInfo,
@@ -54,7 +68,10 @@ export function findObjectLike(
   return dbinfo[objectTypeField]?.find(x => equalStringLike(x.pureName, pureName));
 }
 
-export function findForeignKeyForColumn(table: TableInfo, column: ColumnInfo) {
+export function findForeignKeyForColumn(table: TableInfo, column: ColumnInfo | string) {
+  if (_isString(column)) {
+    return (table.foreignKeys || []).find(fk => fk.columns.find(col => col.columnName == column));
+  }
   return (table.foreignKeys || []).find(fk => fk.columns.find(col => col.columnName == column.columnName));
 }
 
@@ -76,7 +93,7 @@ function columnsConstraintName(prefix: string, table: TableInfo, columns: Column
 
 export function fillConstraintNames(table: TableInfo, dialect: SqlDialect) {
   if (!table) return table;
-  const res = _.cloneDeep(table);
+  const res = _cloneDeep(table);
   if (res.primaryKey && !res.primaryKey.constraintName && !dialect.anonymousPrimaryKey) {
     res.primaryKey.constraintName = `PK_${res.pureName}`;
   }

packages/tools/src/packageTools.ts

@@ -1,4 +1,4 @@
-import { EngineDriver, ExtensionsDirectory } from 'dbgate-types';
+import type { EngineDriver, ExtensionsDirectory } from 'dbgate-types';
 import _camelCase from 'lodash/camelCase';
 import _isString from 'lodash/isString';
 import _isPlainObject from 'lodash/isPlainObject';

packages/tools/src/preloadedRowsTools.ts

@@ -1,5 +1,5 @@
 import _ from 'lodash';
-import { DatabaseInfo, EngineDriver } from 'dbgate-types';
+import type { DatabaseInfo, EngineDriver } from 'dbgate-types';
 
 export async function enrichWithPreloadedRows(
   dbModel: DatabaseInfo,

packages/tools/src/schemaEditorTools.ts

@@ -1,6 +1,6 @@
 import uuidv1 from 'uuid/v1';
 import _omit from 'lodash/omit';
-import {
+import type {
   ColumnInfo,
   ConstraintInfo,
   ForeignKeyInfo,

packages/tools/src/stringTools.ts

@@ -46,6 +46,9 @@ export function stringifyCellValue(value) {
 }
 
 export function safeJsonParse(json, defaultValue?, logError = false) {
+  if (_isArray(json) || _isPlainObject(json)) {
+    return json;
+  }
   try {
     return JSON.parse(json);
   } catch (err) {
@@ -93,3 +96,22 @@ export function isWktGeometry(s) {
     /^POINT\s*\(|^LINESTRING\s*\(|^POLYGON\s*\(|^MULTIPOINT\s*\(|^MULTILINESTRING\s*\(|^MULTIPOLYGON\s*\(|^GEOMCOLLECTION\s*\(|^GEOMETRYCOLLECTION\s*\(/
   );
 }
+
+export function arrayBufferToBase64(buffer) {
+  var binary = '';
+  var bytes = [].slice.call(new Uint8Array(buffer));
+  bytes.forEach(b => (binary += String.fromCharCode(b)));
+  return btoa(binary);
+}
+
+export function getAsImageSrc(obj) {
+  if (obj?.type == 'Buffer' && _isArray(obj?.data)) {
+    return `data:image/png;base64, ${arrayBufferToBase64(obj?.data)}`;
+  }
+
+  if (_isString(obj) && (obj.startsWith('http://') || obj.startsWith('https://'))) {
+    return obj;
+  }
+
+  return null;
+}

packages/tools/src/structureTools.ts

@@ -1,4 +1,4 @@
-import { DatabaseInfo, TableInfo, ApplicationDefinition } from 'dbgate-types';
+import type { DatabaseInfo, TableInfo, ApplicationDefinition, ViewInfo, CollectionInfo } from 'dbgate-types';
 import _flatten from 'lodash/flatten';
 
 export function addTableDependencies(db: DatabaseInfo): DatabaseInfo {
@@ -118,3 +118,15 @@ export function isTableColumnUnique(table: TableInfo, column: string) {
   }
   return false;
 }
+
+export function isTableInfo(obj: { objectTypeField?: string }): obj is TableInfo {
+  return obj.objectTypeField == 'tables';
+}
+
+export function isViewInfo(obj: { objectTypeField?: string }): obj is ViewInfo {
+  return obj.objectTypeField == 'views';
+}
+
+export function isCollectionInfo(obj: { objectTypeField?: string }): obj is CollectionInfo {
+  return obj.objectTypeField == 'collections';
+}

packages/tools/src/tableTransforms.ts

@@ -1,4 +1,4 @@
-import { TableInfo } from 'dbgate-types';
+import type { TableInfo } from 'dbgate-types';
 import _cloneDeep from 'lodash/cloneDeep';
 import _fromPairs from 'lodash/fromPairs';
 import _get from 'lodash/get';

packages/types/dbinfo.d.ts

@@ -1,6 +1,8 @@
 export interface NamedObjectInfo {
   pureName: string;
   schemaName?: string;
+  contentHash?: string;
+  engine?: string;
 }
 
 export interface ColumnReference {
@@ -31,7 +33,8 @@ export interface ForeignKeyInfo extends ColumnsConstraintInfo {
 
 export interface IndexInfo extends ColumnsConstraintInfo {
   isUnique: boolean;
-  indexType: 'normal' | 'clustered' | 'xml' | 'spatial' | 'fulltext';
+  // indexType: 'normal' | 'clustered' | 'xml' | 'spatial' | 'fulltext';
+  indexType: string;
 }
 
 export interface UniqueInfo extends ColumnsConstraintInfo {}
@@ -43,8 +46,8 @@ export interface CheckInfo extends ConstraintInfo {
 export interface ColumnInfo extends NamedObjectInfo {
   pairingId?: string;
   columnName: string;
-  notNull: boolean;
-  autoIncrement: boolean;
+  notNull?: boolean;
+  autoIncrement?: boolean;
   dataType: string;
   precision?: number;
   scale?: number;
@@ -119,7 +122,7 @@ export interface DatabaseInfoObjects {
 }
 
 export interface DatabaseInfo extends DatabaseInfoObjects {
-  schemas: SchemaInfo[];
+  schemas?: SchemaInfo[];
   engine?: string;
   defaultSchema?: string;
 }

packages/types/dialect.d.ts

@@ -9,6 +9,7 @@ export interface SqlDialect {
   fallbackDataType?: string;
   explicitDropConstraint?: boolean;
   anonymousPrimaryKey?: boolean;
+  defaultSchemaName?: string;
   enableConstraintsPerTable?: boolean;
 
   dropColumnDependencies?: string[];

packages/types/dumper.d.ts

@@ -14,9 +14,12 @@ export interface SqlDumper extends AlterProcessor {
   putValue(value: string | number | Date);
   putCollection<T>(delimiter: string, collection: T[], lambda: (item: T) => void);
   transform(type: TransformType, dumpExpr: () => void);
+  createDatabase(name: string);
+  dropDatabase(name: string);
 
   endCommand();
   allowIdentityInsert(table: NamedObjectInfo, allow: boolean);
+  truncateTable(table: NamedObjectInfo);
   beginTransaction();
   commitTransaction();
 }

packages/types/engines.d.ts

@@ -72,6 +72,8 @@ export interface EngineDriver {
   beforeConnectionSave?: (values: any) => any;
   databaseUrlPlaceholder?: string;
   defaultAuthTypeName?: string;
+  defaultSocketPath?: string;
+  authTypeLabel?: string;
   importExportArgs?: any[];
   connect({ server, port, user, password, database }): Promise<any>;
   close(pool): Promise<any>;
@@ -87,9 +89,7 @@ export interface EngineDriver {
   ): Promise<TableInfo | ViewInfo | ProcedureInfo | FunctionInfo | TriggerInfo>;
   analyseSingleTable(pool: any, name: NamedObjectInfo): Promise<TableInfo>;
   getVersion(pool: any): Promise<{ version: string }>;
-  listDatabases(
-    pool: any
-  ): Promise<
+  listDatabases(pool: any): Promise<
     {
       name: string;
     }[]
@@ -110,7 +110,8 @@ export interface EngineDriver {
   updateCollection(pool: any, changeSet: any): Promise<any>;
   getCollectionUpdateScript(changeSet: any): string;
   createDatabase(pool: any, name: string): Promise;
-  getQuerySplitterOptions(usage: 'stream' | 'script'): any;
+  dropDatabase(pool: any, name: string): Promise;
+  getQuerySplitterOptions(usage: 'stream' | 'script' | 'editor'): any;
   script(pool: any, sql: string): Promise;
   getNewObjectTemplates(): NewObjectTemplate[];
   // direct call of pool method, only some methods could be supported, on only some drivers

packages/web/package.json

@@ -24,7 +24,7 @@
     "chartjs-adapter-moment": "^1.0.0",
     "cross-env": "^7.0.3",
     "dbgate-datalib": "^5.0.0-alpha.1",
-    "dbgate-query-splitter": "^4.9.0",
+    "dbgate-query-splitter": "^4.9.2",
     "dbgate-sqltree": "^5.0.0-alpha.1",
     "dbgate-tools": "^5.0.0-alpha.1",
     "dbgate-types": "^5.0.0-alpha.1",
@@ -57,6 +57,7 @@
   "dependencies": {
     "chartjs-plugin-zoom": "^1.2.0",
     "date-fns": "^2.28.0",
+    "debug": "^4.3.4",
     "interval-operations": "^1.0.7",
     "leaflet": "^1.8.0",
     "wellknown": "^0.5.0"

packages/web/public/global.css

@@ -167,3 +167,33 @@ textarea {
   color: var(--theme-font-1);
   border: 1px solid var(--theme-border);
 }
+
+.ace_gutter-cell.ace-gutter-sql-run {
+  background-repeat: no-repeat;
+  background-position: 2px center;
+
+  /* content: '▶';
+  margin-right: 3px; */
+
+  /* border-radius: 20px 0px 0px 20px; */
+  /* Change the color of the breakpoint if you want 
+  box-shadow: 0px 0px 1px 1px #248c46 inset;  */
+}
+
+.theme-type-light .ace_gutter-cell.ace-gutter-sql-run {
+  background-image: url(data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iaXNvLTg4NTktMSI/Pgo8IS0tIEdlbmVyYXRvcjogQWRvYmUgSWxsdXN0cmF0b3IgMTguMS4xLCBTVkcgRXhwb3J0IFBsdWctSW4gLiBTVkcgVmVyc2lvbjogNi4wMCBCdWlsZCAwKSAgLS0+CjxzdmcgdmVyc2lvbj0iMS4xIiBpZD0iQ2FwYV8xIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKCSB2aWV3Qm94PSIwIDAgMTcuODA0IDE3LjgwNCIgc3R5bGU9ImVuYWJsZS1iYWNrZ3JvdW5kOm5ldyAwIDAgMTcuODA0IDE3LjgwNDsiIHhtbDpzcGFjZT0icHJlc2VydmUiPgo8Zz4KCTxnIGlkPSJjOThfcGxheSI+CgkJPHBhdGggZmlsbD0nIzQ0NCcgZD0iTTIuMDY3LDAuMDQzQzIuMjEtMC4wMjgsMi4zNzItMC4wMDgsMi40OTMsMC4wODVsMTMuMzEyLDguNTAzYzAuMDk0LDAuMDc4LDAuMTU0LDAuMTkxLDAuMTU0LDAuMzEzCgkJCWMwLDAuMTItMC4wNjEsMC4yMzctMC4xNTQsMC4zMTRMMi40OTIsMTcuNzE3Yy0wLjA3LDAuMDU3LTAuMTYyLDAuMDg3LTAuMjUsMC4wODdsLTAuMTc2LTAuMDQKCQkJYy0wLjEzNi0wLjA2NS0wLjIyMi0wLjIwNy0wLjIyMi0wLjM2MVYwLjQwMkMxLjg0NCwwLjI1LDEuOTMsMC4xMDcsMi4wNjcsMC4wNDN6Ii8+Cgk8L2c+Cgk8ZyBpZD0iQ2FwYV8xXzc4XyI+Cgk8L2c+CjwvZz4KPC9zdmc+Cg==);
+}
+
+.theme-type-dark .ace_gutter-cell.ace-gutter-sql-run {
+  background-image: url(data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iaXNvLTg4NTktMSI/Pgo8IS0tIEdlbmVyYXRvcjogQWRvYmUgSWxsdXN0cmF0b3IgMTguMS4xLCBTVkcgRXhwb3J0IFBsdWctSW4gLiBTVkcgVmVyc2lvbjogNi4wMCBCdWlsZCAwKSAgLS0+CjxzdmcgdmVyc2lvbj0iMS4xIiBpZD0iQ2FwYV8xIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKCSB2aWV3Qm94PSIwIDAgMTcuODA0IDE3LjgwNCIgc3R5bGU9ImVuYWJsZS1iYWNrZ3JvdW5kOm5ldyAwIDAgMTcuODA0IDE3LjgwNDsiIHhtbDpzcGFjZT0icHJlc2VydmUiPgo8Zz4KCTxnIGlkPSJjOThfcGxheSI+CgkJPHBhdGggZmlsbD0nI2RkZCcgZD0iTTIuMDY3LDAuMDQzQzIuMjEtMC4wMjgsMi4zNzItMC4wMDgsMi40OTMsMC4wODVsMTMuMzEyLDguNTAzYzAuMDk0LDAuMDc4LDAuMTU0LDAuMTkxLDAuMTU0LDAuMzEzCgkJCWMwLDAuMTItMC4wNjEsMC4yMzctMC4xNTQsMC4zMTRMMi40OTIsMTcuNzE3Yy0wLjA3LDAuMDU3LTAuMTYyLDAuMDg3LTAuMjUsMC4wODdsLTAuMTc2LTAuMDQKCQkJYy0wLjEzNi0wLjA2NS0wLjIyMi0wLjIwNy0wLjIyMi0wLjM2MVYwLjQwMkMxLjg0NCwwLjI1LDEuOTMsMC4xMDcsMi4wNjcsMC4wNDN6Ii8+Cgk8L2c+Cgk8ZyBpZD0iQ2FwYV8xXzc4XyI+Cgk8L2c+CjwvZz4KPC9zdmc+Cg==);
+}
+
+.ace_gutter-cell.ace-gutter-sql-run:hover {
+   background-color: var(--theme-bg-2);
+}
+
+.ace_gutter-cell.ace-gutter-current-part {
+   /* background-color: var(--theme-bg-2); */
+   font-weight: bold;
+   color: var(--theme-font-hover);
+}

packages/web/rollup.config.js

@@ -45,7 +45,7 @@ export default [
       resolve({
         browser: true,
       }),
-      
+
       // If we're building for production (npm run build
       // instead of npm run dev), minify
       production && terser(),
@@ -88,6 +88,20 @@ export default [
           // enable run-time checks when not in production
           dev: !production,
         },
+        onwarn: (warning, handler) => {
+          const ignoreWarnings = [
+            'a11y-click-events-have-key-events',
+            'a11y-missing-attribute',
+            'a11y-invalid-attribute',
+            'a11y-no-noninteractive-tabindex',
+            'a11y-label-has-associated-control',
+            'vite-plugin-svelte-css-no-scopable-elements',
+            'unused-export-let',
+          ];
+          if (ignoreWarnings.includes(warning.code)) return;
+          // console.log('***************************', warning.code);
+          handler(warning);
+        },
       }),
       // we'll extract any component CSS out into
       // a separate file - better for performance

packages/web/src/App.svelte

@@ -19,6 +19,8 @@
   import AppTitleProvider from './utility/AppTitleProvider.svelte';
   import getElectron from './utility/getElectron';
   import AppStartInfo from './widgets/AppStartInfo.svelte';
+  import SettingsListener from './utility/SettingsListener.svelte';
+  import { handleAuthOnStartup, handleOauthCallback } from './clientAuth';
 
   let loadedApi = false;
   let loadedPlugins = false;
@@ -32,9 +34,11 @@
     try {
       // console.log('************** LOADING API');
 
+      const config = await getConfig();
+      await handleAuthOnStartup(config);
+
       const connections = await apiCall('connections/list');
       const settings = await getSettings();
-      const config = await getConfig();
       const apps = await getUsedApps();
       loadedApi = settings && connections && config && apps;
 
@@ -79,6 +83,7 @@
   <AppTitleProvider />
   {#if loadedPlugins}
     <OpenTabsOnStartup />
+    <SettingsListener />
     <Screen />
   {:else}
     <AppStartInfo

packages/web/src/LoginPage.svelte

@@ -0,0 +1,115 @@
+<script lang="ts">
+  import { onMount } from 'svelte';
+  import { internalRedirectTo } from './clientAuth';
+  import FormButton from './forms/FormButton.svelte';
+  import FormPasswordField from './forms/FormPasswordField.svelte';
+  import FormProvider from './forms/FormProvider.svelte';
+  import FormSubmit from './forms/FormSubmit.svelte';
+  import FormTextField from './forms/FormTextField.svelte';
+  import { apiCall, enableApi } from './utility/api';
+
+  onMount(() => {
+    const removed = document.getElementById('starting_dbgate_zero');
+    if (removed) removed.remove();
+  });
+</script>
+
+<div class="root theme-light theme-type-light">
+  <div class="text">DbGate</div>
+  <div class="wrap">
+    <div class="logo">
+      <img class="img" src="logo192.png" />
+    </div>
+    <div class="box">
+      <div class="heading">Log In</div>
+      <FormProvider>
+        <FormTextField label="Username" name="login" autocomplete="username" />
+        <FormPasswordField label="Password" name="password" autocomplete="current-password" />
+
+        <div class="submit">
+          <FormSubmit
+            value="Log In"
+            on:click={async e => {
+              enableApi();
+              const resp = await apiCall('auth/login', e.detail);
+              if (resp.error) {
+                internalRedirectTo(`/?page=not-logged&error=${encodeURIComponent(resp.error)}`);
+                return;
+              }
+              const { accessToken } = resp;
+              if (accessToken) {
+                localStorage.setItem('accessToken', accessToken);
+                internalRedirectTo('/');
+                return;
+              }
+              internalRedirectTo(`/?page=not-logged`);
+            }}
+          />
+        </div>
+      </FormProvider>
+    </div>
+  </div>
+</div>
+
+<style>
+  .logo {
+    display: flex;
+    margin-bottom: 1rem;
+    align-items: center;
+    justify-content: center;
+  }
+  .img {
+    width: 80px;
+  }
+  .text {
+    position: fixed;
+    top: 1rem;
+    left: 1rem;
+    font-size: 30pt;
+    font-family: monospace;
+    color: var(--theme-bg-2);
+    text-transform: uppercase;
+  }
+  .submit {
+    margin: var(--dim-large-form-margin);
+    display: flex;
+  }
+
+  .submit :global(input) {
+    flex: 1;
+    font-size: larger;
+  }
+
+  .root {
+    color: var(--theme-font-1);
+    display: flex;
+    justify-content: center;
+    background-color: var(--theme-bg-1);
+    align-items: baseline;
+    position: fixed;
+    top: 0;
+    left: 0;
+    right: 0;
+    bottom: 0;
+  }
+
+  .box {
+    width: 600px;
+    max-width: 80vw;
+    /* max-width: 600px;
+    width: 40vw; */
+    border: 1px solid var(--theme-border);
+    border-radius: 4px;
+    background-color: var(--theme-bg-0);
+  }
+
+  .wrap {
+    margin-top: 20vh;
+  }
+
+  .heading {
+    text-align: center;
+    margin: 1em;
+    font-size: xx-large;
+  }
+</style>

packages/web/src/NotLoggedPage.svelte

@@ -0,0 +1,52 @@
+<script lang="ts">
+  import { onMount } from 'svelte';
+  import FormStyledButton from './buttons/FormStyledButton.svelte';
+  import { doLogout, redirectToLogin } from './clientAuth';
+
+  onMount(() => {
+    const removed = document.getElementById('starting_dbgate_zero');
+    if (removed) removed.remove();
+  });
+
+  const params = new URLSearchParams(location.search);
+  const error = params.get('error');
+
+  function handleLogin() {
+    redirectToLogin(undefined, true);
+  }
+</script>
+
+<div class="root theme-light theme-type-light">
+  <div class="title">Sorry, you are not authorized to run DbGate</div>
+  {#if error}
+    <div class="error">{error}</div>
+  {/if}
+
+  <div class="button">
+    <FormStyledButton value="Log In" on:click={handleLogin} />
+    <FormStyledButton value="Log Out" on:click={doLogout} />
+  </div>
+</div>
+
+<style>
+  .root {
+    color: var(--theme-font-1);
+  }
+
+  .title {
+    font-size: x-large;
+    margin-top: 20vh;
+    text-align: center;
+  }
+
+  .error {
+    margin-top: 1em;
+    text-align: center;
+  }
+
+  .button {
+    display: flex;
+    justify-content: center;
+    margin-top: 1em;
+  }
+</style>

packages/web/src/Screen.svelte

@@ -8,6 +8,7 @@
     leftPanelWidth,
     openedSnackbars,
     selectedWidget,
+    visibleWidgetSideBar,
     visibleCommandPalette,
     visibleTitleBar,
     visibleToolbar,
@@ -29,7 +30,7 @@
 
   $: currentThemeType = $currentThemeDefinition?.themeType == 'dark' ? 'theme-type-dark' : 'theme-type-light';
 
-  $: themeStyle = `<style id="themePlugin">${$currentThemeDefinition?.themeCss}</style>`;
+  $: themeStyle = `<st` + `yle id="themePlugin">${$currentThemeDefinition?.themeCss}</st` + `yle>`;
 
   const isElectron = !!getElectron();
 </script>
@@ -63,7 +64,7 @@
   <div class="statusbar">
     <StatusBar />
   </div>
-  {#if $selectedWidget}
+  {#if $selectedWidget && $visibleWidgetSideBar}
     <div class="leftpanel">
       <WidgetContainer />
     </div>
@@ -74,7 +75,7 @@
   <div class="content">
     <TabRegister />
   </div>
-  {#if $selectedWidget}
+  {#if $selectedWidget && $visibleWidgetSideBar}
     <div
       class="horizontal-split-handle splitter"
       use:splitterDrag={'clientX'}

packages/web/src/appobj/AppFileAppObject.svelte

@@ -16,7 +16,7 @@
         tabComponent,
         tooltip,
         props: {
-          savedFile:fileName + '.' + fileType,
+          savedFile: fileName + '.' + fileType,
           savedFolder: 'app:' + folderName,
           savedFormat: 'text',
           appFolder: folderName,
@@ -28,7 +28,10 @@
   }
 
   export const extractKey = data => data.fileName;
-  export const createMatcher = ({ fileName }) => filter => filterName(filter, fileName);
+  export const createMatcher =
+    ({ fileName }) =>
+    filter =>
+      filterName(filter, fileName);
   const APP_ICONS = {
     'config.json': 'img json',
     'command.sql': 'img app-command',
@@ -50,7 +53,6 @@
   import InputTextModal from '../modals/InputTextModal.svelte';
   import ConfirmModal from '../modals/ConfirmModal.svelte';
   import { apiCall } from '../utility/api';
-  import { currentDatabase, currentDatabase } from '../stores';
 
   export let data;