v5.1.7-alpha.14

Make maximize button reflect window state

.github/workflows/build-npm.yaml

@@ -138,3 +138,8 @@ jobs:
         working-directory: plugins/dbgate-plugin-redis
         run: |
           npm publish
+
+      - name: Publish dbgate-plugin-oracle
+        working-directory: plugins/dbgate-plugin-oracle
+        run: |
+          npm publish

README.md

@@ -22,6 +22,7 @@ DbGate is licensed under MIT license and is completely free.
 * MySQL
 * PostgreSQL
 * SQL Server
+* Oracle
 * MongoDB
 * Redis
 * SQLite
@@ -66,13 +67,13 @@ DbGate is licensed under MIT license and is completely free.
 * Mongo JavaScript editor, execute Mongo script (with NodeJs syntax)
 * Redis tree view, generate script from keys, run Redis script
 * Runs as application for Windows, Linux and Mac. Or in Docker container on server and in web Browser on client.
-* Import, export from/to CSV, Excel, JSON, XML
+* Import, export from/to CSV, Excel, JSON, NDJSON, XML
 * Free table editor - quick table data editing (cleanup data after import/before export, prototype tables etc.)
-* Archives - backup your data in JSON files on local filesystem (or on DbGate server, when using web application)
+* Archives - backup your data in NDJSON files on local filesystem (or on DbGate server, when using web application)
 * Charts, export chart to HTML page
 * For detailed info, how to run DbGate in docker container, visit [docker hub](https://hub.docker.com/r/dbgate/dbgate)
 * Extensible plugin architecture
-* Perspectives - nested table view over complex relational data
+* Perspectives - nested table view over complex relational data, query designer on MongoDB databases
 
 ## How to contribute
 Any contributions are welcome. If you want to contribute without coding, consider following:

app/src/electron.js

@@ -154,6 +154,10 @@ ipcMain.on('app-started', async (event, arg) => {
     mainWindow.webContents.send('run-command', runCommandOnLoad);
     runCommandOnLoad = null;
   }
+
+  if (initialConfig['winIsMaximized']) {
+    mainWindow.webContents.send('setIsMaximized', true);
+  }
 });
 ipcMain.on('window-action', async (event, arg) => {
   if (!mainWindow) {
@@ -166,8 +170,10 @@ ipcMain.on('window-action', async (event, arg) => {
     case 'maximize':
       if (mainWindow.isMaximized()) {
         mainWindow.unmaximize();
+        mainWindow.webContents.send('setIsMaximized', false);
       } else {
         mainWindow.maximize();
+        mainWindow.webContents.send('setIsMaximized', true);
       }
       break;
     case 'close':

app/src/mainMenuDefinition.js

@@ -21,6 +21,7 @@ module.exports = ({ editMenu }) => [
       { divider: true },
       { command: 'file.exit', hideDisabled: true },
       { command: 'app.logout', hideDisabled: true, skipInApp: true },
+      { command: 'app.disconnect', hideDisabled: true, skipInApp: true },
     ],
   },
   {

package.json

@@ -1,6 +1,6 @@
 {
   "private": true,
-  "version": "5.1.7-beta.7",
+  "version": "5.1.7-alpha.14",
   "name": "dbgate-all",
   "workspaces": [
     "packages/*",
@@ -17,6 +17,7 @@
     "start:api:portal": "yarn workspace dbgate-api start:portal",
     "start:api:singledb": "yarn workspace dbgate-api start:singledb",
     "start:api:auth": "yarn workspace dbgate-api start:auth",
+    "start:api:dblogin": "yarn workspace dbgate-api start:dblogin",
     "start:web": "yarn workspace dbgate-web dev",
     "start:sqltree": "yarn workspace dbgate-sqltree start",
     "start:tools": "yarn workspace dbgate-tools start",

packages/api/env/dblogin/.env

@@ -0,0 +1,14 @@
+DEVMODE=1
+
+CONNECTIONS=mysql
+SINGLE_CONNECTION=mysql
+# SINGLE_DATABASE=Chinook
+
+LABEL_mysql=MySql localhost
+SERVER_mysql=localhost
+# USER_mysql=root
+PORT_mysql=3306
+# PASSWORD_mysql=Pwd2020Db
+ENGINE_mysql=mysql@dbgate-plugin-mysql
+# PASSWORD_MODE_mysql=askPassword
+PASSWORD_MODE_mysql=askUser

packages/api/env/singledb/.env

@@ -5,8 +5,8 @@ CONNECTIONS=mysql
 LABEL_mysql=MySql localhost
 SERVER_mysql=localhost
 USER_mysql=root
-PASSWORD_mysql=test
-PORT_mysql=3307
+PASSWORD_mysql=Pwd2020Db
+PORT_mysql=3306
 ENGINE_mysql=mysql@dbgate-plugin-mysql
 DBCONFIG_mysql=[{"name":"Chinook","connectionColor":"cyan"}]
 

packages/api/package.json

@@ -26,7 +26,7 @@
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-query-splitter": "^4.9.2",
+    "dbgate-query-splitter": "^4.9.3",
     "dbgate-sqltree": "^5.0.0-alpha.1",
     "dbgate-tools": "^5.0.0-alpha.1",
     "debug": "^4.3.4",
@@ -60,6 +60,7 @@
     "start:portal": "env-cmd -f env/portal/.env node src/index.js --listen-api",
     "start:singledb": "env-cmd -f env/singledb/.env node src/index.js --listen-api",
     "start:auth": "env-cmd -f env/auth/.env node src/index.js --listen-api",
+    "start:dblogin": "env-cmd -f env/dblogin/.env node src/index.js --listen-api",
     "start:filedb": "env-cmd node src/index.js /home/jena/test/chinook/Chinook.db --listen-api",
     "start:singleconn": "env-cmd node src/index.js --server localhost --user root --port 3307 --engine mysql@dbgate-plugin-mysql --password test --listen-api",
     "ts": "tsc",

packages/api/src/controllers/apps.js

@@ -58,7 +58,7 @@ module.exports = {
 
   refreshFiles_meta: true,
   async refreshFiles({ folder }) {
-    socket.emitChanged(`app-files-changed-${folder}`);
+    socket.emitChanged('app-files-changed', { app: folder });
   },
 
   refreshFolders_meta: true,
@@ -69,7 +69,7 @@ module.exports = {
   deleteFile_meta: true,
   async deleteFile({ folder, file, fileType }) {
     await fs.unlink(path.join(appdir(), folder, `${file}.${fileType}`));
-    socket.emitChanged(`app-files-changed-${folder}`);
+    socket.emitChanged('app-files-changed', { app: folder });
     this.emitChangedDbApp(folder);
   },
 
@@ -79,7 +79,7 @@ module.exports = {
       path.join(path.join(appdir(), folder), `${file}.${fileType}`),
       path.join(path.join(appdir(), folder), `${newFile}.${fileType}`)
     );
-    socket.emitChanged(`app-files-changed-${folder}`);
+    socket.emitChanged('app-files-changed', { app: folder });
     this.emitChangedDbApp(folder);
   },
 
@@ -95,7 +95,7 @@ module.exports = {
     if (!folder) throw new Error('Missing folder parameter');
     await fs.rmdir(path.join(appdir(), folder), { recursive: true });
     socket.emitChanged(`app-folders-changed`);
-    socket.emitChanged(`app-files-changed-${folder}`);
+    socket.emitChanged('app-files-changed', { app: folder });
     socket.emitChanged('used-apps-changed');
   },
 
@@ -219,7 +219,7 @@ module.exports = {
 
     await fs.writeFile(file, JSON.stringify(json, undefined, 2));
 
-    socket.emitChanged(`app-files-changed-${appFolder}`);
+    socket.emitChanged('app-files-changed', { app: appFolder });
     socket.emitChanged('used-apps-changed');
   },
 
@@ -271,7 +271,7 @@ module.exports = {
     const file = path.join(appdir(), appFolder, fileName);
     if (!(await fs.exists(file))) {
       await fs.writeFile(file, JSON.stringify(content, undefined, 2));
-      socket.emitChanged(`app-files-changed-${appFolder}`);
+      socket.emitChanged('app-files-changed', { app: appFolder });
       socket.emitChanged('used-apps-changed');
       return true;
     }

packages/api/src/controllers/archive.js

@@ -5,6 +5,7 @@ const { archivedir, clearArchiveLinksCache, resolveArchiveFolder } = require('..
 const socket = require('../utility/socket');
 const { saveFreeTableData } = require('../utility/freeTableStorage');
 const loadFilesRecursive = require('../utility/loadFilesRecursive');
+const getJslFileName = require('../utility/getJslFileName');
 
 module.exports = {
   folders_meta: true,
@@ -74,7 +75,7 @@ module.exports = {
 
   refreshFiles_meta: true,
   async refreshFiles({ folder }) {
-    socket.emitChanged(`archive-files-changed-${folder}`);
+    socket.emitChanged('archive-files-changed', { folder });
   },
 
   refreshFolders_meta: true,
@@ -85,7 +86,7 @@ module.exports = {
   deleteFile_meta: true,
   async deleteFile({ folder, file, fileType }) {
     await fs.unlink(path.join(resolveArchiveFolder(folder), `${file}.${fileType}`));
-    socket.emitChanged(`archive-files-changed-${folder}`);
+    socket.emitChanged(`archive-files-changed`, { folder });
   },
 
   renameFile_meta: true,
@@ -94,7 +95,7 @@ module.exports = {
       path.join(resolveArchiveFolder(folder), `${file}.${fileType}`),
       path.join(resolveArchiveFolder(folder), `${newFile}.${fileType}`)
     );
-    socket.emitChanged(`archive-files-changed-${folder}`);
+    socket.emitChanged(`archive-files-changed`, { folder });
   },
 
   renameFolder_meta: true,
@@ -118,7 +119,7 @@ module.exports = {
   saveFreeTable_meta: true,
   async saveFreeTable({ folder, file, data }) {
     await saveFreeTableData(path.join(resolveArchiveFolder(folder), `${file}.jsonl`), data);
-    socket.emitChanged(`archive-files-changed-${folder}`);
+    socket.emitChanged(`archive-files-changed`, { folder });
     return true;
   },
 
@@ -146,7 +147,16 @@ module.exports = {
   saveText_meta: true,
   async saveText({ folder, file, text }) {
     await fs.writeFile(path.join(resolveArchiveFolder(folder), `${file}.jsonl`), text);
-    socket.emitChanged(`archive-files-changed-${folder}`);
+    socket.emitChanged(`archive-files-changed`, { folder });
+    return true;
+  },
+
+  saveJslData_meta: true,
+  async saveJslData({ folder, file, jslid }) {
+    const source = getJslFileName(jslid);
+    const target = path.join(resolveArchiveFolder(folder), `${file}.jsonl`);
+    await fs.copyFile(source, target);
+    socket.emitChanged(`archive-files-changed`, { folder });
     return true;
   },
 

packages/api/src/controllers/config.js

@@ -38,7 +38,8 @@ module.exports = {
 
     return {
       runAsPortal: !!connections.portalConnections,
-      singleDatabase: connections.singleDatabase,
+      singleDbConnection: connections.singleDbConnection,
+      singleConnection: connections.singleConnection,
       // hideAppEditor: !!process.env.HIDE_APP_EDITOR,
       allowShellConnection: platformInfo.allowShellConnection,
       allowShellScripting: platformInfo.allowShellScripting,

packages/api/src/controllers/connections.js

@@ -2,6 +2,7 @@ const path = require('path');
 const { fork } = require('child_process');
 const _ = require('lodash');
 const fs = require('fs-extra');
+const crypto = require('crypto');
 
 const { datadir, filesdir } = require('../utility/directories');
 const socket = require('../utility/socket');
@@ -15,6 +16,8 @@ const { safeJsonParse } = require('dbgate-tools');
 const platformInfo = require('../utility/platformInfo');
 const { connectionHasPermission, testConnectionPermission } = require('../utility/hasPermission');
 
+let volatileConnections = {};
+
 function getNamedArgs() {
   const res = {};
   for (let i = 0; i < process.argv.length; i++) {
@@ -49,6 +52,7 @@ function getPortalCollections() {
       server: process.env[`SERVER_${id}`],
       user: process.env[`USER_${id}`],
       password: process.env[`PASSWORD_${id}`],
+      passwordMode: process.env[`PASSWORD_MODE_${id}`],
       port: process.env[`PORT_${id}`],
       databaseUrl: process.env[`URL_${id}`],
       useDatabaseUrl: !!process.env[`URL_${id}`],
@@ -126,9 +130,10 @@ function getPortalCollections() {
 
   return null;
 }
+
 const portalConnections = getPortalCollections();
 
-function getSingleDatabase() {
+function getSingleDbConnection() {
   if (process.env.SINGLE_CONNECTION && process.env.SINGLE_DATABASE) {
     // @ts-ignore
     const connection = portalConnections.find(x => x._id == process.env.SINGLE_CONNECTION);
@@ -152,12 +157,31 @@ function getSingleDatabase() {
   return null;
 }
 
-const singleDatabase = getSingleDatabase();
+function getSingleConnection() {
+  if (getSingleDbConnection()) return null;
+  if (process.env.SINGLE_CONNECTION) {
+    // @ts-ignore
+    const connection = portalConnections.find(x => x._id == process.env.SINGLE_CONNECTION);
+    if (connection) {
+      return connection;
+    }
+  }
+  // @ts-ignore
+  const arg0 = (portalConnections || []).find(x => x._id == 'argv');
+  if (arg0) {
+    return arg0;
+  }
+  return null;
+}
+
+const singleDbConnection = getSingleDbConnection();
+const singleConnection = getSingleConnection();
 
 module.exports = {
   datastore: null,
   opened: [],
-  singleDatabase,
+  singleDbConnection,
+  singleConnection,
   portalConnections,
 
   async _init() {
@@ -199,6 +223,36 @@ module.exports = {
     });
   },
 
+  saveVolatile_meta: true,
+  async saveVolatile({ conid, user, password, test }) {
+    const old = await this.getCore({ conid });
+    const res = {
+      ...old,
+      _id: crypto.randomUUID(),
+      password,
+      passwordMode: undefined,
+      unsaved: true,
+    };
+    if (old.passwordMode == 'askUser') {
+      res.user = user;
+    }
+
+    if (test) {
+      const testRes = await this.test(res);
+      if (testRes.msgtype == 'connected') {
+        volatileConnections[res._id] = res;
+        return {
+          ...res,
+          msgtype: 'connected',
+        };
+      }
+      return testRes;
+    } else {
+      volatileConnections[res._id] = res;
+      return res;
+    }
+  },
+
   save_meta: true,
   async save(connection) {
     if (portalConnections) return;
@@ -229,6 +283,14 @@ module.exports = {
     return res;
   },
 
+  batchChangeFolder_meta: true,
+  async batchChangeFolder({ folder, newFolder }, req) {
+    // const updated = await this.datastore.find(x => x.parent == folder);
+    const res = await this.datastore.updateAll(x => (x.parent == folder ? { ...x, parent: newFolder } : x));
+    socket.emitChanged('connection-list-changed');
+    return res;
+  },
+
   updateDatabase_meta: true,
   async updateDatabase({ conid, database, values }, req) {
     if (portalConnections) return;
@@ -258,6 +320,10 @@ module.exports = {
 
   async getCore({ conid, mask = false }) {
     if (!conid) return null;
+    const volatile = volatileConnections[conid];
+    if (volatile) {
+      return volatile;
+    }
     if (portalConnections) {
       const res = portalConnections.find(x => x._id == conid) || null;
       return mask && !platformInfo.allowShellConnection ? maskConnection(res) : res;

packages/api/src/controllers/databaseConnections.js

@@ -27,6 +27,7 @@ const { createTwoFilesPatch } = require('diff');
 const diff2htmlPage = require('../utility/diff2htmlPage');
 const processArgs = require('../utility/processArgs');
 const { testConnectionPermission } = require('../utility/hasPermission');
+const { MissingCredentialsError } = require('../utility/exceptions');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedDatabaseConnection[]} */
@@ -42,19 +43,19 @@ module.exports = {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (!existing) return;
     existing.structure = structure;
-    socket.emitChanged(`database-structure-changed-${conid}-${database}`);
+    socket.emitChanged('database-structure-changed', { conid, database });
   },
   handle_structureTime(conid, database, { analysedTime }) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (!existing) return;
     existing.analysedTime = analysedTime;
-    socket.emitChanged(`database-status-changed-${conid}-${database}`);
+    socket.emitChanged(`database-status-changed`, { conid, database });
   },
   handle_version(conid, database, { version }) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (!existing) return;
     existing.serverVersion = version;
-    socket.emitChanged(`database-server-version-changed-${conid}-${database}`);
+    socket.emitChanged(`database-server-version-changed`, { conid, database });
   },
 
   handle_error(conid, database, props) {
@@ -72,7 +73,7 @@ module.exports = {
     if (!existing) return;
     if (existing.status && status && existing.status.counter > status.counter) return;
     existing.status = status;
-    socket.emitChanged(`database-status-changed-${conid}-${database}`);
+    socket.emitChanged(`database-status-changed`, { conid, database });
   },
 
   handle_ping() {},
@@ -81,6 +82,9 @@ module.exports = {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) return existing;
     const connection = await connections.getCore({ conid });
+    if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') {
+      throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode });
+    }
     const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
       '--is-forked-api',
       '--start-process',
@@ -313,7 +317,7 @@ module.exports = {
         },
         structure: existing.structure,
       };
-      socket.emitChanged(`database-status-changed-${conid}-${database}`);
+      socket.emitChanged(`database-status-changed`, { conid, database });
     }
   },
 

packages/api/src/controllers/files.js

@@ -49,7 +49,7 @@ module.exports = {
   async delete({ folder, file }, req) {
     if (!hasPermission(`files/${folder}/write`, req)) return false;
     await fs.unlink(path.join(filesdir(), folder, file));
-    socket.emitChanged(`files-changed-${folder}`);
+    socket.emitChanged(`files-changed`, { folder });
     socket.emitChanged(`all-files-changed`);
     return true;
   },
@@ -58,7 +58,7 @@ module.exports = {
   async rename({ folder, file, newFile }, req) {
     if (!hasPermission(`files/${folder}/write`, req)) return false;
     await fs.rename(path.join(filesdir(), folder, file), path.join(filesdir(), folder, newFile));
-    socket.emitChanged(`files-changed-${folder}`);
+    socket.emitChanged(`files-changed`, { folder });
     socket.emitChanged(`all-files-changed`);
     return true;
   },
@@ -66,7 +66,7 @@ module.exports = {
   refresh_meta: true,
   async refresh({ folders }, req) {
     for (const folder of folders) {
-      socket.emitChanged(`files-changed-${folder}`);
+      socket.emitChanged(`files-changed`, { folder });
       socket.emitChanged(`all-files-changed`);
     }
     return true;
@@ -76,7 +76,7 @@ module.exports = {
   async copy({ folder, file, newFile }, req) {
     if (!hasPermission(`files/${folder}/write`, req)) return false;
     await fs.copyFile(path.join(filesdir(), folder, file), path.join(filesdir(), folder, newFile));
-    socket.emitChanged(`files-changed-${folder}`);
+    socket.emitChanged(`files-changed`, { folder });
     socket.emitChanged(`all-files-changed`);
     return true;
   },
@@ -112,13 +112,13 @@ module.exports = {
       if (!hasPermission(`archive/write`, req)) return false;
       const dir = resolveArchiveFolder(folder.substring('archive:'.length));
       await fs.writeFile(path.join(dir, file), serialize(format, data));
-      socket.emitChanged(`archive-files-changed-${folder.substring('archive:'.length)}`);
+      socket.emitChanged(`archive-files-changed`, { folder: folder.substring('archive:'.length) });
       return true;
     } else if (folder.startsWith('app:')) {
       if (!hasPermission(`apps/write`, req)) return false;
       const app = folder.substring('app:'.length);
       await fs.writeFile(path.join(appdir(), app, file), serialize(format, data));
-      socket.emitChanged(`app-files-changed-${app}`);
+      socket.emitChanged(`app-files-changed`, { app });
       socket.emitChanged('used-apps-changed');
       apps.emitChangedDbApp(folder);
       return true;
@@ -129,7 +129,7 @@ module.exports = {
         await fs.mkdir(dir);
       }
       await fs.writeFile(path.join(dir, file), serialize(format, data));
-      socket.emitChanged(`files-changed-${folder}`);
+      socket.emitChanged(`files-changed`, { folder });
       socket.emitChanged(`all-files-changed`);
       if (folder == 'shell') {
         scheduler.reload();

packages/api/src/controllers/jsldata.js

@@ -7,6 +7,7 @@ const DatastoreProxy = require('../utility/DatastoreProxy');
 const { saveFreeTableData } = require('../utility/freeTableStorage');
 const getJslFileName = require('../utility/getJslFileName');
 const JsonLinesDatastore = require('../utility/JsonLinesDatastore');
+const requirePluginFunction = require('../utility/requirePluginFunction');
 const socket = require('../utility/socket');
 
 function readFirstLine(file) {
@@ -99,10 +100,13 @@ module.exports = {
   //   return readerInfo;
   // },
 
-  async ensureDatastore(jslid) {
+  async ensureDatastore(jslid, formatterFunction) {
     let datastore = this.datastores[jslid];
-    if (!datastore) {
-      datastore = new JsonLinesDatastore(getJslFileName(jslid));
+    if (!datastore || datastore.formatterFunction != formatterFunction) {
+      if (datastore) {
+        datastore._closeReader();
+      }
+      datastore = new JsonLinesDatastore(getJslFileName(jslid), formatterFunction);
       // datastore = new DatastoreProxy(getJslFileName(jslid));
       this.datastores[jslid] = datastore;
     }
@@ -131,8 +135,8 @@ module.exports = {
   },
 
   getRows_meta: true,
-  async getRows({ jslid, offset, limit, filters }) {
-    const datastore = await this.ensureDatastore(jslid);
+  async getRows({ jslid, offset, limit, filters, formatterFunction }) {
+    const datastore = await this.ensureDatastore(jslid, formatterFunction);
     return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters);
   },
 
@@ -150,8 +154,8 @@ module.exports = {
   },
 
   loadFieldValues_meta: true,
-  async loadFieldValues({ jslid, field, search }) {
-    const datastore = await this.ensureDatastore(jslid);
+  async loadFieldValues({ jslid, field, search, formatterFunction }) {
+    const datastore = await this.ensureDatastore(jslid, formatterFunction);
     const res = new Set();
     await datastore.enumRows(row => {
       if (!filterName(search, row[field])) return true;
@@ -188,4 +192,85 @@ module.exports = {
     await fs.promises.writeFile(getJslFileName(jslid), text);
     return true;
   },
+
+  extractTimelineChart_meta: true,
+  async extractTimelineChart({ jslid, timestampFunction, aggregateFunction, measures }) {
+    const timestamp = requirePluginFunction(timestampFunction);
+    const aggregate = requirePluginFunction(aggregateFunction);
+    const datastore = new JsonLinesDatastore(getJslFileName(jslid));
+    let mints = null;
+    let maxts = null;
+    // pass 1 - counts stats, time range
+    await datastore.enumRows(row => {
+      const ts = timestamp(row);
+      if (!mints || ts < mints) mints = ts;
+      if (!maxts || ts > maxts) maxts = ts;
+      return true;
+    });
+    const minTime = new Date(mints).getTime();
+    const maxTime = new Date(maxts).getTime();
+    const duration = maxTime - minTime;
+    const STEPS = 100;
+    let stepCount = duration > 100 * 1000 ? STEPS : Math.round((maxTime - minTime) / 1000);
+    if (stepCount < 2) {
+      stepCount = 2;
+    }
+    const stepDuration = duration / stepCount;
+    const labels = _.range(stepCount).map(i => new Date(minTime + stepDuration / 2 + stepDuration * i));
+
+    // const datasets = measures.map(m => ({
+    //   label: m.label,
+    //   data: Array(stepCount).fill(0),
+    // }));
+
+    const mproc = measures.map(m => ({
+      ...m,
+    }));
+
+    const data = Array(stepCount)
+      .fill(0)
+      .map(() => ({}));
+
+    // pass 2 - count measures
+    await datastore.enumRows(row => {
+      const ts = timestamp(row);
+      let part = Math.round((new Date(ts).getTime() - minTime) / stepDuration);
+      if (part < 0) part = 0;
+      if (part >= stepCount) part - stepCount - 1;
+      if (data[part]) {
+        data[part] = aggregate(data[part], row, stepDuration);
+      }
+      return true;
+    });
+
+    datastore._closeReader();
+
+    // const measureByField = _.fromPairs(measures.map((m, i) => [m.field, i]));
+
+    // for (let mindex = 0; mindex < measures.length; mindex++) {
+    //   for (let stepIndex = 0; stepIndex < stepCount; stepIndex++) {
+    //     const measure = measures[mindex];
+    //     if (measure.perSecond) {
+    //       datasets[mindex].data[stepIndex] /= stepDuration / 1000;
+    //     }
+    //     if (measure.perField) {
+    //       datasets[mindex].data[stepIndex] /= datasets[measureByField[measure.perField]].data[stepIndex];
+    //     }
+    //   }
+    // }
+
+    // for (let i = 0; i < measures.length; i++) {
+    //   if (measures[i].hidden) {
+    //     datasets[i] = null;
+    //   }
+    // }
+
+    return {
+      labels,
+      datasets: mproc.map(m => ({
+        label: m.label,
+        data: data.map(d => d[m.field] || 0),
+      })),
+    };
+  },
 };

packages/api/src/controllers/serverConnections.js

@@ -1,6 +1,7 @@
 const connections = require('./connections');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
+const uuidv1 = require('uuid/v1');
 const _ = require('lodash');
 const AsyncLock = require('async-lock');
 const { handleProcessCommunication } = require('../utility/processComm');
@@ -8,23 +9,25 @@ const lock = new AsyncLock();
 const config = require('./config');
 const processArgs = require('../utility/processArgs');
 const { testConnectionPermission } = require('../utility/hasPermission');
+const { MissingCredentialsError } = require('../utility/exceptions');
 
 module.exports = {
   opened: [],
   closed: {},
   lastPinged: {},
+  requests: {},
 
   handle_databases(conid, { databases }) {
     const existing = this.opened.find(x => x.conid == conid);
     if (!existing) return;
     existing.databases = databases;
-    socket.emitChanged(`database-list-changed-${conid}`);
+    socket.emitChanged(`database-list-changed`, { conid });
   },
   handle_version(conid, { version }) {
     const existing = this.opened.find(x => x.conid == conid);
     if (!existing) return;
     existing.version = version;
-    socket.emitChanged(`server-version-changed-${conid}`);
+    socket.emitChanged(`server-version-changed`, { conid });
   },
   handle_status(conid, { status }) {
     const existing = this.opened.find(x => x.conid == conid);
@@ -33,12 +36,20 @@ module.exports = {
     socket.emitChanged(`server-status-changed`);
   },
   handle_ping() {},
+  handle_response(conid, { msgid, ...response }) {
+    const [resolve, reject] = this.requests[msgid];
+    resolve(response);
+    delete this.requests[msgid];
+  },
 
   async ensureOpened(conid) {
     const res = await lock.acquire(conid, async () => {
       const existing = this.opened.find(x => x.conid == conid);
       if (existing) return existing;
       const connection = await connections.getCore({ conid });
+      if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') {
+        throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode });
+      }
       const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
         '--is-forked-api',
         '--start-process',
@@ -120,9 +131,9 @@ module.exports = {
   },
 
   ping_meta: true,
-  async ping({ connections }) {
+  async ping({ conidArray }) {
     await Promise.all(
-      _.uniq(connections).map(async conid => {
+      _.uniq(conidArray).map(async conid => {
         const last = this.lastPinged[conid];
         if (last && new Date().getTime() - last < 30 * 1000) {
           return Promise.resolve();
@@ -161,4 +172,41 @@ module.exports = {
     opened.subprocess.send({ msgtype: 'dropDatabase', name });
     return { status: 'ok' };
   },
+
+  sendRequest(conn, message) {
+    const msgid = uuidv1();
+    const promise = new Promise((resolve, reject) => {
+      this.requests[msgid] = [resolve, reject];
+      conn.subprocess.send({ msgid, ...message });
+    });
+    return promise;
+  },
+
+  async loadDataCore(msgtype, { conid, ...args }, req) {
+    testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid);
+    const res = await this.sendRequest(opened, { msgtype, ...args });
+    if (res.errorMessage) {
+      console.error(res.errorMessage);
+
+      return {
+        errorMessage: res.errorMessage,
+      };
+    }
+    return res.result || null;
+  },
+
+  serverSummary_meta: true,
+  async serverSummary({ conid }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('serverSummary', { conid });
+  },
+
+  summaryCommand_meta: true,
+  async summaryCommand({ conid, command, row }, req) {
+    testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid);
+    if (opened.connection.isReadOnly) return false;
+    return this.loadDataCore('summaryCommand', { conid, command, row });
+  },
 };

packages/api/src/controllers/sessions.js

@@ -150,6 +150,31 @@ module.exports = {
     return true;
   },
 
+  startProfiler_meta: true,
+  async startProfiler({ sesid }) {
+    const jslid = uuidv1();
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (!session) {
+      throw new Error('Invalid session');
+    }
+
+    console.log(`Starting profiler, sesid=${sesid}`);
+    session.loadingReader_jslid = jslid;
+    session.subprocess.send({ msgtype: 'startProfiler', jslid });
+
+    return { state: 'ok', jslid };
+  },
+
+  stopProfiler_meta: true,
+  async stopProfiler({ sesid }) {
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (!session) {
+      throw new Error('Invalid session');
+    }
+    session.subprocess.send({ msgtype: 'stopProfiler' });
+    return { state: 'ok' };
+  },
+
   // cancel_meta: true,
   // async cancel({ sesid }) {
   //   const session = this.opened.find((x) => x.sesid == sesid);

packages/api/src/proc/serverConnectionProcess.js

@@ -10,6 +10,7 @@ let storedConnection;
 let lastDatabases = null;
 let lastStatus = null;
 let lastPing = null;
+let afterConnectCallbacks = [];
 
 async function handleRefresh() {
   const driver = requireEngineDriver(storedConnection);
@@ -74,6 +75,18 @@ async function handleConnect(connection) {
     // console.error(err);
     setTimeout(() => process.exit(1), 1000);
   }
+
+  for (const [resolve] of afterConnectCallbacks) {
+    resolve();
+  }
+  afterConnectCallbacks = [];
+}
+
+function waitConnected() {
+  if (systemConnection) return Promise.resolve();
+  return new Promise((resolve, reject) => {
+    afterConnectCallbacks.push([resolve, reject]);
+  });
 }
 
 function handlePing() {
@@ -94,9 +107,30 @@ async function handleDatabaseOp(op, { name }) {
   await handleRefresh();
 }
 
+async function handleDriverDataCore(msgid, callMethod) {
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+  try {
+    const result = await callMethod(driver);
+    process.send({ msgtype: 'response', msgid, result });
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+  }
+}
+
+async function handleServerSummary({ msgid }) {
+  return handleDriverDataCore(msgid, driver => driver.serverSummary(systemConnection));
+}
+
+async function handleSummaryCommand({ msgid, command, row }) {
+  return handleDriverDataCore(msgid, driver => driver.summaryCommand(systemConnection, command, row));
+}
+
 const messageHandlers = {
   connect: handleConnect,
   ping: handlePing,
+  serverSummary: handleServerSummary,
+  summaryCommand: handleSummaryCommand,
   createDatabase: props => handleDatabaseOp('createDatabase', props),
   dropDatabase: props => handleDatabaseOp('dropDatabase', props),
 };

packages/api/src/proc/sessionProcess.js

@@ -16,6 +16,7 @@ let storedConnection;
 let afterConnectCallbacks = [];
 // let currentHandlers = [];
 let lastPing = null;
+let currentProfiler = null;
 
 class TableWriter {
   constructor() {
@@ -210,6 +211,31 @@ function waitConnected() {
   });
 }
 
+async function handleStartProfiler({ jslid }) {
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+
+  if (!allowExecuteCustomScript(driver)) {
+    process.send({ msgtype: 'done' });
+    return;
+  }
+
+  const writer = new TableWriter();
+  writer.initializeFromReader(jslid);
+
+  currentProfiler = await driver.startProfiler(systemConnection, {
+    row: data => writer.rowFromReader(data),
+  });
+  currentProfiler.writer = writer;
+}
+
+async function handleStopProfiler({ jslid }) {
+  const driver = requireEngineDriver(storedConnection);
+  currentProfiler.writer.close();
+  driver.stopProfiler(systemConnection, currentProfiler);
+  currentProfiler = null;
+}
+
 async function handleExecuteQuery({ sql }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
@@ -280,6 +306,8 @@ const messageHandlers = {
   connect: handleConnect,
   executeQuery: handleExecuteQuery,
   executeReader: handleExecuteReader,
+  startProfiler: handleStartProfiler,
+  stopProfiler: handleStopProfiler,
   ping: handlePing,
   // cancel: handleCancel,
 };

packages/api/src/utility/JsonLinesDatabase.js

@@ -90,6 +90,12 @@ class JsonLinesDatabase {
     return obj;
   }
 
+  async updateAll(mapFunction) {
+    await this._ensureLoaded();
+    this.data = this.data.map(mapFunction);
+    await this._save();
+  }
+
   async patch(id, values) {
     await this._ensureLoaded();
     this.data = this.data.map(x => (x._id == id ? { ...x, ...values } : x));

packages/api/src/utility/JsonLinesDatastore.js

@@ -3,6 +3,7 @@ const AsyncLock = require('async-lock');
 const lock = new AsyncLock();
 const stableStringify = require('json-stable-stringify');
 const { evaluateCondition } = require('dbgate-sqltree');
+const requirePluginFunction = require('./requirePluginFunction');
 
 function fetchNextLineFromReader(reader) {
   return new Promise((resolve, reject) => {
@@ -22,14 +23,16 @@ function fetchNextLineFromReader(reader) {
 }
 
 class JsonLinesDatastore {
-  constructor(file) {
+  constructor(file, formatterFunction) {
     this.file = file;
+    this.formatterFunction = formatterFunction;
     this.reader = null;
     this.readedDataRowCount = 0;
     this.readedSchemaRow = false;
     // this.firstRowToBeReturned = null;
     this.notifyChangedCallback = null;
     this.currentFilter = null;
+    this.rowFormatter = requirePluginFunction(formatterFunction);
   }
 
   _closeReader() {
@@ -62,6 +65,11 @@ class JsonLinesDatastore {
     );
   }
 
+  parseLine(line) {
+    const res = JSON.parse(line);
+    return this.rowFormatter ? this.rowFormatter(res) : res;
+  }
+
   async _readLine(parse) {
     // if (this.firstRowToBeReturned) {
     //   const res = this.firstRowToBeReturned;
@@ -84,14 +92,14 @@ class JsonLinesDatastore {
         }
       }
       if (this.currentFilter) {
-        const parsedLine = JSON.parse(line);
+        const parsedLine = this.parseLine(line);
         if (evaluateCondition(this.currentFilter, parsedLine)) {
           this.readedDataRowCount += 1;
           return parse ? parsedLine : true;
         }
       } else {
         this.readedDataRowCount += 1;
-        return parse ? JSON.parse(line) : true;
+        return parse ? this.parseLine(line) : true;
       }
     }
 

packages/api/src/utility/exceptions.js

@@ -0,0 +1,9 @@
+class MissingCredentialsError {
+  constructor(detail) {
+    this.detail = detail;
+  }
+}
+
+module.exports = {
+  MissingCredentialsError,
+};

packages/api/src/utility/requirePluginFunction.js

@@ -0,0 +1,16 @@
+const _ = require('lodash');
+const requirePlugin = require('../shell/requirePlugin');
+
+function requirePluginFunction(functionName) {
+  if (!functionName) return null;
+  if (functionName.includes('@')) {
+    const [shortName, packageName] = functionName.split('@');
+    const plugin = requirePlugin(packageName);
+    if (plugin.functions) {
+      return plugin.functions[shortName];
+    }
+  }
+  return null;
+}
+
+module.exports = requirePluginFunction;

packages/api/src/utility/socket.js

@@ -1,4 +1,5 @@
 const _ = require('lodash');
+const stableStringify = require('json-stable-stringify');
 
 const sseResponses = [];
 let electronSender = null;
@@ -27,12 +28,12 @@ module.exports = {
       electronSender.send(message, data == null ? null : data);
     }
     for (const res of sseResponses) {
-      res.write(`event: ${message}\ndata: ${JSON.stringify(data == null ? null : data)}\n\n`);
+      res.write(`event: ${message}\ndata: ${stableStringify(data == null ? null : data)}\n\n`);
     }
   },
-  emitChanged(key) {
+  emitChanged(key, params = undefined) {
     // console.log('EMIT CHANGED', key);
-    this.emit('changed-cache', key);
+    this.emit('changed-cache', { key, ...params });
     // this.emit(key);
   },
 };

packages/api/src/utility/useController.js

@@ -1,6 +1,7 @@
 const _ = require('lodash');
 const express = require('express');
 const getExpressPath = require('./getExpressPath');
+const { MissingCredentialsError } = require('./exceptions');
 
 /**
  * @param {string} route
@@ -37,6 +38,13 @@ module.exports = function useController(app, electron, route, controller) {
             if (data === undefined) return null;
             return data;
           } catch (err) {
+            if (err instanceof MissingCredentialsError) {
+              return {
+                missingCredentials: true,
+                apiErrorMessage: 'Missing credentials',
+                detail: err.detail,
+              };
+            }
             return { apiErrorMessage: err.message };
           }
         });
@@ -69,7 +77,15 @@ module.exports = function useController(app, electron, route, controller) {
           res.json(data);
         } catch (e) {
           console.log(e);
-          res.status(500).json({ apiErrorMessage: e.message });
+          if (e instanceof MissingCredentialsError) {
+            res.json({
+              missingCredentials: true,
+              apiErrorMessage: 'Missing credentials',
+              detail: e.detail,
+            });
+          } else {
+            res.status(500).json({ apiErrorMessage: e.message });
+          }
         }
       });
     }

packages/datalib/src/PerspectiveCache.ts

@@ -5,6 +5,7 @@ import _difference from 'lodash/difference';
 import debug from 'debug';
 import stableStringify from 'json-stable-stringify';
 import { PerspectiveDataPattern } from './PerspectiveDataPattern';
+import { perspectiveValueMatcher } from './perspectiveTools';
 
 const dbg = debug('dbgate:PerspectiveCache');
 
@@ -17,7 +18,9 @@ export class PerspectiveBindingGroup {
   bindingValues: any[];
 
   matchRow(row) {
-    return this.table.bindingColumns.every((column, index) => row[column] == this.bindingValues[index]);
+    return this.table.bindingColumns.every((column, index) =>
+    perspectiveValueMatcher(row[column], this.bindingValues[index])
+    );
   }
 }
 
@@ -69,7 +72,11 @@ export class PerspectiveCacheTable {
   }
 
   storeGroupSize(props: PerspectiveDataLoadProps, bindingValues: any[], count: number) {
-    const originalBindingValue = props.bindingValues.find(v => _zip(v, bindingValues).every(([x, y]) => x == y));
+    const originalBindingValue = props.bindingValues.find(v =>
+      _zip(v, bindingValues).every(([x, y]) => perspectiveValueMatcher(x, y))
+    );
+    // console.log('storeGroupSize NEW', bindingValues);
+    // console.log('storeGroupSize ORIGINAL', originalBindingValue);
     if (originalBindingValue) {
       const key = stableStringify(originalBindingValue);
       // console.log('SET SIZE', originalBindingValue, bindingValues, key, count);

packages/datalib/src/PerspectiveDataPattern.ts

@@ -51,7 +51,7 @@ function addObjectToColumns(columns: PerspectiveDataPatternColumn[], row) {
       if (!column.types.includes(type)) {
         column.types.push(type);
       }
-      if (_isPlainObject(value)) {
+      if (_isPlainObject(value) && type != 'oid') {
         addObjectToColumns(column.columns, value);
       }
       if (_isArray(value)) {

packages/datalib/src/PerspectiveDataProvider.ts

@@ -47,6 +47,7 @@ export class PerspectiveDataProvider {
 
   async loadDataNested(props: PerspectiveDataLoadProps): Promise<{ rows: any[]; incomplete: boolean }> {
     const tableCache = this.cache.getTableCache(props);
+    // console.log('loadDataNested', props);
 
     const uncached = tableCache.getUncachedBindingGroups(props);
     if (uncached.length > 0) {
@@ -54,7 +55,7 @@ export class PerspectiveDataProvider {
         ...props,
         bindingValues: uncached,
       });
-      // console.log('COUNTS', counts);
+      // console.log('loadDataNested COUNTS', counts);
       for (const resetItem of uncached) {
         tableCache.storeGroupSize(props, resetItem, 0);
       }
@@ -196,6 +197,14 @@ export class PerspectiveDataProvider {
       },
     });
 
+    if (!nextRows) {
+      // return tableCache.getRowsResult(props);
+      return {
+        rows: [],
+        incomplete: false,
+      };
+    }
+
     if (nextRows.errorMessage) {
       throw new Error(nextRows.errorMessage);
     }

packages/datalib/src/PerspectiveTreeNode.ts

@@ -22,6 +22,7 @@ import {
   PerspectiveReferenceConfig,
 } from './PerspectiveConfig';
 import _isEqual from 'lodash/isEqual';
+import _isArray from 'lodash/isArray';
 import _cloneDeep from 'lodash/cloneDeep';
 import _compact from 'lodash/compact';
 import _uniq from 'lodash/uniq';
@@ -38,6 +39,11 @@ import { Condition, Expression, Select } from 'dbgate-sqltree';
 // import { getPerspectiveDefaultColumns } from './getPerspectiveDefaultColumns';
 import uuidv1 from 'uuid/v1';
 import { PerspectiveDataPatternColumn } from './PerspectiveDataPattern';
+import {
+  getPerspectiveMostNestedChildColumnName,
+  getPerspectiveParentColumnName,
+  perspectiveValueMatcher,
+} from './perspectiveTools';
 
 export interface PerspectiveDataLoadPropsWithNode {
   props: PerspectiveDataLoadProps;
@@ -137,6 +143,16 @@ export abstract class PerspectiveTreeNode {
   get generatesDataGridColumn() {
     return this.isCheckedColumn;
   }
+  get validParentDesignerId() {
+    if (this.designerId) return this.designerId;
+    return this.parentNode?.validParentDesignerId;
+  }
+  get preloadedLevelData() {
+    return false;
+  }
+  get findByDesignerIdWithoutDesignerId() {
+    return false;
+  }
   matchChildRow(parentRow: any, childRow: any): boolean {
     return true;
   }
@@ -210,7 +226,7 @@ export abstract class PerspectiveTreeNode {
 
   get hasUncheckedNodeInPath() {
     if (!this.parentNode) return false;
-    if (!this.isCheckedNode) return true;
+    if (this.designerId && !this.isCheckedNode) return true;
     return this.parentNode.hasUncheckedNodeInPath;
   }
 
@@ -404,7 +420,7 @@ export abstract class PerspectiveTreeNode {
   // }
 
   findNodeByDesignerId(designerId: string): PerspectiveTreeNode {
-    if (!this.designerId) {
+    if (!this.designerId && !this.findByDesignerIdWithoutDesignerId) {
       return null;
     }
     if (!designerId) {
@@ -758,15 +774,22 @@ export class PerspectivePatternColumnNode extends PerspectiveTreeNode {
   ) {
     super(dbs, config, setConfig, parentNode, dataProvider, databaseConfig, designerId);
     this.parentNodeConfig = this.tableNodeOrParent?.nodeConfig;
+    // console.log('PATTERN COLUMN', column);
   }
 
   get isChildColumn() {
     return this.parentNode instanceof PerspectivePatternColumnNode;
   }
 
+  get findByDesignerIdWithoutDesignerId() {
+    return this.isExpandable;
+  }
+
   // matchChildRow(parentRow: any, childRow: any): boolean {
-  //   if (!this.foreignKey) return false;
-  //   return parentRow[this.foreignKey.columns[0].columnName] == childRow[this.foreignKey.columns[0].refColumnName];
+  //   console.log('MATCH PATTENR ROW', parentRow, childRow);
+  //   return false;
+  //   // if (!this.foreignKey) return false;
+  //   // return parentRow[this.foreignKey.columns[0].columnName] == childRow[this.foreignKey.columns[0].refColumnName];
   // }
 
   // getChildMatchColumns() {
@@ -808,12 +831,19 @@ export class PerspectivePatternColumnNode extends PerspectiveTreeNode {
   // }
 
   getNodeLoadProps(parentRows: any[]): PerspectiveDataLoadProps {
+    // console.log('GETTING PATTERN', parentRows);
     return null;
   }
 
-  get generatesHiearchicGridColumn() {
+  get generatesHiearchicGridColumn(): boolean {
+    // return true;
     // console.log('generatesHiearchicGridColumn', this.parentTableNode?.nodeConfig?.checkedColumns, this.codeName + '::');
-    return !!this.tableNodeOrParent?.nodeConfig?.checkedColumns?.find(x => x.startsWith(this.codeName + '::'));
+    if (this.tableNodeOrParent?.nodeConfig?.checkedColumns?.find(x => x.startsWith(this.codeName + '::'))) {
+      return true;
+    }
+    // return false;
+
+    return this.hasCheckedJoinChild();
   }
 
   // get generatesHiearchicGridColumn() {
@@ -859,7 +889,11 @@ export class PerspectivePatternColumnNode extends PerspectiveTreeNode {
     return 'mongo';
   }
 
-  generateChildNodes(): PerspectiveTreeNode[] {
+  get preloadedLevelData() {
+    return true;
+  }
+
+  generatePatternChildNodes(): PerspectivePatternColumnNode[] {
     return this.column.columns.map(
       column =>
         new PerspectivePatternColumnNode(
@@ -875,7 +909,92 @@ export class PerspectivePatternColumnNode extends PerspectiveTreeNode {
           null
         )
     );
-    return [];
+  }
+
+  hasCheckedJoinChild() {
+    for (const node of this.childNodes) {
+      if (node instanceof PerspectivePatternColumnNode) {
+        if (node.hasCheckedJoinChild()) return true;
+      }
+      if (node.isCheckedNode) return true;
+    }
+    return false;
+  }
+
+  generateChildNodes(): PerspectiveTreeNode[] {
+    const patternChildren = this.generatePatternChildNodes();
+
+    const customs = [];
+    // console.log('GETTING CHILDREN', this.config.nodes, this.config.references);
+    for (const node of this.config.nodes) {
+      for (const ref of this.config.references) {
+        const validDesignerId = this.validParentDesignerId;
+        if (
+          (ref.sourceId == validDesignerId && ref.targetId == node.designerId) ||
+          (ref.targetId == validDesignerId && ref.sourceId == node.designerId)
+        ) {
+          // console.log('TESTING REF', ref, this.codeName);
+          if (ref.columns.length != 1) continue;
+          // console.log('CP1');
+          if (
+            ref.sourceId == validDesignerId &&
+            this.codeName == getPerspectiveParentColumnName(ref.columns[0].source)
+          ) {
+            if (ref.columns[0].target.includes('::')) continue;
+          } else if (
+            ref.targetId == validDesignerId &&
+            this.codeName == getPerspectiveParentColumnName(ref.columns[0].target)
+          ) {
+            if (ref.columns[0].source.includes('::')) continue;
+          } else {
+            continue;
+          }
+          // console.log('CP2');
+
+          const newConfig = { ...this.databaseConfig };
+          if (node.conid) newConfig.conid = node.conid;
+          if (node.database) newConfig.database = node.database;
+          const db = this.dbs?.[newConfig.conid]?.[newConfig.database];
+          const table = db?.tables?.find(x => x.pureName == node.pureName && x.schemaName == node.schemaName);
+          const view = db?.views?.find(x => x.pureName == node.pureName && x.schemaName == node.schemaName);
+          const collection = db?.collections?.find(x => x.pureName == node.pureName && x.schemaName == node.schemaName);
+
+          const join: PerspectiveCustomJoinConfig = {
+            refNodeDesignerId: node.designerId,
+            referenceDesignerId: ref.designerId,
+            baseDesignerId: validDesignerId,
+            joinName: node.alias,
+            refTableName: node.pureName,
+            refSchemaName: node.schemaName,
+            conid: node.conid,
+            database: node.database,
+            columns:
+              ref.sourceId == validDesignerId
+                ? ref.columns.map(col => ({ baseColumnName: col.source, refColumnName: col.target }))
+                : ref.columns.map(col => ({ baseColumnName: col.target, refColumnName: col.source })),
+          };
+
+          if (table || view || collection) {
+            customs.push(
+              new PerspectiveCustomJoinTreeNode(
+                join,
+                table || view || collection,
+                this.dbs,
+                this.config,
+                this.setConfig,
+                this.dataProvider,
+                newConfig,
+                this,
+                node.designerId
+              )
+            );
+          }
+        }
+      }
+    }
+
+    return [...patternChildren, ...customs];
+    // return [];
     // if (!this.foreignKey) return [];
     // const tbl = this?.db?.tables?.find(
     //   x => x.pureName == this.foreignKey?.refTableName && x.schemaName == this.foreignKey?.refSchemaName
@@ -1153,8 +1272,14 @@ export class PerspectiveCustomJoinTreeNode extends PerspectiveTableNode {
   }
 
   matchChildRow(parentRow: any, childRow: any): boolean {
+    // console.log('MATCH ROW', parentRow, childRow);
     for (const column of this.customJoin.columns) {
-      if (parentRow[column.baseColumnName] != childRow[column.refColumnName]) {
+      if (
+        !perspectiveValueMatcher(
+          parentRow[getPerspectiveMostNestedChildColumnName(column.baseColumnName)],
+          childRow[column.refColumnName]
+        )
+      ) {
         return false;
       }
     }
@@ -1171,17 +1296,68 @@ export class PerspectiveCustomJoinTreeNode extends PerspectiveTableNode {
 
   getNodeLoadProps(parentRows: any[]): PerspectiveDataLoadProps {
     // console.log('CUSTOM JOIN', this.customJoin);
+    // console.log('PARENT ROWS', parentRows);
+
     // console.log('this.getDataLoadColumns()', this.getDataLoadColumns());
     const isMongo = isCollectionInfo(this.table);
 
+    // const bindingValues = [];
+
+    // for (const row of parentRows) {
+    //   const rowBindingValueArrays = [];
+    //   for (const col of this.customJoin.columns) {
+    //     const path = col.baseColumnName.split('::');
+    //     const values = [];
+
+    //     function processSubpath(parent, subpath) {
+    //       if (subpath.length == 0) {
+    //         values.push(parent);
+    //         return;
+    //       }
+    //       if (parent == null) {
+    //         return;
+    //       }
+
+    //       const obj = parent[subpath[0]];
+    //       if (_isArray(obj)) {
+    //         for (const elem of obj) {
+    //           processSubpath(elem, subpath.slice(1));
+    //         }
+    //       } else {
+    //         processSubpath(obj, subpath.slice(1));
+    //       }
+    //     }
+
+    //     processSubpath(row, path);
+
+    //     rowBindingValueArrays.push(values);
+    //   }
+
+    //   const valueCount = Math.max(...rowBindingValueArrays.map(x => x.length));
+
+    //   for (let i = 0; i < valueCount; i += 1) {
+    //     const value = Array(this.customJoin.columns.length);
+    //     for (let col = 0; col < this.customJoin.columns.length; col++) {
+    //       value[col] = rowBindingValueArrays[col][i % rowBindingValueArrays[col].length];
+    //     }
+    //     bindingValues.push(value);
+    //   }
+    // }
+    const bindingValues = parentRows.map(row =>
+      this.customJoin.columns.map(x => row[getPerspectiveMostNestedChildColumnName(x.baseColumnName)])
+    );
+
+    // console.log('bindingValues', bindingValues);
+    // console.log(
+    //   'bindingValues UNIQ',
+    //   _uniqBy(bindingValues, x => JSON.stringify(x))
+    // );
+
     return {
       schemaName: this.table.schemaName,
       pureName: this.table.pureName,
       bindingColumns: this.getParentMatchColumns(),
-      bindingValues: _uniqBy(
-        parentRows.map(row => this.customJoin.columns.map(x => row[x.baseColumnName])),
-        stableStringify
-      ),
+      bindingValues: _uniqBy(bindingValues, x => JSON.stringify(x)),
       dataColumns: this.getDataLoadColumns(),
       allColumns: isMongo,
       databaseConfig: this.databaseConfig,
@@ -1412,6 +1588,9 @@ export function getTableChildPerspectiveNodes(
         (ref.sourceId == parentNode.designerId && ref.targetId == node.designerId) ||
         (ref.targetId == parentNode.designerId && ref.sourceId == node.designerId)
       ) {
+        if (ref.columns.find(x => x.source.includes('::') || x.target.includes('::'))) {
+          continue;
+        }
         const newConfig = { ...databaseConfig };
         if (node.conid) newConfig.conid = node.conid;
         if (node.database) newConfig.database = node.database;

packages/datalib/src/index.ts

@@ -21,3 +21,4 @@ export * from './PerspectiveConfig';
 export * from './processPerspectiveDefaultColunns';
 export * from './PerspectiveDataPattern';
 export * from './PerspectiveDataLoader';
+export * from './perspectiveTools';

packages/datalib/src/perspectiveTools.ts

@@ -0,0 +1,22 @@
+export function getPerspectiveParentColumnName(columnName: string) {
+  const path = columnName.split('::');
+  if (path.length >= 2) return path.slice(0, -1).join('::');
+  return null;
+}
+
+export function getPerspectiveMostNestedChildColumnName(columnName: string) {
+  const path = columnName.split('::');
+  return path[path.length - 1];
+}
+
+// export function perspectiveValueMatcher(value1, value2): boolean {
+//   if (value1?.$oid && value2?.$oid) return value1.$oid == value2.$oid;
+//   if (Array.isArray(value1)) return !!value1.find(x => perspectiveValueMatcher(x, value2));
+//   if (Array.isArray(value2)) return !!value2.find(x => perspectiveValueMatcher(value1, x));
+//   return value1 == value2;
+// }
+
+export function perspectiveValueMatcher(value1, value2): boolean {
+  if (value1?.$oid && value2?.$oid) return value1.$oid == value2.$oid;
+  return value1 == value2;
+}

packages/filterparser/src/mongoParser.ts

@@ -44,6 +44,10 @@ const testCondition = (operator, value) => () => ({
   },
 });
 
+const multiTestCondition = condition => () => ({
+  __placeholder__: condition,
+});
+
 const compoudCondition = conditionType => conditions => {
   if (conditions.length == 1) return conditions[0];
   return {
@@ -85,7 +89,15 @@ const createParser = () => {
 
     comma: () => word(','),
     not: () => word('NOT'),
+    empty: () => word('EMPTY'),
+    array: () => word('ARRAY'),
     notExists: r => r.not.then(r.exists).map(testCondition('$exists', false)),
+    notEmptyArray: r =>
+      r.not
+        .then(r.empty)
+        .then(r.array)
+        .map(multiTestCondition({ $exists: true, $type: 'array', $ne: [] })),
+    emptyArray: r => r.empty.then(r.array).map(multiTestCondition({ $exists: true, $eq: [] })),
     exists: () => word('EXISTS').map(testCondition('$exists', true)),
     true: () => word('TRUE').map(testCondition('$eq', true)),
     false: () => word('FALSE').map(testCondition('$eq', false)),
@@ -117,6 +129,8 @@ const createParser = () => {
         r.gt,
         r.le,
         r.ge,
+        r.notEmptyArray,
+        r.emptyArray,
         r.startsWith,
         r.endsWith,
         r.contains,

packages/serve/README.md

@@ -14,7 +14,7 @@ After installing, you can run dbgate with command:
 dbgate-serve
 ```
 
-Then open http://localhost:5000 in your browser
+Then open http://localhost:3000 in your browser
 
 ## Download electron app
 You can also download binary packages from https://dbgate.org . Or run from source code, as described on [github](https://github.com/dbgate/dbgate)

packages/sqltree/src/evaluateExpression.ts

@@ -1,4 +1,4 @@
-import _ from 'lodash';
+import _get from 'lodash/get';
 import { SqlDumper } from 'dbgate-types';
 import { Expression, ColumnRefExpression } from './types';
 import { dumpSqlSourceRef } from './dumpSqlSource';
@@ -6,7 +6,7 @@ import { dumpSqlSourceRef } from './dumpSqlSource';
 export function evaluateExpression(expr: Expression, values) {
   switch (expr.exprType) {
     case 'column':
-      return values[expr.columnName];
+      return _get(values, expr.columnName);
 
     case 'placeholder':
       return values.__placeholder;

packages/tools/package.json

@@ -31,7 +31,7 @@
     "typescript": "^4.4.3"
   },
   "dependencies": {
-    "dbgate-query-splitter": "^4.9.2",
+    "dbgate-query-splitter": "^4.9.3",
     "dbgate-sqltree": "^5.0.0-alpha.1",
     "debug": "^4.3.4",
     "json-stable-stringify": "^1.0.1",

packages/tools/src/filterName.ts

@@ -1,62 +1,23 @@
 import _compact from 'lodash/compact';
 import _isString from 'lodash/isString';
+import _startCase from 'lodash/startCase';
 
 export interface FilterNameDefinition {
   childName: string;
 }
 
-// original C# variant
-// public bool Match(string value)
-// {
-//     if (String.IsNullOrEmpty(Filter)) return false;
-//     if (String.IsNullOrEmpty(value)) return true;
+function camelMatch(filter: string, text: string): boolean {
+  if (!text) return false;
+  if (!filter) return true;
 
-//     var camelVariants = new HashSet<string>();
-//     camelVariants.Add(new String(value.Where(Char.IsUpper).ToArray()));
-//     if (value.All(x => Char.IsUpper(x) || x == '_'))
-//     {
-//         var sb = new StringBuilder();
-//         for (int i = 0; i < value.Length; i++)
-//         {
-//             if (Char.IsUpper(value[i]) && (i == 0 || value[i - 1] == '_')) sb.Append(value[i]);
-//         }
-//         camelVariants.Add(sb.ToString());
-//     }
-//     else
-//     {
-//         string s = value, s0;
-//         do
-//         {
-//             s0 = s;
-//             s = Regex.Replace(s, "([A-Z])([A-Z])([A-Z])", "$1$3");
-//         } while (s0 != s);
-//         camelVariants.Add(new String(s.Where(Char.IsUpper).ToArray()));
-//     }
-
-//     bool camelMatch = camelVariants.Any(x => DoMatch(Filter, x));
-//     if (Filter.All(Char.IsUpper)) return camelMatch;
-//     return DoMatch(Filter, value) || camelMatch;
-// }
-
-function fuzzysearch(needle, haystack) {
-  var hlen = haystack.length;
-  var nlen = needle.length;
-  if (nlen > hlen) {
-    return false;
+  if (filter.replace(/[A-Z]/g, '').length == 0) {
+    const textCapitals = _startCase(text).replace(/[^A-Z]/g, '');
+    const pattern = '.*' + filter.split('').join('.*') + '.*';
+    const re = new RegExp(pattern);
+    return re.test(textCapitals);
+  } else {
+    return text.toUpperCase().includes(filter.toUpperCase());
   }
-  if (nlen === hlen) {
-    return needle === haystack;
-  }
-  outer: for (var i = 0, j = 0; i < nlen; i++) {
-    var nch = needle.charCodeAt(i);
-    while (j < hlen) {
-      if (haystack.charCodeAt(j++) === nch) {
-        continue outer;
-      }
-    }
-    return false;
-  }
-  return true;
 }
 
 export function filterName(filter: string, ...names: (string | FilterNameDefinition)[]) {
@@ -73,13 +34,13 @@ export function filterName(filter: string, ...names: (string | FilterNameDefinit
   const namesChild: string[] = namesCompacted.filter(x => x.childName).map(x => x.childName);
 
   for (const token of tokens) {
-    const tokenUpper = token.toUpperCase();
-    if (tokenUpper.startsWith('#')) {
-      const tokenUpperSub = tokenUpper.substring(1);
-      const found = namesChild.find(name => fuzzysearch(tokenUpperSub, name.toUpperCase()));
+    // const tokenUpper = token.toUpperCase();
+    if (token.startsWith('#')) {
+      // const tokenUpperSub = tokenUpper.substring(1);
+      const found = namesChild.find(name => camelMatch(token.substring(1), name));
       if (!found) return false;
     } else {
-      const found = namesOwn.find(name => fuzzysearch(tokenUpper, name.toUpperCase()));
+      const found = namesOwn.find(name => camelMatch(token, name));
       if (!found) return false;
     }
   }

packages/types/engines.d.ts

@@ -55,6 +55,17 @@ export interface SqlBackupDumper {
   run();
 }
 
+export interface SummaryColumn {
+  fieldName: string;
+  header: string;
+  dataType: 'string' | 'number' | 'bytes';
+}
+export interface ServerSummaryDatabase {}
+export interface ServerSummary {
+  columns: SummaryColumn[];
+  databases: ServerSummaryDatabase[];
+}
+
 export interface EngineDriver {
   engine: string;
   title: string;
@@ -65,6 +76,12 @@ export interface EngineDriver {
   supportedKeyTypes: SupportedDbKeyType[];
   supportsDatabaseUrl?: boolean;
   supportsDatabaseDump?: boolean;
+  supportsServerSummary?: boolean;
+  supportsDatabaseProfiler?: boolean;
+  profilerFormatterFunction?: string;
+  profilerTimestampFunction?: string;
+  profilerChartAggregateFunction?: string;
+  profilerChartMeasures?: { label: string; field: string }[];
   isElectronOnly?: boolean;
   supportedCreateDatabase?: boolean;
   showConnectionField?: (field: string, values: any) => boolean;
@@ -81,7 +98,7 @@ export interface EngineDriver {
   stream(pool: any, sql: string, options: StreamOptions);
   readQuery(pool: any, sql: string, structure?: TableInfo): Promise<stream.Readable>;
   readJsonQuery(pool: any, query: any, structure?: TableInfo): Promise<stream.Readable>;
-  writeTable(pool: any, name: NamedObjectInfo, options: WriteTableOptions): Promise<stream.Writeable>;
+  writeTable(pool: any, name: NamedObjectInfo, options: WriteTableOptions): Promise<stream.Writable>;
   analyseSingleObject(
     pool: any,
     name: NamedObjectInfo,
@@ -116,6 +133,10 @@ export interface EngineDriver {
   getNewObjectTemplates(): NewObjectTemplate[];
   // direct call of pool method, only some methods could be supported, on only some drivers
   callMethod(pool, method, args);
+  serverSummary(pool): Promise<ServerSummary>;
+  summaryCommand(pool, command, row): Promise<void>;
+  startProfiler(pool, options): Promise<any>;
+  stopProfiler(pool, profiler): Promise<void>;
 
   analyserClass?: any;
   dumperClass?: any;

packages/web/package.json

@@ -13,7 +13,7 @@
   ],
   "devDependencies": {
     "@ant-design/colors": "^5.0.0",
-    "@mdi/font": "^5.9.55",
+    "@mdi/font": "^7.1.96",
     "@rollup/plugin-commonjs": "^20.0.0",
     "@rollup/plugin-node-resolve": "^13.0.5",
     "@rollup/plugin-replace": "^3.0.0",
@@ -24,7 +24,7 @@
     "chartjs-adapter-moment": "^1.0.0",
     "cross-env": "^7.0.3",
     "dbgate-datalib": "^5.0.0-alpha.1",
-    "dbgate-query-splitter": "^4.9.2",
+    "dbgate-query-splitter": "^4.9.3",
     "dbgate-sqltree": "^5.0.0-alpha.1",
     "dbgate-tools": "^5.0.0-alpha.1",
     "dbgate-types": "^5.0.0-alpha.1",
@@ -58,6 +58,7 @@
     "chartjs-plugin-zoom": "^1.2.0",
     "date-fns": "^2.28.0",
     "debug": "^4.3.4",
+    "fuzzy": "^0.1.3",
     "interval-operations": "^1.0.7",
     "leaflet": "^1.8.0",
     "wellknown": "^0.5.0"

packages/web/public/dimensions.css

@@ -2,8 +2,8 @@
   --dim-widget-icon-size: 60px;
   --dim-statusbar-height: 22px;
   --dim-left-panel-width: 300px;
-  --dim-tabs-panel-height: 53px;
   --dim-tabs-height: 33px;
+  --dim-tabs-panel-height: calc( var(--dim-visible-tabs-databases) * 20px + var(--dim-tabs-height) );
   --dim-splitter-thickness: 3px;
 
   --dim-visible-left-panel: 1; /* set from JS */

packages/web/src/appobj/AppObjectGroup.svelte

@@ -4,6 +4,7 @@
   import { plusExpandIcon } from '../icons/expandIcons';
 
   import FontIcon from '../icons/FontIcon.svelte';
+  import contextMenu from '../utility/contextMenu';
 
   import AppObjectListItem from './AppObjectListItem.svelte';
 
@@ -16,8 +17,10 @@
   export let disableContextMenu = false;
   export let passProps;
   export let onDropOnGroup = undefined;
+  export let groupContextMenu = null;
+  export let collapsedGroupNames;
 
-  let isExpanded = true;
+  $: isExpanded = !$collapsedGroupNames.includes(group);
 
   $: filtered = items.filter(x => x.isMatched);
   $: countText = filtered.length < items.length ? `${filtered.length}/${items.length}` : `${items.length}`;
@@ -45,7 +48,16 @@
   }
 </script>
 
-<div class="group" on:click={() => (isExpanded = !isExpanded)} on:drop={handleDrop}>
+<div
+  class="group"
+  on:click={() =>
+    collapsedGroupNames.update(names => {
+      if (names.includes(group)) return names.filter(x => x != group);
+      return [...names, group];
+    })}
+  on:drop={handleDrop}
+  use:contextMenu={() => groupContextMenu(group)}
+>
   <span class="expand-icon">
     <FontIcon icon={groupIconFunc(isExpanded)} />
   </span>

packages/web/src/appobj/AppObjectList.svelte

@@ -5,6 +5,7 @@
   import { plusExpandIcon } from '../icons/expandIcons';
 
   import AppObjectListItem from './AppObjectListItem.svelte';
+  import { writable } from 'svelte/store';
 
   export let list;
   export let module;
@@ -19,12 +20,15 @@
   export let getIsExpanded = null;
   export let setIsExpanded = null;
   export let sortGroups = false;
+  export let groupContextMenu = null;
 
   export let groupIconFunc = plusExpandIcon;
   export let groupFunc = undefined;
   export let onDropOnGroup = undefined;
   export let emptyGroupNames = [];
 
+  export let collapsedGroupNames = writable([]);
+
   $: filtered = !groupFunc
     ? list.filter(data => {
         const matcher = module.createMatcher && module.createMatcher(data);
@@ -98,6 +102,8 @@
       {getIsExpanded}
       {setIsExpanded}
       {onDropOnGroup}
+      {groupContextMenu}
+      {collapsedGroupNames}
     />
   {/each}
 {:else}

packages/web/src/appobj/ArchiveFileAppObject.svelte

@@ -41,7 +41,10 @@
   }
 
   export const extractKey = data => data.fileName;
-  export const createMatcher = ({ fileName }) => filter => filterName(filter, fileName);
+  export const createMatcher =
+    ({ fileName }) =>
+    filter =>
+      filterName(filter, fileName);
   const ARCHIVE_ICONS = {
     'table.yaml': 'img table',
     'view.sql': 'img view',
@@ -67,7 +70,7 @@
   import ImportExportModal from '../modals/ImportExportModal.svelte';
   import { showModal } from '../modals/modalTools';
 
-  import { archiveFilesAsDataSheets, currentArchive, extensions, getCurrentDatabase } from '../stores';
+  import { archiveFilesAsDataSheets, currentArchive, extensions, getCurrentDatabase, getExtensions } from '../stores';
 
   import createQuickExportMenu from '../utility/createQuickExportMenu';
   import { exportQuickExportFile } from '../utility/exportFileTools';
@@ -198,6 +201,29 @@
         ),
       data.fileType.endsWith('.sql') && { text: 'Open SQL', onClick: handleOpenSqlFile },
       data.fileType.endsWith('.yaml') && { text: 'Open YAML', onClick: handleOpenYamlFile },
+      data.fileType == 'jsonl' && {
+        text: 'Open in profiler',
+        submenu: getExtensions()
+          .drivers.filter(eng => eng.profilerFormatterFunction)
+          .map(eng => ({
+            text: eng.title,
+            onClick: () => {
+              openNewTab({
+                title: 'Profiler',
+                icon: 'img profiler',
+                tabComponent: 'ProfilerTab',
+                props: {
+                  jslidLoad: `archive://${data.folderName}/${data.fileName}`,
+                  engine: eng.engine,
+                  // profilerFormatterFunction: eng.profilerFormatterFunction,
+                  // profilerTimestampFunction: eng.profilerTimestampFunction,
+                  // profilerChartAggregateFunction: eng.profilerChartAggregateFunction,
+                  // profilerChartMeasures: eng.profilerChartMeasures,
+                },
+              });
+            },
+          })),
+      },
     ];
   }
 </script>

packages/web/src/appobj/ConnectionAppObject.svelte

@@ -52,6 +52,7 @@
     const electron = getElectron();
     const currentDb = getCurrentDatabase();
     openedConnections.update(list => list.filter(x => x != conid));
+    removeVolatileMapping(conid);
     if (electron) {
       apiCall('server-connections/disconnect', { conid });
     }
@@ -100,11 +101,11 @@
   import getConnectionLabel from '../utility/getConnectionLabel';
   import { getDatabaseList, useUsedApps } from '../utility/metadataLoaders';
   import { getLocalStorage } from '../utility/storageCache';
-  import { apiCall } from '../utility/api';
+  import { apiCall, removeVolatileMapping } from '../utility/api';
   import ImportDatabaseDumpModal from '../modals/ImportDatabaseDumpModal.svelte';
   import { closeMultipleTabs } from '../widgets/TabsPanel.svelte';
   import AboutModal from '../modals/AboutModal.svelte';
-import { tick } from 'svelte';
+  import { tick } from 'svelte';
 
   export let data;
   export let passProps;
@@ -195,6 +196,16 @@ import { tick } from 'svelte';
           }),
       });
     };
+    const handleServerSummary = () => {
+      openNewTab({
+        title: getConnectionLabel(data),
+        icon: 'img server',
+        tabComponent: 'ServerSummaryTab',
+        props: {
+          conid: data._id,
+        },
+      });
+    };
     const handleNewQuery = () => {
       const tooltip = `${getConnectionLabel(data)}`;
       openNewTab({
@@ -244,6 +255,10 @@ import { tick } from 'svelte';
             text: 'Create database',
             onClick: handleCreateDatabase,
           },
+        driver?.supportsServerSummary && {
+          text: 'Server summary',
+          onClick: handleServerSummary,
+        },
       ],
       data.singleDatabase && [
         { divider: true },

packages/web/src/appobj/DatabaseAppObject.svelte

@@ -254,6 +254,18 @@
       });
     };
 
+    const handleDatabaseProfiler = () => {
+      openNewTab({
+        title: 'Profiler',
+        icon: 'img profiler',
+        tabComponent: 'ProfilerTab',
+        props: {
+          conid: connection._id,
+          database: name,
+        },
+      });
+    };
+
     async function handleConfirmSql(sql) {
       saveScriptToDatabase({ conid: connection._id, database: name }, sql, false);
     }
@@ -284,7 +296,8 @@
         !connection.singleDatabase && { onClick: handleDropDatabase, text: 'Drop database' },
       { divider: true },
       driver?.databaseEngineTypes?.includes('sql') && { onClick: handleShowDiagram, text: 'Show diagram' },
-      isSqlOrDoc && { onClick: handleSqlGenerator, text: 'SQL Generator' },
+      driver?.databaseEngineTypes?.includes('sql') && { onClick: handleSqlGenerator, text: 'SQL Generator' },
+      driver?.supportsDatabaseProfiler && { onClick: handleDatabaseProfiler, text: 'Database profiler' },
       isSqlOrDoc && { onClick: handleOpenJsonModel, text: 'Open model as JSON' },
       isSqlOrDoc && { onClick: handleExportModel, text: 'Export DB model - experimental' },
       isSqlOrDoc &&

packages/web/src/appobj/DatabaseObjectAppObject.svelte

@@ -191,6 +191,10 @@
         label: 'SQL: CREATE VIEW',
         scriptTemplate: 'CREATE OBJECT',
       },
+      {
+        label: 'SQL: ALTER VIEW',
+        scriptTemplate: 'ALTER OBJECT',
+      },
       {
         label: 'SQL: CREATE TABLE',
         scriptTemplate: 'CREATE TABLE',
@@ -253,6 +257,10 @@
         label: 'SQL: CREATE MATERIALIZED VIEW',
         scriptTemplate: 'CREATE OBJECT',
       },
+      {
+        label: 'SQL: ALTER MATERIALIZED VIEW',
+        scriptTemplate: 'ALTER OBJECT',
+      },
       {
         label: 'SQL: CREATE TABLE',
         scriptTemplate: 'CREATE TABLE',
@@ -290,6 +298,10 @@
         label: 'SQL: CREATE PROCEDURE',
         scriptTemplate: 'CREATE OBJECT',
       },
+      {
+        label: 'SQL: ALTER PROCEDURE',
+        scriptTemplate: 'ALTER OBJECT',
+      },
       {
         label: 'SQL: EXECUTE',
         scriptTemplate: 'EXECUTE PROCEDURE',
@@ -316,6 +328,10 @@
         label: 'SQL: CREATE FUNCTION',
         scriptTemplate: 'CREATE OBJECT',
       },
+      {
+        label: 'SQL: ALTER FUNCTION',
+        scriptTemplate: 'ALTER OBJECT',
+      },
       {
         label: 'SQL Generator: CREATE FUNCTION',
         sqlGeneratorProps: {
@@ -359,10 +375,17 @@
       {
         label: 'Drop collection',
         isDropCollection: true,
+        requiresWriteAccess: true,
       },
       {
         label: 'Rename collection',
         isRenameCollection: true,
+        requiresWriteAccess: true,
+      },
+      {
+        label: 'Create collection backup',
+        isDuplicateCollection: true,
+        requiresWriteAccess: true,
       },
       {
         divider: true,
@@ -516,8 +539,8 @@
       showModal(ConfirmModal, {
         message: `Really drop collection ${data.pureName}?`,
         onConfirm: async () => {
-          saveScriptToDatabase(_.pick(data, ['conid', 'database']), `db.dropCollection('${data.pureName}')`);
           const dbid = _.pick(data, ['conid', 'database']);
+          saveScriptToDatabase(dbid, `db.dropCollection('${data.pureName}')`);
         },
       });
     } else if (menu.isRenameCollection) {
@@ -534,6 +557,16 @@
           apiCall('database-connections/sync-model', dbid);
         },
       });
+    } else if (menu.isDuplicateCollection) {
+      const newName = `_${data.pureName}_${dateFormat(new Date(), 'yyyy-MM-dd-hh-mm-ss')}`;
+
+      showModal(ConfirmModal, {
+        message: `Really create collection copy named ${newName}?`,
+        onConfirm: async () => {
+          const dbid = _.pick(data, ['conid', 'database']);
+          saveScriptToDatabase(dbid, `db.collection('${data.pureName}').aggregate([{$out: '${newName}'}]).toArray()`);
+        },
+      });
     } else if (menu.isDuplicateTable) {
       const driver = await getDriver();
       const dmp = driver.createDumper();

packages/web/src/celldata/MapCellView.svelte

@@ -1,7 +1,60 @@
 <script lang="ts">
-  import MapView from '../elements/MapView.svelte';
+  import _ from 'lodash';
+  import SelectionMapView, { findAllObjectPaths, findLatPaths, findLonPaths } from '../elements/SelectionMapView.svelte';
+  import SelectField from '../forms/SelectField.svelte';
 
   export let selection;
+
+  $: latitudeFields = _.uniq(_.flatten(selection.map(x => findLatPaths(x.rowData)))) as string[];
+  $: longitudeFields = _.uniq(_.flatten(selection.map(x => findLonPaths(x.rowData)))) as string[];
+  $: allFields = _.uniq(_.flatten(selection.map(x => findAllObjectPaths(x.rowData)))) as string[];
+
+  let latitudeField = '';
+  let longitudeField = '';
+
+  $: {
+    if (latitudeFields.length > 0 && !allFields.includes(latitudeField)) {
+      latitudeField = latitudeFields[0];
+    }
+  }
+  $: {
+    if (longitudeFields.length > 0 && !allFields.includes(longitudeField)) {
+      longitudeField = longitudeFields[0];
+    }
+  }
 </script>
 
-<MapView {selection} />
+<div class="container">
+  {#if allFields.length >= 2}
+    <div>
+      Lat:
+      <SelectField
+        isNative
+        options={allFields.map(x => ({ label: x, value: x }))}
+        value={latitudeField}
+        on:change={e => {
+          latitudeField = e.detail;
+        }}
+      />
+      Lon:
+      <SelectField
+        isNative
+        options={allFields.map(x => ({ label: x, value: x }))}
+        value={longitudeField}
+        on:change={e => {
+          longitudeField = e.detail;
+        }}
+      />
+    </div>
+  {/if}
+
+  <SelectionMapView {selection} {latitudeField} {longitudeField} />
+</div>
+
+<style>
+  .container {
+    display: flex;
+    flex: 1;
+    flex-direction: column;
+  }
+</style>

packages/web/src/commands/CommandPalette.svelte

@@ -40,7 +40,7 @@
     for (const connection of connectionList || []) {
       const conid = connection._id;
       if (connection.singleDatabase) continue;
-      if (getCurrentConfig()?.singleDatabase) continue;
+      if (getCurrentConfig()?.singleDbConnection) continue;
       const databases = getLocalStorage(`database_list_${conid}`) || [];
       for (const db of databases) {
         databaseList.push({
@@ -64,6 +64,7 @@
 
   import _ from 'lodash';
   import { onMount } from 'svelte';
+  import fuzzy from 'fuzzy';
   import { databaseObjectIcons, handleDatabaseObjectClick } from '../appobj/DatabaseObjectAppObject.svelte';
   import FontIcon from '../icons/FontIcon.svelte';
   import {
@@ -80,6 +81,7 @@
   import { useConnectionList, useDatabaseInfo } from '../utility/metadataLoaders';
   import { getLocalStorage } from '../utility/storageCache';
   import registerCommand from './registerCommand';
+  import { formatKeyText } from '../utility/common';
 
   let domInput;
   let filter = '';
@@ -106,12 +108,25 @@
   $: databaseInfo = useDatabaseInfo({ conid, database });
   $: connectionList = useConnectionList();
 
-  $: filteredItems = ($visibleCommandPalette == 'database'
-    ? extractDbItems($databaseInfo, { conid, database }, $connectionList)
-    : parentCommand
-    ? parentCommand.getSubCommands()
-    : sortedComands
-  ).filter(x => !x.isGroupCommand && filterName(filter, x.text));
+  $: filteredItems = fuzzy
+    .filter(
+      filter,
+      ($visibleCommandPalette == 'database'
+        ? extractDbItems($databaseInfo, { conid, database }, $connectionList)
+        : parentCommand
+        ? parentCommand.getSubCommands()
+        : sortedComands
+      ).filter(x => !x.isGroupCommand),
+      {
+        extract: x => x.text,
+        pre: '<b>',
+        post: '</b>',
+      }
+    )
+    .map(x => ({
+      ...x.original,
+      text: x.string,
+    }));
 
   function handleCommand(command) {
     if (command.getSubCommands) {
@@ -194,10 +209,10 @@
             {#if command.icon}
               <span class="mr-1"><FontIcon icon={command.icon} /></span>
             {/if}
-            {command.text}
+            {@html command.text}
           </div>
           {#if command.keyText}
-            <div class="shortcut">{command.keyText}</div>
+            <div class="shortcut">{formatKeyText(command.keyText)}</div>
           {/if}
         </div>
       {/each}

packages/web/src/commands/stdCommands.ts

@@ -37,6 +37,7 @@ import { openWebLink } from '../utility/exportFileTools';
 import { getSettings } from '../utility/metadataLoaders';
 import { isMac } from '../utility/common';
 import { doLogout, internalRedirectTo } from '../clientAuth';
+import { disconnectServerConnection } from '../appobj/ConnectionAppObject.svelte';
 
 // function themeCommand(theme: ThemeDefinition) {
 //   return {
@@ -552,6 +553,14 @@ registerCommand({
   onClick: doLogout,
 });
 
+registerCommand({
+  id: 'app.disconnect',
+  category: 'App',
+  name: 'Disconnect',
+  testEnabled: () => getCurrentConfig()?.singleConnection != null,
+  onClick: () => disconnectServerConnection(getCurrentConfig()?.singleConnection?._id),
+});
+
 export function registerFileCommands({
   idPrefix,
   category,

packages/web/src/datagrid/CellValue.svelte

@@ -75,11 +75,17 @@
 {:else if value.$oid}
   <span class="value">ObjectId("{value.$oid}")</span>
 {:else if _.isPlainObject(value)}
-  <span class="null" title={JSON.stringify(value, undefined, 2)}>(JSON)</span>
+  {@const svalue = JSON.stringify(value, undefined, 2)}
+  <span class="null" title={svalue}
+    >{#if svalue.length < 100}{JSON.stringify(value)}{:else}(JSON){/if}</span
+  >
 {:else if _.isArray(value)}
   <span class="null" title={value.map(x => JSON.stringify(x)).join('\n')}>[{value.length} items]</span>
 {:else if _.isPlainObject(jsonParsedValue)}
-  <span class="null" title={JSON.stringify(jsonParsedValue, undefined, 2)}>(JSON)</span>
+  {@const svalue = JSON.stringify(jsonParsedValue, undefined, 2)}
+  <span class="null" title={svalue}
+    >{#if svalue.length < 100}{JSON.stringify(jsonParsedValue)}{:else}(JSON){/if}</span
+  >
 {:else if _.isArray(jsonParsedValue)}
   <span class="null" title={jsonParsedValue.map(x => JSON.stringify(x)).join('\n')}
     >[{jsonParsedValue.length} items]</span

packages/web/src/datagrid/CollectionDataGridCore.svelte

@@ -235,5 +235,6 @@
   bind:loadedRows
   bind:selectedCellsPublished
   frameSelection={!!macroPreview}
+  onOpenQuery={openQuery}
   {grider}
 />

packages/web/src/datagrid/DataFilterControl.svelte

@@ -33,6 +33,7 @@
   export let customCommandIcon = null;
   export let onCustomCommand = null;
   export let customCommandTooltip = null;
+  export let formatterFunction = null;
 
   export let pureName = null;
   export let schemaName = null;
@@ -168,6 +169,8 @@
           { onClick: () => openFilterWindow('<>'), text: 'Does Not Equal...' },
           { onClick: () => setFilter('EXISTS'), text: 'Field exists' },
           { onClick: () => setFilter('NOT EXISTS'), text: 'Field does not exist' },
+          { onClick: () => setFilter('NOT EMPTY ARRAY'), text: 'Array is not empty' },
+          { onClick: () => setFilter('EMPTY ARRAY'), text: 'Array is empty' },
           { onClick: () => openFilterWindow('>'), text: 'Greater Than...' },
           { onClick: () => openFilterWindow('>='), text: 'Greater Than Or Equal To...' },
           { onClick: () => openFilterWindow('<'), text: 'Less Than...' },
@@ -274,6 +277,7 @@
       schemaName,
       pureName,
       field: columnName || uniqueName,
+      formatterFunction,
       onConfirm: keys => setFilter(keys.map(x => getFilterValueExpression(x)).join(',')),
     });
   }

packages/web/src/datagrid/DataGridCore.svelte

@@ -13,6 +13,18 @@
     onClick: () => getCurrentDataGrid().refresh(),
   });
 
+  registerCommand({
+    id: 'dataGrid.deepRefresh',
+    category: 'Data grid',
+    name: 'Refresh with structure',
+    keyText: 'Ctrl+F5',
+    toolbar: true,
+    isRelatedToTab: true,
+    icon: 'icon reload',
+    testEnabled: () => getCurrentDataGrid()?.canDeepRefresh(),
+    onClick: () => getCurrentDataGrid().deepRefresh(),
+  });
+
   registerCommand({
     id: 'dataGrid.revertRowChanges',
     category: 'Data grid',
@@ -282,7 +294,7 @@
 <script lang="ts">
   import { GridDisplay } from 'dbgate-datalib';
   import { driverBase, parseCellValue } from 'dbgate-tools';
-  import { getContext } from 'svelte';
+  import { getContext, onDestroy } from 'svelte';
   import _ from 'lodash';
   import registerCommand from '../commands/registerCommand';
   import ColumnHeaderControl from './ColumnHeaderControl.svelte';
@@ -333,9 +345,10 @@
   import { apiCall } from '../utility/api';
   import getElectron from '../utility/getElectron';
   import { isCtrlOrCommandKey, isMac } from '../utility/common';
-  import { selectionCouldBeShownOnMap } from '../elements/MapView.svelte';
+  import { selectionCouldBeShownOnMap } from '../elements/SelectionMapView.svelte';
   import ErrorMessageModal from '../modals/ErrorMessageModal.svelte';
   import EditCellDataModal, { shouldOpenMultilineDialog } from '../modals/EditCellDataModal.svelte';
+  import { getDatabaseInfo, useDatabaseStatus } from '../utility/metadataLoaders';
 
   export let onLoadNextData = undefined;
   export let grider = undefined;
@@ -355,6 +368,7 @@
   export let pureName = undefined;
   export let schemaName = undefined;
   export let allowDefineVirtualReferences = false;
+  export let formatterFunction;
 
   export let isLoadedAll;
   export let loadedTime;
@@ -395,6 +409,26 @@
 
   const tabid = getContext('tabid');
 
+  let unsubscribeDbRefresh;
+
+  onDestroy(callUnsubscribeDbRefresh);
+
+  function callUnsubscribeDbRefresh() {
+    if (unsubscribeDbRefresh) {
+      unsubscribeDbRefresh();
+      unsubscribeDbRefresh = null;
+    }
+  }
+
+  async function refreshAndUnsubscribe(status) {
+    if (status?.name != 'pending' && status?.name != 'checkStructure' && status?.name != 'loadStructure') {
+      callUnsubscribeDbRefresh();
+      // ensure new structure is loaded
+      await getDatabaseInfo({ conid, database });
+      refresh();
+    }
+  }
+
   export function refresh() {
     if (onCustomGridRefresh) onCustomGridRefresh();
     else display.reload();
@@ -405,6 +439,16 @@
     return getDisplay()?.supportsReload;
   }
 
+  export function canDeepRefresh() {
+    return canRefresh() && !!conid && !!database;
+  }
+
+  export async function deepRefresh() {
+    callUnsubscribeDbRefresh();
+    await apiCall('database-connections/sync-model', { conid, database });
+    unsubscribeDbRefresh = useDatabaseStatus({ conid, database }).subscribe(refreshAndUnsubscribe);
+  }
+
   export function getGrider() {
     return grider;
   }
@@ -461,6 +505,7 @@
         for (const column of display.columns) {
           if (column.uniquePath.length > 1) continue;
           if (column.autoIncrement) continue;
+          if (column.columnName == '_id' && isDynamicStructure) continue;
 
           grider.setCellValue(rowIndex, column.uniqueName, grider.getRowData(index)[column.uniqueName]);
         }
@@ -1634,6 +1679,9 @@
     {#if grider.editable}
       <FormStyledButton value="Add document" on:click={addJsonDocument} />
     {/if}
+    {#if onOpenQuery}
+      <FormStyledButton value="Open Query" on:click={onOpenQuery} />
+    {/if}
   </div>
 {:else if grider.errors && grider.errors.length > 0}
   <div>
@@ -1740,6 +1788,7 @@
                   {conid}
                   {database}
                   {jslid}
+                  {formatterFunction}
                   driver={display?.driver}
                   filterType={useEvalFilters ? 'eval' : col.filterType || getFilterType(col.dataType)}
                   filter={display.getFilter(col.uniqueName)}

packages/web/src/datagrid/InplaceEditor.svelte

@@ -1,12 +1,3 @@
-<script lang="ts" context="module">
-  function getEditedValue(value) {
-    if (value?.type == 'Buffer' && _.isArray(value.data)) return '0x' + arrayToHexString(value.data);
-    if (value?.$oid) return `ObjectId("${value?.$oid}")`;
-    if (_.isPlainObject(value) || _.isArray(value)) return JSON.stringify(value);
-    return value;
-  }
-</script>
-
 <script lang="ts">
   import keycodes from '../utility/keycodes';
   import { onMount, tick } from 'svelte';

packages/web/src/datagrid/JslDataGridCore.svelte

@@ -12,12 +12,13 @@
   });
 
   async function loadDataPage(props, offset, limit) {
-    const { jslid, display } = props;
+    const { jslid, display, formatterFunction } = props;
 
     const response = await apiCall('jsldata/get-rows', {
       jslid,
       offset,
       limit,
+      formatterFunction,
       filters: display ? display.compileFilters() : null,
     });
 
@@ -34,6 +35,9 @@
     const response = await apiCall('jsldata/get-stats', { jslid });
     return response.rowCount;
   }
+
+  export let formatterPlugin;
+  export let formatterFunction;
 </script>
 
 <script lang="ts">
@@ -56,6 +60,7 @@
 
   export let jslid;
   export let display;
+  export let formatterFunction;
 
   export const activator = createActivator('JslDataGridCore', false);
 

packages/web/src/designer/ColumnLine.svelte

@@ -6,6 +6,7 @@
   import ColumnLabel from '../elements/ColumnLabel.svelte';
 
   import CheckboxField from '../forms/CheckboxField.svelte';
+  import { plusExpandIcon } from '../icons/expandIcons';
   import FontIcon from '../icons/FontIcon.svelte';
   import contextMenu from '../utility/contextMenu';
   import SortOrderIcon from './SortOrderIcon.svelte';
@@ -21,6 +22,11 @@
   export let onAddReferenceByColumn;
   export let onSelectColumn;
   export let settings;
+  export let nestingSupported = null;
+  export let isExpandable = false;
+  export let isExpanded = false;
+  export let expandLevel = 0;
+  export let toggleExpanded = null;
 
   $: designerColumn = (designer.columns || []).find(
     x => x.designerId == designerId && x.columnName == column.columnName
@@ -115,16 +121,27 @@
     })}
   use:contextMenu={settings?.canSelectColumns ? createMenu : '__no_menu'}
 >
+  {#if nestingSupported}
+    <span class="expandColumnIcon" style={`margin-right: ${5 + expandLevel * 10}px`}>
+      <FontIcon
+        icon={isExpandable ? plusExpandIcon(isExpanded) : 'icon invisible-box'}
+        on:click={() => {
+          toggleExpanded(!isExpanded);
+        }}
+      />
+    </span>
+  {/if}
+
   {#if settings?.allowColumnOperations}
     <CheckboxField
       checked={settings?.isColumnChecked
-        ? settings?.isColumnChecked(designerId, column.columnName)
+        ? settings?.isColumnChecked(designerId, column)
         : !!(designer.columns || []).find(
             x => x.designerId == designerId && x.columnName == column.columnName && x.isOutput
           )}
       on:change={e => {
         if (settings?.setColumnChecked) {
-          settings?.setColumnChecked(designerId, column.columnName, e.target.checked);
+          settings?.setColumnChecked(designerId, column, e.target.checked);
         } else {
           if (e.target.checked) {
             onChangeColumn(
@@ -147,7 +164,13 @@
       }}
     />
   {/if}
-  <ColumnLabel {...column} {foreignKey} forceIcon {iconOverride} />
+  <ColumnLabel
+    {...column}
+    columnName={settings?.getColumnDisplayName ? settings?.getColumnDisplayName(column) : column.columnName}
+    {foreignKey}
+    forceIcon
+    {iconOverride}
+  />
   {#if designerColumn?.filter}
     <FontIcon icon="img filter" />
   {/if}

packages/web/src/designer/Designer.svelte

@@ -48,6 +48,9 @@
   import ChooseColorModal from '../modals/ChooseColorModal.svelte';
   import { currentThemeDefinition } from '../stores';
   import { extendDatabaseInfoFromApps } from 'dbgate-tools';
+  import SearchInput from '../elements/SearchInput.svelte';
+  import CloseSearchButton from '../buttons/CloseSearchButton.svelte';
+  import DragColumnMemory from './DragColumnMemory.svelte';
 
   export let value;
   export let onChange;
@@ -64,6 +67,7 @@
   let canvasHeight = 3000;
   let dragStartPoint = null;
   let dragCurrentPoint = null;
+  let columnFilter;
 
   const sourceDragColumn$ = writable(null);
   const targetDragColumn$ = writable(null);
@@ -80,7 +84,15 @@
 
   const tableRefs = {};
   const referenceRefs = {};
-  $: domTables = _.pickBy(_.mapValues(tableRefs, (tbl: any) => tbl?.getDomTable()));
+  let domTables;
+  $: {
+    tableRefs;
+    recomputeDomTables();
+  }
+
+  function recomputeDomTables() {
+    domTables = _.pickBy(_.mapValues(tableRefs, (tbl: any) => tbl?.getDomTable()));
+  }
 
   function fixPositions(tables) {
     const minLeft = _.min(tables.map(x => x.left));
@@ -835,6 +847,14 @@
       ],
     ];
   }
+
+  $: {
+    columnFilter;
+    tick().then(() => {
+      recomputeReferencePositions();
+      recomputeDomTables();
+    });
+  }
 </script>
 
 <div class="wrapper noselect" use:contextMenu={createMenu}>
@@ -896,6 +916,7 @@
         onAddAllReferences={handleAddTableReferences}
         onChangeTableColor={handleChangeTableColor}
         onMoveReferences={recomputeReferencePositions}
+        {columnFilter}
         {table}
         {conid}
         {database}
@@ -930,6 +951,15 @@
       </svg>
     {/if}
   </div>
+  {#if tables?.length > 0}
+    <div class="panel">
+      <DragColumnMemory {settings} {sourceDragColumn$} {targetDragColumn$} />
+      <div class="searchbox">
+        <SearchInput bind:value={columnFilter} placeholder="Filter columns" />
+        <CloseSearchButton bind:filter={columnFilter} />
+      </div>
+    </div>
+  {/if}
 </div>
 
 <style>
@@ -946,6 +976,17 @@
   .canvas {
     position: relative;
   }
+  .panel {
+    position: absolute;
+    right: 16px;
+    top: 0;
+    display: flex;
+  }
+  .searchbox {
+    width: 200px;
+    display: flex;
+    margin-left: 1px;
+  }
 
   svg.drag-rect {
     visibility: hidden;

packages/web/src/designer/DesignerTable.svelte

@@ -1,6 +1,6 @@
 <script lang="ts">
   import { presetDarkPalettes, presetPalettes } from '@ant-design/colors';
-  import { computeDbDiffRows } from 'dbgate-tools';
+  import { filterName } from 'dbgate-tools';
 
   import { tick } from 'svelte';
   import { createDatabaseObjectMenu } from '../appobj/DatabaseObjectAppObject.svelte';
@@ -51,6 +51,7 @@
   export let designer;
   export let onMoveReferences;
   export let settings;
+  export let columnFilter;
 
   let movingPosition = null;
   let domWrapper;
@@ -68,6 +69,27 @@
   $: specificDb = settings?.tableSpecificDb ? settings?.tableSpecificDb(designerId) : null;
   $: filterParentRows = settings?.hasFilterParentRowsFlag ? settings?.hasFilterParentRowsFlag(designerId) : false;
   $: isGrayed = settings?.isGrayedTable ? settings?.isGrayedTable(designerId) : false;
+  $: flatColumns = getFlatColumns(columns, columnFilter, 0);
+
+  function getFlatColumns(columns, filter, level) {
+    if (!columns) return [];
+    const res = [];
+    for (const col of columns) {
+      if (filterName(filter, col.columnName)) {
+        res.push({ ...col, expandLevel: level });
+        if (col.isExpanded) {
+          res.push(...getFlatColumns(col.getChildColumns ? col.getChildColumns() : null, filter, level + 1));
+        }
+      } else if (col.isExpanded) {
+        const children = getFlatColumns(col.getChildColumns ? col.getChildColumns() : null, filter, level + 1);
+        if (children.length > 0) {
+          res.push({ ...col, expandLevel: level });
+          res.push(...children);
+        }
+      }
+    }
+    return res;
+  }
 
   export function isSelected() {
     return table?.isSelectedTable;
@@ -156,7 +178,7 @@
   export function getDomTable() {
     const domRefs = { ...columnRefs };
     domRefs[''] = domWrapper;
-    return new DomTableRef(table, domRefs, domCanvas);
+    return new DomTableRef(table, domRefs, domCanvas, settings);
   }
 
   const handleSetTableAlias = () => {
@@ -214,7 +236,7 @@
     ];
   }
 
-  // $: console.log('COLUMNS', columns);
+  // $: console.log('COLUMNS', flatColumns);
 </script>
 
 <div
@@ -279,8 +301,13 @@
     {/if}
   </div>
   <div class="columns" on:scroll={() => tick().then(onMoveReferences)} class:scroll={settings?.allowScrollColumns}>
-    {#each columns || [] as column}
+    {#each flatColumns || [] as column (column.columnName)}
       <ColumnLine
+        nestingSupported={!!settings?.isColumnExpandable && columns.find(x => settings?.isColumnExpandable(x))}
+        isExpandable={settings?.isColumnExpandable && settings?.isColumnExpandable(column)}
+        isExpanded={settings?.isColumnExpanded && settings?.isColumnExpanded(column)}
+        expandLevel={settings?.columnExpandLevel ? settings?.columnExpandLevel(column) : 0}
+        toggleExpanded={value => settings?.toggleExpandedColumn(column, value)}
         {column}
         {table}
         {designer}

packages/web/src/designer/DomTableRef.ts

@@ -6,13 +6,15 @@ export default class DomTableRef {
   table: DesignerTableInfo;
   designerId: string;
   domRefs: { [column: string]: Element };
+  settings: any;
 
-  constructor(table: DesignerTableInfo, domRefs, domWrapper: Element) {
+  constructor(table: DesignerTableInfo, domRefs, domWrapper: Element, settings) {
     this.domTable = domRefs[''];
     this.domWrapper = domWrapper;
     this.table = table;
     this.designerId = table.designerId;
     this.domRefs = domRefs;
+    this.settings = settings;
   }
 
   getRect() {
@@ -31,10 +33,14 @@ export default class DomTableRef {
 
   getColumnY(columnName: string) {
     let col = this.domRefs[columnName];
-    if (!col) return null;
+    while (col == null && this.settings?.getParentColumnName && this.settings?.getParentColumnName(columnName)) {
+      columnName = this.settings?.getParentColumnName(columnName);
+      col = this.domRefs[columnName];
+    }
+    const tableRect = this.getRect();
+    if (!col) return tableRect.top + 12;
     const rect = col.getBoundingClientRect();
     const wrap = this.domWrapper.getBoundingClientRect();
-    const tableRect = this.getRect();
     let res = (rect.top + rect.bottom) / 2 - wrap.top;
     if (res < tableRect.top) res = tableRect.top;
     if (res > tableRect.bottom) res = tableRect.bottom;

packages/web/src/designer/DragColumnMemory.svelte

@@ -0,0 +1,54 @@
+<script lang="ts">
+  export let sourceDragColumn$;
+  export let targetDragColumn$;
+  export let settings;
+
+  let memory;
+</script>
+
+{#if settings?.allowCreateRefByDrag}
+  <div
+    class="wrapper"
+    draggable={!!memory}
+    title={memory ? 'Drag this column to other column for creating JOIN' : 'Drag column here for creating JOIN'}
+    on:dragstart={e => {
+      if (!settings?.allowCreateRefByDrag) return;
+      if (!memory) return;
+
+      const dragData = { ...memory };
+      sourceDragColumn$.set(dragData);
+      e.dataTransfer.setData('designer_column_drag_data', JSON.stringify(dragData));
+    }}
+    on:dragend={e => {
+      sourceDragColumn$.set(null);
+      targetDragColumn$.set(null);
+    }}
+    on:dragover={e => {
+      if ($sourceDragColumn$) {
+        e.preventDefault();
+      }
+    }}
+    on:drop={e => {
+      var data = e.dataTransfer.getData('designer_column_drag_data');
+      e.preventDefault();
+      if (!data) return;
+      memory = $sourceDragColumn$;
+      sourceDragColumn$.set(null);
+      targetDragColumn$.set(null);
+    }}
+  >
+    {#if memory}
+      {memory.columnName}
+    {:else}
+      Drag & drop column here
+    {/if}
+  </div>
+{/if}
+
+<style>
+  .wrapper {
+    border: 1px solid var(--theme-border);
+    padding: 3px;
+    color: var(--theme-font-2);
+  }
+</style>

packages/web/src/elements/MapView.svelte

@@ -1,102 +1,27 @@
-<script lang="ts" context="module">
-  export function selectionCouldBeShownOnMap(selection) {
-    if (selection.length > 0 && _.find(selection, x => isWktGeometry(x.value))) {
-      return true;
-    }
-
-    if (
-      selection.find(x => x.column.toLowerCase().includes('lat')) &&
-      (selection.find(x => x.column.toLowerCase().includes('lon')) ||
-        selection.find(x => x.column.toLowerCase().includes('lng')))
-    ) {
-      return true;
-    }
-    return false;
-  }
-</script>
-
 <script lang="ts">
   import _ from 'lodash';
   import { onMount, tick } from 'svelte';
   import 'leaflet/dist/leaflet.css';
   import leaflet from 'leaflet';
-  import wellknown from 'wellknown';
-  import { isWktGeometry, ScriptWriter, ScriptWriterJson } from 'dbgate-tools';
   import resizeObserver from '../utility/resizeObserver';
   import openNewTab from '../utility/openNewTab';
   import contextMenu from '../utility/contextMenu';
-  import { saveExportedFile, saveFileToDisk } from '../utility/exportFileTools';
-  import { getCurrentConfig } from '../stores';
+  import { saveFileToDisk } from '../utility/exportFileTools';
   import { apiCall } from '../utility/api';
 
-  export let selection;
-
   let refContainer;
   let map;
 
-  let selectionLayers = [];
-  let geoJson;
+  let layers = [];
+  export let geoJson;
 
-  function createColumnsTable(cells) {
-    if (cells.length == 0) return '';
-    return `<table>${cells.map(cell => `<tr><td>${cell.column}</td><td>${cell.value}</td></tr>`).join('\n')}</table>`;
-  }
-
-  function addSelectionToMap() {
+  function addObjectToMap() {
     if (!map) return;
-    if (!selection) return;
 
-    for (const selectionLayer of selectionLayers) {
-      selectionLayer.remove();
+    for (const layer of layers) {
+      layer.remove();
     }
-    selectionLayers = [];
-
-    const selectedRows = _.groupBy(selection || [], 'row');
-
-    const features = [];
-
-    for (const rowKey of _.keys(selectedRows)) {
-      const cells = selectedRows[rowKey];
-      const lat = cells.find(x => x.column.toLowerCase().includes('lat'));
-      const lon = cells.find(x => x.column.toLowerCase().includes('lon') || x.column.toLowerCase().includes('lng'));
-
-      const geoValues = cells.map(x => x.value).filter(isWktGeometry);
-
-      if (lat && lon) {
-        features.push({
-          type: 'Feature',
-          properties: {
-            popupContent: createColumnsTable(cells),
-          },
-          geometry: {
-            type: 'Point',
-            coordinates: [lon.value, lat.value],
-          },
-        });
-      }
-
-      if (geoValues.length > 0) {
-        // parse WKT to geoJSON array
-        features.push(
-          ...geoValues.map(wellknown).map(geometry => ({
-            type: 'Feature',
-            properties: {
-              popupContent: createColumnsTable(cells.filter(x => !isWktGeometry(x.value))),
-            },
-            geometry,
-          }))
-        );
-      }
-    }
-
-    if (features.length == 0) {
-      return;
-    }
-
-    geoJson = {
-      type: 'FeatureCollection',
-      features,
-    };
+    layers = [];
 
     const geoJsonObj = leaflet
       .geoJSON(geoJson, {
@@ -130,7 +55,7 @@
       .addTo(map);
     // geoJsonObj.bindPopup('This is the Transamerica Pyramid'); //.openPopup();
     map.fitBounds(geoJsonObj.getBounds());
-    selectionLayers.push(geoJsonObj);
+    layers.push(geoJsonObj);
   }
 
   onMount(() => {
@@ -143,20 +68,12 @@
       })
       .addTo(map);
 
-    addSelectionToMap();
-    // map.fitBounds([
-    //   [50, 15],
-    //   [50.1, 15],
-    //   [50, 15.1],
-    // ]);
-
-    // const marker = leaflet.marker([50, 15]).addTo(map);
-    // <div bind:this={refContainer} class="flex1 map-container" />
+    addObjectToMap();
   });
 
   $: {
-    selection;
-    addSelectionToMap();
+    geoJson;
+    addObjectToMap();
   }
 
   function createMenu() {
@@ -170,7 +87,7 @@
               icon: 'img map',
               tabComponent: 'MapTab',
             },
-            { editor: selection.map(x => _.omit(x, ['engine'])) }
+            { editor: geoJson }
           );
         },
       },

packages/web/src/elements/ObjectListControl.svelte

@@ -1,8 +1,7 @@
 <script lang="ts">
+  import _ from 'lodash';
   import FontIcon from '../icons/FontIcon.svelte';
-
   import Link from './Link.svelte';
-
   import TableControl from './TableControl.svelte';
 
   export let title;
@@ -10,6 +9,7 @@
   export let columns;
   export let showIfEmpty = false;
   export let emptyMessage = null;
+  export let hideDisplayName = false;
   export let clickable;
   export let onAddNew;
 </script>
@@ -31,43 +31,43 @@
       <div class="body">
         <TableControl
           rows={collection || []}
-          columns={[
-            {
+          columns={_.compact([
+            !hideDisplayName && {
               fieldName: 'displayName',
               header: 'Name',
               slot: -1,
             },
             ...columns,
-          ]}
+          ])}
           {clickable}
           on:clickrow
         >
-          <svelte:fragment slot="-1" let:row>
-            <slot name="name" {row} />
+          <svelte:fragment slot="-1" let:row let:col>
+            <slot name="name" {row} {col} />
           </svelte:fragment>
-          <svelte:fragment slot="0" let:row>
-            <slot name="0" {row} />
+          <svelte:fragment slot="0" let:row let:col>
+            <slot name="0" {row} {col} />
           </svelte:fragment>
-          <svelte:fragment slot="1" let:row>
-            <slot name="1" {row} />
+          <svelte:fragment slot="1" let:row let:col>
+            <slot name="1" {row} {col} />
           </svelte:fragment>
-          <svelte:fragment slot="2" let:row>
-            <slot name="2" {row} />
+          <svelte:fragment slot="2" let:row let:col>
+            <slot name="2" {row} {col} />
           </svelte:fragment>
-          <svelte:fragment slot="3" let:row>
-            <slot name="3" {row} />
+          <svelte:fragment slot="3" let:row let:col>
+            <slot name="3" {row} {col} />
           </svelte:fragment>
-          <svelte:fragment slot="4" let:row>
-            <slot name="4" {row} />
+          <svelte:fragment slot="4" let:row let:col>
+            <slot name="4" {row} {col} />
           </svelte:fragment>
-          <svelte:fragment slot="5" let:row>
-            <slot name="5" {row} />
+          <svelte:fragment slot="5" let:row let:col>
+            <slot name="5" {row} {col} />
           </svelte:fragment>
-          <svelte:fragment slot="6" let:row>
-            <slot name="6" {row} />
+          <svelte:fragment slot="6" let:row let:col>
+            <slot name="6" {row} {col} />
           </svelte:fragment>
-          <svelte:fragment slot="7" let:row>
-            <slot name="7" {row} />
+          <svelte:fragment slot="7" let:row let:col>
+            <slot name="7" {row} {col} />
           </svelte:fragment>
         </TableControl>
       </div>

packages/web/src/elements/SelectionMapView.svelte

@@ -0,0 +1,117 @@
+<script lang="ts" context="module">
+  function findLatLonPaths(obj, attrTest, res = [], prefix = '') {
+    for (const key of Object.keys(obj)) {
+      if (attrTest(key, obj[key])) {
+        res.push(prefix + key);
+      }
+      if (_.isPlainObject(obj[key])) {
+        findLatLonPaths(obj[key], attrTest, res, prefix + key + '.');
+      }
+    }
+    return res;
+  }
+  export function findLatPaths(obj) {
+    return findLatLonPaths(obj, x => x.includes('lat'));
+  }
+  export function findLonPaths(obj) {
+    return findLatLonPaths(obj, x => x.includes('lon') || x.includes('lng'));
+  }
+  export function findAllObjectPaths(obj) {
+    return findLatLonPaths(obj, (_k, v) => v != null && !_.isNaN(Number(v)));
+  }
+
+  export function selectionCouldBeShownOnMap(selection) {
+    if (selection.length > 0 && _.find(selection, x => isWktGeometry(x.value))) {
+      return true;
+    }
+
+    if (
+      selection.length > 0 &&
+      _.find(selection, x => findLatPaths(x.rowData).length > 0 && findLonPaths(x.rowData).length > 0)
+    ) {
+      return true;
+    }
+    return false;
+  }
+</script>
+
+<script lang="ts">
+  import _ from 'lodash';
+  import 'leaflet/dist/leaflet.css';
+  import wellknown from 'wellknown';
+  import { isWktGeometry, stringifyCellValue } from 'dbgate-tools';
+  import MapView from './MapView.svelte';
+
+  export let selection;
+
+  export let latitudeField = '';
+  export let longitudeField = '';
+
+  let geoJson;
+
+  function createColumnsTable(cells) {
+    if (cells.length == 0) return '';
+    return `<table>${cells
+      .map(cell => `<tr><td>${cell.column}</td><td>${stringifyCellValue(cell.value)}</td></tr>`)
+      .join('\n')}</table>`;
+  }
+
+  function createGeoJson() {
+    const selectedRows = _.groupBy(selection || [], 'row');
+
+    const features = [];
+
+    for (const rowKey of _.keys(selectedRows)) {
+      const cells = selectedRows[rowKey];
+
+      const geoValues = cells.map(x => x.value).filter(isWktGeometry);
+
+      const lat = latitudeField ? Number(_.get(cells[0].rowData, latitudeField)) : NaN;
+      const lon = longitudeField ? Number(_.get(cells[0].rowData, longitudeField)) : NaN;
+
+      if (!_.isNaN(lat) && !_.isNaN(lon)) {
+        features.push({
+          type: 'Feature',
+          properties: {
+            popupContent: createColumnsTable(cells),
+          },
+          geometry: {
+            type: 'Point',
+            coordinates: [Number(lon), Number(lat)],
+          },
+        });
+      }
+
+      if (geoValues.length > 0) {
+        // parse WKT to geoJSON array
+        features.push(
+          ...geoValues.map(wellknown).map(geometry => ({
+            type: 'Feature',
+            properties: {
+              popupContent: createColumnsTable(cells.filter(x => !isWktGeometry(x.value))),
+            },
+            geometry,
+          }))
+        );
+      }
+    }
+
+    if (features.length == 0) {
+      return;
+    }
+
+    geoJson = {
+      type: 'FeatureCollection',
+      features,
+    };
+  }
+
+  $: {
+    selection;
+    latitudeField;
+    longitudeField;
+    createGeoJson();
+  }
+</script>
+
+<MapView {geoJson} />

packages/web/src/elements/TableControl.svelte

@@ -86,17 +86,17 @@
             {:else if col.formatter}
               {col.formatter(row)}
             {:else if col.slot != null}
-              {#if col.slot == -1}<slot name="-1" {row} {index} />
-              {:else if col.slot == 0}<slot name="0" {row} {index} {...rowProps} />
-              {:else if col.slot == 1}<slot name="1" {row} {index} {...rowProps} />
-              {:else if col.slot == 2}<slot name="2" {row} {index} {...rowProps} />
-              {:else if col.slot == 3}<slot name="3" {row} {index} {...rowProps} />
-              {:else if col.slot == 4}<slot name="4" {row} {index} {...rowProps} />
-              {:else if col.slot == 5}<slot name="5" {row} {index} {...rowProps} />
-              {:else if col.slot == 6}<slot name="6" {row} {index} {...rowProps} />
-              {:else if col.slot == 7}<slot name="7" {row} {index} {...rowProps} />
-              {:else if col.slot == 8}<slot name="8" {row} {index} {...rowProps} />
-              {:else if col.slot == 9}<slot name="9" {row} {index} {...rowProps} />
+              {#if col.slot == -1}<slot name="-1" {row} {col} {index} />
+              {:else if col.slot == 0}<slot name="0" {row} {col} {index} {...rowProps} />
+              {:else if col.slot == 1}<slot name="1" {row} {col} {index} {...rowProps} />
+              {:else if col.slot == 2}<slot name="2" {row} {col} {index} {...rowProps} />
+              {:else if col.slot == 3}<slot name="3" {row} {col} {index} {...rowProps} />
+              {:else if col.slot == 4}<slot name="4" {row} {col} {index} {...rowProps} />
+              {:else if col.slot == 5}<slot name="5" {row} {col} {index} {...rowProps} />
+              {:else if col.slot == 6}<slot name="6" {row} {col} {index} {...rowProps} />
+              {:else if col.slot == 7}<slot name="7" {row} {col} {index} {...rowProps} />
+              {:else if col.slot == 8}<slot name="8" {row} {col} {index} {...rowProps} />
+              {:else if col.slot == 9}<slot name="9" {row} {col} {index} {...rowProps} />
               {/if}
             {:else}
               {row[col.fieldName] || ''}

packages/web/src/forms/FormPasswordFieldRaw.svelte

@@ -8,6 +8,7 @@
 
   export let name;
   export let disabled = false;
+  export let saveOnInput = false;
 
   const { values, setFieldValue } = getFormContext();
 
@@ -23,6 +24,11 @@
     {disabled}
     value={isCrypted ? '' : value}
     on:change={e => setFieldValue(name, e.target['value'])}
+    on:input={e => {
+      if (saveOnInput) {
+        setFieldValue(name, e.target['value']);
+      }
+    }}
     placeholder={isCrypted ? '(Password is encrypted)' : undefined}
     type={isCrypted || showPassword ? 'text' : 'password'}
   />

packages/web/src/forms/FormTextFieldRaw.svelte

@@ -4,6 +4,7 @@
 
   export let name;
   export let defaultValue;
+  export let saveOnInput = false;
 
   const { values, setFieldValue } = getFormContext();
 </script>
@@ -12,4 +13,9 @@
   {...$$restProps}
   value={$values[name] ?? defaultValue}
   on:input={e => setFieldValue(name, e.target['value'])}
+  on:input={e => {
+    if (saveOnInput) {
+      setFieldValue(name, e.target['value']);
+    }
+  }}
 />

packages/web/src/freetable/FreeTableGrider.ts

@@ -10,10 +10,10 @@ export default class FreeTableGrider extends Grider {
     this.model = modelState && modelState.value;
   }
   getRowData(index: any) {
-    return this.model.rows[index];
+    return this.model.rows?.[index];
   }
   get rowCount() {
-    return this.model.rows.length;
+    return this.model.rows?.length;
   }
   get currentModel(): FreeTableModel {
     return this.batchModel || this.model;

packages/web/src/icons/FontIcon.svelte

@@ -32,6 +32,7 @@
     'icon add-folder': 'mdi mdi-folder-plus-outline',
 
     'icon window-restore': 'mdi mdi-window-restore',
+    'icon window-maximize': 'mdi mdi-window-maximize',
     'icon window-close': 'mdi mdi-window-close',
     'icon window-minimize': 'mdi mdi-window-minimize',
     'img dbgate': 'mdi mdi-database color-icon-gold',
@@ -39,6 +40,9 @@
     'icon columns': 'mdi mdi-view-column',
     'icon columns-outline': 'mdi mdi-view-column-outline',
 
+    'icon locked-database-mode': 'mdi mdi-database-lock',
+    'icon unlocked-database-mode': 'mdi mdi-database-eye',
+
     'icon database': 'mdi mdi-database',
     'icon server': 'mdi mdi-server',
     'icon table': 'mdi mdi-table',
@@ -49,6 +53,9 @@
     'icon close': 'mdi mdi-close',
     'icon unsaved': 'mdi mdi-record',
     'icon stop': 'mdi mdi-close-octagon',
+    'icon play': 'mdi mdi-play',
+    'icon play-stop': 'mdi mdi-stop',
+    'icon pause': 'mdi mdi-pause',
     'icon filter': 'mdi mdi-filter',
     'icon filter-off': 'mdi mdi-filter-off',
     'icon reload': 'mdi mdi-reload',
@@ -176,6 +183,7 @@
     'img app-command': 'mdi mdi-flash color-icon-green',
     'img app-query': 'mdi mdi-view-comfy color-icon-magenta',
     'img connection': 'mdi mdi-connection color-icon-blue',
+    'img profiler': 'mdi mdi-gauge color-icon-blue',
 
     'img add': 'mdi mdi-plus-circle color-icon-green',
     'img minus': 'mdi mdi-minus-circle color-icon-red',

packages/web/src/modals/DatabaseLoginModal.svelte

@@ -0,0 +1,148 @@
+<script lang="ts" context="module">
+  let currentModalConid = null;
+
+  export function isDatabaseLoginVisible() {
+    return !!currentModalConid;
+  }
+</script>
+
+<script lang="ts">
+  import _ from 'lodash';
+  import { onDestroy, onMount } from 'svelte';
+  import { writable } from 'svelte/store';
+  import FormStyledButton from '../buttons/FormStyledButton.svelte';
+  import Link from '../elements/Link.svelte';
+  import FormPasswordField from '../forms/FormPasswordField.svelte';
+  import FormProviderCore from '../forms/FormProviderCore.svelte';
+  import FormSubmit from '../forms/FormSubmit.svelte';
+  import FormTextField from '../forms/FormTextField.svelte';
+  import FontIcon from '../icons/FontIcon.svelte';
+  import { apiCall, setVolatileConnectionRemapping } from '../utility/api';
+  import { batchDispatchCacheTriggers, dispatchCacheChange } from '../utility/cache';
+  import createRef from '../utility/createRef';
+
+  import { getConnectionInfo } from '../utility/metadataLoaders';
+  import ErrorMessageModal from './ErrorMessageModal.svelte';
+  import ModalBase from './ModalBase.svelte';
+  import { closeCurrentModal, showModal } from './modalTools';
+
+  export let conid;
+  export let passwordMode;
+
+  const values = writable({});
+  let connection;
+
+  let isTesting;
+  let sqlConnectResult;
+  const testIdRef = createRef(0);
+
+  let engineTitle;
+
+  currentModalConid = conid;
+
+  onMount(async () => {
+    connection = await getConnectionInfo({ conid });
+    if (passwordMode == 'askPassword') {
+      $values = {
+        ...$values,
+        user: connection.user,
+        server: connection.server,
+      };
+    }
+    if (passwordMode == 'askUser') {
+      $values = {
+        ...$values,
+        server: connection.server,
+      };
+    }
+
+    const match = (connection.engine || '').match(/^([^@]*)@/);
+    engineTitle = match ? match[1] : connection.engine;
+  });
+
+  onDestroy(() => {
+    currentModalConid = null;
+  });
+
+  function handleCancelTest() {
+    testIdRef.update(x => x + 1); // invalidate current test
+    isTesting = false;
+  }
+
+  async function handleSubmit(ev) {
+    isTesting = true;
+    testIdRef.update(x => x + 1);
+    const testid = testIdRef.get();
+    const resp = await apiCall('connections/save-volatile', {
+      conid,
+      user: $values['user'],
+      password: $values['password'],
+      test: true,
+    });
+    if (testIdRef.get() != testid) return;
+    isTesting = false;
+    if (resp.msgtype == 'connected') {
+      setVolatileConnectionRemapping(conid, resp._id);
+      dispatchCacheChange({ key: `server-status-changed` });
+      batchDispatchCacheTriggers(x => x.conid == conid);
+      closeCurrentModal();
+    } else {
+      sqlConnectResult = resp;
+    }
+  }
+</script>
+
+<FormProviderCore {values}>
+  <ModalBase {...$$restProps} simple>
+    <svelte:fragment slot="header">Database Log In ({engineTitle})</svelte:fragment>
+
+    <FormTextField label="Server" name="server" disabled />
+    <FormTextField
+      label="Username"
+      name="user"
+      autocomplete="username"
+      disabled={passwordMode == 'askPassword'}
+      focused={passwordMode == 'askUser'}
+      saveOnInput
+    />
+    <FormPasswordField
+      label="Password"
+      name="password"
+      autocomplete="current-password"
+      focused={passwordMode == 'askPassword'}
+      saveOnInput
+    />
+
+    {#if isTesting}
+      <div>
+        <FontIcon icon="icon loading" /> Testing connection
+      </div>
+    {/if}
+
+    {#if !isTesting && sqlConnectResult && sqlConnectResult.msgtype == 'error'}
+      <div class="error-result">
+        Connect failed: <FontIcon icon="img error" />
+        {sqlConnectResult.error}
+        <Link
+          onClick={() =>
+            showModal(ErrorMessageModal, {
+              message: sqlConnectResult.detail,
+              showAsCode: true,
+              title: 'Database connection error',
+            })}
+        >
+          Show detail
+        </Link>
+      </div>
+    {/if}
+
+    <svelte:fragment slot="footer">
+      {#if isTesting}
+        <FormStyledButton value="Stop connecting" on:click={handleCancelTest} />
+      {:else}
+        <FormSubmit value="Connect" on:click={handleSubmit} />
+      {/if}
+      <FormStyledButton value="Close" on:click={closeCurrentModal} />
+    </svelte:fragment>
+  </ModalBase>
+</FormProviderCore>

packages/web/src/modals/ExportDatabaseDumpModal.svelte

@@ -66,7 +66,7 @@
     const files = await electron.showSaveDialog({
       properties: ['showOverwriteConfirmation'],
       filters: [
-        { name: 'SQL Files', extensions: ['*.sql'] },
+        { name: 'SQL Files', extensions: ['sql'] },
         { name: 'All Files', extensions: ['*'] },
       ],
     });

packages/web/src/modals/ImportDatabaseDumpModal.svelte

@@ -58,7 +58,7 @@
     const files = await electron.showOpenDialog({
       properties: ['openFile'],
       filters: [
-        { name: 'SQL Files', extensions: ['*.sql'] },
+        { name: 'SQL Files', extensions: ['sql'] },
         { name: 'All Files', extensions: ['*'] },
       ],
     });

packages/web/src/modals/ValueLookupModal.svelte

@@ -25,6 +25,7 @@
   export let driver;
   export let multiselect = false;
   export let jslid;
+  export let formatterFunction;
 
   // console.log('ValueLookupModal', conid, database, pureName, schemaName, columnName, driver);
 
@@ -42,6 +43,7 @@
         jslid,
         search,
         field,
+        formatterFunction,
       });
     } else {
       rows = await apiCall('database-connections/load-field-values', {

packages/web/src/perspectives/PerspectiveCell.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-  import { getAsImageSrc, safeJsonParse } from 'dbgate-tools';
+  import { getAsImageSrc, safeJsonParse, stringifyCellValue } from 'dbgate-tools';
   import _ from 'lodash';
 
   import CellValue from '../datagrid/CellValue.svelte';
@@ -10,6 +10,13 @@
   export let rowData;
   export let columnIndex;
   export let displayType;
+
+  function getValueAsText(value, force) {
+    if (force && value?.type == 'Buffer' && _.isArray(value.data)) {
+      return String.fromCharCode.apply(String, value.data);
+    }
+    return stringifyCellValue(value);
+  }
 </script>
 
 <td rowspan={rowSpan} data-column={columnIndex} class:isEmpty={value === undefined}>
@@ -23,6 +30,10 @@
       {:else}
         <span class="null"> (no image)</span>
       {/if}
+    {:else if displayType == 'text'}
+      {getValueAsText(value, false)}
+    {:else if displayType == 'forceText'}
+      {getValueAsText(value, true)}
     {:else if !value?.$oid && (_.isArray(value) || _.isPlainObject(value))}
       <JSONTree {value} slicedKeyCount={1} disableContextMenu />
     {:else}

packages/web/src/perspectives/PerspectiveDesigner.svelte

@@ -1,6 +1,10 @@
 <script lang="ts">
   import {
+    ChangePerspectiveConfigFunc,
     createPerspectiveNodeConfig,
+    getPerspectiveParentColumnName,
+    PerspectiveDataPatternColumn,
+    PerspectiveNodeConfig,
     perspectiveNodesHaveStructure,
     PerspectiveTreeNode,
     switchPerspectiveReferenceDirection,
@@ -28,6 +32,32 @@
 
   export let onClickTableHeader = null;
 
+  function mapDataPatternColumn(
+    column: PerspectiveDataPatternColumn,
+    node: PerspectiveNodeConfig,
+    codeNamePrefix: string
+  ) {
+    return {
+      columnName: codeNamePrefix + column.name,
+      shortName: column.name,
+      getChildColumns:
+        column.columns?.length > 0
+          ? () => column.columns.map(x => mapDataPatternColumn(x, node, codeNamePrefix + column.name + '::'))
+          : null,
+      isExpanded: node.expandedColumns.includes(codeNamePrefix + column.name),
+      toggleExpanded: value =>
+        setConfig(cfg => ({
+          ...cfg,
+          nodes: cfg.nodes.map(node => ({
+            ...node,
+            expandedColumns: value
+              ? [...(node.expandedColumns || []), codeNamePrefix + column.name]
+              : (node.expandedColumns || []).filter(x => x != codeNamePrefix + column.name),
+          })),
+        })),
+    };
+  }
+
   function createDesignerModel(
     config: PerspectiveConfig,
     dbInfos: MultipleDatabaseInfo,
@@ -49,10 +79,7 @@
             if (!pattern) return null;
             collection = {
               ...collection,
-              columns:
-                pattern?.columns.map(x => ({
-                  columnName: x.name,
-                })) || [],
+              columns: pattern?.columns.map(x => mapDataPatternColumn(x, node, '')) || [],
             };
           }
 
@@ -200,10 +227,10 @@
       ];
     },
     createReferenceText: reference => (reference.isAutoGenerated ? 'FK' : 'Custom'),
-    isColumnChecked: (designerId, columnName) => {
-      return config.nodes.find(x => x.designerId == designerId)?.checkedColumns?.includes(columnName);
+    isColumnChecked: (designerId, column) => {
+      return config.nodes.find(x => x.designerId == designerId)?.checkedColumns?.includes(column.columnName);
     },
-    setColumnChecked: (designerId, columnName, value) => {
+    setColumnChecked: (designerId, column, value) => {
       setConfig(cfg => ({
         ...cfg,
         nodes: cfg.nodes.map(node =>
@@ -211,8 +238,8 @@
             ? {
                 ...node,
                 checkedColumns: value
-                  ? [...(node.checkedColumns || []), columnName]
-                  : (node.checkedColumns || []).filter(x => x != columnName),
+                  ? [...(node.checkedColumns || []), column.columnName]
+                  : (node.checkedColumns || []).filter(x => x != column.columnName),
               }
             : node
         ),
@@ -301,6 +328,12 @@
       return false;
     },
     onClickTableHeader,
+    isColumnExpandable: column => !!column.getChildColumns,
+    isColumnExpanded: column => column.isExpanded,
+    columnExpandLevel: column => column.expandLevel,
+    toggleExpandedColumn: (column, value) => column.toggleExpanded(value),
+    getColumnDisplayName: column => column.shortName || column.columnName,
+    getParentColumnName: getPerspectiveParentColumnName,
   }}
   referenceComponent={QueryDesignerReference}
   value={createDesignerModel(config, dbInfos, dataPatterns)}

packages/web/src/perspectives/PerspectiveTable.svelte

@@ -79,7 +79,12 @@
   const lastVisibleRowIndexRef = createRef(0);
   const disableLoadNextRef = createRef(false);
 
+  // Essential function !!
+  // Fills nested data into parentRows (assigns into array parentRows[i][node.fieldName])
+  // eg. when node is CustomJoinTreeNode, loads data from data provider
   async function loadLevelData(node: PerspectiveTreeNode, parentRows: any[], counts) {
+    // console.log('loadLevelData', node.codeName, node.fieldName, parentRows);
+    // console.log('COUNTS', node.codeName, counts);
     dbg('load level data', counts);
     // const loadProps: PerspectiveDataLoadPropsWithNode[] = [];
     const loadChildNodes = [];
@@ -100,7 +105,8 @@
           incompleteRowsIndicator: [node.designerId],
         });
       }
-    } else {
+    } else if (!node.preloadedLevelData) {
+      // console.log('LOADED ROWS', rows);
       let lastRowWithChildren = null;
       for (const parentRow of parentRows) {
         const childRows = rows.filter(row => node.matchChildRow(parentRow, row));
@@ -114,11 +120,38 @@
           incompleteRowsIndicator: [node.designerId],
         });
       }
+    } else {
+      // this is needed for nested call
+      rows = _.compact(_.flatten(parentRows.map(x => x[node.fieldName])));
     }
 
+    // console.log('TESTING NODE', node);
+    // console.log('ROWS', rows);
     for (const child of node.childNodes) {
-      if (child.isExpandable && child.isCheckedNode) {
-        await loadLevelData(child, rows, counts);
+      // console.log('TEST CHILD FOR LOAD', child);
+      // console.log(child.isExpandable, child.isCheckedNode, child.preloadedLevelData);
+      if (child.isExpandable && (child.isCheckedNode || child.preloadedLevelData)) {
+        // console.log('TESTED OK');
+        // if (child.preloadedLevelData) console.log('LOADING CHILD DATA', rows);
+        // console.log(child.preloadedLevelData, child);
+        // console.log('LOADING FOR CHILD', child.codeName, child.columnName, child);
+        // console.log('CALL CHILD', child.codeName, rows, parentRows);
+        await loadLevelData(
+          child,
+          rows,
+          // node.preloadedLevelData
+          //   ? _.compact(_.flatten(parentRows.map(x => x[child.columnName])))
+          //   : child.preloadedLevelData
+          //   ? parentRows
+          //   : rows,
+          // child.preloadedLevelData
+          //   ? _.compact(_.flatten(parentRows.map(x => x[child.columnName])))
+          //   : node.preloadedLevelData
+          //   ? parentRows
+          //   : rows,
+
+          counts
+        );
         // loadProps.push(child.getNodeLoadProps());
       }
     }
@@ -268,11 +301,11 @@
 
       if (isHeader && !tableNode?.headerTableAttributes) {
         res.push({
-          text: `Change display (${
+          text: `Change display (${_.startCase(
             config.nodes.find(x => x.designerId == column?.dataNode?.parentNode?.designerId)?.columnDisplays?.[
               column.dataNode.columnName
             ] || 'default'
-          })`,
+          )})`,
           submenu: [
             {
               text: 'Default',
@@ -286,6 +319,14 @@
               text: 'Image',
               onClick: () => setColumnDisplay('image'),
             },
+            {
+              text: 'Text',
+              onClick: () => setColumnDisplay('text'),
+            },
+            {
+              text: 'Force Text',
+              onClick: () => setColumnDisplay('forceText'),
+            },
           ],
         });
       }

packages/web/src/query/AceEditor.svelte

@@ -447,6 +447,7 @@
 
     editor.container.addEventListener('contextmenu', handleContextMenu);
     editor.keyBinding.addKeyboardHandler(handleKeyDown);
+    editor.renderer.setScrollMargin(2, 0);
     changedQueryParts();
 
     // editor.session.addGutterDecoration(0, 'ace-gutter-sql-run');

packages/web/src/query/codeCompletion.ts

@@ -173,7 +173,7 @@ export function mountCodeCompletion({ conid, database, editor, getText }) {
                 ? []
                 : _.flatten(
                     sourceObjects.map(obj =>
-                      obj.columns.map(col => ({
+                      (obj.columns || []).map(col => ({
                         name: col.columnName,
                         value: obj.alias ? `${obj.alias}.${col.columnName}` : col.columnName,
                         caption: obj.alias ? `${obj.alias}.${col.columnName}` : col.columnName,

packages/web/src/settings/ConnectionDriverFields.svelte

@@ -28,8 +28,12 @@
   $: driver = $extensions.drivers.find(x => x.engine == engine);
   $: defaultDatabase = $values.defaultDatabase;
 
-  $: showUser = driver?.showConnectionField('user', $values);
-  $: showPassword = driver?.showConnectionField('password', $values);
+  $: showUser = driver?.showConnectionField('user', $values) && $values.passwordMode != 'askUser';
+  $: showPassword =
+    driver?.showConnectionField('password', $values) &&
+    $values.passwordMode != 'askPassword' &&
+    $values.passwordMode != 'askUser';
+  $: showPasswordMode = driver?.showConnectionField('password', $values);
   $: isConnected = $openedConnections.includes($values._id) || $openedSingleDatabaseConnections.includes($values._id);
 </script>
 
@@ -159,7 +163,7 @@
   <FormPasswordField label="Password" name="password" disabled={isConnected || disabledFields.includes('password')} />
 {/if}
 
-{#if !disabledFields.includes('password') && showPassword}
+{#if !disabledFields.includes('password') && showPasswordMode}
   <FormSelectField
     label="Password mode"
     isNative
@@ -169,6 +173,8 @@
     options={[
       { value: 'saveEncrypted', label: 'Save and encrypt' },
       { value: 'saveRaw', label: 'Save raw (UNSAFE!!)' },
+      { value: 'askPassword', label: "Don't save, ask for password" },
+      { value: 'askUser', label: "Don't save, ask for login and password" },
     ]}
   />
 {/if}

packages/web/src/settings/SettingsModal.svelte

@@ -3,6 +3,7 @@
   import FormStyledButton from '../buttons/FormStyledButton.svelte';
   import Link from '../elements/Link.svelte';
   import TabControl from '../elements/TabControl.svelte';
+  import CheckboxField from '../forms/CheckboxField.svelte';
 
   import FormCheckboxField from '../forms/FormCheckboxField.svelte';
   import FormFieldTemplateLarge from '../forms/FormFieldTemplateLarge.svelte';
@@ -23,6 +24,7 @@
     currentEditorTheme,
     extensions,
     selectedWidget,
+    lockedDatabaseMode,
     visibleWidgetSideBar,
   } from '../stores';
   import { isMac } from '../utility/common';
@@ -107,6 +109,19 @@ ORDER BY
           />
 
           <div class="heading">Connection</div>
+
+          <FormFieldTemplateLarge
+            label="Show only tabs from selected database"
+            type="checkbox"
+            labelProps={{
+              onClick: () => {
+                $lockedDatabaseMode = !$lockedDatabaseMode;
+              },
+            }}
+          >
+            <CheckboxField checked={$lockedDatabaseMode} on:change={e => ($lockedDatabaseMode = e.target.checked)} />
+          </FormFieldTemplateLarge>
+
           <FormCheckboxField
             name="connection.autoRefresh"
             label="Automatic refresh of database model on background"

packages/web/src/stores.ts

@@ -50,12 +50,14 @@ function subscribeCssVariable(store, transform, cssVariable) {
 }
 
 export const selectedWidget = writableWithStorage('database', 'selectedWidget');
+export const lockedDatabaseMode = writableWithStorage<boolean>(false, 'lockedDatabaseMode');
 export const visibleWidgetSideBar = writableWithStorage(true, 'visibleWidgetSideBar');
 export const visibleSelectedWidget = derived(
   [selectedWidget, visibleWidgetSideBar],
   ([$selectedWidget, $visibleWidgetSideBar]) => ($visibleWidgetSideBar ? $selectedWidget : null)
 );
 export const emptyConnectionGroupNames = writableWithStorage([], 'emptyConnectionGroupNames');
+export const collapsedConnectionGroupNames = writableWithStorage([], 'collapsedConnectionGroupNames');
 export const openedConnections = writable([]);
 export const openedSingleDatabaseConnections = writable([]);
 export const expandedConnections = writable([]);
@@ -137,6 +139,7 @@ subscribeCssVariable(visibleSelectedWidget, x => (x ? 1 : 0), '--dim-visible-lef
 // subscribeCssVariable(visibleToolbar, x => (x ? 1 : 0), '--dim-visible-toolbar');
 subscribeCssVariable(leftPanelWidth, x => `${x}px`, '--dim-left-panel-width');
 subscribeCssVariable(visibleTitleBar, x => (x ? 1 : 0), '--dim-visible-titlebar');
+subscribeCssVariable(lockedDatabaseMode, x => (x ? 0 : 1), '--dim-visible-tabs-databases');
 
 let activeTabIdValue = null;
 activeTabId.subscribe(value => {
@@ -198,6 +201,12 @@ pinnedDatabases.subscribe(value => {
 });
 export const getPinnedDatabases = () => _.compact(pinnedDatabasesValue);
 
+let lockedDatabaseModeValue = null;
+lockedDatabaseMode.subscribe(value => {
+  lockedDatabaseModeValue = value;
+});
+export const getLockedDatabaseMode = () => lockedDatabaseModeValue;
+
 let currentDatabaseValue = null;
 currentDatabase.subscribe(value => {
   currentDatabaseValue = value;
@@ -216,7 +225,7 @@ export const getCurrentDatabase = () => currentDatabaseValue;
 let currentSettingsValue = null;
 export const getCurrentSettings = () => currentSettingsValue || {};
 
-let extensionsValue = null;
+let extensionsValue: ExtensionsDirectory = null;
 extensions.subscribe(value => {
   extensionsValue = value;
 });
@@ -238,8 +247,8 @@ export function subscribeApiDependendStores() {
   useConfig().subscribe(value => {
     currentConfigValue = value;
     invalidateCommands();
-    if (value.singleDatabase) {
-      currentDatabase.set(value.singleDatabase);
+    if (value.singleDbConnection) {
+      currentDatabase.set(value.singleDbConnection);
     }
   });
 }

packages/web/src/tabs/MapTab.svelte

@@ -4,16 +4,16 @@
   import useEditorData from '../query/useEditorData';
 
   export let tabid;
-  let selection;
+  let geoJson;
 
   useEditorData({
     tabid,
     onInitialData: value => {
-      selection = value;
+      geoJson = value;
     },
   });
 </script>
 
-{#if selection}
-  <MapView {selection} />
+{#if geoJson}
+  <MapView {geoJson} />
 {/if}

packages/web/src/tabs/PerspectiveTab.svelte

@@ -111,11 +111,12 @@
     )
   );
 
-  const cache = new PerspectiveCache();
+  let cache = new PerspectiveCache();
   const loadedCounts = writable({});
 
   export function refresh() {
-    cache.clear();
+    cache = new PerspectiveCache();
+    // cache.clear();
     loadedCounts.set({});
   }
 </script>

packages/web/src/tabs/ProfilerTab.svelte

@@ -0,0 +1,250 @@
+<script lang="ts" context="module">
+  const getCurrentEditor = () => getActiveComponent('ProfilerTab');
+
+  registerCommand({
+    id: 'profiler.start',
+    category: 'Profiler',
+    name: 'Start profiling',
+    icon: 'icon play',
+    testEnabled: () => getCurrentEditor()?.startProfilingEnabled(),
+    onClick: () => getCurrentEditor().startProfiling(),
+  });
+
+  registerCommand({
+    id: 'profiler.stop',
+    category: 'Profiler',
+    name: 'Stop profiling',
+    icon: 'icon play-stop',
+    testEnabled: () => getCurrentEditor()?.stopProfilingEnabled(),
+    onClick: () => getCurrentEditor().stopProfiling(),
+  });
+
+  registerCommand({
+    id: 'profiler.save',
+    category: 'Profiler',
+    name: 'Save',
+    icon: 'icon save',
+    testEnabled: () => getCurrentEditor()?.saveEnabled(),
+    onClick: () => getCurrentEditor().save(),
+  });
+</script>
+
+<script>
+  import { findEngineDriver } from 'dbgate-tools';
+
+  import { onDestroy, onMount } from 'svelte';
+
+  import ToolStripCommandButton from '../buttons/ToolStripCommandButton.svelte';
+  import ToolStripContainer from '../buttons/ToolStripContainer.svelte';
+  import invalidateCommands from '../commands/invalidateCommands';
+  import registerCommand from '../commands/registerCommand';
+  import JslDataGrid from '../datagrid/JslDataGrid.svelte';
+  import ErrorInfo from '../elements/ErrorInfo.svelte';
+  import VerticalSplitter from '../elements/VerticalSplitter.svelte';
+  import { showModal } from '../modals/modalTools';
+  import SaveArchiveModal from '../modals/SaveArchiveModal.svelte';
+  import { currentArchive, selectedWidget } from '../stores';
+  import { apiCall } from '../utility/api';
+  import createActivator, { getActiveComponent } from '../utility/createActivator';
+  import { useConnectionInfo } from '../utility/metadataLoaders';
+  import { extensions } from '../stores';
+  import ChartCore from '../charts/ChartCore.svelte';
+  import LoadingInfo from '../elements/LoadingInfo.svelte';
+  import randomcolor from 'randomcolor';
+
+  export const activator = createActivator('ProfilerTab', true);
+
+  export let conid;
+  export let database;
+  export let engine;
+  export let jslidLoad;
+
+  let jslidSession;
+
+  let isProfiling = false;
+  let sessionId;
+  let isLoadingChart = false;
+
+  let intervalId;
+  let chartData;
+
+  $: connection = useConnectionInfo({ conid });
+  $: driver = findEngineDriver(engine || $connection, $extensions);
+  $: jslid = jslidSession || jslidLoad;
+
+  onMount(() => {
+    intervalId = setInterval(() => {
+      if (sessionId) {
+        apiCall('sessions/ping', {
+          sesid: sessionId,
+        });
+      }
+    }, 15 * 1000);
+  });
+
+  $: {
+    if (jslidLoad && driver) {
+      loadChart();
+    }
+  }
+
+  onDestroy(() => {
+    clearInterval(intervalId);
+  });
+
+  export async function startProfiling() {
+    isProfiling = true;
+
+    let sesid = sessionId;
+    if (!sesid) {
+      const resp = await apiCall('sessions/create', {
+        conid,
+        database,
+      });
+      sesid = resp.sesid;
+      sessionId = sesid;
+    }
+
+    const resp = await apiCall('sessions/start-profiler', {
+      sesid,
+    });
+    jslidSession = resp.jslid;
+
+    invalidateCommands();
+  }
+
+  export function startProfilingEnabled() {
+    return conid && database && !isProfiling;
+  }
+
+  async function loadChart() {
+    isLoadingChart = true;
+
+    const colors = randomcolor({
+      count: driver.profilerChartMeasures.length,
+      seed: 5,
+    });
+
+    const data = await apiCall('jsldata/extract-timeline-chart', {
+      jslid,
+      timestampFunction: driver.profilerTimestampFunction,
+      aggregateFunction: driver.profilerChartAggregateFunction,
+      measures: driver.profilerChartMeasures,
+    });
+    chartData = {
+      ...data,
+      labels: data.labels.map(x => new Date(x)),
+      datasets: data.datasets.map((x, i) => ({
+        ...x,
+        borderColor: colors[i],
+      })),
+    };
+    isLoadingChart = false;
+  }
+
+  export async function stopProfiling() {
+    isProfiling = false;
+    await apiCall('sessions/stop-profiler', { sesid: sessionId });
+    await apiCall('sessions/kill', { sesid: sessionId });
+    sessionId = null;
+
+    invalidateCommands();
+
+    loadChart();
+  }
+
+  export function stopProfilingEnabled() {
+    return conid && database && isProfiling;
+  }
+
+  export function saveEnabled() {
+    return !!jslidSession;
+  }
+
+  async function doSave(folder, file) {
+    await apiCall('archive/save-jsl-data', { folder, file, jslid });
+    currentArchive.set(folder);
+    selectedWidget.set('archive');
+  }
+
+  export function save() {
+    showModal(SaveArchiveModal, {
+      //   folder: archiveFolder,
+      //   file: archiveFile,
+      onSave: doSave,
+    });
+  }
+
+  // const data = [
+  //   { year: 2010, count: 10 },
+  //   { year: 2011, count: 20 },
+  //   { year: 2012, count: 15 },
+  //   { year: 2013, count: 25 },
+  //   { year: 2014, count: 22 },
+  //   { year: 2015, count: 30 },
+  //   { year: 2016, count: 28 },
+  // ];
+  // {
+  //   labels: data.map(row => row.year),
+  //   datasets: [
+  //     {
+  //       label: 'Acquisitions by year',
+  //       data: data.map(row => row.count),
+  //     },
+  //   ],
+  // }
+</script>
+
+<ToolStripContainer>
+  {#if jslid}
+    <VerticalSplitter allowCollapseChild1 allowCollapseChild2>
+      <svelte:fragment slot="1">
+        {#key jslid}
+          <JslDataGrid {jslid} listenInitializeFile formatterFunction={driver?.profilerFormatterFunction} />
+        {/key}
+      </svelte:fragment>
+      <svelte:fragment slot="2">
+        {#if isLoadingChart}
+          <LoadingInfo wrapper message="Loading chart" />
+        {:else}
+          <ChartCore
+            title="Profile data"
+            data={chartData}
+            options={{
+              maintainAspectRatio: false,
+              scales: {
+                x: {
+                  type: 'time',
+                  distribution: 'linear',
+
+                  time: {
+                    tooltipFormat: 'D. M. YYYY HH:mm',
+                    displayFormats: {
+                      millisecond: 'HH:mm:ss.SSS',
+                      second: 'HH:mm:ss',
+                      minute: 'HH:mm',
+                      hour: 'D.M hA',
+                      day: 'D. M.',
+                      week: 'D. M. YYYY',
+                      month: 'MM-YYYY',
+                      quarter: '[Q]Q - YYYY',
+                      year: 'YYYY',
+                    },
+                  },
+                },
+              },
+            }}
+          />
+        {/if}
+      </svelte:fragment>
+    </VerticalSplitter>
+  {:else}
+    <ErrorInfo message="Profiler not yet started" alignTop />
+  {/if}
+
+  <svelte:fragment slot="toolstrip">
+    <ToolStripCommandButton command="profiler.start" />
+    <ToolStripCommandButton command="profiler.stop" />
+    <ToolStripCommandButton command="profiler.save" />
+  </svelte:fragment>
+</ToolStripContainer>

packages/web/src/tabs/ServerSummaryTab.svelte

@@ -0,0 +1,122 @@
+<script lang="ts" context="module">
+  const getCurrentEditor = () => getActiveComponent('ServerSummaryTab');
+
+  registerCommand({
+    id: 'serverSummary.refresh',
+    category: 'Server sumnmary',
+    name: 'Refresh',
+    keyText: 'F5 | CtrlOrCommand+R',
+    toolbar: true,
+    isRelatedToTab: true,
+    icon: 'icon reload',
+    onClick: () => getCurrentEditor().refresh(),
+    testEnabled: () => getCurrentEditor() != null,
+  });
+</script>
+
+<script>
+  import ToolStripCommandButton from '../buttons/ToolStripCommandButton.svelte';
+  import ToolStripContainer from '../buttons/ToolStripContainer.svelte';
+  import registerCommand from '../commands/registerCommand';
+  import Link from '../elements/Link.svelte';
+  import LoadingInfo from '../elements/LoadingInfo.svelte';
+
+  import ObjectListControl from '../elements/ObjectListControl.svelte';
+  import { apiCall } from '../utility/api';
+  import createActivator, { getActiveComponent } from '../utility/createActivator';
+  import formatFileSize from '../utility/formatFileSize';
+  import openNewTab from '../utility/openNewTab';
+
+  export let conid;
+
+  let refreshToken = 0;
+
+  export const activator = createActivator('ServerSummaryTab', true);
+
+  export function refresh() {
+    refreshToken += 1;
+  }
+
+  async function runAction(action, row) {
+    const { command, openQuery, openTab, addDbProps } = action;
+    if (command) {
+      await apiCall('server-connections/summary-command', { conid, refreshToken, command, row });
+      refresh();
+    }
+    if (openQuery) {
+      openNewTab({
+        title: action.tabTitle || row.name,
+        icon: 'img query-data',
+        tabComponent: 'QueryDataTab',
+        props: {
+          conid,
+          database: row.name,
+          sql: openQuery,
+        },
+      });
+    }
+    if (openTab) {
+      const props = {};
+      if (addDbProps) {
+        props['conid'] = conid;
+        props['database'] = row.name;
+      }
+      openNewTab({
+        ...openTab,
+        props: {
+          ...openTab.props,
+          ...props,
+        },
+      });
+    }
+  }
+</script>
+
+<ToolStripContainer>
+  {#await apiCall('server-connections/server-summary', { conid, refreshToken })}
+    <LoadingInfo message="Loading server details" wrapper />
+  {:then summary}
+    <div class="wrapper">
+      <ObjectListControl
+        collection={summary.databases}
+        hideDisplayName
+        title={`Databases (${summary.databases.length})`}
+        emptyMessage={'No databases'}
+        columns={summary.columns.map(col => ({
+          ...col,
+          slot: col.columnType == 'bytes' ? 1 : col.columnType == 'actions' ? 2 : null,
+        }))}
+      >
+        <svelte:fragment slot="1" let:row let:col>{formatFileSize(row?.[col.fieldName])}</svelte:fragment>
+        <svelte:fragment slot="2" let:row let:col>
+          {#each col.actions as action, index}
+            {#if index > 0}
+              <span class="action-separator">|</span>
+            {/if}
+            <Link onClick={() => runAction(action, row)}>{action.header}</Link>
+          {/each}
+        </svelte:fragment>
+      </ObjectListControl>
+    </div>
+  {/await}
+
+  <svelte:fragment slot="toolstrip">
+    <ToolStripCommandButton command="serverSummary.refresh" />
+  </svelte:fragment>
+</ToolStripContainer>
+
+<style>
+  .wrapper {
+    position: absolute;
+    left: 0;
+    top: 0;
+    right: 0;
+    bottom: 0;
+    background-color: var(--theme-bg-0);
+    overflow: auto;
+  }
+
+  .action-separator {
+    margin: 0 5px;
+  }
+</style>

packages/web/src/tabs/TableDataTab.svelte

@@ -211,6 +211,7 @@
   function createAutoRefreshMenu() {
     return [
       { divider: true },
+      { command: 'dataGrid.deepRefresh', hideDisabled: true },
       { command: 'tableData.stopAutoRefresh', hideDisabled: true },
       { command: 'tableData.startAutoRefresh', hideDisabled: true },
       'tableData.setAutoRefresh.1',

packages/web/src/tabs/index.js

@@ -26,6 +26,8 @@ import * as QueryDataTab from './QueryDataTab.svelte';
 import * as ConnectionTab from './ConnectionTab.svelte';
 import * as MapTab from './MapTab.svelte';
 import * as PerspectiveTab from './PerspectiveTab.svelte';
+import * as ServerSummaryTab from './ServerSummaryTab.svelte';
+import * as ProfilerTab from './ProfilerTab.svelte';
 
 export default {
   TableDataTab,
@@ -56,4 +58,6 @@ export default {
   ConnectionTab,
   MapTab,
   PerspectiveTab,
+  ServerSummaryTab,
+  ProfilerTab,
 };

packages/web/src/utility/api.ts

@@ -5,6 +5,9 @@ import getElectron from './getElectron';
 // import socket from './socket';
 import { showSnackbarError } from '../utility/snackbar';
 import { isOauthCallback, redirectToLogin } from '../clientAuth';
+import { showModal } from '../modals/modalTools';
+import DatabaseLoginModal, { isDatabaseLoginVisible } from '../modals/DatabaseLoginModal.svelte';
+import _ from 'lodash';
 
 let eventSource;
 let apiLogging = false;
@@ -12,6 +15,9 @@ let apiLogging = false;
 let apiDisabled = false;
 const disabledOnOauth = isOauthCallback();
 
+const volatileConnectionMap = {};
+const volatileConnectionMapInv = {};
+
 export function disableApi() {
   apiDisabled = true;
 }
@@ -20,6 +26,27 @@ export function enableApi() {
   apiDisabled = false;
 }
 
+export function setVolatileConnectionRemapping(existingConnectionId, volatileConnectionId) {
+  volatileConnectionMap[existingConnectionId] = volatileConnectionId;
+  volatileConnectionMapInv[volatileConnectionId] = existingConnectionId;
+}
+
+export function getVolatileRemapping(conid) {
+  return volatileConnectionMap[conid] || conid;
+}
+
+export function getVolatileRemappingInv(conid) {
+  return volatileConnectionMapInv[conid] || conid;
+}
+
+export function removeVolatileMapping(conid) {
+  const mapped = volatileConnectionMap[conid];
+  if (mapped) {
+    delete volatileConnectionMap[conid];
+    delete volatileConnectionMapInv[mapped];
+  }
+}
+
 function wantEventSource() {
   if (!eventSource) {
     eventSource = new EventSource(`${resolveApi()}/stream`);
@@ -32,7 +59,16 @@ function processApiResponse(route, args, resp) {
   //   console.log('<<< API RESPONSE', route, args, resp);
   // }
 
-  if (resp?.apiErrorMessage) {
+  if (resp?.missingCredentials) {
+    if (!isDatabaseLoginVisible()) {
+      showModal(DatabaseLoginModal, resp.detail);
+    }
+    return null;
+    // return {
+    //   errorMessage: resp.apiErrorMessage,
+    //   missingCredentials: true,
+    // };
+  } else if (resp?.apiErrorMessage) {
     showSnackbarError('API error:' + resp?.apiErrorMessage);
     return {
       errorMessage: resp.apiErrorMessage,
@@ -42,6 +78,22 @@ function processApiResponse(route, args, resp) {
   return resp;
 }
 
+export function transformApiArgs(args) {
+  return _.mapValues(args, (v, k) => {
+    if (k == 'conid' && v && volatileConnectionMap[v]) return volatileConnectionMap[v];
+    if (k == 'conidArray' && _.isArray(v)) return v.map(x => volatileConnectionMap[x] || x);
+    return v;
+  });
+}
+
+export function transformApiArgsInv(args) {
+  return _.mapValues(args, (v, k) => {
+    if (k == 'conid' && v && volatileConnectionMapInv[v]) return volatileConnectionMapInv[v];
+    if (k == 'conidArray' && _.isArray(v)) return v.map(x => volatileConnectionMapInv[x] || x);
+    return v;
+  });
+}
+
 export async function apiCall(route: string, args: {} = undefined) {
   if (apiLogging) {
     console.log('>>> API CALL', route, args);
@@ -55,6 +107,8 @@ export async function apiCall(route: string, args: {} = undefined) {
     return;
   }
 
+  args = transformApiArgs(args);
+
   const electron = getElectron();
   if (electron) {
     const resp = await electron.invoke(route.replace('/', '-'), args);

packages/web/src/utility/applyScriptTemplate.ts

@@ -36,6 +36,17 @@ export default async function applyScriptTemplate(scriptTemplate, extensions, pr
       else return objectInfo.createSql;
     }
   }
+  if (scriptTemplate == 'ALTER OBJECT') {
+    const objectInfo = await getSqlObjectInfo(props);
+    if (objectInfo) {
+      const createSql =
+        objectInfo.requiresFormat && objectInfo.createSql
+          ? sqlFormatter.format(objectInfo.createSql)
+          : objectInfo.createSql || '';
+      const alterPrefix = createSql.trimStart().startsWith('CREATE ') ? 'ALTER ' : 'alter ';
+      return createSql.replace(/^\s*create\s+/i, alterPrefix);
+    }
+  }
   if (scriptTemplate == 'EXECUTE PROCEDURE') {
     const procedureInfo = await getSqlObjectInfo(props);
     const connection = await getConnectionInfo(props);

packages/web/src/utility/cache.ts

@@ -1,5 +1,6 @@
-import { apiOn } from './api';
+import { apiOn, transformApiArgsInv } from './api';
 import getAsArray from './getAsArray';
+import stableStringify from 'json-stable-stringify';
 
 const cachedByKey = {};
 const cachedPromisesByKey = {};
@@ -15,10 +16,11 @@ function cacheGet(key) {
 
 function addCacheKeyToReloadTrigger(cacheKey, reloadTrigger) {
   for (const item of getAsArray(reloadTrigger)) {
-    if (!(item in cachedKeysByReloadTrigger)) {
-      cachedKeysByReloadTrigger[item] = [];
+    const itemString = stableStringify(item);
+    if (!(itemString in cachedKeysByReloadTrigger)) {
+      cachedKeysByReloadTrigger[itemString] = [];
     }
-    cachedKeysByReloadTrigger[item].push(cacheKey);
+    cachedKeysByReloadTrigger[itemString].push(cacheKey);
   }
 }
 
@@ -32,7 +34,8 @@ function cacheSet(cacheKey, value, reloadTrigger, generation) {
 function cacheClean(reloadTrigger) {
   cacheGeneration += 1;
   for (const item of getAsArray(reloadTrigger)) {
-    const keys = cachedKeysByReloadTrigger[item];
+    const itemString = stableStringify(transformApiArgsInv(item));
+    const keys = cachedKeysByReloadTrigger[itemString];
     if (keys) {
       for (const key of keys) {
         delete cachedByKey[key];
@@ -40,7 +43,7 @@ function cacheClean(reloadTrigger) {
         cacheGenerationByKey[key] = cacheGeneration;
       }
     }
-    delete cachedKeysByReloadTrigger[item];
+    delete cachedKeysByReloadTrigger[itemString];
   }
 }
 
@@ -77,7 +80,8 @@ export async function loadCachedValue(reloadTrigger, cacheKey, func) {
       }
     } catch (err) {
       console.error('Error when using cached promise', err);
-      cacheClean(cacheKey);
+      // cacheClean(cacheKey);
+      cacheClean(reloadTrigger);
       const res = await func();
       cacheSet(cacheKey, res, reloadTrigger, generation);
       return res;
@@ -87,35 +91,48 @@ export async function loadCachedValue(reloadTrigger, cacheKey, func) {
 
 export async function subscribeCacheChange(reloadTrigger, cacheKey, reloadHandler) {
   for (const item of getAsArray(reloadTrigger)) {
-    if (!subscriptionsByReloadTrigger[item]) {
-      subscriptionsByReloadTrigger[item] = [];
+    const itemString = stableStringify(item);
+    if (!subscriptionsByReloadTrigger[itemString]) {
+      subscriptionsByReloadTrigger[itemString] = [];
     }
-    subscriptionsByReloadTrigger[item].push(reloadHandler);
+    subscriptionsByReloadTrigger[itemString].push(reloadHandler);
   }
 }
 
 export async function unsubscribeCacheChange(reloadTrigger, cacheKey, reloadHandler) {
   for (const item of getAsArray(reloadTrigger)) {
-    if (subscriptionsByReloadTrigger[item]) {
-      subscriptionsByReloadTrigger[item] = subscriptionsByReloadTrigger[item].filter(x => x != reloadHandler);
+    const itemString = stableStringify(item);
+    if (subscriptionsByReloadTrigger[itemString]) {
+      subscriptionsByReloadTrigger[itemString] = subscriptionsByReloadTrigger[itemString].filter(
+        x => x != reloadHandler
+      );
     }
-    if (subscriptionsByReloadTrigger[item].length == 0) {
-      delete subscriptionsByReloadTrigger[item];
+    if (subscriptionsByReloadTrigger[itemString].length == 0) {
+      delete subscriptionsByReloadTrigger[itemString];
     }
   }
 }
 
-function dispatchCacheChange(reloadTrigger) {
-  // console.log('CHANGE', reloadTrigger);
+export function dispatchCacheChange(reloadTrigger) {
   cacheClean(reloadTrigger);
 
   for (const item of getAsArray(reloadTrigger)) {
-    if (subscriptionsByReloadTrigger[item]) {
-      for (const handler of subscriptionsByReloadTrigger[item]) {
+    const itemString = stableStringify(transformApiArgsInv(item));
+    if (subscriptionsByReloadTrigger[itemString]) {
+      for (const handler of subscriptionsByReloadTrigger[itemString]) {
         handler();
       }
     }
   }
 }
 
+export function batchDispatchCacheTriggers(predicate) {
+  for (const key in subscriptionsByReloadTrigger) {
+    const relaodTrigger = JSON.parse(key);
+    if (predicate(relaodTrigger)) {
+      dispatchCacheChange(relaodTrigger);
+    }
+  }
+}
+
 apiOn('changed-cache', reloadTrigger => dispatchCacheChange(reloadTrigger));

packages/web/src/utility/changeCurrentDbByTab.ts

@@ -1,6 +1,7 @@
 import _ from 'lodash';
-import { currentDatabase, openedTabs } from '../stores';
-import { callWhenAppLoaded } from './appLoadManager';
+import { currentDatabase, getCurrentDatabase, getLockedDatabaseMode, openedTabs } from '../stores';
+import { shouldShowTab } from '../widgets/TabsPanel.svelte';
+import { callWhenAppLoaded, getAppLoaded } from './appLoadManager';
 import { getConnectionInfo } from './metadataLoaders';
 
 let lastCurrentTab = null;
@@ -8,6 +9,7 @@ let lastCurrentTab = null;
 openedTabs.subscribe(value => {
   const newCurrentTab = (value || []).find(x => x.selected);
   if (newCurrentTab == lastCurrentTab) return;
+  if (getLockedDatabaseMode() && getCurrentDatabase()) return;
 
   const lastTab = lastCurrentTab;
   lastCurrentTab = newCurrentTab;
@@ -27,3 +29,23 @@ openedTabs.subscribe(value => {
     }
   }
 });
+
+currentDatabase.subscribe(currentDb => {
+  if (!getLockedDatabaseMode()) return;
+  if (!currentDb && !getAppLoaded()) return;
+  openedTabs.update(tabs => {
+    const newTabs = tabs.map(tab => ({
+      ...tab,
+      selected: tab.selected && shouldShowTab(tab, true, currentDb),
+    }));
+
+    if (newTabs.find(x => x.selected)) return newTabs;
+
+    const selectedIndex = _.findLastIndex(newTabs, x => shouldShowTab(x));
+
+    return newTabs.map((x, index) => ({
+      ...x,
+      selected: index == selectedIndex,
+    }));
+  });
+});

packages/web/src/utility/connectionsPinger.js

@@ -10,7 +10,7 @@ import { getConnectionList } from './metadataLoaders';
 // };
 
 const doServerPing = value => {
-  apiCall('server-connections/ping', { connections: value });
+  apiCall('server-connections/ping', { conidArray: value });
 };
 
 const doDatabasePing = value => {
@@ -29,12 +29,12 @@ export function subscribeConnectionPingers() {
   openedConnections.subscribe(value => {
     doServerPing(value);
     if (openedConnectionsHandle) window.clearInterval(openedConnectionsHandle);
-    openedConnectionsHandle = window.setInterval(() => doServerPing(value), 30 * 1000);
+    openedConnectionsHandle = window.setInterval(() => doServerPing(value), 20 * 1000);
   });
 
   currentDatabase.subscribe(value => {
     doDatabasePing(value);
     if (currentDatabaseHandle) window.clearInterval(currentDatabaseHandle);
-    currentDatabaseHandle = window.setInterval(() => doDatabasePing(value), 30 * 1000);
+    currentDatabaseHandle = window.setInterval(() => doDatabasePing(value), 20 * 1000);
   });
 }

packages/web/src/utility/metadataLoaders.ts

@@ -9,7 +9,7 @@ import { apiCall, apiOff, apiOn } from './api';
 const databaseInfoLoader = ({ conid, database }) => ({
   url: 'database-connections/structure',
   params: { conid, database },
-  reloadTrigger: `database-structure-changed-${conid}-${database}`,
+  reloadTrigger: { key: `database-structure-changed`, conid, database },
   transform: extendDatabaseInfo,
 });
 
@@ -28,31 +28,31 @@ const databaseInfoLoader = ({ conid, database }) => ({
 const connectionInfoLoader = ({ conid }) => ({
   url: 'connections/get',
   params: { conid },
-  reloadTrigger: 'connection-list-changed',
+  reloadTrigger: { key: 'connection-list-changed' },
 });
 
 const configLoader = () => ({
   url: 'config/get',
   params: {},
-  reloadTrigger: 'config-changed',
+  reloadTrigger: { key: 'config-changed' },
 });
 
 const settingsLoader = () => ({
   url: 'config/get-settings',
   params: {},
-  reloadTrigger: 'settings-changed',
+  reloadTrigger: { key: 'settings-changed' },
 });
 
 const platformInfoLoader = () => ({
   url: 'config/platform-info',
   params: {},
-  reloadTrigger: 'platform-info-changed',
+  reloadTrigger: { key: 'platform-info-changed' },
 });
 
 const favoritesLoader = () => ({
   url: 'files/favorites',
   params: {},
-  reloadTrigger: 'files-changed-favorites',
+  reloadTrigger: { key: 'files-changed-favorites' },
 });
 
 // const sqlObjectListLoader = ({ conid, database }) => ({
@@ -64,13 +64,13 @@ const favoritesLoader = () => ({
 const databaseStatusLoader = ({ conid, database }) => ({
   url: 'database-connections/status',
   params: { conid, database },
-  reloadTrigger: `database-status-changed-${conid}-${database}`,
+  reloadTrigger: { key: `database-status-changed`, conid, database },
 });
 
 const databaseListLoader = ({ conid }) => ({
   url: 'server-connections/list-databases',
   params: { conid },
-  reloadTrigger: `database-list-changed-${conid}`,
+  reloadTrigger: { key: `database-list-changed`, conid },
   onLoaded: value => {
     if (value?.length > 0) setLocalStorage(`database_list_${conid}`, value);
   },
@@ -85,37 +85,37 @@ const databaseListLoader = ({ conid }) => ({
 const serverVersionLoader = ({ conid }) => ({
   url: 'server-connections/version',
   params: { conid },
-  reloadTrigger: `server-version-changed-${conid}`,
+  reloadTrigger: { key: `server-version-changed`, conid },
 });
 
 const databaseServerVersionLoader = ({ conid, database }) => ({
   url: 'database-connections/server-version',
   params: { conid, database },
-  reloadTrigger: `database-server-version-changed-${conid}-${database}`,
+  reloadTrigger: { key: `database-server-version-changed`, conid, database },
 });
 
 const archiveFoldersLoader = () => ({
   url: 'archive/folders',
   params: {},
-  reloadTrigger: `archive-folders-changed`,
+  reloadTrigger: { key: `archive-folders-changed` },
 });
 
 const archiveFilesLoader = ({ folder }) => ({
   url: 'archive/files',
   params: { folder },
-  reloadTrigger: `archive-files-changed-${folder}`,
+  reloadTrigger: { key: `archive-files-changed`, folder },
 });
 
 const appFoldersLoader = () => ({
   url: 'apps/folders',
   params: {},
-  reloadTrigger: `app-folders-changed`,
+  reloadTrigger: { key: `app-folders-changed` },
 });
 
 const appFilesLoader = ({ folder }) => ({
   url: 'apps/files',
   params: { folder },
-  reloadTrigger: `app-files-changed-${folder}`,
+  reloadTrigger: { key: `app-files-changed`, app: folder },
 });
 
 // const dbAppsLoader = ({ conid, database }) => ({
@@ -127,41 +127,41 @@ const appFilesLoader = ({ folder }) => ({
 const usedAppsLoader = ({ conid, database }) => ({
   url: 'apps/get-used-apps',
   params: {},
-  reloadTrigger: `used-apps-changed`,
+  reloadTrigger: { key: `used-apps-changed` },
 });
 
 const serverStatusLoader = () => ({
   url: 'server-connections/server-status',
   params: {},
-  reloadTrigger: `server-status-changed`,
+  reloadTrigger: { key: `server-status-changed` },
 });
 
 const connectionListLoader = () => ({
   url: 'connections/list',
   params: {},
-  reloadTrigger: `connection-list-changed`,
+  reloadTrigger: { key: `connection-list-changed` },
 });
 
 const installedPluginsLoader = () => ({
   url: 'plugins/installed',
   params: {},
-  reloadTrigger: `installed-plugins-changed`,
+  reloadTrigger: { key: `installed-plugins-changed` },
 });
 
 const filesLoader = ({ folder }) => ({
   url: 'files/list',
   params: { folder },
-  reloadTrigger: `files-changed-${folder}`,
+  reloadTrigger: { key: `files-changed`, folder },
 });
 const allFilesLoader = () => ({
   url: 'files/list-all',
   params: {},
-  reloadTrigger: `all-files-changed`,
+  reloadTrigger: { key: `all-files-changed` },
 });
 const authTypesLoader = ({ engine }) => ({
   url: 'plugins/auth-types',
   params: { engine },
-  reloadTrigger: `installed-plugins-changed`,
+  reloadTrigger: { key: `installed-plugins-changed` },
   errorValue: null,
 });
 

packages/web/src/widgets/CellDataWidget.svelte

@@ -47,14 +47,14 @@
   ];
 
   function autodetect(selection) {
-    if (selection[0]?.engine?.databaseEngineTypes?.includes('document')) {
-      return 'jsonRow';
-    }
-
     if (selectionCouldBeShownOnMap(selection)) {
       return 'map';
     }
 
+    if (selection[0]?.engine?.databaseEngineTypes?.includes('document')) {
+      return 'jsonRow';
+    }
+
     const value = selection.length == 1 ? selection[0].value : null;
     if (_.isString(value)) {
       if (value.startsWith('[') || value.startsWith('{')) return 'json';
@@ -80,7 +80,7 @@
   import TextCellViewNoWrap from '../celldata/TextCellViewNoWrap.svelte';
   import TextCellViewWrap from '../celldata/TextCellViewWrap.svelte';
   import ErrorInfo from '../elements/ErrorInfo.svelte';
-  import { selectionCouldBeShownOnMap } from '../elements/MapView.svelte';
+  import { selectionCouldBeShownOnMap } from '../elements/SelectionMapView.svelte';
   import SelectField from '../forms/SelectField.svelte';
   import { selectedCellsCallback } from '../stores';
   import WidgetTitle from './WidgetTitle.svelte';

packages/web/src/widgets/ConnectionList.svelte

@@ -3,7 +3,7 @@
   import InlineButton from '../buttons/InlineButton.svelte';
   import SearchInput from '../elements/SearchInput.svelte';
   import WidgetsInnerContainer from './WidgetsInnerContainer.svelte';
-  import { useConnectionList, useServerStatus } from '../utility/metadataLoaders';
+  import { useConfig, useConnectionList, useServerStatus } from '../utility/metadataLoaders';
   import SearchBoxWrapper from '../elements/SearchBoxWrapper.svelte';
   import AppObjectList from '../appobj/AppObjectList.svelte';
   import * as connectionAppObject from '../appobj/ConnectionAppObject.svelte';
@@ -15,16 +15,20 @@
     openedSingleDatabaseConnections,
     openedTabs,
     emptyConnectionGroupNames,
+    collapsedConnectionGroupNames,
   } from '../stores';
   import runCommand from '../commands/runCommand';
   import getConnectionLabel from '../utility/getConnectionLabel';
   import { useConnectionColorFactory } from '../utility/useConnectionColor';
   import FontIcon from '../icons/FontIcon.svelte';
   import CloseSearchButton from '../buttons/CloseSearchButton.svelte';
-  import { apiCall } from '../utility/api';
+  import { apiCall, getVolatileRemapping } from '../utility/api';
   import LargeButton from '../buttons/LargeButton.svelte';
   import { plusExpandIcon, chevronExpandIcon } from '../icons/expandIcons';
   import { safeJsonParse } from 'dbgate-tools';
+  import { showModal } from '../modals/modalTools';
+  import InputTextModal from '../modals/InputTextModal.svelte';
+  import ConfirmModal from '../modals/ConfirmModal.svelte';
 
   const connections = useConnectionList();
   const serverStatus = useServerStatus();
@@ -33,7 +37,7 @@
 
   $: connectionsWithStatus =
     $connections && $serverStatus
-      ? $connections.map(conn => ({ ...conn, status: $serverStatus[conn._id] }))
+      ? $connections.map(conn => ({ ...conn, status: $serverStatus[getVolatileRemapping(conn._id)] }))
       : $connections;
 
   $: connectionsWithStatusFiltered = connectionsWithStatus?.filter(
@@ -56,9 +60,9 @@
   const handleDropOnGroup = (data, group) => {
     const json = safeJsonParse(data);
     if (json?._id) {
-      if (json.parent) {
-        emptyConnectionGroupNames.update(x => x.filter(y => y != json.parent));
-      }
+      // if (json.parent) {
+      //   emptyConnectionGroupNames.update(x => x.filter(y => y != json.parent));
+      // }
       apiCall('connections/update', {
         _id: json?._id,
         values: { parent: group },
@@ -67,6 +71,41 @@
   };
 
   const connectionColorFactory = useConnectionColorFactory(3);
+
+  function createGroupContextMenu(folder) {
+    const handleRename = () => {
+      showModal(InputTextModal, {
+        value: folder,
+        label: 'New folder name',
+        header: 'Rename folder',
+        onConfirm: async newFolder => {
+          emptyConnectionGroupNames.update(folders => _.uniq(folders.map(fld => (fld == folder ? newFolder : fld))));
+          apiCall('connections/batch-change-folder', {
+            folder,
+            newFolder,
+          });
+        },
+      });
+    };
+
+    const handleDelete = () => {
+      showModal(ConfirmModal, {
+        message: `Really delete folder ${folder}? Connections in folder will be moved into root folder.`,
+        onConfirm: () => {
+          emptyConnectionGroupNames.update(folders => folders.filter(fld => fld != folder));
+          apiCall('connections/batch-change-folder', {
+            folder,
+            newFolder: '',
+          });
+        },
+      });
+    };
+
+    return [
+      { text: 'Rename', onClick: handleRename },
+      { text: 'Delete', onClick: handleDelete },
+    ];
+  }
 </script>
 
 <SearchBoxWrapper>
@@ -110,6 +149,8 @@
     onDropOnGroup={handleDropOnGroup}
     emptyGroupNames={$emptyConnectionGroupNames}
     sortGroups
+    groupContextMenu={createGroupContextMenu}
+    collapsedGroupNames={collapsedConnectionGroupNames}
   />
   {#if (connectionsWithParent?.length > 0 && connectionsWithoutParent?.length > 0) || ($emptyConnectionGroupNames.length > 0 && connectionsWithoutParent?.length > 0)}
     <div class="br" />

packages/web/src/widgets/DatabaseWidget.svelte

@@ -12,6 +12,7 @@
   import WidgetColumnBarItem from './WidgetColumnBarItem.svelte';
   import SqlObjectList from './SqlObjectList.svelte';
   import DbKeysTree from './DbKeysTree.svelte';
+  import SingleConnectionDatabaseList from './SingleConnectionDatabaseList.svelte';
 
   export let hidden = false;
 
@@ -24,7 +25,11 @@
 </script>
 
 <WidgetColumnBar {hidden}>
-  {#if !$config?.singleDatabase}
+  {#if $config?.singleConnection}
+    <WidgetColumnBarItem title="Databases" name="databases" height="35%" storageName="databasesWidget">
+      <SingleConnectionDatabaseList connection={$config?.singleConnection} />
+    </WidgetColumnBarItem>
+  {:else if !$config?.singleDbConnection}
     <WidgetColumnBarItem title="Connections" name="connections" height="35%" storageName="connectionsWidget">
       <ConnectionList />
     </WidgetColumnBarItem>

packages/web/src/widgets/SingleConnectionDatabaseList.svelte

@@ -0,0 +1,32 @@
+<script lang="ts">
+  import _ from 'lodash';
+  import InlineButton from '../buttons/InlineButton.svelte';
+  import SearchInput from '../elements/SearchInput.svelte';
+  import WidgetsInnerContainer from './WidgetsInnerContainer.svelte';
+  import SearchBoxWrapper from '../elements/SearchBoxWrapper.svelte';
+  import SubDatabaseList from '../appobj/SubDatabaseList.svelte';
+  import { openedConnections } from '../stores';
+  import { useConnectionColorFactory } from '../utility/useConnectionColor';
+  import FontIcon from '../icons/FontIcon.svelte';
+  import CloseSearchButton from '../buttons/CloseSearchButton.svelte';
+  import { apiCall } from '../utility/api';
+
+  export let connection;
+
+  let filter = '';
+
+  const handleRefreshDatabases = () => {
+    apiCall('server-connections/refresh', { conid: connection._id });
+  };
+</script>
+
+<SearchBoxWrapper>
+  <SearchInput placeholder="Search connection or database" bind:value={filter} />
+  <CloseSearchButton bind:filter />
+  <InlineButton on:click={handleRefreshDatabases} title="Refresh database list">
+    <FontIcon icon="icon refresh" />
+  </InlineButton>
+</SearchBoxWrapper>
+<WidgetsInnerContainer>
+  <SubDatabaseList data={connection} {filter} passProps={{}} />
+</WidgetsInnerContainer>

packages/web/src/widgets/TabsPanel.svelte

@@ -1,4 +1,23 @@
 <script lang="ts" context="module">
+  const getCurrentValueMarker: any = {};
+
+  export function shouldShowTab(tab, lockedDbMode = getCurrentValueMarker, currentDb = getCurrentValueMarker) {
+    if (lockedDbMode == getCurrentValueMarker) {
+      lockedDbMode = getLockedDatabaseMode();
+    }
+    if (lockedDbMode) {
+      if (currentDb == getCurrentValueMarker) {
+        currentDb = getCurrentDatabase();
+      }
+      return (
+        tab.closedTime == null &&
+        (!tab.props?.conid || tab.props?.conid == currentDb?.connection?._id) &&
+        (!tab.props?.database || tab.props?.database == currentDb?.name)
+      );
+    }
+    return tab.closedTime == null;
+  }
+
   function allowCloseTabs(tabs) {
     if (tabs.length == 0) return Promise.resolve(true);
     return new Promise(resolve => {
@@ -14,7 +33,8 @@
     const activeCandidate = getOpenedTabs().find(x => x.tabid == tabid);
     const closeCandidates = getOpenedTabs()
       .filter(x => closeCondition(x, activeCandidate))
-      .filter(x => x.unsaved && x.closedTime == null);
+      .filter(x => x.unsaved)
+      .filter(shouldShowTab);
 
     if (!(await allowCloseTabs(closeCandidates))) return;
 
@@ -24,14 +44,14 @@
 
       const newFiles = files.map(x => ({
         ...x,
-        closedTime: x.closedTime || (closeCondition(x, active) ? new Date().getTime() : undefined),
+        closedTime: shouldShowTab(x) && closeCondition(x, active) ? new Date().getTime() : x.closedTime,
       }));
 
-      if (newFiles.find(x => x.selected && x.closedTime == null)) {
+      if (newFiles.find(x => x.selected && shouldShowTab(x))) {
         return newFiles;
       }
 
-      const selectedIndex = _.findLastIndex(newFiles, x => x.closedTime == null);
+      const selectedIndex = _.findLastIndex(newFiles, x => shouldShowTab(x));
 
       return newFiles.map((x, index) => ({
         ...x,
@@ -43,7 +63,8 @@
   export const closeMultipleTabs = async (closeCondition, deleteFromHistory = false) => {
     const closeCandidates = getOpenedTabs()
       .filter(x => closeCondition(x))
-      .filter(x => x.unsaved && x.closedTime == null);
+      .filter(x => x.unsaved)
+      .filter(shouldShowTab);
 
     if (!(await allowCloseTabs(closeCandidates))) return;
 
@@ -52,14 +73,14 @@
         ? files.filter(x => !closeCondition(x))
         : files.map(x => ({
             ...x,
-            closedTime: x.closedTime || (closeCondition(x) ? new Date().getTime() : undefined),
+            closedTime: shouldShowTab(x) && closeCondition(x) ? new Date().getTime() : x.closedTime,
           }));
 
-      if (newFiles.find(x => x.selected && x.closedTime == null)) {
+      if (newFiles.find(x => x.selected && shouldShowTab(x))) {
         return newFiles;
       }
 
-      const selectedIndex = _.findLastIndex(newFiles, x => x.closedTime == null);
+      const selectedIndex = _.findLastIndex(newFiles, x => shouldShowTab(x));
 
       return newFiles.map((x, index) => ({
         ...x,
@@ -70,7 +91,9 @@
 
   const closeTab = closeTabFunc((x, active) => x.tabid == active.tabid);
   const closeAll = async () => {
-    const closeCandidates = getOpenedTabs().filter(x => x.unsaved && x.closedTime == null);
+    const closeCandidates = getOpenedTabs()
+      .filter(x => x.unsaved)
+      .filter(shouldShowTab);
 
     if (!(await allowCloseTabs(closeCandidates))) return;
 
@@ -78,7 +101,7 @@
     openedTabs.update(tabs =>
       tabs.map(tab => ({
         ...tab,
-        closedTime: tab.closedTime || closedTime,
+        closedTime: shouldShowTab(tab) ? closedTime : tab.closedTime,
         selected: false,
       }))
     );
@@ -133,10 +156,7 @@
   }
 
   function switchTabByOrder(reverse) {
-    const tabs = _.sortBy(
-      get(openedTabs).filter(x => x.closedTime == null),
-      'tabOrder'
-    );
+    const tabs = _.sortBy(get(openedTabs).filter(shouldShowTab), 'tabOrder');
     if (reverse) tabs.reverse();
     const selectedTab = tabs.find(x => x.selected);
     if (!selectedTab) return;
@@ -212,8 +232,7 @@
 </script>
 
 <script lang="ts">
-  import { LogarithmicScale } from 'chart.js';
-  import _, { map, slice, sortBy } from 'lodash';
+  import _ from 'lodash';
   import { tick } from 'svelte';
   import { derived, get } from 'svelte/store';
   import registerCommand from '../commands/registerCommand';
@@ -231,6 +250,8 @@
     activeTabId,
     getActiveTabId,
     getCurrentDatabase,
+    lockedDatabaseMode,
+    getLockedDatabaseMode,
   } from '../stores';
   import tabs from '../tabs';
   import { setSelectedTab } from '../utility/common';
@@ -243,6 +264,7 @@
   import TabCloseButton from '../elements/TabCloseButton.svelte';
   import CloseTabModal from '../modals/CloseTabModal.svelte';
 
+  $: showTabFilterFunc = tab => shouldShowTab(tab, $lockedDatabaseMode, $currentDatabase);
   $: connectionList = useConnectionList();
 
   $: currentDbKey =
@@ -252,13 +274,11 @@
       ? `server://${$currentDatabase.connection._id}`
       : '_no';
 
-  $: tabsWithDb = $openedTabs
-    .filter(x => !x.closedTime)
-    .map(tab => ({
-      ...tab,
-      tabDbName: getTabDbName(tab, $connectionList),
-      tabDbKey: getTabDbKey(tab),
-    }));
+  $: tabsWithDb = $openedTabs.filter(showTabFilterFunc).map(tab => ({
+    ...tab,
+    tabDbName: getTabDbName(tab, $connectionList),
+    tabDbKey: getTabDbKey(tab),
+  }));
 
   $: groupedTabs = groupTabs(tabsWithDb);
 
@@ -423,54 +443,56 @@
   <div class="tabs" on:wheel={handleTabsWheel} bind:this={domTabs}>
     {#each groupedTabs as tabGroup}
       <div class="db-wrapper">
-        <div
-          class="db-name"
-          class:selected={draggingDbGroup
-            ? tabGroup.grpid == draggingDbGroupTarget?.grpid
-            : tabGroup.tabDbKey == currentDbKey}
-          on:mouseup={e => {
-            if (e.button == 1) {
-              closeMultipleTabs(tab => tabGroup.tabs.find(x => x.tabid == tab.tabid));
-            } else {
-              handleSetDb(tabGroup.tabs[0].props);
-            }
-          }}
-          use:contextMenu={getDatabaseContextMenu(tabGroup.tabs)}
-          style={$connectionColorFactory(
-            tabGroup.tabs[0].props,
-            (draggingDbGroup ? tabGroup.grpid == draggingDbGroupTarget?.grpid : tabGroup.tabDbKey == currentDbKey)
-              ? 2
-              : 3
-          )}
-          draggable={true}
-          on:dragstart={e => {
-            draggingDbGroup = tabGroup;
-          }}
-          on:dragenter={e => {
-            draggingDbGroupTarget = tabGroup;
-          }}
-          on:drop={e => {
-            dragDropTabs(draggingDbGroup.tabs, tabGroup.tabs);
-          }}
-          on:dragend={e => {
-            draggingDbGroup = null;
-            draggingDbGroupTarget = null;
-          }}
-        >
-          <div class="db-name-inner">
-            <FontIcon icon={getDbIcon(tabGroup.tabDbKey)} />
-            {tabGroup.tabDbName}
-            {#if $connectionList?.find(x => x._id == tabGroup.tabs[0]?.props?.conid)?.isReadOnly}
-              <FontIcon icon="icon lock" />
-            {/if}
-          </div>
+        {#if !$lockedDatabaseMode}
           <div
-            class="close-button-right tabCloseButton"
-            on:click={e => closeMultipleTabs(tab => tabGroup.tabs.find(x => x.tabid == tab.tabid))}
+            class="db-name"
+            class:selected={draggingDbGroup
+              ? tabGroup.grpid == draggingDbGroupTarget?.grpid
+              : tabGroup.tabDbKey == currentDbKey}
+            on:mouseup={e => {
+              if (e.button == 1) {
+                closeMultipleTabs(tab => tabGroup.tabs.find(x => x.tabid == tab.tabid));
+              } else {
+                handleSetDb(tabGroup.tabs[0].props);
+              }
+            }}
+            use:contextMenu={getDatabaseContextMenu(tabGroup.tabs)}
+            style={$connectionColorFactory(
+              tabGroup.tabs[0].props,
+              (draggingDbGroup ? tabGroup.grpid == draggingDbGroupTarget?.grpid : tabGroup.tabDbKey == currentDbKey)
+                ? 2
+                : 3
+            )}
+            draggable={true}
+            on:dragstart={e => {
+              draggingDbGroup = tabGroup;
+            }}
+            on:dragenter={e => {
+              draggingDbGroupTarget = tabGroup;
+            }}
+            on:drop={e => {
+              dragDropTabs(draggingDbGroup.tabs, tabGroup.tabs);
+            }}
+            on:dragend={e => {
+              draggingDbGroup = null;
+              draggingDbGroupTarget = null;
+            }}
           >
-            <FontIcon icon="icon close" />
+            <div class="db-name-inner">
+              <FontIcon icon={getDbIcon(tabGroup.tabDbKey)} />
+              {tabGroup.tabDbName}
+              {#if $connectionList?.find(x => x._id == tabGroup.tabs[0]?.props?.conid)?.isReadOnly}
+                <FontIcon icon="icon lock" />
+              {/if}
+            </div>
+            <div
+              class="close-button-right tabCloseButton"
+              on:click={e => closeMultipleTabs(tab => tabGroup.tabs.find(x => x.tabid == tab.tabid))}
+            >
+              <FontIcon icon="icon close" />
+            </div>
           </div>
-        </div>
+        {/if}
         <div class="db-group">
           {#each tabGroup.tabs as tab}
             <div

packages/web/src/widgets/TitleBar.svelte

@@ -7,9 +7,18 @@
   import { activeTab, currentDatabase } from '../stores';
   import { isMac } from '../utility/common';
   import getElectron from '../utility/getElectron';
+  import { apiOn } from '../utility/api';
 
   $: title = _.compact([$activeTab?.title, $currentDatabase?.name, 'DbGate']).join(' - ');
   const electron = getElectron();
+
+  let isMaximized = false;
+
+  if (electron) {
+    apiOn('setIsMaximized', (maximized: boolean) => {
+      isMaximized = maximized;
+    });
+  }
 </script>
 
 <div class="container" on:dblclick|stopPropagation|preventDefault={() => electron.send('window-action', 'maximize')}>
@@ -27,7 +36,7 @@
         <FontIcon icon="icon window-minimize" />
       </div>
       <div class="button">
-        <FontIcon icon="icon window-restore" on:click={() => electron.send('window-action', 'maximize')} />
+        <FontIcon icon={`icon ${isMaximized ? 'window-restore' : 'window-maximize'}`} on:click={() => electron.send('window-action', 'maximize')} />
       </div>
       <div class="button close-button" on:click={() => electron.send('window-action', 'close')}>
         <FontIcon icon="icon window-close" />

packages/web/src/widgets/WidgetIconPanel.svelte

@@ -7,9 +7,9 @@
     visibleSelectedWidget,
     visibleWidgetSideBar,
     visibleHamburgerMenuWidget,
+    lockedDatabaseMode,
   } from '../stores';
   import mainMenuDefinition from '../../../../app/src/mainMenuDefinition';
-  import { useConfig } from '../utility/metadataLoaders';
   import hasPermission from '../utility/hasPermission';
 
   let domSettings;
@@ -90,8 +90,6 @@
     const items = mainMenuDefinition({ editMenu: false });
     currentDropDownMenu.set({ left, top, items });
   }
-
-  $: config = useConfig();
 </script>
 
 <div class="main">
@@ -112,6 +110,15 @@
 
   <div class="flex1">&nbsp;</div>
 
+  <div
+    class="wrapper"
+    title={`Toggle whether tabs from all databases are visible. Currently - ${$lockedDatabaseMode ? 'NO' : 'YES'}`}
+    on:click={() => {
+      $lockedDatabaseMode = !$lockedDatabaseMode;
+    }}
+  >
+    <FontIcon icon={$lockedDatabaseMode ? 'icon locked-database-mode' : 'icon unlocked-database-mode'} />
+  </div>
   <div class="wrapper" on:click={handleSettingsMenu} bind:this={domSettings}>
     <FontIcon icon="icon settings" />
   </div>

plugins/dbgate-plugin-csv/src/frontend/index.js

@@ -57,7 +57,7 @@ export default {
       }),
     },
     {
-      label: 'CSV file (semicolor separated)',
+      label: 'CSV file (semicolon separated)',
       extension: 'csv',
       createWriter: (fileName) => ({
         functionName: 'writer@dbgate-plugin-csv',

plugins/dbgate-plugin-mongo/package.json

@@ -32,7 +32,7 @@
   },
   "devDependencies": {
     "dbgate-plugin-tools": "^1.0.7",
-    "dbgate-query-splitter": "^4.9.2",
+    "dbgate-query-splitter": "^4.9.3",
     "webpack": "^4.42.0",
     "webpack-cli": "^3.3.11",
     "dbgate-tools": "^5.0.0-alpha.1",