v5.2.2-beta.8

Update sql-formatter, fixes #450

.github/workflows/build-app-beta.yaml

@@ -62,18 +62,13 @@ jobs:
           APPLE_ID: ${{ secrets.APPLE_ID }}
           APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
 
-      - name: Save snap login
-        if: matrix.os == 'ubuntu-18.04'
-        run: 'echo "$SNAPCRAFT_LOGIN" > snapcraft.login'
-        shell: bash
-        env:
-          SNAPCRAFT_LOGIN: ${{secrets.SNAPCRAFT_LOGIN}}
-
       - name: publishSnap
         if: matrix.os == 'ubuntu-18.04'
         run: |
-          snapcraft login --with snapcraft.login
+          snapcraft login
           snapcraft upload --release=beta app/dist/*.snap
+        env:
+          SNAPCRAFT_STORE_CREDENTIALS: ${{secrets.SNAPCRAFT_LOGIN}}
 
       - name: Copy artifacts
         run: |

.github/workflows/build-app.yaml

@@ -72,18 +72,13 @@ jobs:
         run: |
           yarn generatePadFile
 
-      - name: Save snap login
-        if: matrix.os == 'ubuntu-18.04'
-        run: 'echo "$SNAPCRAFT_LOGIN" > snapcraft.login'
-        shell: bash
-        env:
-          SNAPCRAFT_LOGIN: ${{secrets.SNAPCRAFT_LOGIN}}
-
       - name: publishSnap
         if: matrix.os == 'ubuntu-18.04'
         run: |
-          snapcraft login --with snapcraft.login
+          snapcraft login
           snapcraft upload --release=stable app/dist/*.snap
+        env:
+          SNAPCRAFT_STORE_CREDENTIALS: ${{secrets.SNAPCRAFT_LOGIN}}
 
       - name: Copy artifacts
         run: |

.github/workflows/build-npm.yaml

@@ -138,3 +138,8 @@ jobs:
         working-directory: plugins/dbgate-plugin-redis
         run: |
           npm publish
+
+      - name: Publish dbgate-plugin-oracle
+        working-directory: plugins/dbgate-plugin-oracle
+        run: |
+          npm publish

CHANGELOG.md

@@ -8,6 +8,45 @@ Builds:
  - linux - application for linux
  - win - application for Windows
 
+### 5.2.1
+- FIXED: client_id param in OAuth
+- ADDED: OAuth scope parameter
+- FIXED: login page - password was not sent, when submitting by pressing ENTER
+- FIXED: Used permissions fix
+- FIXED: Export modal - fixed crash when selecting different database
+
+### 5.2.0
+- ADDED: Oracle database support #380
+- ADDED: OAuth authentification #407
+- ADDED: Active directory (Windows) authentification #261
+- ADDED: Ask database credentials when login to DB
+- ADDED: Login form instead of simple authorization (simple auth is possible with special configuration)
+- FIXED: MongoDB - connection uri regression
+- ADDED: MongoDB server summary tab
+- FIXED: Broken versioned tables in MariaDB #433
+- CHANGED: Improved editor margin #422
+- ADDED: Implemented camel case search in all search boxes
+- ADDED: MonhoDB filter empty array, not empty array
+- ADDED: Maximize button reflects window state
+- ADDED: MongoDB - database profiler
+- CHANGED: Short JSON values are shown directly in grid
+- FIXED: Fixed filtering nested fields in NDJSON viewer
+- CHANGED: Improved fuzzy search after Ctrl+P #246
+- ADDED: MongoDB: Create collection backup
+- ADDED: Single database mode
+- ADDED: Perspective designer supports joins from MongoDB nested documents and arrays
+- FIXED: Perspective designer joins on MongoDB ObjectId fields
+- ADDED: Filtering columns in designer (query designer, diagram designer, perspective designer)
+- FIXED: Clone MongoDB rows without _id attribute #404
+- CHANGED: Improved cell view with GPS latitude, longitude fields
+- ADDED: SQL: ALTER VIEW and SQL:ALTER PROCEDURE scripts
+- ADDED: Ctrl+F5 refreshes data grid also with database structure #428
+- ADDED: Perspective display modes: text, force text #439
+- FIXED: Fixed file filters #445
+- ADDED: Rename, remove connection folder, memoize opened state after app restart #425
+- FIXED: Show SQLServer alter store procedure #435
+
+
 ### 5.1.6
 - ADDED: Connection folders support #274
 - ADDED: Keyboard shortcut to hide result window and show/hide the side toolbar #406

README.md

@@ -22,6 +22,7 @@ DbGate is licensed under MIT license and is completely free.
 * MySQL
 * PostgreSQL
 * SQL Server
+* Oracle
 * MongoDB
 * Redis
 * SQLite
@@ -66,13 +67,13 @@ DbGate is licensed under MIT license and is completely free.
 * Mongo JavaScript editor, execute Mongo script (with NodeJs syntax)
 * Redis tree view, generate script from keys, run Redis script
 * Runs as application for Windows, Linux and Mac. Or in Docker container on server and in web Browser on client.
-* Import, export from/to CSV, Excel, JSON, XML
+* Import, export from/to CSV, Excel, JSON, NDJSON, XML
 * Free table editor - quick table data editing (cleanup data after import/before export, prototype tables etc.)
-* Archives - backup your data in JSON files on local filesystem (or on DbGate server, when using web application)
+* Archives - backup your data in NDJSON files on local filesystem (or on DbGate server, when using web application)
 * Charts, export chart to HTML page
 * For detailed info, how to run DbGate in docker container, visit [docker hub](https://hub.docker.com/r/dbgate/dbgate)
 * Extensible plugin architecture
-* Perspectives - nested table view over complex relational data
+* Perspectives - nested table view over complex relational data, query designer on MongoDB databases
 
 ## How to contribute
 Any contributions are welcome. If you want to contribute without coding, consider following:
@@ -174,4 +175,8 @@ cd dbgate-plugin-my-new-plugin # this directory is created by wizard, edit, what
 yarn plugin # this compiles plugin and copies it into existing DbGate installation
 ```
 
-After restarting DbGate, you could use your new plugin from DbGate.
+After restarting DbGate, you could use your new plugin from DbGate.
+
+## Logging
+DbGate uses [pinomin logger](https://github.com/dbgate/pinomin). So by default, it produces JSON log messages into console and log files. If you want to see formatted logs, please use [pino-pretty](https://github.com/pinojs/pino-pretty) log formatter.
+

app/package.json

@@ -113,7 +113,7 @@
   },
   "optionalDependencies": {
     "better-sqlite3": "7.6.2",
-    "oracledb": "^5.5.0",
-    "msnodesqlv8": "^2.6.0"
+    "msnodesqlv8": "^2.6.0",
+    "oracledb": "^5.5.0"
   }
 }

app/src/electron.js

@@ -1,6 +1,8 @@
 const electron = require('electron');
 const os = require('os');
 const fs = require('fs');
+// const unhandled = require('electron-unhandled');
+// const { openNewGitHubIssue, debugInfo } = require('electron-util');
 const { Menu, ipcMain } = require('electron');
 const { autoUpdater } = require('electron-updater');
 const log = require('electron-log');
@@ -22,9 +24,25 @@ const configRootPath = path.join(app.getPath('userData'), 'config-root.json');
 let initialConfig = {};
 let apiLoaded = false;
 let mainModule;
+// let getLogger;
+// let loadLogsContent;
 
 const isMac = () => os.platform() == 'darwin';
 
+// unhandled({
+//   showDialog: true,
+//   reportButton: error => {
+//     openNewGitHubIssue({
+//       user: 'dbgate',
+//       repo: 'dbgate',
+//       body: `PLEASE DELETE SENSITIVE INFO BEFORE POSTING ISSUE!!!\n\n\`\`\`\n${
+//         error.stack
+//       }\n\`\`\`\n\n---\n\n${debugInfo()}\n\n\`\`\`\n${loadLogsContent ? loadLogsContent(50) : ''}\n\`\`\``,
+//     });
+//   },
+//   logger: error => (getLogger ? getLogger('electron').fatal(error) : console.error(error)),
+// });
+
 try {
   initialConfig = JSON.parse(fs.readFileSync(configRootPath, { encoding: 'utf-8' }));
 } catch (err) {
@@ -333,9 +351,12 @@ function createWindow() {
     //     path.join(__dirname, process.env.DEVMODE ? '../../packages/api/src/index' : '../packages/api/dist/bundle.js')
     //   )
     // );
+    api.configureLogger();
     const main = api.getMainModule();
     main.useAllControllers(null, electron);
     mainModule = main;
+    // getLogger = api.getLogger;
+    // loadLogsContent = api.loadLogsContent;
     apiLoaded = true;
   }
   mainModule.setElectronSender(mainWindow.webContents);

app/src/mainMenuDefinition.js

@@ -86,6 +86,9 @@ module.exports = ({ editMenu }) => [
       { command: 'sql.generator', hideDisabled: true },
       { command: 'file.import', hideDisabled: true },
       { command: 'new.modelCompare', hideDisabled: true },
+      { divider: true },
+      { command: 'folder.showLogs', hideDisabled: true },
+      { command: 'folder.showData', hideDisabled: true },
     ],
   },
   {

package.json

@@ -1,6 +1,6 @@
 {
   "private": true,
-  "version": "5.1.7-beta.9",
+  "version": "5.2.2-beta.8",
   "name": "dbgate-all",
   "workspaces": [
     "packages/*",
@@ -8,16 +8,17 @@
     "integration-tests"
   ],
   "scripts": {
-    "start:api": "yarn workspace dbgate-api start",
-    "start:app": "cd app && yarn start",
+    "start:api": "yarn workspace dbgate-api start | pino-pretty",
+    "start:api:json": "yarn workspace dbgate-api start",
+    "start:app": "cd app && yarn start | pino-pretty",
     "start:api:debug": "cross-env DEBUG=* yarn workspace dbgate-api start",
     "start:app:debug": "cd app && cross-env DEBUG=* yarn start",
     "start:api:debug:ssh": "cross-env DEBUG=ssh yarn workspace dbgate-api start",
     "start:app:debug:ssh": "cd app && cross-env DEBUG=ssh yarn start",
-    "start:api:portal": "yarn workspace dbgate-api start:portal",
-    "start:api:singledb": "yarn workspace dbgate-api start:singledb",
-    "start:api:auth": "yarn workspace dbgate-api start:auth",
-    "start:api:dblogin": "yarn workspace dbgate-api start:dblogin",
+    "start:api:portal": "yarn workspace dbgate-api start:portal | pino-pretty",
+    "start:api:singledb": "yarn workspace dbgate-api start:singledb | pino-pretty",
+    "start:api:auth": "yarn workspace dbgate-api start:auth | pino-pretty",
+    "start:api:dblogin": "yarn workspace dbgate-api start:dblogin | pino-pretty",
     "start:web": "yarn workspace dbgate-web dev",
     "start:sqltree": "yarn workspace dbgate-sqltree start",
     "start:tools": "yarn workspace dbgate-tools start",
@@ -57,7 +58,8 @@
   },
   "dependencies": {
     "concurrently": "^5.1.0",
-    "patch-package": "^6.2.1"
+    "patch-package": "^6.2.1",
+    "pino-pretty": "^9.1.1"
   },
   "devDependencies": {
     "copyfiles": "^2.2.0",

packages/api/package.json

@@ -49,6 +49,7 @@
     "ncp": "^2.0.0",
     "node-cron": "^2.0.3",
     "on-finished": "^2.4.1",
+    "pinomin": "^1.0.1",
     "portfinder": "^1.0.28",
     "simple-encryptor": "^4.0.0",
     "ssh2": "^1.11.0",
@@ -79,7 +80,7 @@
   },
   "optionalDependencies": {
     "better-sqlite3": "7.6.2",
-    "oracledb": "^5.5.0",
-    "msnodesqlv8": "^2.6.0"
+    "msnodesqlv8": "^2.6.0",
+    "oracledb": "^5.5.0"
   }
 }

packages/api/src/controllers/archive.js

@@ -6,6 +6,9 @@ const socket = require('../utility/socket');
 const { saveFreeTableData } = require('../utility/freeTableStorage');
 const loadFilesRecursive = require('../utility/loadFilesRecursive');
 const getJslFileName = require('../utility/getJslFileName');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('archive');
 
 module.exports = {
   folders_meta: true,
@@ -68,7 +71,7 @@ module.exports = {
         ...fileType('.matview.sql', 'matview.sql'),
       ];
     } catch (err) {
-      console.log('Error reading archive files', err.message);
+      logger.error({ err }, 'Error reading archive files');
       return [];
     }
   },

packages/api/src/controllers/auth.js

@@ -3,8 +3,11 @@ const jwt = require('jsonwebtoken');
 const getExpressPath = require('../utility/getExpressPath');
 const uuidv1 = require('uuid/v1');
 const { getLogins } = require('../utility/hasPermission');
+const { getLogger } = require('dbgate-tools');
 const AD = require('activedirectory2').promiseWrapper;
 
+const logger = getLogger('auth');
+
 const tokenSecret = uuidv1();
 
 function shouldAuthorizeApi() {
@@ -51,7 +54,7 @@ function authMiddleware(req, res, next) {
       return next();
     }
 
-    console.log('Sending invalid token error', err.message);
+    logger.error({ err }, 'Sending invalid token error');
 
     return unauthorizedResponse(req, res, 'invalid token');
   }
@@ -62,20 +65,24 @@ module.exports = {
   async oauthToken(params) {
     const { redirectUri, code } = params;
 
+    const scopeParam = process.env.OAUTH_SCOPE ? `&scope=${process.env.OAUTH_SCOPE}` : '';
     const resp = await axios.default.post(
       `${process.env.OAUTH_TOKEN}`,
       `grant_type=authorization_code&code=${encodeURIComponent(code)}&redirect_uri=${encodeURIComponent(
         redirectUri
-      )}&client_id=${process.env.OAUTH_CLIENT_ID}&client_secret=${process.env.OAUTH_CLIENT_SECRET}`
+      )}&client_id=${process.env.OAUTH_CLIENT_ID}&client_secret=${process.env.OAUTH_CLIENT_SECRET}${scopeParam}`
     );
 
     const { access_token, refresh_token } = resp.data;
 
     const payload = jwt.decode(access_token);
 
-    console.log('User payload returned from OAUTH:', payload);
+    logger.info({ payload }, 'User payload returned from OAUTH');
 
-    const login = process.env.OAUTH_LOGIN_FIELD ? payload[process.env.OAUTH_LOGIN_FIELD] : 'oauth';
+    const login =
+      process.env.OAUTH_LOGIN_FIELD && payload && payload[process.env.OAUTH_LOGIN_FIELD]
+        ? payload[process.env.OAUTH_LOGIN_FIELD]
+        : 'oauth';
 
     if (
       process.env.OAUTH_ALLOWED_LOGINS &&
@@ -113,12 +120,12 @@ module.exports = {
           !process.env.AD_ALLOWED_LOGINS.split(',').find(x => x.toLowerCase().trim() == login.toLowerCase().trim())
         ) {
           return { error: `Username ${login} not allowed to log in` };
-        }        
+        }
         return {
           accessToken: jwt.sign({ login }, tokenSecret, { expiresIn: getTokenLifetime() }),
         };
       } catch (err) {
-        console.log('Failed active directory authentization', err.message);
+        logger.error({ err }, 'Failed active directory authentization');
         return {
           error: err.message,
         };
@@ -129,7 +136,7 @@ module.exports = {
     if (!logins) {
       return { error: 'Logins not configured' };
     }
-    const foundLogin = logins.find(x => x.login == login)
+    const foundLogin = logins.find(x => x.login == login);
     if (foundLogin && foundLogin.password == password) {
       return {
         accessToken: jwt.sign({ login }, tokenSecret, { expiresIn: getTokenLifetime() }),

packages/api/src/controllers/config.js

@@ -2,7 +2,7 @@ const fs = require('fs-extra');
 const os = require('os');
 const path = require('path');
 const axios = require('axios');
-const { datadir } = require('../utility/directories');
+const { datadir, getLogsFilePath } = require('../utility/directories');
 const { hasPermission, getLogins } = require('../utility/hasPermission');
 const socket = require('../utility/socket');
 const _ = require('lodash');
@@ -28,12 +28,9 @@ module.exports = {
   get_meta: true,
   async get(_params, req) {
     const logins = getLogins();
-    const login =
-      req && req.user
-        ? req.user.login
-        : logins
-        ? logins.find(x => x.login == (req && req.auth && req.auth.user))
-        : null;
+    const loginName =
+      req && req.user && req.user.login ? req.user.login : req && req.auth && req.auth.user ? req.auth.user : null;
+    const login = logins && loginName ? logins.find(x => x.login == loginName) : null;
     const permissions = login ? login.permissions : process.env.PERMISSIONS;
 
     return {
@@ -47,8 +44,12 @@ module.exports = {
       permissions,
       login,
       oauth: process.env.OAUTH_AUTH,
+      oauthClient: process.env.OAUTH_CLIENT_ID,
+      oauthScope: process.env.OAUTH_SCOPE,
       oauthLogout: process.env.OAUTH_LOGOUT,
       isLoginForm: !!process.env.AD_URL || (!!logins && !process.env.BASIC_AUTH),
+      logsFilePath: getLogsFilePath(),
+      connectionsFilePath: path.join(datadir(), 'connections.jsonl'),
       ...currentVersion,
     };
   },

packages/api/src/controllers/connections.js

@@ -12,9 +12,12 @@ const { pickSafeConnectionInfo } = require('../utility/crypting');
 const JsonLinesDatabase = require('../utility/JsonLinesDatabase');
 
 const processArgs = require('../utility/processArgs');
-const { safeJsonParse } = require('dbgate-tools');
+const { safeJsonParse, getLogger } = require('dbgate-tools');
 const platformInfo = require('../utility/platformInfo');
 const { connectionHasPermission, testConnectionPermission } = require('../utility/hasPermission');
+const pipeForkLogs = require('../utility/pipeForkLogs');
+
+const logger = getLogger('connections');
 
 let volatileConnections = {};
 
@@ -86,13 +89,13 @@ function getPortalCollections() {
       sslKeyFile: process.env[`SSL_KEY_FILE_${id}`],
       sslRejectUnauthorized: process.env[`SSL_REJECT_UNAUTHORIZED_${id}`],
     }));
-    console.log('Using connections from ENV variables:');
-    console.log(JSON.stringify(connections.map(pickSafeConnectionInfo), undefined, 2));
+
+    logger.info({ connections: connections.map(pickSafeConnectionInfo) }, 'Using connections from ENV variables');
     const noengine = connections.filter(x => !x.engine);
     if (noengine.length > 0) {
-      console.log(
-        'Warning: Invalid CONNECTIONS configutation, missing ENGINE for connection ID:',
-        noengine.map(x => x._id)
+      logger.warn(
+        { connections: noengine.map(x => x._id) },
+        'Invalid CONNECTIONS configutation, missing ENGINE for connection ID'
       );
     }
     return connections;
@@ -203,13 +206,20 @@ module.exports = {
 
   test_meta: true,
   test(connection) {
-    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
-      '--is-forked-api',
-      '--start-process',
-      'connectProcess',
-      ...processArgs.getPassArgs(),
-      // ...process.argv.slice(3),
-    ]);
+    const subprocess = fork(
+      global['API_PACKAGE'] || process.argv[1],
+      [
+        '--is-forked-api',
+        '--start-process',
+        'connectProcess',
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
+      ],
+      {
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+      }
+    );
+    pipeForkLogs(subprocess);
     subprocess.send(connection);
     return new Promise(resolve => {
       subprocess.on('message', resp => {
@@ -283,6 +293,14 @@ module.exports = {
     return res;
   },
 
+  batchChangeFolder_meta: true,
+  async batchChangeFolder({ folder, newFolder }, req) {
+    // const updated = await this.datastore.find(x => x.parent == folder);
+    const res = await this.datastore.updateAll(x => (x.parent == folder ? { ...x, parent: newFolder } : x));
+    socket.emitChanged('connection-list-changed');
+    return res;
+  },
+
   updateDatabase_meta: true,
   async updateDatabase({ conid, database, values }, req) {
     if (portalConnections) return;

packages/api/src/controllers/databaseConnections.js

@@ -12,6 +12,7 @@ const {
   matchPairedObjects,
   extendDatabaseInfo,
   modelCompareDbDiffOptions,
+  getLogger,
 } = require('dbgate-tools');
 const { html, parse } = require('diff2html');
 const { handleProcessCommunication } = require('../utility/processComm');
@@ -28,6 +29,9 @@ const diff2htmlPage = require('../utility/diff2htmlPage');
 const processArgs = require('../utility/processArgs');
 const { testConnectionPermission } = require('../utility/hasPermission');
 const { MissingCredentialsError } = require('../utility/exceptions');
+const pipeForkLogs = require('../utility/pipeForkLogs');
+
+const logger = getLogger('databaseConnections');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedDatabaseConnection[]} */
@@ -60,7 +64,7 @@ module.exports = {
 
   handle_error(conid, database, props) {
     const { error } = props;
-    console.log(`Error in database connection ${conid}, database ${database}: ${error}`);
+    logger.error(`Error in database connection ${conid}, database ${database}: ${error}`);
   },
   handle_response(conid, database, { msgid, ...response }) {
     const [resolve, reject] = this.requests[msgid];
@@ -85,13 +89,20 @@ module.exports = {
     if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') {
       throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode });
     }
-    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
-      '--is-forked-api',
-      '--start-process',
-      'databaseConnectionProcess',
-      ...processArgs.getPassArgs(),
-      // ...process.argv.slice(3),
-    ]);
+    const subprocess = fork(
+      global['API_PACKAGE'] || process.argv[1],
+      [
+        '--is-forked-api',
+        '--start-process',
+        'databaseConnectionProcess',
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
+      ],
+      {
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+      }
+    );
+    pipeForkLogs(subprocess);
     const lastClosed = this.closed[`${conid}/${database}`];
     const newOpened = {
       conid,
@@ -129,7 +140,12 @@ module.exports = {
     const msgid = uuidv1();
     const promise = new Promise((resolve, reject) => {
       this.requests[msgid] = [resolve, reject];
-      conn.subprocess.send({ msgid, ...message });
+      try {
+        conn.subprocess.send({ msgid, ...message });
+      } catch (err) {
+        logger.error({ err }, 'Error sending request do process');
+        this.close(conn.conid, conn.database);
+      }
     });
     return promise;
   },
@@ -137,7 +153,7 @@ module.exports = {
   queryData_meta: true,
   async queryData({ conid, database, sql }, req) {
     testConnectionPermission(conid, req);
-    console.log(`Processing query, conid=${conid}, database=${database}, sql=${sql}`);
+    logger.info({ conid, database, sql }, 'Processing query');
     const opened = await this.ensureOpened(conid, database);
     // if (opened && opened.status && opened.status.name == 'error') {
     //   return opened.status;
@@ -157,7 +173,7 @@ module.exports = {
   runScript_meta: true,
   async runScript({ conid, database, sql }, req) {
     testConnectionPermission(conid, req);
-    console.log(`Processing script, conid=${conid}, database=${database}, sql=${sql}`);
+    logger.info({ conid, database, sql }, 'Processing script');
     const opened = await this.ensureOpened(conid, database);
     const res = await this.sendRequest(opened, { msgtype: 'runScript', sql });
     return res;
@@ -278,6 +294,7 @@ module.exports = {
     if (existing) {
       existing.subprocess.send({ msgtype: 'ping' });
     } else {
+      // @ts-ignore
       existing = await this.ensureOpened(conid, database);
     }
 
@@ -308,7 +325,13 @@ module.exports = {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) {
       existing.disconnected = true;
-      if (kill) existing.subprocess.kill();
+      if (kill) {
+        try {
+          existing.subprocess.kill();
+        } catch (err) {
+          logger.error({ err }, 'Error killing subprocess');
+        }
+      }
       this.opened = this.opened.filter(x => x.conid != conid || x.database != database);
       this.closed[`${conid}/${database}`] = {
         status: {

packages/api/src/controllers/runners.js

@@ -6,10 +6,17 @@ const byline = require('byline');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories');
-const { extractShellApiPlugins, extractShellApiFunctionName, jsonScriptToJavascript } = require('dbgate-tools');
+const {
+  extractShellApiPlugins,
+  extractShellApiFunctionName,
+  jsonScriptToJavascript,
+  getLogger,
+  safeJsonParse,
+} = require('dbgate-tools');
 const { handleProcessCommunication } = require('../utility/processComm');
 const processArgs = require('../utility/processArgs');
 const platformInfo = require('../utility/platformInfo');
+const logger = getLogger('runners');
 
 function extractPlugins(script) {
   const requireRegex = /\s*\/\/\s*@require\s+([^\s]+)\s*\n/g;
@@ -29,13 +36,14 @@ const requirePluginsTemplate = (plugins, isExport) =>
 
 const scriptTemplate = (script, isExport) => `
 const dbgateApi = require(${isExport ? `'dbgate-api'` : 'process.env.DBGATE_API'});
+const logger = dbgateApi.getLogger('script');
 dbgateApi.initializeApiEnvironment();
 ${requirePluginsTemplate(extractPlugins(script), isExport)}
 require=null;
 async function run() {
 ${script}
 await dbgateApi.finalizer.run();
-console.log('Finished job script');
+logger.info('Finished job script');
 }
 dbgateApi.runScript(run);
 `;
@@ -59,19 +67,17 @@ module.exports = {
   requests: {},
 
   dispatchMessage(runid, message) {
-    if (message) console.log('...', message.message);
-    if (_.isString(message)) {
-      socket.emit(`runner-info-${runid}`, {
-        message,
-        time: new Date(),
-        severity: 'info',
-      });
-    }
-    if (_.isPlainObject(message)) {
+    if (message) {
+      const json = safeJsonParse(message.message);
+
+      if (json) logger.info(json);
+      else logger.info(message.message);
+
       socket.emit(`runner-info-${runid}`, {
         time: new Date(),
         severity: 'info',
         ...message,
+        message: json ? json.msg : message.message,
       });
     }
   },
@@ -98,13 +104,15 @@ module.exports = {
     fs.writeFileSync(`${scriptFile}`, scriptText);
     fs.mkdirSync(directory);
     const pluginNames = _.union(fs.readdirSync(pluginsdir()), packagedPluginList);
-    console.log(`RUNNING SCRIPT ${scriptFile}`);
+    logger.info({ scriptFile }, 'Running script');
     // const subprocess = fork(scriptFile, ['--checkParent', '--max-old-space-size=8192'], {
     const subprocess = fork(
       scriptFile,
       [
         '--checkParent', // ...process.argv.slice(3)
         '--is-forked-api',
+        '--process-display-name',
+        'script',
         ...processArgs.getPassArgs(),
       ],
       {
@@ -124,7 +132,7 @@ module.exports = {
     byline(subprocess.stderr).on('data', pipeDispatcher('error'));
     subprocess.on('exit', code => {
       this.rejectRequest(runid, { message: 'No data retured, maybe input data source is too big' });
-      console.log('... EXIT process', code);
+      logger.info({ code, pid: subprocess.pid }, 'Exited process');
       socket.emit(`runner-done-${runid}`, code);
     });
     subprocess.on('error', error => {

packages/api/src/controllers/scheduler.js

@@ -4,6 +4,9 @@ const path = require('path');
 const cron = require('node-cron');
 const runners = require('./runners');
 const { hasPermission } = require('../utility/hasPermission');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('scheduler');
 
 const scheduleRegex = /\s*\/\/\s*@schedule\s+([^\n]+)\n/;
 
@@ -21,7 +24,7 @@ module.exports = {
     if (!match) return;
     const pattern = match[1];
     if (!cron.validate(pattern)) return;
-    console.log(`Schedule script ${file} with pattern ${pattern}`);
+    logger.info(`Schedule script ${file} with pattern ${pattern}`);
     const task = cron.schedule(pattern, () => runners.start({ script: text }));
     this.tasks.push(task);
   },

packages/api/src/controllers/serverConnections.js

@@ -10,6 +10,10 @@ const config = require('./config');
 const processArgs = require('../utility/processArgs');
 const { testConnectionPermission } = require('../utility/hasPermission');
 const { MissingCredentialsError } = require('../utility/exceptions');
+const pipeForkLogs = require('../utility/pipeForkLogs');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('serverConnection');
 
 module.exports = {
   opened: [],
@@ -47,16 +51,26 @@ module.exports = {
       const existing = this.opened.find(x => x.conid == conid);
       if (existing) return existing;
       const connection = await connections.getCore({ conid });
+      if (!connection) {
+        throw new Error(`Connection with conid="${conid}" not fund`);
+      }
       if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') {
         throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode });
       }
-      const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
-        '--is-forked-api',
-        '--start-process',
-        'serverConnectionProcess',
-        ...processArgs.getPassArgs(),
-        // ...process.argv.slice(3),
-      ]);
+      const subprocess = fork(
+        global['API_PACKAGE'] || process.argv[1],
+        [
+          '--is-forked-api',
+          '--start-process',
+          'serverConnectionProcess',
+          ...processArgs.getPassArgs(),
+          // ...process.argv.slice(3),
+        ],
+        {
+          stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+        }
+      );
+      pipeForkLogs(subprocess);
       const newOpened = {
         conid,
         subprocess,
@@ -91,7 +105,13 @@ module.exports = {
     const existing = this.opened.find(x => x.conid == conid);
     if (existing) {
       existing.disconnected = true;
-      if (kill) existing.subprocess.kill();
+      if (kill) {
+        try {
+          existing.subprocess.kill();
+        } catch (err) {
+          logger.error({ err }, 'Error killing subprocess');
+        }
+      }
       this.opened = this.opened.filter(x => x.conid != conid);
       this.closed[conid] = {
         ...existing.status,
@@ -110,6 +130,7 @@ module.exports = {
 
   listDatabases_meta: true,
   async listDatabases({ conid }, req) {
+    if (!conid) return [];
     testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid);
     return opened.databases;
@@ -140,7 +161,12 @@ module.exports = {
         }
         this.lastPinged[conid] = new Date().getTime();
         const opened = await this.ensureOpened(conid);
-        opened.subprocess.send({ msgtype: 'ping' });
+        try {
+          opened.subprocess.send({ msgtype: 'ping' });
+        } catch (err) {
+          logger.error({ err }, 'Error calling ping');
+          this.close(conid);
+        }
       })
     );
     return { status: 'ok' };
@@ -177,7 +203,12 @@ module.exports = {
     const msgid = uuidv1();
     const promise = new Promise((resolve, reject) => {
       this.requests[msgid] = [resolve, reject];
-      conn.subprocess.send({ msgid, ...message });
+      try {
+        conn.subprocess.send({ msgid, ...message });
+      } catch (err) {
+        logger.error({ err }, 'Error sending request');
+        this.close(conn.conid);
+      }
     });
     return promise;
   },

packages/api/src/controllers/sessions.js

@@ -8,6 +8,11 @@ const path = require('path');
 const { handleProcessCommunication } = require('../utility/processComm');
 const processArgs = require('../utility/processArgs');
 const { appdir } = require('../utility/directories');
+const { getLogger } = require('dbgate-tools');
+const pipeForkLogs = require('../utility/pipeForkLogs');
+const config = require('./config');
+
+const logger = getLogger('sessions');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedSession[]} */
@@ -82,13 +87,20 @@ module.exports = {
   async create({ conid, database }) {
     const sesid = uuidv1();
     const connection = await connections.getCore({ conid });
-    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
-      '--is-forked-api',
-      '--start-process',
-      'sessionProcess',
-      ...processArgs.getPassArgs(),
-      // ...process.argv.slice(3),
-    ]);
+    const subprocess = fork(
+      global['API_PACKAGE'] || process.argv[1],
+      [
+        '--is-forked-api',
+        '--start-process',
+        'sessionProcess',
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
+      ],
+      {
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+      }
+    );
+    pipeForkLogs(subprocess);
     const newOpened = {
       conid,
       database,
@@ -109,7 +121,12 @@ module.exports = {
       socket.emit(`session-closed-${sesid}`);
     });
 
-    subprocess.send({ msgtype: 'connect', ...connection, database });
+    subprocess.send({
+      msgtype: 'connect',
+      ...connection,
+      database,
+      globalSettings: await config.getSettings(),
+    });
     return _.pick(newOpened, ['conid', 'database', 'sesid']);
   },
 
@@ -120,7 +137,7 @@ module.exports = {
       throw new Error('Invalid session');
     }
 
-    console.log(`Processing query, sesid=${sesid}, sql=${sql}`);
+    logger.info({ sesid, sql }, 'Processing query');
     this.dispatchMessage(sesid, 'Query execution started');
     session.subprocess.send({ msgtype: 'executeQuery', sql });
 
@@ -158,7 +175,7 @@ module.exports = {
       throw new Error('Invalid session');
     }
 
-    console.log(`Starting profiler, sesid=${sesid}`);
+    logger.info({ sesid }, 'Starting profiler');
     session.loadingReader_jslid = jslid;
     session.subprocess.send({ msgtype: 'startProfiler', jslid });
 

packages/api/src/controllers/uploads.js

@@ -1,6 +1,8 @@
 const path = require('path');
 const { uploadsdir } = require('../utility/directories');
 const uuidv1 = require('uuid/v1');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('uploads');
 
 module.exports = {
   upload_meta: {
@@ -15,7 +17,7 @@ module.exports = {
     }
     const uploadName = uuidv1();
     const filePath = path.join(uploadsdir(), uploadName);
-    console.log(`Uploading file ${data.name}, size=${data.size}`);
+    logger.info(`Uploading file ${data.name}, size=${data.size}`);
 
     data.mv(filePath, () => {
       res.json({

packages/api/src/index.js

@@ -1,5 +1,96 @@
-const shell = require('./shell');
+const { setLogger, getLogger, setLoggerName } = require('dbgate-tools');
 const processArgs = require('./utility/processArgs');
+const fs = require('fs');
+const moment = require('moment');
+const path = require('path');
+const { logsdir, setLogsFilePath, getLogsFilePath } = require('./utility/directories');
+const { createLogger } = require('pinomin');
+
+if (processArgs.startProcess) {
+  setLoggerName(processArgs.startProcess.replace(/Process$/, ''));
+}
+if (processArgs.processDisplayName) {
+  setLoggerName(processArgs.processDisplayName);
+}
+
+// function loadLogsContent(maxLines) {
+//   const text = fs.readFileSync(getLogsFilePath(), { encoding: 'utf8' });
+//   if (maxLines) {
+//     const lines = text
+//       .split('\n')
+//       .map(x => x.trim())
+//       .filter(x => x);
+//     return lines.slice(-maxLines).join('\n');
+//   }
+//   return text;
+// }
+
+function configureLogger() {
+  const logsFilePath = path.join(logsdir(), `${moment().format('YYYY-MM-DD-HH-mm')}-${process.pid}.ndjson`);
+  setLogsFilePath(logsFilePath);
+  setLoggerName('main');
+
+  const logger = createLogger({
+    base: { pid: process.pid },
+    targets: [
+      {
+        type: 'console',
+        // @ts-ignore
+        level: process.env.CONSOLE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+      },
+      {
+        type: 'stream',
+        // @ts-ignore
+        level: process.env.FILE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+        stream: fs.createWriteStream(logsFilePath, { flags: 'a' }),
+      },
+    ],
+  });
+
+  // const streams = [];
+  // if (!platformInfo.isElectron) {
+  //   streams.push({
+  //     stream: process.stdout,
+  //     level: process.env.CONSOLE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  //   });
+  // }
+
+  // streams.push({
+  //   stream: fs.createWriteStream(logsFilePath),
+  //   level: process.env.FILE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  // });
+
+  // let logger = pinoms({
+  //   redact: { paths: ['hostname'], remove: true },
+  //   streams,
+  // });
+
+  // // @ts-ignore
+  // let logger = pino({
+  //   redact: { paths: ['hostname'], remove: true },
+  //   transport: {
+  //     targets: [
+  //       {
+  //         level: process.env.CONSOLE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  //         target: 'pino/file',
+  //       },
+  //       {
+  //         level: process.env.FILE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  //         target: 'pino/file',
+  //         options: { destination: logsFilePath },
+  //       },
+  //     ],
+  //   },
+  // });
+
+  setLogger(logger);
+}
+
+if (processArgs.listenApi) {
+  configureLogger();
+}
+
+const shell = require('./shell');
 const dbgateTools = require('dbgate-tools');
 
 global['DBGATE_TOOLS'] = dbgateTools;
@@ -8,7 +99,7 @@ if (processArgs.startProcess) {
   const proc = require('./proc');
   const module = proc[processArgs.startProcess];
   module.start();
-} 
+}
 
 if (processArgs.listenApi) {
   const main = require('./main');
@@ -17,5 +108,8 @@ if (processArgs.listenApi) {
 
 module.exports = {
   ...shell,
+  getLogger,
+  configureLogger,
+  // loadLogsContent,
   getMainModule: () => require('./main'),
 };

packages/api/src/main.js

@@ -33,6 +33,9 @@ const platformInfo = require('./utility/platformInfo');
 const getExpressPath = require('./utility/getExpressPath');
 const { getLogins } = require('./utility/hasPermission');
 const _ = require('lodash');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('main');
 
 function start() {
   // console.log('process.argv', process.argv);
@@ -60,8 +63,8 @@ function start() {
   } else if (platformInfo.isNpmDist) {
     app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../dbgate-web/public')));
   } else if (process.env.DEVWEB) {
-    console.log('__dirname', __dirname);
-    console.log(path.join(__dirname, '../../web/public/build'));
+    // console.log('__dirname', __dirname);
+    // console.log(path.join(__dirname, '../../web/public/build'));
     app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../web/public')));
   } else {
     app.get(getExpressPath('/'), (req, res) => {
@@ -109,7 +112,7 @@ function start() {
 
   if (platformInfo.isDocker) {
     const port = process.env.PORT || 3000;
-    console.log('DbGate API listening on port (docker build)', port);
+    logger.info(`DbGate API listening on port ${port} (docker build)`);
     server.listen(port);
   } else if (platformInfo.isNpmDist) {
     getPort({
@@ -119,27 +122,27 @@ function start() {
       ),
     }).then(port => {
       server.listen(port, () => {
-        console.log(`DbGate API listening on port ${port} (NPM build)`);
+        logger.info(`DbGate API listening on port ${port} (NPM build)`);
       });
     });
   } else if (process.env.DEVWEB) {
     const port = process.env.PORT || 3000;
-    console.log('DbGate API & web listening on port (dev web build)', port);
+    logger.info(`DbGate API & web listening on port ${port} (dev web build)`);
     server.listen(port);
   } else {
     const port = process.env.PORT || 3000;
-    console.log('DbGate API listening on port (dev API build)', port);
+    logger.info(`DbGate API listening on port ${port} (dev API build)`);
     server.listen(port);
   }
 
   function shutdown() {
-    console.log('\nShutting down DbGate API server');
+    logger.info('\nShutting down DbGate API server');
     server.close(() => {
-      console.log('Server shut down, terminating');
+      logger.info('Server shut down, terminating');
       process.exit(0);
     });
     setTimeout(() => {
-      console.log('Server close timeout, terminating');
+      logger.info('Server close timeout, terminating');
       process.exit(0);
     }, 1000);
   }

packages/api/src/proc/databaseConnectionProcess.js

@@ -1,7 +1,7 @@
 const stableStringify = require('json-stable-stringify');
 const { splitQuery } = require('dbgate-query-splitter');
 const childProcessChecker = require('../utility/childProcessChecker');
-const { extractBoolSettingsValue, extractIntSettingsValue } = require('dbgate-tools');
+const { extractBoolSettingsValue, extractIntSettingsValue, getLogger } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
@@ -9,6 +9,8 @@ const { SqlGenerator } = require('dbgate-tools');
 const generateDeploySql = require('../shell/generateDeploySql');
 const { dumpSqlSelect } = require('dbgate-sqltree');
 
+const logger = getLogger('dbconnProcess');
+
 let systemConnection;
 let storedConnection;
 let afterConnectCallbacks = [];
@@ -269,7 +271,7 @@ async function handleSqlPreview({ msgid, objects, options }) {
     process.send({ msgtype: 'response', msgid, sql: dmp.s, isTruncated: generator.isTruncated });
     if (generator.isUnhandledException) {
       setTimeout(() => {
-        console.log('Exiting because of unhandled exception');
+        getLogger.info('Exiting because of unhandled exception');
         process.exit(0);
       }, 500);
     }
@@ -336,7 +338,7 @@ function start() {
   setInterval(() => {
     const time = new Date().getTime();
     if (time - lastPing > 40 * 1000) {
-      console.log('Database connection not alive, exiting');
+      logger.info('Database connection not alive, exiting');
       process.exit(0);
     }
   }, 10 * 1000);
@@ -345,9 +347,9 @@ function start() {
     if (handleProcessCommunication(message)) return;
     try {
       await handleMessage(message);
-    } catch (e) {
-      console.error('Error in DB connection', e);
-      process.send({ msgtype: 'error', error: e.message });
+    } catch (err) {
+      logger.error({ err }, 'Error in DB connection');
+      process.send({ msgtype: 'error', error: err.message });
     }
   });
 }

packages/api/src/proc/serverConnectionProcess.js

@@ -1,9 +1,10 @@
 const stableStringify = require('json-stable-stringify');
-const { extractBoolSettingsValue, extractIntSettingsValue } = require('dbgate-tools');
+const { extractBoolSettingsValue, extractIntSettingsValue, getLogger } = require('dbgate-tools');
 const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
+const logger = getLogger('srvconnProcess');
 
 let systemConnection;
 let storedConnection;
@@ -101,7 +102,7 @@ async function handleDatabaseOp(op, { name }) {
   } else {
     const dmp = driver.createDumper();
     dmp[op](name);
-    console.log(`RUNNING SCRIPT: ${dmp.s}`);
+    logger.info({ sql: dmp.s }, 'Running script');
     await driver.query(systemConnection, dmp.s);
   }
   await handleRefresh();
@@ -146,7 +147,7 @@ function start() {
   setInterval(() => {
     const time = new Date().getTime();
     if (time - lastPing > 40 * 1000) {
-      console.log('Server connection not alive, exiting');
+      logger.info('Server connection not alive, exiting');
       process.exit(0);
     }
   }, 10 * 1000);

packages/api/src/proc/sessionProcess.js

@@ -10,12 +10,16 @@ const requireEngineDriver = require('../utility/requireEngineDriver');
 const { decryptConnection } = require('../utility/crypting');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
+const { getLogger, extractIntSettingsValue, extractBoolSettingsValue } = require('dbgate-tools');
+
+const logger = getLogger('sessionProcess');
 
 let systemConnection;
 let storedConnection;
 let afterConnectCallbacks = [];
 // let currentHandlers = [];
 let lastPing = null;
+let lastActivity = null;
 let currentProfiler = null;
 
 class TableWriter {
@@ -212,6 +216,8 @@ function waitConnected() {
 }
 
 async function handleStartProfiler({ jslid }) {
+  lastActivity = new Date().getTime();
+
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
 
@@ -230,6 +236,8 @@ async function handleStartProfiler({ jslid }) {
 }
 
 async function handleStopProfiler({ jslid }) {
+  lastActivity = new Date().getTime();
+
   const driver = requireEngineDriver(storedConnection);
   currentProfiler.writer.close();
   driver.stopProfiler(systemConnection, currentProfiler);
@@ -237,6 +245,8 @@ async function handleStopProfiler({ jslid }) {
 }
 
 async function handleExecuteQuery({ sql }) {
+  lastActivity = new Date().getTime();
+
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
 
@@ -270,6 +280,8 @@ async function handleExecuteQuery({ sql }) {
 }
 
 async function handleExecuteReader({ jslid, sql, fileName }) {
+  lastActivity = new Date().getTime();
+
   await waitConnected();
 
   const driver = requireEngineDriver(storedConnection);
@@ -325,7 +337,20 @@ function start() {
   setInterval(() => {
     const time = new Date().getTime();
     if (time - lastPing > 25 * 1000) {
-      console.log('Session not alive, exiting');
+      logger.info('Session not alive, exiting');
+      process.exit(0);
+    }
+
+    const useSessionTimeout =
+      storedConnection && storedConnection.globalSettings
+        ? extractBoolSettingsValue(storedConnection.globalSettings, 'session.autoClose', true)
+        : false;
+    const sessionTimeout =
+      storedConnection && storedConnection.globalSettings
+        ? extractIntSettingsValue(storedConnection.globalSettings, 'session.autoCloseTimeout', 15, 1, 120)
+        : 15;
+    if (useSessionTimeout && time - lastActivity > sessionTimeout * 60 * 1000 && !currentProfiler) {
+      logger.info('Session not active, exiting');
       process.exit(0);
     }
   }, 10 * 1000);

packages/api/src/proc/sshForwardProcess.js

@@ -3,6 +3,9 @@ const platformInfo = require('../utility/platformInfo');
 const childProcessChecker = require('../utility/childProcessChecker');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { SSHConnection } = require('../utility/SSHConnection');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('sshProcess');
 
 async function getSshConnection(connection) {
   const sshConfig = {
@@ -35,8 +38,8 @@ async function handleStart({ connection, tunnelConfig }) {
       tunnelConfig,
     });
   } catch (err) {
-    console.log('Error creating SSH tunnel connection:', err.message);
-    
+    logger.error({ err }, 'Error creating SSH tunnel connection:');
+
     process.send({
       msgtype: 'error',
       connection,

packages/api/src/shell/archiveWriter.js

@@ -3,11 +3,14 @@ const fs = require('fs');
 const { archivedir, resolveArchiveFolder } = require('../utility/directories');
 // const socket = require('../utility/socket');
 const jsonLinesWriter = require('./jsonLinesWriter');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger();
 
 function archiveWriter({ folderName, fileName }) {
   const dir = resolveArchiveFolder(folderName);
   if (!fs.existsSync(dir)) {
-    console.log(`Creating directory ${dir}`);
+    logger.info(`Creating directory ${dir}`);
     fs.mkdirSync(dir);
   }
   const jsonlFile = path.join(dir, `${fileName}.jsonl`);

packages/api/src/shell/dumpDatabase.js

@@ -1,5 +1,8 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('dumpDb');
 
 function doDump(dumper) {
   return new Promise((resolve, reject) => {
@@ -21,11 +24,11 @@ async function dumpDatabase({
   databaseName,
   schemaName,
 }) {
-  console.log(`Dumping database`);
+  logger.info(`Dumping database`);
 
   if (!driver) driver = requireEngineDriver(connection);
   const pool = systemConnection || (await connectUtility(driver, connection, 'read', { forceRowsAsObjects: true }));
-  console.log(`Connected.`);
+  logger.info(`Connected.`);
 
   const dumper = await driver.createBackupDumper(pool, {
     outputFile,

packages/api/src/shell/executeQuery.js

@@ -1,12 +1,15 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('execQuery');
 
 async function executeQuery({ connection = undefined, systemConnection = undefined, driver = undefined, sql }) {
-  console.log(`Execute query ${sql}`);
+  logger.info({ sql }, `Execute query`);
 
   if (!driver) driver = requireEngineDriver(connection);
   const pool = systemConnection || (await connectUtility(driver, connection, 'script'));
-  console.log(`Connected.`);
+  logger.info(`Connected.`);
 
   await driver.script(pool, sql);
 }

packages/api/src/shell/importDatabase.js

@@ -4,6 +4,9 @@ const connectUtility = require('../utility/connectUtility');
 const { splitQueryStream } = require('dbgate-query-splitter/lib/splitQueryStream');
 const download = require('./download');
 const stream = require('stream');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('importDb');
 
 class ImportStream extends stream.Transform {
   constructor(pool, driver) {
@@ -38,11 +41,11 @@ function awaitStreamEnd(stream) {
 }
 
 async function importDatabase({ connection = undefined, systemConnection = undefined, driver = undefined, inputFile }) {
-  console.log(`Importing database`);
+  logger.info(`Importing database`);
 
   if (!driver) driver = requireEngineDriver(connection);
   const pool = systemConnection || (await connectUtility(driver, connection, 'write'));
-  console.log(`Connected.`);
+  logger.info(`Connected.`);
 
   const downloadedFile = await download(inputFile);
 

packages/api/src/shell/jsonArrayWriter.js

@@ -1,6 +1,9 @@
+const { getLogger } = require('dbgate-tools');
 const fs = require('fs');
 const stream = require('stream');
 
+const logger = getLogger('jsonArrayWriter');
+
 class StringifyStream extends stream.Transform {
   constructor() {
     super({ objectMode: true });
@@ -38,7 +41,7 @@ class StringifyStream extends stream.Transform {
 }
 
 async function jsonArrayWriter({ fileName, encoding = 'utf-8' }) {
-  console.log(`Writing file ${fileName}`);
+  logger.info(`Writing file ${fileName}`);
   const stringify = new StringifyStream();
   const fileStream = fs.createWriteStream(fileName, encoding);
   stringify.pipe(fileStream);

packages/api/src/shell/jsonLinesReader.js

@@ -1,6 +1,8 @@
 const fs = require('fs');
 const stream = require('stream');
 const byline = require('byline');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('jsonLinesReader');
 
 class ParseStream extends stream.Transform {
   constructor({ limitRows }) {
@@ -31,7 +33,7 @@ class ParseStream extends stream.Transform {
 }
 
 async function jsonLinesReader({ fileName, encoding = 'utf-8', limitRows = undefined }) {
-  console.log(`Reading file ${fileName}`);
+  logger.info(`Reading file ${fileName}`);
 
   const fileStream = fs.createReadStream(fileName, encoding);
   const liner = byline(fileStream);

packages/api/src/shell/jsonLinesWriter.js

@@ -1,5 +1,7 @@
+const { getLogger } = require('dbgate-tools');
 const fs = require('fs');
 const stream = require('stream');
+const logger = getLogger('jsonLinesWriter');
 
 class StringifyStream extends stream.Transform {
   constructor({ header }) {
@@ -21,7 +23,7 @@ class StringifyStream extends stream.Transform {
 }
 
 async function jsonLinesWriter({ fileName, encoding = 'utf-8', header = true }) {
-  console.log(`Writing file ${fileName}`);
+  logger.info(`Writing file ${fileName}`);
   const stringify = new StringifyStream({ header });
   const fileStream = fs.createWriteStream(fileName, encoding);
   stringify.pipe(fileStream);

packages/api/src/shell/queryReader.js

@@ -1,5 +1,7 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('queryReader');
 
 async function queryReader({
   connection,
@@ -14,12 +16,12 @@ async function queryReader({
   // if (!sql && !json) {
   //   throw new Error('One of sql or json must be set');
   // }
-  console.log(`Reading query ${query || sql}`);
+  logger.info({ sql: query || sql }, `Reading query`);
   // else console.log(`Reading query ${JSON.stringify(json)}`);
 
   const driver = requireEngineDriver(connection);
   const pool = await connectUtility(driver, connection, queryType == 'json' ? 'read' : 'script');
-  console.log(`Connected.`);
+  logger.info(`Connected.`);
   const reader =
     queryType == 'json' ? await driver.readJsonQuery(pool, query) : await driver.readQuery(pool, query || sql);
   return reader;

packages/api/src/shell/requirePlugin.js

@@ -3,6 +3,8 @@ const fs = require('fs');
 const { pluginsdir, packagedPluginsDir, getPluginBackendPath } = require('../utility/directories');
 const nativeModules = require('../nativeModules');
 const platformInfo = require('../utility/platformInfo');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('requirePlugin');
 
 const loadedPlugins = {};
 
@@ -17,7 +19,7 @@ function requirePlugin(packageName, requiredPlugin = null) {
   if (requiredPlugin == null) {
     let module;
     const modulePath = getPluginBackendPath(packageName);
-    console.log(`Loading module ${packageName} from ${modulePath}`);
+    logger.info(`Loading module ${packageName} from ${modulePath}`);
     try {
       // @ts-ignore
       module = __non_webpack_require__(modulePath);

packages/api/src/shell/runScript.js

@@ -1,5 +1,7 @@
+const { getLogger } = require('dbgate-tools');
 const childProcessChecker = require('../utility/childProcessChecker');
 const processArgs = require('../utility/processArgs');
+const logger = getLogger();
 
 async function runScript(func) {
   if (processArgs.checkParent) {
@@ -9,7 +11,7 @@ async function runScript(func) {
     await func();
     process.exit(0);
   } catch (err) {
-    console.log(err);
+    logger.error('Error running script', err);
     process.exit(1);
   }
 }

packages/api/src/shell/sqlDataWriter.js

@@ -1,8 +1,9 @@
 const fs = require('fs');
 const stream = require('stream');
 const path = require('path');
-const { driverBase } = require('dbgate-tools');
+const { driverBase, getLogger } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
+const logger = getLogger('sqlDataWriter');
 
 class SqlizeStream extends stream.Transform {
   constructor({ fileName, dataName }) {
@@ -40,7 +41,7 @@ class SqlizeStream extends stream.Transform {
 }
 
 async function sqlDataWriter({ fileName, dataName, driver, encoding = 'utf-8' }) {
-  console.log(`Writing file ${fileName}`);
+  logger.info(`Writing file ${fileName}`);
   const stringify = new SqlizeStream({ fileName, dataName });
   const fileStream = fs.createWriteStream(fileName, encoding);
   stringify.pipe(fileStream);

packages/api/src/shell/tableReader.js

@@ -1,17 +1,18 @@
-const { quoteFullName, fullNameToString } = require('dbgate-tools');
+const { quoteFullName, fullNameToString, getLogger } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
+const logger = getLogger('tableReader');
 
 async function tableReader({ connection, pureName, schemaName }) {
   const driver = requireEngineDriver(connection);
   const pool = await connectUtility(driver, connection, 'read');
-  console.log(`Connected.`);
+  logger.info(`Connected.`);
 
   const fullName = { pureName, schemaName };
 
   if (driver.databaseEngineTypes.includes('document')) {
     // @ts-ignore
-    console.log(`Reading collection ${fullNameToString(fullName)}`);
+    logger.info(`Reading collection ${fullNameToString(fullName)}`);
     // @ts-ignore
     return await driver.readQuery(pool, JSON.stringify(fullName));
   }
@@ -20,14 +21,14 @@ async function tableReader({ connection, pureName, schemaName }) {
   const query = `select * from ${quoteFullName(driver.dialect, fullName)}`;
   if (table) {
     // @ts-ignore
-    console.log(`Reading table ${fullNameToString(table)}`);
+    logger.info(`Reading table ${fullNameToString(table)}`);
     // @ts-ignore
     return await driver.readQuery(pool, query, table);
   }
   const view = await driver.analyseSingleObject(pool, fullName, 'views');
   if (view) {
     // @ts-ignore
-    console.log(`Reading view ${fullNameToString(view)}`);
+    logger.info(`Reading view ${fullNameToString(view)}`);
     // @ts-ignore
     return await driver.readQuery(pool, query, view);
   }

packages/api/src/shell/tableWriter.js

@@ -1,16 +1,17 @@
-const { fullNameToString } = require('dbgate-tools');
+const { fullNameToString, getLogger } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
+const logger = getLogger('tableWriter');
 
 async function tableWriter({ connection, schemaName, pureName, driver, systemConnection, ...options }) {
-  console.log(`Writing table ${fullNameToString({ schemaName, pureName })}`);
+  logger.info(`Writing table ${fullNameToString({ schemaName, pureName })}`);
 
   if (!driver) {
     driver = requireEngineDriver(connection);
   }
   const pool = systemConnection || (await connectUtility(driver, connection, 'write'));
 
-  console.log(`Connected.`);
+  logger.info(`Connected.`);
   return await driver.writeTable(pool, { schemaName, pureName }, options);
 }
 

packages/api/src/utility/DatastoreProxy.js

@@ -2,6 +2,9 @@ const { fork } = require('child_process');
 const uuidv1 = require('uuid/v1');
 const { handleProcessCommunication } = require('./processComm');
 const processArgs = require('../utility/processArgs');
+const pipeForkLogs = require('./pipeForkLogs');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('DatastoreProxy');
 
 class DatastoreProxy {
   constructor(file) {
@@ -30,13 +33,20 @@ class DatastoreProxy {
 
   async ensureSubprocess() {
     if (!this.subprocess) {
-      this.subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
-        '--is-forked-api',
-        '--start-process',
-        'jslDatastoreProcess',
-        ...processArgs.getPassArgs(),
-        // ...process.argv.slice(3),
-      ]);
+      this.subprocess = fork(
+        global['API_PACKAGE'] || process.argv[1],
+        [
+          '--is-forked-api',
+          '--start-process',
+          'jslDatastoreProcess',
+          ...processArgs.getPassArgs(),
+          // ...process.argv.slice(3),
+        ],
+        {
+          stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+        }
+      );
+      pipeForkLogs(this.subprocess);
 
       this.subprocess.on('message', message => {
         // @ts-ignore
@@ -60,7 +70,12 @@ class DatastoreProxy {
     const msgid = uuidv1();
     const promise = new Promise((resolve, reject) => {
       this.requests[msgid] = [resolve, reject];
-      this.subprocess.send({ msgtype: 'read', msgid, offset, limit });
+      try {
+        this.subprocess.send({ msgtype: 'read', msgid, offset, limit });
+      } catch (err) {
+        logger.error({ err }, 'Error getting rows');
+        this.subprocess = null;
+      }
     });
     return promise;
   }
@@ -69,7 +84,12 @@ class DatastoreProxy {
     const msgid = uuidv1();
     const promise = new Promise((resolve, reject) => {
       this.requests[msgid] = [resolve, reject];
-      this.subprocess.send({ msgtype: 'notify', msgid });
+      try {
+        this.subprocess.send({ msgtype: 'notify', msgid });
+      } catch (err) {
+        logger.error({ err }, 'Error notifying subprocess');
+        this.subprocess = null;
+      }
     });
     return promise;
   }

packages/api/src/utility/JsonLinesDatabase.js

@@ -90,6 +90,12 @@ class JsonLinesDatabase {
     return obj;
   }
 
+  async updateAll(mapFunction) {
+    await this._ensureLoaded();
+    this.data = this.data.map(mapFunction);
+    await this._save();
+  }
+
   async patch(id, values) {
     await this._ensureLoaded();
     this.data = this.data.map(x => (x._id == id ? { ...x, ...values } : x));

packages/api/src/utility/childProcessChecker.js

@@ -1,14 +1,18 @@
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('childProcessChecked');
+
 let counter = 0;
 
 function childProcessChecker() {
   setInterval(() => {
     try {
       process.send({ msgtype: 'ping', counter: counter++ });
-    } catch (ex) {
+    } catch (err) {
       // This will come once parent dies.
       // One way can be to check for error code ERR_IPC_CHANNEL_CLOSED
       //     and call process.exit()
-      console.log('parent died', ex.toString());
+      logger.error({ err }, 'parent died');
       process.exit(1);
     }
   }, 1000);

packages/api/src/utility/cleanDirectory.js

@@ -2,7 +2,7 @@ const fs = require('fs-extra');
 const path = require('path');
 const ageSeconds = 3600;
 
-async function cleanDirectory(directory) {
+async function cleanDirectory(directory, age = undefined) {
   const files = await fs.readdir(directory);
   const now = new Date().getTime();
 
@@ -10,7 +10,7 @@ async function cleanDirectory(directory) {
     const full = path.join(directory, file);
     const stat = await fs.stat(full);
     const mtime = stat.mtime.getTime();
-    const expirationTime = mtime + ageSeconds * 1000;
+    const expirationTime = mtime + (age || ageSeconds) * 1000;
     if (now > expirationTime) {
       if (stat.isDirectory()) {
         await fs.rmdir(full, { recursive: true });

packages/api/src/utility/directories.js

@@ -1,20 +1,24 @@
 const os = require('os');
 const path = require('path');
 const fs = require('fs');
+const _ = require('lodash');
 const cleanDirectory = require('./cleanDirectory');
 const platformInfo = require('./platformInfo');
 const processArgs = require('./processArgs');
 const consoleObjectWriter = require('../shell/consoleObjectWriter');
+const { getLogger } = require('dbgate-tools');
+
+let logsFilePath;
 
 const createDirectories = {};
 const ensureDirectory = (dir, clean) => {
   if (!createDirectories[dir]) {
     if (clean && fs.existsSync(dir) && !platformInfo.isForkedApi) {
-      console.log(`Cleaning directory ${dir}`);
-      cleanDirectory(dir);
+      getLogger('directories').info(`Cleaning directory ${dir}`);
+      cleanDirectory(dir, _.isNumber(clean) ? clean : null);
     }
     if (!fs.existsSync(dir)) {
-      console.log(`Creating directory ${dir}`);
+      getLogger('directories').info(`Creating directory ${dir}`);
       fs.mkdirSync(dir);
     }
     createDirectories[dir] = true;
@@ -38,7 +42,7 @@ function datadir() {
   return dir;
 }
 
-const dirFunc = (dirname, clean = false) => () => {
+const dirFunc = (dirname, clean) => () => {
   const dir = path.join(datadir(), dirname);
   ensureDirectory(dir, clean);
 
@@ -52,6 +56,7 @@ const pluginsdir = dirFunc('plugins');
 const archivedir = dirFunc('archive');
 const appdir = dirFunc('apps');
 const filesdir = dirFunc('files');
+const logsdir = dirFunc('logs', 3600 * 24 * 7);
 
 function packagedPluginsDir() {
   // console.log('CALL DIR FROM', new Error('xxx').stack);
@@ -127,11 +132,19 @@ function migrateDataDir() {
     if (fs.existsSync(oldDir) && !fs.existsSync(newDir)) {
       fs.renameSync(oldDir, newDir);
     }
-  } catch (e) {
-    console.log('Error migrating data dir:', e.message);
+  } catch (err) {
+    getLogger('directories').error({ err }, 'Error migrating data dir');
   }
 }
 
+function setLogsFilePath(value) {
+  logsFilePath = value;
+}
+
+function getLogsFilePath() {
+  return logsFilePath;
+}
+
 migrateDataDir();
 
 module.exports = {
@@ -144,9 +157,12 @@ module.exports = {
   ensureDirectory,
   pluginsdir,
   filesdir,
+  logsdir,
   packagedPluginsDir,
   packagedPluginList,
   getPluginBackendPath,
   resolveArchiveFolder,
   clearArchiveLinksCache,
+  getLogsFilePath,
+  setLogsFilePath,
 };

packages/api/src/utility/pipeForkLogs.js

@@ -0,0 +1,19 @@
+const byline = require('byline');
+const { safeJsonParse, getLogger } = require('dbgate-tools');
+const logger = getLogger();
+
+const logDispatcher = method => data => {
+  const json = safeJsonParse(data.toString());
+  if (json && json.level) {
+    logger.log(json);
+  } else {
+    logger[method](json || data.toString());
+  }
+};
+
+function pipeForkLogs(subprocess) {
+  byline(subprocess.stdout).on('data', logDispatcher('info'));
+  byline(subprocess.stderr).on('data', logDispatcher('error'));
+}
+
+module.exports = pipeForkLogs;

packages/api/src/utility/processArgs.js

@@ -11,6 +11,7 @@ const startProcess = getNamedArg('--start-process');
 const isForkedApi = process.argv.includes('--is-forked-api');
 const pluginsDir = getNamedArg('--plugins-dir');
 const workspaceDir = getNamedArg('--workspace-dir');
+const processDisplayName = getNamedArg('--process-display-name');
 const listenApi = process.argv.includes('--listen-api');
 const listenApiChild = process.argv.includes('--listen-api-child') || listenApi;
 
@@ -37,4 +38,5 @@ module.exports = {
   workspaceDir,
   listenApi,
   listenApiChild,
+  processDisplayName,
 };

packages/api/src/utility/sshTunnel.js

@@ -5,6 +5,9 @@ const AsyncLock = require('async-lock');
 const lock = new AsyncLock();
 const { fork } = require('child_process');
 const processArgs = require('../utility/processArgs');
+const { getLogger } = require('dbgate-tools');
+const pipeForkLogs = require('./pipeForkLogs');
+const logger = getLogger('sshTunnel');
 
 const sshTunnelCache = {};
 
@@ -21,18 +24,24 @@ const CONNECTION_FIELDS = [
 const TUNNEL_FIELDS = [...CONNECTION_FIELDS, 'server', 'port'];
 
 function callForwardProcess(connection, tunnelConfig, tunnelCacheKey) {
-  let subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
-    '--is-forked-api',
-    '--start-process',
-    'sshForwardProcess',
-    ...processArgs.getPassArgs(),
-  ]);
+  let subprocess = fork(
+    global['API_PACKAGE'] || process.argv[1],
+    ['--is-forked-api', '--start-process', 'sshForwardProcess', ...processArgs.getPassArgs()],
+    {
+      stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+    }
+  );
+  pipeForkLogs(subprocess);
 
-  subprocess.send({
-    msgtype: 'connect',
-    connection,
-    tunnelConfig,
-  });
+  try {
+    subprocess.send({
+      msgtype: 'connect',
+      connection,
+      tunnelConfig,
+    });
+  } catch (err) {
+    logger.error({ err }, 'Error connecting SSH');
+  }
   return new Promise((resolve, reject) => {
     subprocess.on('message', resp => {
       // @ts-ignore
@@ -45,7 +54,7 @@ function callForwardProcess(connection, tunnelConfig, tunnelCacheKey) {
       }
     });
     subprocess.on('exit', code => {
-      console.log('SSH forward process exited');
+      logger.info('SSH forward process exited');
       delete sshTunnelCache[tunnelCacheKey];
     });
   });
@@ -65,13 +74,13 @@ async function getSshTunnel(connection) {
       toHost: connection.server,
     };
     try {
-      console.log(
+      logger.info(
         `Creating SSH tunnel to ${connection.sshHost}-${connection.server}:${connection.port}, using local port ${localPort}`
       );
 
       const subprocess = await callForwardProcess(connection, tunnelConfig, tunnelCacheKey);
 
-      console.log(
+      logger.info(
         `Created SSH tunnel to ${connection.sshHost}-${connection.server}:${connection.port}, using local port ${localPort}`
       );
 

packages/api/src/utility/sshTunnelProxy.js

@@ -1,11 +1,17 @@
+const { getLogger } = require('dbgate-tools');
 const uuidv1 = require('uuid/v1');
 const { getSshTunnel } = require('./sshTunnel');
+const logger = getLogger('sshTunnelProxy');
 
 const dispatchedMessages = {};
 
 async function handleGetSshTunnelRequest({ msgid, connection }, subprocess) {
   const response = await getSshTunnel(connection);
-  subprocess.send({ msgtype: 'getsshtunnel-response', msgid, response });
+  try {
+    subprocess.send({ msgtype: 'getsshtunnel-response', msgid, response });
+  } catch (err) {
+    logger.error({ err }, 'Error sending to SSH tunnel');
+  }
 }
 
 function handleGetSshTunnelResponse({ msgid, response }, subprocess) {

packages/api/src/utility/useController.js

@@ -2,7 +2,9 @@ const _ = require('lodash');
 const express = require('express');
 const getExpressPath = require('./getExpressPath');
 const { MissingCredentialsError } = require('./exceptions');
+const { getLogger } = require('dbgate-tools');
 
+const logger = getLogger('useController');
 /**
  * @param {string} route
  */
@@ -10,11 +12,11 @@ module.exports = function useController(app, electron, route, controller) {
   const router = express.Router();
 
   if (controller._init) {
-    console.log(`Calling init controller for controller ${route}`);
+    logger.info(`Calling init controller for controller ${route}`);
     try {
       controller._init();
     } catch (err) {
-      console.log(`Error initializing controller, exiting application`, err);
+      logger.error({ err }, `Error initializing controller, exiting application`);
       process.exit(1);
     }
   }
@@ -75,16 +77,16 @@ module.exports = function useController(app, electron, route, controller) {
         try {
           const data = await controller[key]({ ...req.body, ...req.query }, req);
           res.json(data);
-        } catch (e) {
-          console.log(e);
-          if (e instanceof MissingCredentialsError) {
+        } catch (err) {
+          logger.error({ err }, `Error when processing route ${route}/${key}`);
+          if (err instanceof MissingCredentialsError) {
             res.json({
               missingCredentials: true,
               apiErrorMessage: 'Missing credentials',
-              detail: e.detail,
+              detail: err.detail,
             });
           } else {
-            res.status(500).json({ apiErrorMessage: e.message });
+            res.status(500).json({ apiErrorMessage: err.message });
           }
         }
       });

packages/datalib/src/FormViewDisplay.ts

@@ -1,120 +0,0 @@
-import _ from 'lodash';
-import { GridConfig, GridCache, GridConfigColumns, createGridCache, GroupFunc } from './GridConfig';
-import type { TableInfo, EngineDriver, DatabaseInfo, SqlDialect } from 'dbgate-types';
-import { getFilterValueExpression } from 'dbgate-filterparser';
-import { ChangeCacheFunc, ChangeConfigFunc, DisplayColumn } from './GridDisplay';
-
-export class FormViewDisplay {
-  isLoadedCorrectly = true;
-  columns: DisplayColumn[];
-  public baseTable: TableInfo;
-  dialect: SqlDialect;
-
-  constructor(
-    public config: GridConfig,
-    protected setConfig: ChangeConfigFunc,
-    public cache: GridCache,
-    protected setCache: ChangeCacheFunc,
-    public driver?: EngineDriver,
-    public dbinfo: DatabaseInfo = null,
-    public serverVersion = null
-  ) {
-    this.dialect = (driver?.dialectByVersion && driver?.dialectByVersion(serverVersion)) || driver?.dialect;
-  }
-
-  addFilterColumn(column) {
-    if (!column) return;
-    this.setConfig(cfg => ({
-      ...cfg,
-      formFilterColumns: [...(cfg.formFilterColumns || []), column.uniqueName],
-    }));
-  }
-
-  filterCellValue(column, rowData) {
-    if (!column || !rowData) return;
-    const value = rowData[column.uniqueName];
-    const expr = getFilterValueExpression(value, column.dataType);
-    if (expr) {
-      this.setConfig(cfg => ({
-        ...cfg,
-        filters: {
-          ...cfg.filters,
-          [column.uniqueName]: expr,
-        },
-        addedColumns: cfg.addedColumns.includes(column.uniqueName)
-          ? cfg.addedColumns
-          : [...cfg.addedColumns, column.uniqueName],
-      }));
-      this.reload();
-    }
-  }
-
-  setFilter(uniqueName, value) {
-    this.setConfig(cfg => ({
-      ...cfg,
-      filters: {
-        ...cfg.filters,
-        [uniqueName]: value,
-      },
-    }));
-    this.reload();
-  }
-
-  removeFilter(uniqueName) {
-    const reloadRequired = !!this.config.filters[uniqueName];
-    this.setConfig(cfg => ({
-      ...cfg,
-      formFilterColumns: (cfg.formFilterColumns || []).filter(x => x != uniqueName),
-      filters: _.omit(cfg.filters || [], uniqueName),
-    }));
-    if (reloadRequired) this.reload();
-  }
-
-  reload() {
-    this.setCache(cache => ({
-      // ...cache,
-      ...createGridCache(),
-      refreshTime: new Date().getTime(),
-    }));
-  }
-
-  getKeyValue(columnName) {
-    const { formViewKey, formViewKeyRequested } = this.config;
-    if (formViewKeyRequested && formViewKeyRequested[columnName]) return formViewKeyRequested[columnName];
-    if (formViewKey && formViewKey[columnName]) return formViewKey[columnName];
-    return null;
-  }
-
-  requestKeyValue(columnName, value) {
-    if (this.getKeyValue(columnName) == value) return;
-
-    this.setConfig(cfg => ({
-      ...cfg,
-      formViewKeyRequested: {
-        ...cfg.formViewKey,
-        ...cfg.formViewKeyRequested,
-        [columnName]: value,
-      },
-    }));
-    this.reload();
-  }
-
-  extractKey(row) {
-    if (!row || !this.baseTable || !this.baseTable.primaryKey) {
-      return null;
-    }
-    const formViewKey = _.pick(
-      row,
-      this.baseTable.primaryKey.columns.map(x => x.columnName)
-    );
-    return formViewKey;
-  }
-
-  cancelRequestKey(rowData) {
-    this.setConfig(cfg => ({
-      ...cfg,
-      formViewKeyRequested: null,
-      formViewKey: rowData ? this.extractKey(rowData) : cfg.formViewKey,
-    }));
-  }
-}

packages/datalib/src/GridConfig.ts

@@ -27,8 +27,7 @@ export interface GridConfig extends GridConfigColumns {
   childConfig?: GridConfig;
   reference?: GridReferenceDefinition;
   isFormView?: boolean;
-  formViewKey?: { [uniqueName: string]: string };
-  formViewKeyRequested?: { [uniqueName: string]: string };
+  formViewRecordNumber?: number;
   formFilterColumns: string[];
   formColumnFilterText?: string;
 }

packages/datalib/src/GridDisplay.ts

@@ -70,6 +70,7 @@ export abstract class GridDisplay {
   }
   dialect: SqlDialect;
   columns: DisplayColumn[];
+  formColumns: DisplayColumn[] = [];
   baseTable?: TableInfo;
   baseView?: ViewInfo;
   baseCollection?: CollectionInfo;
@@ -329,6 +330,7 @@ export abstract class GridDisplay {
         ...cfg.filters,
         [uniqueName]: value,
       },
+      formViewRecordNumber: 0,
     }));
     this.reload();
   }
@@ -351,6 +353,7 @@ export abstract class GridDisplay {
     this.setConfig(cfg => ({
       ...cfg,
       filters: _.omit(cfg.filters, [uniqueName]),
+      formFilterColumns: (cfg.formFilterColumns || []).filter(x => x != uniqueName),
     }));
     this.reload();
   }
@@ -718,22 +721,11 @@ export abstract class GridDisplay {
     };
   }
 
-  switchToFormView(rowData) {
-    if (!this.baseTable) return;
-    const { primaryKey } = this.baseTable;
-    if (!primaryKey) return;
-    const { columns } = primaryKey;
-
+  switchToFormView(rowIndex) {
     this.setConfig(cfg => ({
       ...cfg,
       isFormView: true,
-      formViewKey: rowData
-        ? _.pick(
-            rowData,
-            columns.map(x => x.columnName)
-          )
-        : null,
-      formViewKeyRequested: null,
+      formViewRecordNumber: rowIndex,
     }));
   }
 
@@ -743,6 +735,36 @@ export abstract class GridDisplay {
       isJsonView: true,
     }));
   }
+
+  formViewNavigate(command, allRowCount) {
+    switch (command) {
+      case 'begin':
+        this.setConfig(cfg => ({
+          ...cfg,
+          formViewRecordNumber: 0,
+        }));
+        break;
+      case 'previous':
+        this.setConfig(cfg => ({
+          ...cfg,
+          formViewRecordNumber: Math.max((cfg.formViewRecordNumber || 0) - 1, 0),
+        }));
+        break;
+      case 'next':
+        this.setConfig(cfg => ({
+          ...cfg,
+          formViewRecordNumber: Math.max((cfg.formViewRecordNumber || 0) + 1, 0),
+        }));
+        break;
+      case 'end':
+        this.setConfig(cfg => ({
+          ...cfg,
+          formViewRecordNumber: Math.max(allRowCount - 1, 0),
+        }));
+        break;
+    }
+    this.reload();
+  }
 }
 
 export function reloadDataCacheFunc(cache: GridCache): GridCache {

packages/datalib/src/JslGridDisplay.ts

@@ -48,5 +48,7 @@ export class JslGridDisplay extends GridDisplay {
     }
 
     if (!this.columns) this.columns = [];
+
+    this.formColumns = this.columns;
   }
 }

packages/datalib/src/PerspectiveCache.ts

@@ -5,6 +5,7 @@ import _difference from 'lodash/difference';
 import debug from 'debug';
 import stableStringify from 'json-stable-stringify';
 import { PerspectiveDataPattern } from './PerspectiveDataPattern';
+import { perspectiveValueMatcher } from './perspectiveTools';
 
 const dbg = debug('dbgate:PerspectiveCache');
 
@@ -17,7 +18,9 @@ export class PerspectiveBindingGroup {
   bindingValues: any[];
 
   matchRow(row) {
-    return this.table.bindingColumns.every((column, index) => row[column] == this.bindingValues[index]);
+    return this.table.bindingColumns.every((column, index) =>
+    perspectiveValueMatcher(row[column], this.bindingValues[index])
+    );
   }
 }
 
@@ -69,7 +72,11 @@ export class PerspectiveCacheTable {
   }
 
   storeGroupSize(props: PerspectiveDataLoadProps, bindingValues: any[], count: number) {
-    const originalBindingValue = props.bindingValues.find(v => _zip(v, bindingValues).every(([x, y]) => x == y));
+    const originalBindingValue = props.bindingValues.find(v =>
+      _zip(v, bindingValues).every(([x, y]) => perspectiveValueMatcher(x, y))
+    );
+    // console.log('storeGroupSize NEW', bindingValues);
+    // console.log('storeGroupSize ORIGINAL', originalBindingValue);
     if (originalBindingValue) {
       const key = stableStringify(originalBindingValue);
       // console.log('SET SIZE', originalBindingValue, bindingValues, key, count);

packages/datalib/src/PerspectiveDataPattern.ts

@@ -51,7 +51,7 @@ function addObjectToColumns(columns: PerspectiveDataPatternColumn[], row) {
       if (!column.types.includes(type)) {
         column.types.push(type);
       }
-      if (_isPlainObject(value)) {
+      if (_isPlainObject(value) && type != 'oid') {
         addObjectToColumns(column.columns, value);
       }
       if (_isArray(value)) {

packages/datalib/src/PerspectiveDataProvider.ts

@@ -47,6 +47,7 @@ export class PerspectiveDataProvider {
 
   async loadDataNested(props: PerspectiveDataLoadProps): Promise<{ rows: any[]; incomplete: boolean }> {
     const tableCache = this.cache.getTableCache(props);
+    // console.log('loadDataNested', props);
 
     const uncached = tableCache.getUncachedBindingGroups(props);
     if (uncached.length > 0) {
@@ -54,7 +55,7 @@ export class PerspectiveDataProvider {
         ...props,
         bindingValues: uncached,
       });
-      // console.log('COUNTS', counts);
+      // console.log('loadDataNested COUNTS', counts);
       for (const resetItem of uncached) {
         tableCache.storeGroupSize(props, resetItem, 0);
       }
@@ -196,6 +197,14 @@ export class PerspectiveDataProvider {
       },
     });
 
+    if (!nextRows) {
+      // return tableCache.getRowsResult(props);
+      return {
+        rows: [],
+        incomplete: false,
+      };
+    }
+
     if (nextRows.errorMessage) {
       throw new Error(nextRows.errorMessage);
     }

packages/datalib/src/PerspectiveTreeNode.ts

@@ -22,6 +22,7 @@ import {
   PerspectiveReferenceConfig,
 } from './PerspectiveConfig';
 import _isEqual from 'lodash/isEqual';
+import _isArray from 'lodash/isArray';
 import _cloneDeep from 'lodash/cloneDeep';
 import _compact from 'lodash/compact';
 import _uniq from 'lodash/uniq';
@@ -38,6 +39,11 @@ import { Condition, Expression, Select } from 'dbgate-sqltree';
 // import { getPerspectiveDefaultColumns } from './getPerspectiveDefaultColumns';
 import uuidv1 from 'uuid/v1';
 import { PerspectiveDataPatternColumn } from './PerspectiveDataPattern';
+import {
+  getPerspectiveMostNestedChildColumnName,
+  getPerspectiveParentColumnName,
+  perspectiveValueMatcher,
+} from './perspectiveTools';
 
 export interface PerspectiveDataLoadPropsWithNode {
   props: PerspectiveDataLoadProps;
@@ -137,6 +143,16 @@ export abstract class PerspectiveTreeNode {
   get generatesDataGridColumn() {
     return this.isCheckedColumn;
   }
+  get validParentDesignerId() {
+    if (this.designerId) return this.designerId;
+    return this.parentNode?.validParentDesignerId;
+  }
+  get preloadedLevelData() {
+    return false;
+  }
+  get findByDesignerIdWithoutDesignerId() {
+    return false;
+  }
   matchChildRow(parentRow: any, childRow: any): boolean {
     return true;
   }
@@ -210,7 +226,7 @@ export abstract class PerspectiveTreeNode {
 
   get hasUncheckedNodeInPath() {
     if (!this.parentNode) return false;
-    if (!this.isCheckedNode) return true;
+    if (this.designerId && !this.isCheckedNode) return true;
     return this.parentNode.hasUncheckedNodeInPath;
   }
 
@@ -404,7 +420,7 @@ export abstract class PerspectiveTreeNode {
   // }
 
   findNodeByDesignerId(designerId: string): PerspectiveTreeNode {
-    if (!this.designerId) {
+    if (!this.designerId && !this.findByDesignerIdWithoutDesignerId) {
       return null;
     }
     if (!designerId) {
@@ -758,15 +774,22 @@ export class PerspectivePatternColumnNode extends PerspectiveTreeNode {
   ) {
     super(dbs, config, setConfig, parentNode, dataProvider, databaseConfig, designerId);
     this.parentNodeConfig = this.tableNodeOrParent?.nodeConfig;
+    // console.log('PATTERN COLUMN', column);
   }
 
   get isChildColumn() {
     return this.parentNode instanceof PerspectivePatternColumnNode;
   }
 
+  get findByDesignerIdWithoutDesignerId() {
+    return this.isExpandable;
+  }
+
   // matchChildRow(parentRow: any, childRow: any): boolean {
-  //   if (!this.foreignKey) return false;
-  //   return parentRow[this.foreignKey.columns[0].columnName] == childRow[this.foreignKey.columns[0].refColumnName];
+  //   console.log('MATCH PATTENR ROW', parentRow, childRow);
+  //   return false;
+  //   // if (!this.foreignKey) return false;
+  //   // return parentRow[this.foreignKey.columns[0].columnName] == childRow[this.foreignKey.columns[0].refColumnName];
   // }
 
   // getChildMatchColumns() {
@@ -808,12 +831,19 @@ export class PerspectivePatternColumnNode extends PerspectiveTreeNode {
   // }
 
   getNodeLoadProps(parentRows: any[]): PerspectiveDataLoadProps {
+    // console.log('GETTING PATTERN', parentRows);
     return null;
   }
 
-  get generatesHiearchicGridColumn() {
+  get generatesHiearchicGridColumn(): boolean {
+    // return true;
     // console.log('generatesHiearchicGridColumn', this.parentTableNode?.nodeConfig?.checkedColumns, this.codeName + '::');
-    return !!this.tableNodeOrParent?.nodeConfig?.checkedColumns?.find(x => x.startsWith(this.codeName + '::'));
+    if (this.tableNodeOrParent?.nodeConfig?.checkedColumns?.find(x => x.startsWith(this.codeName + '::'))) {
+      return true;
+    }
+    // return false;
+
+    return this.hasCheckedJoinChild();
   }
 
   // get generatesHiearchicGridColumn() {
@@ -859,7 +889,11 @@ export class PerspectivePatternColumnNode extends PerspectiveTreeNode {
     return 'mongo';
   }
 
-  generateChildNodes(): PerspectiveTreeNode[] {
+  get preloadedLevelData() {
+    return true;
+  }
+
+  generatePatternChildNodes(): PerspectivePatternColumnNode[] {
     return this.column.columns.map(
       column =>
         new PerspectivePatternColumnNode(
@@ -875,7 +909,92 @@ export class PerspectivePatternColumnNode extends PerspectiveTreeNode {
           null
         )
     );
-    return [];
+  }
+
+  hasCheckedJoinChild() {
+    for (const node of this.childNodes) {
+      if (node instanceof PerspectivePatternColumnNode) {
+        if (node.hasCheckedJoinChild()) return true;
+      }
+      if (node.isCheckedNode) return true;
+    }
+    return false;
+  }
+
+  generateChildNodes(): PerspectiveTreeNode[] {
+    const patternChildren = this.generatePatternChildNodes();
+
+    const customs = [];
+    // console.log('GETTING CHILDREN', this.config.nodes, this.config.references);
+    for (const node of this.config.nodes) {
+      for (const ref of this.config.references) {
+        const validDesignerId = this.validParentDesignerId;
+        if (
+          (ref.sourceId == validDesignerId && ref.targetId == node.designerId) ||
+          (ref.targetId == validDesignerId && ref.sourceId == node.designerId)
+        ) {
+          // console.log('TESTING REF', ref, this.codeName);
+          if (ref.columns.length != 1) continue;
+          // console.log('CP1');
+          if (
+            ref.sourceId == validDesignerId &&
+            this.codeName == getPerspectiveParentColumnName(ref.columns[0].source)
+          ) {
+            if (ref.columns[0].target.includes('::')) continue;
+          } else if (
+            ref.targetId == validDesignerId &&
+            this.codeName == getPerspectiveParentColumnName(ref.columns[0].target)
+          ) {
+            if (ref.columns[0].source.includes('::')) continue;
+          } else {
+            continue;
+          }
+          // console.log('CP2');
+
+          const newConfig = { ...this.databaseConfig };
+          if (node.conid) newConfig.conid = node.conid;
+          if (node.database) newConfig.database = node.database;
+          const db = this.dbs?.[newConfig.conid]?.[newConfig.database];
+          const table = db?.tables?.find(x => x.pureName == node.pureName && x.schemaName == node.schemaName);
+          const view = db?.views?.find(x => x.pureName == node.pureName && x.schemaName == node.schemaName);
+          const collection = db?.collections?.find(x => x.pureName == node.pureName && x.schemaName == node.schemaName);
+
+          const join: PerspectiveCustomJoinConfig = {
+            refNodeDesignerId: node.designerId,
+            referenceDesignerId: ref.designerId,
+            baseDesignerId: validDesignerId,
+            joinName: node.alias,
+            refTableName: node.pureName,
+            refSchemaName: node.schemaName,
+            conid: node.conid,
+            database: node.database,
+            columns:
+              ref.sourceId == validDesignerId
+                ? ref.columns.map(col => ({ baseColumnName: col.source, refColumnName: col.target }))
+                : ref.columns.map(col => ({ baseColumnName: col.target, refColumnName: col.source })),
+          };
+
+          if (table || view || collection) {
+            customs.push(
+              new PerspectiveCustomJoinTreeNode(
+                join,
+                table || view || collection,
+                this.dbs,
+                this.config,
+                this.setConfig,
+                this.dataProvider,
+                newConfig,
+                this,
+                node.designerId
+              )
+            );
+          }
+        }
+      }
+    }
+
+    return [...patternChildren, ...customs];
+    // return [];
     // if (!this.foreignKey) return [];
     // const tbl = this?.db?.tables?.find(
     //   x => x.pureName == this.foreignKey?.refTableName && x.schemaName == this.foreignKey?.refSchemaName
@@ -1153,8 +1272,14 @@ export class PerspectiveCustomJoinTreeNode extends PerspectiveTableNode {
   }
 
   matchChildRow(parentRow: any, childRow: any): boolean {
+    // console.log('MATCH ROW', parentRow, childRow);
     for (const column of this.customJoin.columns) {
-      if (parentRow[column.baseColumnName] != childRow[column.refColumnName]) {
+      if (
+        !perspectiveValueMatcher(
+          parentRow[getPerspectiveMostNestedChildColumnName(column.baseColumnName)],
+          childRow[column.refColumnName]
+        )
+      ) {
         return false;
       }
     }
@@ -1171,17 +1296,68 @@ export class PerspectiveCustomJoinTreeNode extends PerspectiveTableNode {
 
   getNodeLoadProps(parentRows: any[]): PerspectiveDataLoadProps {
     // console.log('CUSTOM JOIN', this.customJoin);
+    // console.log('PARENT ROWS', parentRows);
+
     // console.log('this.getDataLoadColumns()', this.getDataLoadColumns());
     const isMongo = isCollectionInfo(this.table);
 
+    // const bindingValues = [];
+
+    // for (const row of parentRows) {
+    //   const rowBindingValueArrays = [];
+    //   for (const col of this.customJoin.columns) {
+    //     const path = col.baseColumnName.split('::');
+    //     const values = [];
+
+    //     function processSubpath(parent, subpath) {
+    //       if (subpath.length == 0) {
+    //         values.push(parent);
+    //         return;
+    //       }
+    //       if (parent == null) {
+    //         return;
+    //       }
+
+    //       const obj = parent[subpath[0]];
+    //       if (_isArray(obj)) {
+    //         for (const elem of obj) {
+    //           processSubpath(elem, subpath.slice(1));
+    //         }
+    //       } else {
+    //         processSubpath(obj, subpath.slice(1));
+    //       }
+    //     }
+
+    //     processSubpath(row, path);
+
+    //     rowBindingValueArrays.push(values);
+    //   }
+
+    //   const valueCount = Math.max(...rowBindingValueArrays.map(x => x.length));
+
+    //   for (let i = 0; i < valueCount; i += 1) {
+    //     const value = Array(this.customJoin.columns.length);
+    //     for (let col = 0; col < this.customJoin.columns.length; col++) {
+    //       value[col] = rowBindingValueArrays[col][i % rowBindingValueArrays[col].length];
+    //     }
+    //     bindingValues.push(value);
+    //   }
+    // }
+    const bindingValues = parentRows.map(row =>
+      this.customJoin.columns.map(x => row[getPerspectiveMostNestedChildColumnName(x.baseColumnName)])
+    );
+
+    // console.log('bindingValues', bindingValues);
+    // console.log(
+    //   'bindingValues UNIQ',
+    //   _uniqBy(bindingValues, x => JSON.stringify(x))
+    // );
+
     return {
       schemaName: this.table.schemaName,
       pureName: this.table.pureName,
       bindingColumns: this.getParentMatchColumns(),
-      bindingValues: _uniqBy(
-        parentRows.map(row => this.customJoin.columns.map(x => row[x.baseColumnName])),
-        stableStringify
-      ),
+      bindingValues: _uniqBy(bindingValues, x => JSON.stringify(x)),
       dataColumns: this.getDataLoadColumns(),
       allColumns: isMongo,
       databaseConfig: this.databaseConfig,
@@ -1412,6 +1588,9 @@ export function getTableChildPerspectiveNodes(
         (ref.sourceId == parentNode.designerId && ref.targetId == node.designerId) ||
         (ref.targetId == parentNode.designerId && ref.sourceId == node.designerId)
       ) {
+        if (ref.columns.find(x => x.source.includes('::') || x.target.includes('::'))) {
+          continue;
+        }
         const newConfig = { ...databaseConfig };
         if (node.conid) newConfig.conid = node.conid;
         if (node.database) newConfig.database = node.database;

packages/datalib/src/TableFormViewDisplay.ts

@@ -1,272 +0,0 @@
-import { FormViewDisplay } from './FormViewDisplay';
-import _ from 'lodash';
-import { ChangeCacheFunc, DisplayColumn, ChangeConfigFunc } from './GridDisplay';
-import type { EngineDriver, NamedObjectInfo, DatabaseInfo } from 'dbgate-types';
-import { GridConfig, GridCache } from './GridConfig';
-import { mergeConditions, Condition, OrderByExpression } from 'dbgate-sqltree';
-import { TableGridDisplay } from './TableGridDisplay';
-import stableStringify from 'json-stable-stringify';
-import { ChangeSetFieldDefinition, ChangeSetRowDefinition } from './ChangeSet';
-import { DictionaryDescriptionFunc } from '.';
-
-export class TableFormViewDisplay extends FormViewDisplay {
-  // use utility functions from GridDisplay and publish result in FromViewDisplay interface
-  private gridDisplay: TableGridDisplay;
-
-  constructor(
-    public tableName: NamedObjectInfo,
-    driver: EngineDriver,
-    config: GridConfig,
-    setConfig: ChangeConfigFunc,
-    cache: GridCache,
-    setCache: ChangeCacheFunc,
-    dbinfo: DatabaseInfo,
-    displayOptions,
-    serverVersion,
-    getDictionaryDescription: DictionaryDescriptionFunc = null,
-    isReadOnly = false
-  ) {
-    super(config, setConfig, cache, setCache, driver, dbinfo, serverVersion);
-    this.gridDisplay = new TableGridDisplay(
-      tableName,
-      driver,
-      config,
-      setConfig,
-      cache,
-      setCache,
-      dbinfo,
-      displayOptions,
-      serverVersion,
-      getDictionaryDescription,
-      isReadOnly
-    );
-    this.gridDisplay.addAllExpandedColumnsToSelected = true;
-
-    this.isLoadedCorrectly = this.gridDisplay.isLoadedCorrectly && !!this.driver;
-    this.columns = [];
-    this.addDisplayColumns(this.gridDisplay.columns);
-    this.baseTable = this.gridDisplay.baseTable;
-    this.gridDisplay.hintBaseColumns = this.columns;
-  }
-
-  addDisplayColumns(columns: DisplayColumn[]) {
-    for (const col of columns) {
-      this.columns.push(col);
-      if (this.gridDisplay.isExpandedColumn(col.uniqueName)) {
-        const table = this.gridDisplay.getFkTarget(col);
-        if (table) {
-          const subcolumns = this.gridDisplay.getDisplayColumns(table, col.uniquePath);
-          this.addDisplayColumns(subcolumns);
-        }
-      }
-    }
-  }
-
-  getPrimaryKeyEqualCondition(row = null): Condition {
-    if (!row) row = this.config.formViewKeyRequested || this.config.formViewKey;
-    if (!row) return null;
-    const { primaryKey } = this.gridDisplay.baseTable;
-    if (!primaryKey) return null;
-    return {
-      conditionType: 'and',
-      conditions: primaryKey.columns.map(({ columnName }) => ({
-        conditionType: 'binary',
-        operator: '=',
-        left: {
-          exprType: 'column',
-          columnName,
-          source: {
-            alias: 'basetbl',
-          },
-        },
-        right: {
-          exprType: 'value',
-          value: row[columnName],
-        },
-      })),
-    };
-  }
-
-  getPrimaryKeyOperatorCondition(operator): Condition {
-    if (!this.config.formViewKey) return null;
-    const conditions = [];
-
-    const { primaryKey } = this.gridDisplay.baseTable;
-    if (!primaryKey) return null;
-    for (let index = 0; index < primaryKey.columns.length; index++) {
-      conditions.push({
-        conditionType: 'and',
-        conditions: [
-          ...primaryKey.columns.slice(0, index).map(({ columnName }) => ({
-            conditionType: 'binary',
-            operator: '=',
-            left: {
-              exprType: 'column',
-              columnName,
-              source: {
-                alias: 'basetbl',
-              },
-            },
-            right: {
-              exprType: 'value',
-              value: this.config.formViewKey[columnName],
-            },
-          })),
-          ...primaryKey.columns.slice(index).map(({ columnName }) => ({
-            conditionType: 'binary',
-            operator: operator,
-            left: {
-              exprType: 'column',
-              columnName,
-              source: {
-                alias: 'basetbl',
-              },
-            },
-            right: {
-              exprType: 'value',
-              value: this.config.formViewKey[columnName],
-            },
-          })),
-        ],
-      });
-    }
-
-    if (conditions.length == 1) {
-      return conditions[0];
-    }
-
-    return {
-      conditionType: 'or',
-      conditions,
-    };
-  }
-
-  getSelect() {
-    if (!this.driver) return null;
-    const select = this.gridDisplay.createSelect();
-    if (!select) return null;
-    select.topRecords = 1;
-    return select;
-  }
-
-  getCurrentRowQuery() {
-    const select = this.getSelect();
-    if (!select) return null;
-
-    select.where = mergeConditions(select.where, this.getPrimaryKeyEqualCondition());
-    return select;
-  }
-
-  getCountSelect() {
-    const select = this.getSelect();
-    if (!select) return null;
-    select.orderBy = null;
-    select.columns = [
-      {
-        exprType: 'raw',
-        sql: 'COUNT(*)',
-        alias: 'count',
-      },
-    ];
-    select.topRecords = null;
-    return select;
-  }
-
-  getCountQuery() {
-    if (!this.driver) return null;
-    const select = this.getCountSelect();
-    if (!select) return null;
-    return select;
-  }
-
-  getBeforeCountQuery() {
-    if (!this.driver) return null;
-    const select = this.getCountSelect();
-    if (!select) return null;
-    select.where = mergeConditions(select.where, this.getPrimaryKeyOperatorCondition('<'));
-    return select;
-  }
-
-  navigate(row) {
-    const formViewKey = this.extractKey(row);
-    this.setConfig(cfg => ({
-      ...cfg,
-      formViewKey,
-    }));
-  }
-
-  isLoadedCurrentRow(row) {
-    if (!row) return false;
-    const formViewKey = this.extractKey(row);
-    return stableStringify(formViewKey) == stableStringify(this.config.formViewKey);
-  }
-
-  navigateRowQuery(commmand: 'begin' | 'previous' | 'next' | 'end') {
-    if (!this.driver) return null;
-    const select = this.gridDisplay.createSelect();
-    if (!select) return null;
-    const { primaryKey } = this.gridDisplay.baseTable;
-
-    function getOrderBy(direction): OrderByExpression[] {
-      return primaryKey.columns.map(({ columnName }) => ({
-        exprType: 'column',
-        columnName,
-        direction,
-      }));
-    }
-
-    select.topRecords = 1;
-    switch (commmand) {
-      case 'begin':
-        select.orderBy = getOrderBy('ASC');
-        break;
-      case 'end':
-        select.orderBy = getOrderBy('DESC');
-        break;
-      case 'previous':
-        select.orderBy = getOrderBy('DESC');
-        select.where = mergeConditions(select.where, this.getPrimaryKeyOperatorCondition('<'));
-        break;
-      case 'next':
-        select.orderBy = getOrderBy('ASC');
-        select.where = mergeConditions(select.where, this.getPrimaryKeyOperatorCondition('>'));
-        break;
-    }
-
-    return select;
-  }
-
-  getChangeSetRow(row): ChangeSetRowDefinition {
-    if (!this.baseTable) return null;
-    return {
-      pureName: this.baseTable.pureName,
-      schemaName: this.baseTable.schemaName,
-      condition: this.extractKey(row),
-    };
-  }
-
-  getChangeSetField(row, uniqueName): ChangeSetFieldDefinition {
-    const col = this.columns.find(x => x.uniqueName == uniqueName);
-    if (!col) return null;
-    if (!this.baseTable) return null;
-    if (this.baseTable.pureName != col.pureName || this.baseTable.schemaName != col.schemaName) return null;
-    return {
-      ...this.getChangeSetRow(row),
-      uniqueName: uniqueName,
-      columnName: col.columnName,
-    };
-  }
-
-  toggleExpandedColumn(uniqueName: string, value?: boolean) {
-    this.gridDisplay.toggleExpandedColumn(uniqueName, value);
-    this.gridDisplay.reload();
-  }
-
-  isExpandedColumn(uniqueName: string) {
-    return this.gridDisplay.isExpandedColumn(uniqueName);
-  }
-
-  get editable() {
-    return this.gridDisplay.editable;
-  }
-}

packages/datalib/src/TableGridDisplay.ts

@@ -51,6 +51,7 @@ export class TableGridDisplay extends GridDisplay {
     }
 
     this.columns = this.getDisplayColumns(this.table, []);
+    this.addFormDisplayColumns(this.getDisplayColumns(this.table, []));
     this.filterable = true;
     this.sortable = true;
     this.groupable = true;
@@ -62,6 +63,24 @@ export class TableGridDisplay extends GridDisplay {
         ? this.table.primaryKey.columns.map(x => x.columnName)
         : this.table.columns.map(x => x.columnName);
     }
+    
+    if (this.config.isFormView) {
+      this.addAllExpandedColumnsToSelected = true;
+      this.hintBaseColumns = this.formColumns;
+    }
+  }
+
+  addFormDisplayColumns(columns) {
+    for (const col of columns) {
+      this.formColumns.push(col);
+      if (this.isExpandedColumn(col.uniqueName)) {
+        const table = this.getFkTarget(col);
+        if (table) {
+          const subcolumns = this.getDisplayColumns(table, col.uniquePath);
+          this.addFormDisplayColumns(subcolumns);
+        }
+      }
+    }
   }
 
   findTable({ schemaName = undefined, pureName }) {

packages/datalib/src/ViewGridDisplay.ts

@@ -15,6 +15,7 @@ export class ViewGridDisplay extends GridDisplay {
   ) {
     super(config, setConfig, cache, setCache, driver, serverVersion);
     this.columns = this.getDisplayColumns(view);
+    this.formColumns = this.columns;
     this.filterable = true;
     this.sortable = true;
     this.groupable = false;

packages/datalib/src/index.ts

@@ -10,8 +10,8 @@ export * from './FreeTableGridDisplay';
 export * from './FreeTableModel';
 export * from './MacroDefinition';
 export * from './runMacro';
-export * from './FormViewDisplay';
-export * from './TableFormViewDisplay';
+// export * from './FormViewDisplay';
+// export * from './TableFormViewDisplay';
 export * from './CollectionGridDisplay';
 export * from './deleteCascade';
 export * from './PerspectiveDisplay';
@@ -21,3 +21,4 @@ export * from './PerspectiveConfig';
 export * from './processPerspectiveDefaultColunns';
 export * from './PerspectiveDataPattern';
 export * from './PerspectiveDataLoader';
+export * from './perspectiveTools';

packages/datalib/src/perspectiveTools.ts

@@ -0,0 +1,22 @@
+export function getPerspectiveParentColumnName(columnName: string) {
+  const path = columnName.split('::');
+  if (path.length >= 2) return path.slice(0, -1).join('::');
+  return null;
+}
+
+export function getPerspectiveMostNestedChildColumnName(columnName: string) {
+  const path = columnName.split('::');
+  return path[path.length - 1];
+}
+
+// export function perspectiveValueMatcher(value1, value2): boolean {
+//   if (value1?.$oid && value2?.$oid) return value1.$oid == value2.$oid;
+//   if (Array.isArray(value1)) return !!value1.find(x => perspectiveValueMatcher(x, value2));
+//   if (Array.isArray(value2)) return !!value2.find(x => perspectiveValueMatcher(value1, x));
+//   return value1 == value2;
+// }
+
+export function perspectiveValueMatcher(value1, value2): boolean {
+  if (value1?.$oid && value2?.$oid) return value1.$oid == value2.$oid;
+  return value1 == value2;
+}

packages/filterparser/src/filterTool.ts

@@ -1,4 +1,4 @@
-import { isTypeDateTime } from 'dbgate-tools';
+import { arrayToHexString, isTypeDateTime } from 'dbgate-tools';
 import moment from 'moment';
 
 export type FilterMultipleValuesMode = 'is' | 'is_not' | 'contains' | 'begins' | 'ends';
@@ -9,6 +9,10 @@ export function getFilterValueExpression(value, dataType?) {
   if (value === true) return 'TRUE';
   if (value === false) return 'FALSE';
   if (value.$oid) return `ObjectId("${value.$oid}")`;
+  if (value.type == 'Buffer' && Array.isArray(value.data)) {
+    return '0x' + arrayToHexString(value.data);
+  }
+
   return `="${value}"`;
 }
 

packages/filterparser/src/parseFilter.ts

@@ -5,6 +5,7 @@ import { Condition } from 'dbgate-sqltree';
 import { interpretEscapes, token, word, whitespace } from './common';
 import { mongoParser } from './mongoParser';
 import { datetimeParser } from './datetimeParser';
+import { hexStringToArray } from 'dbgate-tools';
 
 const binaryCondition = operator => value => ({
   conditionType: 'binary',
@@ -104,6 +105,14 @@ const createParser = (filterType: FilterType) => {
         .map(Number)
         .desc('number'),
 
+    hexstring: () =>
+      token(P.regexp(/0x(([0-9a-fA-F][0-9a-fA-F])+)/, 1))
+        .map(x => ({
+          type: 'Buffer',
+          data: hexStringToArray(x),
+        }))
+        .desc('hex string'),
+
     noQuotedString: () => P.regexp(/[^\s^,^'^"]+/).desc('string unquoted'),
 
     sql: () =>
@@ -113,6 +122,7 @@ const createParser = (filterType: FilterType) => {
 
     value: r => P.alt(...allowedValues.map(x => r[x])),
     valueTestEq: r => r.value.map(binaryCondition('=')),
+    hexTestEq: r => r.hexstring.map(binaryCondition('=')),
     valueTestStr: r => r.value.map(likeCondition('like', '%#VALUE#%')),
 
     comma: () => word(','),
@@ -158,7 +168,7 @@ const createParser = (filterType: FilterType) => {
     allowedElements.push('le', 'ge', 'lt', 'gt');
   }
   if (filterType == 'string') {
-    allowedElements.push('empty', 'notEmpty');
+    allowedElements.push('empty', 'notEmpty', 'hexTestEq');
   }
   if (filterType == 'eval' || filterType == 'string') {
     allowedElements.push('startsWith', 'endsWith', 'contains', 'startsWithNot', 'endsWithNot', 'containsNot');

packages/tools/package.json

@@ -36,6 +36,7 @@
     "debug": "^4.3.4",
     "json-stable-stringify": "^1.0.1",
     "lodash": "^4.17.21",
+    "pinomin": "^1.0.1",
     "uuid": "^3.4.0"
   }
 }

packages/tools/src/DatabaseAnalyser.ts

@@ -3,6 +3,9 @@ import _sortBy from 'lodash/sortBy';
 import _groupBy from 'lodash/groupBy';
 import _pick from 'lodash/pick';
 import _compact from 'lodash/compact';
+import { getLogger } from './getLogger';
+
+const logger = getLogger('dbAnalyser');
 
 const STRUCTURE_FIELDS = ['tables', 'collections', 'views', 'matviews', 'functions', 'procedures', 'triggers'];
 
@@ -107,7 +110,7 @@ export class DatabaseAnalyser {
 
     this.modifications = structureModifications;
     if (structureWithRowCounts) this.structure = structureWithRowCounts;
-    console.log('DB modifications detected:', this.modifications);
+    logger.info({ modifications: this.modifications }, 'DB modifications detected:');
     return this.addEngineField(this.mergeAnalyseResult(await this._runAnalysis()));
   }
 
@@ -170,9 +173,9 @@ export class DatabaseAnalyser {
   //   return this.structure.tables.find((x) => x.objectId == id);
   // }
 
-  containsObjectIdCondition(typeFields) {
-    return this.createQueryCore('=OBJECT_ID_CONDITION', typeFields) != ' is not null';
-  }
+  // containsObjectIdCondition(typeFields) {
+  //   return this.createQueryCore('=OBJECT_ID_CONDITION', typeFields) != ' is not null';
+  // }
 
   createQuery(template, typeFields) {
     return this.createQueryCore(template, typeFields);
@@ -197,7 +200,7 @@ export class DatabaseAnalyser {
       .filter(x => typeFields.includes(x.objectTypeField) && (x.action == 'add' || x.action == 'change'))
       .map(x => x.objectId);
     if (filterIds.length == 0) {
-      return template.replace(/=OBJECT_ID_CONDITION/g, " = '0'");
+      return null;
     }
     return template.replace(/=OBJECT_ID_CONDITION/g, ` in (${filterIds.map(x => `'${x}'`).join(',')})`);
   }
@@ -293,11 +296,18 @@ export class DatabaseAnalyser {
     return [..._compact(res), ...this.getDeletedObjects(snapshot)];
   }
 
-  async safeQuery(sql) {
+  async analyserQuery(template, typeFields) {
+    const sql = this.createQuery(template, typeFields);
+
+    if (!sql) {
+      return {
+        rows: [],
+      };
+    }
     try {
       return await this.driver.query(this.pool, sql);
     } catch (err) {
-      console.log('Error running analyser query', err.message);
+      logger.error({ err }, 'Error running analyser query');
       return {
         rows: [],
       };

packages/tools/src/SqlGenerator.ts

@@ -8,10 +8,13 @@ import type {
   ViewInfo,
 } from 'dbgate-types';
 import _flatten from 'lodash/flatten';
-import _uniqBy from 'lodash/uniqBy'
+import _uniqBy from 'lodash/uniqBy';
+import { getLogger } from './getLogger';
 import { SqlDumper } from './SqlDumper';
 import { extendDatabaseInfo } from './structureTools';
 
+const logger = getLogger('sqlGenerator');
+
 interface SqlGeneratorOptions {
   dropTables: boolean;
   checkIfTableExists: boolean;
@@ -82,7 +85,7 @@ export class SqlGenerator {
   }
 
   private handleException = error => {
-    console.log('Unhandled error', error);
+    logger.error({ error }, 'Unhandled error');
     this.isUnhandledException = true;
   };
 

packages/tools/src/createBulkInsertStreamBase.ts

@@ -1,6 +1,9 @@
 import _intersection from 'lodash/intersection';
+import { getLogger } from './getLogger';
 import { prepareTableForImport } from './tableTransforms';
 
+const logger = getLogger('bulkStreamBase');
+
 export function createBulkInsertStreamBase(driver, stream, pool, name, options): any {
   const fullNameQuoted = name.schemaName
     ? `${driver.dialect.quoteIdentifier(name.schemaName)}.${driver.dialect.quoteIdentifier(name.pureName)}`
@@ -28,14 +31,13 @@ export function createBulkInsertStreamBase(driver, stream, pool, name, options):
     let structure = await driver.analyseSingleTable(pool, name);
     // console.log('ANALYSING', name, structure);
     if (structure && options.dropIfExists) {
-      console.log(`Dropping table ${fullNameQuoted}`);
+      logger.info(`Dropping table ${fullNameQuoted}`);
       await driver.script(pool, `DROP TABLE ${fullNameQuoted}`);
     }
     if (options.createIfNotExists && (!structure || options.dropIfExists)) {
-      console.log(`Creating table ${fullNameQuoted}`);
       const dmp = driver.createDumper();
       dmp.createTable(prepareTableForImport({ ...writable.structure, ...name }));
-      console.log(dmp.s);
+      logger.info({ sql: dmp.s }, `Creating table ${fullNameQuoted}`);
       await driver.script(pool, dmp.s);
       structure = await driver.analyseSingleTable(pool, name);
     }

packages/tools/src/getLogger.ts

@@ -0,0 +1,27 @@
+import pinomin, { Logger } from 'pinomin';
+
+let _logger: Logger;
+let _name: string = null;
+const defaultLogger: Logger = pinomin({
+  base: { pid: global?.process?.pid },
+  targets: [{ type: 'console', level: 'info' }],
+});
+
+export function setLogger(value: Logger) {
+  _logger = value;
+}
+export function setLoggerName(value) {
+  _name = value;
+}
+
+export function getLogger(caller?: string): Logger {
+  let res = _logger || defaultLogger;
+  if (caller) {
+    const props = { caller };
+    if (_name) {
+      props['name'] = _name;
+    }
+    res = res.child(props);
+  }
+  return res;
+}

packages/tools/src/index.ts

@@ -18,3 +18,4 @@ export * from './stringTools';
 export * from './computeDiffRows';
 export * from './preloadedRowsTools';
 export * from './ScriptWriter';
+export * from './getLogger';

packages/web/package.json

@@ -44,7 +44,7 @@
     "rollup-plugin-svelte": "^7.0.0",
     "rollup-plugin-terser": "^7.0.0",
     "sirv-cli": "^1.0.0",
-    "sql-formatter": "^2.3.3",
+    "sql-formatter": "^3.1.0",
     "svelte": "^3.46.4",
     "svelte-check": "^1.0.0",
     "svelte-markdown": "^0.1.4",

packages/web/src/LoginPage.svelte

@@ -23,8 +23,8 @@
     <div class="box">
       <div class="heading">Log In</div>
       <FormProvider>
-        <FormTextField label="Username" name="login" autocomplete="username" />
-        <FormPasswordField label="Password" name="password" autocomplete="current-password" />
+        <FormTextField label="Username" name="login" autocomplete="username" saveOnInput />
+        <FormPasswordField label="Password" name="password" autocomplete="current-password" saveOnInput />
 
         <div class="submit">
           <FormSubmit

packages/web/src/appobj/AppObjectGroup.svelte

@@ -4,6 +4,7 @@
   import { plusExpandIcon } from '../icons/expandIcons';
 
   import FontIcon from '../icons/FontIcon.svelte';
+  import contextMenu from '../utility/contextMenu';
 
   import AppObjectListItem from './AppObjectListItem.svelte';
 
@@ -16,8 +17,10 @@
   export let disableContextMenu = false;
   export let passProps;
   export let onDropOnGroup = undefined;
+  export let groupContextMenu = null;
+  export let collapsedGroupNames;
 
-  let isExpanded = true;
+  $: isExpanded = !$collapsedGroupNames.includes(group);
 
   $: filtered = items.filter(x => x.isMatched);
   $: countText = filtered.length < items.length ? `${filtered.length}/${items.length}` : `${items.length}`;
@@ -45,7 +48,16 @@
   }
 </script>
 
-<div class="group" on:click={() => (isExpanded = !isExpanded)} on:drop={handleDrop}>
+<div
+  class="group"
+  on:click={() =>
+    collapsedGroupNames.update(names => {
+      if (names.includes(group)) return names.filter(x => x != group);
+      return [...names, group];
+    })}
+  on:drop={handleDrop}
+  use:contextMenu={() => groupContextMenu(group)}
+>
   <span class="expand-icon">
     <FontIcon icon={groupIconFunc(isExpanded)} />
   </span>

packages/web/src/appobj/AppObjectList.svelte

@@ -5,6 +5,7 @@
   import { plusExpandIcon } from '../icons/expandIcons';
 
   import AppObjectListItem from './AppObjectListItem.svelte';
+  import { writable } from 'svelte/store';
 
   export let list;
   export let module;
@@ -19,12 +20,15 @@
   export let getIsExpanded = null;
   export let setIsExpanded = null;
   export let sortGroups = false;
+  export let groupContextMenu = null;
 
   export let groupIconFunc = plusExpandIcon;
   export let groupFunc = undefined;
   export let onDropOnGroup = undefined;
   export let emptyGroupNames = [];
 
+  export let collapsedGroupNames = writable([]);
+
   $: filtered = !groupFunc
     ? list.filter(data => {
         const matcher = module.createMatcher && module.createMatcher(data);
@@ -98,6 +102,8 @@
       {getIsExpanded}
       {setIsExpanded}
       {onDropOnGroup}
+      {groupContextMenu}
+      {collapsedGroupNames}
     />
   {/each}
 {:else}

packages/web/src/appobj/DatabaseObjectAppObject.svelte

@@ -191,6 +191,10 @@
         label: 'SQL: CREATE VIEW',
         scriptTemplate: 'CREATE OBJECT',
       },
+      {
+        label: 'SQL: ALTER VIEW',
+        scriptTemplate: 'ALTER OBJECT',
+      },
       {
         label: 'SQL: CREATE TABLE',
         scriptTemplate: 'CREATE TABLE',
@@ -253,6 +257,10 @@
         label: 'SQL: CREATE MATERIALIZED VIEW',
         scriptTemplate: 'CREATE OBJECT',
       },
+      {
+        label: 'SQL: ALTER MATERIALIZED VIEW',
+        scriptTemplate: 'ALTER OBJECT',
+      },
       {
         label: 'SQL: CREATE TABLE',
         scriptTemplate: 'CREATE TABLE',
@@ -290,6 +298,10 @@
         label: 'SQL: CREATE PROCEDURE',
         scriptTemplate: 'CREATE OBJECT',
       },
+      {
+        label: 'SQL: ALTER PROCEDURE',
+        scriptTemplate: 'ALTER OBJECT',
+      },
       {
         label: 'SQL: EXECUTE',
         scriptTemplate: 'EXECUTE PROCEDURE',
@@ -316,6 +328,10 @@
         label: 'SQL: CREATE FUNCTION',
         scriptTemplate: 'CREATE OBJECT',
       },
+      {
+        label: 'SQL: ALTER FUNCTION',
+        scriptTemplate: 'ALTER OBJECT',
+      },
       {
         label: 'SQL Generator: CREATE FUNCTION',
         sqlGeneratorProps: {

packages/web/src/celldata/MapCellView.svelte

@@ -1,7 +1,60 @@
 <script lang="ts">
-  import MapView from '../elements/MapView.svelte';
+  import _ from 'lodash';
+  import SelectionMapView, { findAllObjectPaths, findLatPaths, findLonPaths } from '../elements/SelectionMapView.svelte';
+  import SelectField from '../forms/SelectField.svelte';
 
   export let selection;
+
+  $: latitudeFields = _.uniq(_.flatten(selection.map(x => findLatPaths(x.rowData)))) as string[];
+  $: longitudeFields = _.uniq(_.flatten(selection.map(x => findLonPaths(x.rowData)))) as string[];
+  $: allFields = _.uniq(_.flatten(selection.map(x => findAllObjectPaths(x.rowData)))) as string[];
+
+  let latitudeField = '';
+  let longitudeField = '';
+
+  $: {
+    if (latitudeFields.length > 0 && !allFields.includes(latitudeField)) {
+      latitudeField = latitudeFields[0];
+    }
+  }
+  $: {
+    if (longitudeFields.length > 0 && !allFields.includes(longitudeField)) {
+      longitudeField = longitudeFields[0];
+    }
+  }
 </script>
 
-<MapView {selection} />
+<div class="container">
+  {#if allFields.length >= 2}
+    <div>
+      Lat:
+      <SelectField
+        isNative
+        options={allFields.map(x => ({ label: x, value: x }))}
+        value={latitudeField}
+        on:change={e => {
+          latitudeField = e.detail;
+        }}
+      />
+      Lon:
+      <SelectField
+        isNative
+        options={allFields.map(x => ({ label: x, value: x }))}
+        value={longitudeField}
+        on:change={e => {
+          longitudeField = e.detail;
+        }}
+      />
+    </div>
+  {/if}
+
+  <SelectionMapView {selection} {latitudeField} {longitudeField} />
+</div>
+
+<style>
+  .container {
+    display: flex;
+    flex: 1;
+    flex-direction: column;
+  }
+</style>

packages/web/src/clientAuth.ts

@@ -71,12 +71,13 @@ export async function redirectToLogin(config = null, force = false) {
 
   if (config.oauth) {
     const state = `dbg-oauth:${Math.random().toString().substr(2)}`;
+    const scopeParam = config.oauthScope ? `&scope=${config.oauthScope}` : '';
     sessionStorage.setItem('oauthState', state);
     console.log('Redirecting to OAUTH provider');
     location.replace(
-      `${config.oauth}?client_id=dbgate&response_type=code&redirect_uri=${encodeURIComponent(
+      `${config.oauth}?client_id=${config.oauthClient}&response_type=code&redirect_uri=${encodeURIComponent(
         location.origin + location.pathname
-      )}&state=${encodeURIComponent(state)}`
+      )}&state=${encodeURIComponent(state)}${scopeParam}`
     );
     return;
   }

packages/web/src/commands/stdCommands.ts

@@ -451,6 +451,22 @@ registerCommand({
   onClick: openArchiveFolder,
 });
 
+registerCommand({
+  id: 'folder.showLogs',
+  category: 'Folder',
+  name: 'Open logs',
+  testEnabled: () => getElectron() != null,
+  onClick: () => electron.showItemInFolder(getCurrentConfig().logsFilePath),
+});
+
+registerCommand({
+  id: 'folder.showData',
+  category: 'Folder',
+  name: 'Open data folder',
+  testEnabled: () => getElectron() != null,
+  onClick: () => electron.showItemInFolder(getCurrentConfig().connectionsFilePath),
+});
+
 registerCommand({
   id: 'file.import',
   category: 'File',

packages/web/src/datagrid/DataGrid.svelte

@@ -76,7 +76,7 @@
   export let gridCoreComponent;
   export let formViewComponent = null;
   export let jsonViewComponent = null;
-  export let formDisplay;
+  // export let formDisplay;
   export let display;
   export let changeSetState;
   export let dispatchChangeSet;
@@ -107,7 +107,7 @@
   const collapsedLeftColumnStore =
     getContext('collapsedLeftColumnStore') || writable(getLocalStorage('dataGrid_collapsedLeftColumn', false));
 
-  $: isFormView = !!(formDisplay && formDisplay.config && formDisplay.config.isFormView);
+  $: isFormView = !!config?.isFormView;
   $: isJsonView = !!config?.isJsonView;
 
   const handleExecuteMacro = () => {
@@ -116,14 +116,14 @@
   };
 
   export function switchViewEnabled(view) {
-    if (view == 'form') return !!formViewComponent && !!formDisplay && !isFormView && display?.baseTable?.primaryKey;
+    if (view == 'form') return !!formViewComponent && !isFormView;
     if (view == 'table') return !!(isFormView || isJsonView);
     if (view == 'json') return !!jsonViewComponent && !isJsonView;
   }
 
   export function switchToView(view) {
     if (view == 'form') {
-      display.switchToFormView(selectedCellsPublished()[0]?.rowData);
+      display.switchToFormView(selectedCellsPublished()[0]?.row);
     }
     if (view == 'table') {
       setConfig(cfg => ({
@@ -205,7 +205,7 @@
       </WidgetColumnBarItem>
 
       <WidgetColumnBarItem title="Filters" name="filters" height="30%" show={isFormView}>
-        <FormViewFilters {...$$props} {managerSize} driver={formDisplay?.driver} />
+        <FormViewFilters {...$$props} {managerSize} driver={display?.driver} />
       </WidgetColumnBarItem>
 
       <WidgetColumnBarItem
@@ -235,7 +235,7 @@
             this={gridCoreComponent}
             {...$$props}
             {collapsedLeftColumnStore}
-            formViewAvailable={!!formViewComponent && !!formDisplay}
+            formViewAvailable={!!formViewComponent}
             macroValues={extractMacroValuesForMacro($macroValues, $selectedMacro)}
             macroPreview={$selectedMacro}
             bind:loadedRows

packages/web/src/datagrid/DataGridCore.svelte

@@ -13,6 +13,18 @@
     onClick: () => getCurrentDataGrid().refresh(),
   });
 
+  registerCommand({
+    id: 'dataGrid.deepRefresh',
+    category: 'Data grid',
+    name: 'Refresh with structure',
+    keyText: 'Ctrl+F5',
+    toolbar: true,
+    isRelatedToTab: true,
+    icon: 'icon reload',
+    testEnabled: () => getCurrentDataGrid()?.canDeepRefresh(),
+    onClick: () => getCurrentDataGrid().deepRefresh(),
+  });
+
   registerCommand({
     id: 'dataGrid.revertRowChanges',
     category: 'Data grid',
@@ -282,7 +294,7 @@
 <script lang="ts">
   import { GridDisplay } from 'dbgate-datalib';
   import { driverBase, parseCellValue } from 'dbgate-tools';
-  import { getContext } from 'svelte';
+  import { getContext, onDestroy } from 'svelte';
   import _ from 'lodash';
   import registerCommand from '../commands/registerCommand';
   import ColumnHeaderControl from './ColumnHeaderControl.svelte';
@@ -333,9 +345,10 @@
   import { apiCall } from '../utility/api';
   import getElectron from '../utility/getElectron';
   import { isCtrlOrCommandKey, isMac } from '../utility/common';
-  import { selectionCouldBeShownOnMap } from '../elements/MapView.svelte';
+  import { selectionCouldBeShownOnMap } from '../elements/SelectionMapView.svelte';
   import ErrorMessageModal from '../modals/ErrorMessageModal.svelte';
   import EditCellDataModal, { shouldOpenMultilineDialog } from '../modals/EditCellDataModal.svelte';
+  import { getDatabaseInfo, useDatabaseStatus } from '../utility/metadataLoaders';
 
   export let onLoadNextData = undefined;
   export let grider = undefined;
@@ -396,6 +409,26 @@
 
   const tabid = getContext('tabid');
 
+  let unsubscribeDbRefresh;
+
+  onDestroy(callUnsubscribeDbRefresh);
+
+  function callUnsubscribeDbRefresh() {
+    if (unsubscribeDbRefresh) {
+      unsubscribeDbRefresh();
+      unsubscribeDbRefresh = null;
+    }
+  }
+
+  async function refreshAndUnsubscribe(status) {
+    if (status?.name != 'pending' && status?.name != 'checkStructure' && status?.name != 'loadStructure') {
+      callUnsubscribeDbRefresh();
+      // ensure new structure is loaded
+      await getDatabaseInfo({ conid, database });
+      refresh();
+    }
+  }
+
   export function refresh() {
     if (onCustomGridRefresh) onCustomGridRefresh();
     else display.reload();
@@ -406,6 +439,16 @@
     return getDisplay()?.supportsReload;
   }
 
+  export function canDeepRefresh() {
+    return canRefresh() && !!conid && !!database;
+  }
+
+  export async function deepRefresh() {
+    callUnsubscribeDbRefresh();
+    await apiCall('database-connections/sync-model', { conid, database });
+    unsubscribeDbRefresh = useDatabaseStatus({ conid, database }).subscribe(refreshAndUnsubscribe);
+  }
+
   export function getGrider() {
     return grider;
   }
@@ -462,6 +505,7 @@
         for (const column of display.columns) {
           if (column.uniquePath.length > 1) continue;
           if (column.autoIncrement) continue;
+          if (column.columnName == '_id' && isDynamicStructure) continue;
 
           grider.setCellValue(rowIndex, column.uniqueName, grider.getRowData(index)[column.uniqueName]);
         }

packages/web/src/datagrid/InplaceEditor.svelte

@@ -1,12 +1,3 @@
-<script lang="ts" context="module">
-  function getEditedValue(value) {
-    if (value?.type == 'Buffer' && _.isArray(value.data)) return '0x' + arrayToHexString(value.data);
-    if (value?.$oid) return `ObjectId("${value?.$oid}")`;
-    if (_.isPlainObject(value) || _.isArray(value)) return JSON.stringify(value);
-    return value;
-  }
-</script>
-
 <script lang="ts">
   import keycodes from '../utility/keycodes';
   import { onMount, tick } from 'svelte';

packages/web/src/datagrid/JslDataGrid.svelte

@@ -1,6 +1,7 @@
 <script lang="ts">
   import { createGridCache, createGridConfig, JslGridDisplay } from 'dbgate-datalib';
   import { writable } from 'svelte/store';
+  import JslFormView from '../formview/JslFormView.svelte';
   import { apiOff, apiOn, useApiCall } from '../utility/api';
   import useEffect from '../utility/useEffect';
 
@@ -55,7 +56,10 @@
     {...$$restProps}
     {display}
     {jslid}
+    config={$config}
+    setConfig={config.update}
     gridCoreComponent={JslDataGridCore}
+    formViewComponent={JslFormView}
     bind:loadedRows
     isDynamicStructure={$info?.__isDynamicStructure}
     useEvalFilters

packages/web/src/datagrid/TableDataGrid.svelte

@@ -3,7 +3,7 @@
     createGridCache,
     createGridConfig,
     runMacroOnChangeSet,
-    TableFormViewDisplay,
+    // TableFormViewDisplay,
     TableGridDisplay,
   } from 'dbgate-datalib';
   import { getFilterValueExpression } from 'dbgate-filterparser';
@@ -75,22 +75,22 @@
         )
       : null;
 
-  $: formDisplay =
-    connection && $serverVersion
-      ? new TableFormViewDisplay(
-          { schemaName, pureName },
-          findEngineDriver($connection, $extensions),
-          config,
-          setConfig,
-          cache,
-          setCache,
-          extendedDbInfo,
-          { showHintColumns: getBoolSettingsValue('dataGrid.showHintColumns', true) },
-          $serverVersion,
-          table => getDictionaryDescription(table, conid, database, $apps, $connections),
-          $connection?.isReadOnly
-        )
-      : null;
+  // $: formDisplay =
+  //   connection && $serverVersion
+  //     ? new TableFormViewDisplay(
+  //         { schemaName, pureName },
+  //         findEngineDriver($connection, $extensions),
+  //         config,
+  //         setConfig,
+  //         cache,
+  //         setCache,
+  //         extendedDbInfo,
+  //         { showHintColumns: getBoolSettingsValue('dataGrid.showHintColumns', true) },
+  //         $serverVersion,
+  //         table => getDictionaryDescription(table, conid, database, $apps, $connections),
+  //         $connection?.isReadOnly
+  //       )
+  //     : null;
 
   const setChildConfig = (value, reference = undefined) => {
     if (_.isFunction(value)) {
@@ -157,7 +157,6 @@
       gridCoreComponent={SqlDataGridCore}
       formViewComponent={SqlFormView}
       {display}
-      {formDisplay}
       showReferences
       showMacros
       onRunMacro={handleRunMacro}

packages/web/src/designer/ColumnLine.svelte

@@ -6,6 +6,7 @@
   import ColumnLabel from '../elements/ColumnLabel.svelte';
 
   import CheckboxField from '../forms/CheckboxField.svelte';
+  import { plusExpandIcon } from '../icons/expandIcons';
   import FontIcon from '../icons/FontIcon.svelte';
   import contextMenu from '../utility/contextMenu';
   import SortOrderIcon from './SortOrderIcon.svelte';
@@ -21,6 +22,11 @@
   export let onAddReferenceByColumn;
   export let onSelectColumn;
   export let settings;
+  export let nestingSupported = null;
+  export let isExpandable = false;
+  export let isExpanded = false;
+  export let expandLevel = 0;
+  export let toggleExpanded = null;
 
   $: designerColumn = (designer.columns || []).find(
     x => x.designerId == designerId && x.columnName == column.columnName
@@ -115,16 +121,27 @@
     })}
   use:contextMenu={settings?.canSelectColumns ? createMenu : '__no_menu'}
 >
+  {#if nestingSupported}
+    <span class="expandColumnIcon" style={`margin-right: ${5 + expandLevel * 10}px`}>
+      <FontIcon
+        icon={isExpandable ? plusExpandIcon(isExpanded) : 'icon invisible-box'}
+        on:click={() => {
+          toggleExpanded(!isExpanded);
+        }}
+      />
+    </span>
+  {/if}
+
   {#if settings?.allowColumnOperations}
     <CheckboxField
       checked={settings?.isColumnChecked
-        ? settings?.isColumnChecked(designerId, column.columnName)
+        ? settings?.isColumnChecked(designerId, column)
         : !!(designer.columns || []).find(
             x => x.designerId == designerId && x.columnName == column.columnName && x.isOutput
           )}
       on:change={e => {
         if (settings?.setColumnChecked) {
-          settings?.setColumnChecked(designerId, column.columnName, e.target.checked);
+          settings?.setColumnChecked(designerId, column, e.target.checked);
         } else {
           if (e.target.checked) {
             onChangeColumn(
@@ -147,7 +164,13 @@
       }}
     />
   {/if}
-  <ColumnLabel {...column} {foreignKey} forceIcon {iconOverride} />
+  <ColumnLabel
+    {...column}
+    columnName={settings?.getColumnDisplayName ? settings?.getColumnDisplayName(column) : column.columnName}
+    {foreignKey}
+    forceIcon
+    {iconOverride}
+  />
   {#if designerColumn?.filter}
     <FontIcon icon="img filter" />
   {/if}

packages/web/src/designer/Designer.svelte

@@ -48,6 +48,9 @@
   import ChooseColorModal from '../modals/ChooseColorModal.svelte';
   import { currentThemeDefinition } from '../stores';
   import { extendDatabaseInfoFromApps } from 'dbgate-tools';
+  import SearchInput from '../elements/SearchInput.svelte';
+  import CloseSearchButton from '../buttons/CloseSearchButton.svelte';
+  import DragColumnMemory from './DragColumnMemory.svelte';
 
   export let value;
   export let onChange;
@@ -64,6 +67,7 @@
   let canvasHeight = 3000;
   let dragStartPoint = null;
   let dragCurrentPoint = null;
+  let columnFilter;
 
   const sourceDragColumn$ = writable(null);
   const targetDragColumn$ = writable(null);
@@ -80,7 +84,15 @@
 
   const tableRefs = {};
   const referenceRefs = {};
-  $: domTables = _.pickBy(_.mapValues(tableRefs, (tbl: any) => tbl?.getDomTable()));
+  let domTables;
+  $: {
+    tableRefs;
+    recomputeDomTables();
+  }
+
+  function recomputeDomTables() {
+    domTables = _.pickBy(_.mapValues(tableRefs, (tbl: any) => tbl?.getDomTable()));
+  }
 
   function fixPositions(tables) {
     const minLeft = _.min(tables.map(x => x.left));
@@ -835,6 +847,14 @@
       ],
     ];
   }
+
+  $: {
+    columnFilter;
+    tick().then(() => {
+      recomputeReferencePositions();
+      recomputeDomTables();
+    });
+  }
 </script>
 
 <div class="wrapper noselect" use:contextMenu={createMenu}>
@@ -896,6 +916,7 @@
         onAddAllReferences={handleAddTableReferences}
         onChangeTableColor={handleChangeTableColor}
         onMoveReferences={recomputeReferencePositions}
+        {columnFilter}
         {table}
         {conid}
         {database}
@@ -930,6 +951,15 @@
       </svg>
     {/if}
   </div>
+  {#if tables?.length > 0}
+    <div class="panel">
+      <DragColumnMemory {settings} {sourceDragColumn$} {targetDragColumn$} />
+      <div class="searchbox">
+        <SearchInput bind:value={columnFilter} placeholder="Filter columns" />
+        <CloseSearchButton bind:filter={columnFilter} />
+      </div>
+    </div>
+  {/if}
 </div>
 
 <style>
@@ -946,6 +976,17 @@
   .canvas {
     position: relative;
   }
+  .panel {
+    position: absolute;
+    right: 16px;
+    top: 0;
+    display: flex;
+  }
+  .searchbox {
+    width: 200px;
+    display: flex;
+    margin-left: 1px;
+  }
 
   svg.drag-rect {
     visibility: hidden;

packages/web/src/designer/DesignerTable.svelte

@@ -1,6 +1,6 @@
 <script lang="ts">
   import { presetDarkPalettes, presetPalettes } from '@ant-design/colors';
-  import { computeDbDiffRows } from 'dbgate-tools';
+  import { filterName } from 'dbgate-tools';
 
   import { tick } from 'svelte';
   import { createDatabaseObjectMenu } from '../appobj/DatabaseObjectAppObject.svelte';
@@ -51,6 +51,7 @@
   export let designer;
   export let onMoveReferences;
   export let settings;
+  export let columnFilter;
 
   let movingPosition = null;
   let domWrapper;
@@ -68,6 +69,27 @@
   $: specificDb = settings?.tableSpecificDb ? settings?.tableSpecificDb(designerId) : null;
   $: filterParentRows = settings?.hasFilterParentRowsFlag ? settings?.hasFilterParentRowsFlag(designerId) : false;
   $: isGrayed = settings?.isGrayedTable ? settings?.isGrayedTable(designerId) : false;
+  $: flatColumns = getFlatColumns(columns, columnFilter, 0);
+
+  function getFlatColumns(columns, filter, level) {
+    if (!columns) return [];
+    const res = [];
+    for (const col of columns) {
+      if (filterName(filter, col.columnName)) {
+        res.push({ ...col, expandLevel: level });
+        if (col.isExpanded) {
+          res.push(...getFlatColumns(col.getChildColumns ? col.getChildColumns() : null, filter, level + 1));
+        }
+      } else if (col.isExpanded) {
+        const children = getFlatColumns(col.getChildColumns ? col.getChildColumns() : null, filter, level + 1);
+        if (children.length > 0) {
+          res.push({ ...col, expandLevel: level });
+          res.push(...children);
+        }
+      }
+    }
+    return res;
+  }
 
   export function isSelected() {
     return table?.isSelectedTable;
@@ -156,7 +178,7 @@
   export function getDomTable() {
     const domRefs = { ...columnRefs };
     domRefs[''] = domWrapper;
-    return new DomTableRef(table, domRefs, domCanvas);
+    return new DomTableRef(table, domRefs, domCanvas, settings);
   }
 
   const handleSetTableAlias = () => {
@@ -214,7 +236,7 @@
     ];
   }
 
-  // $: console.log('COLUMNS', columns);
+  // $: console.log('COLUMNS', flatColumns);
 </script>
 
 <div
@@ -279,8 +301,13 @@
     {/if}
   </div>
   <div class="columns" on:scroll={() => tick().then(onMoveReferences)} class:scroll={settings?.allowScrollColumns}>
-    {#each columns || [] as column}
+    {#each flatColumns || [] as column (column.columnName)}
       <ColumnLine
+        nestingSupported={!!settings?.isColumnExpandable && columns.find(x => settings?.isColumnExpandable(x))}
+        isExpandable={settings?.isColumnExpandable && settings?.isColumnExpandable(column)}
+        isExpanded={settings?.isColumnExpanded && settings?.isColumnExpanded(column)}
+        expandLevel={settings?.columnExpandLevel ? settings?.columnExpandLevel(column) : 0}
+        toggleExpanded={value => settings?.toggleExpandedColumn(column, value)}
         {column}
         {table}
         {designer}

packages/web/src/designer/DomTableRef.ts

@@ -6,13 +6,15 @@ export default class DomTableRef {
   table: DesignerTableInfo;
   designerId: string;
   domRefs: { [column: string]: Element };
+  settings: any;
 
-  constructor(table: DesignerTableInfo, domRefs, domWrapper: Element) {
+  constructor(table: DesignerTableInfo, domRefs, domWrapper: Element, settings) {
     this.domTable = domRefs[''];
     this.domWrapper = domWrapper;
     this.table = table;
     this.designerId = table.designerId;
     this.domRefs = domRefs;
+    this.settings = settings;
   }
 
   getRect() {
@@ -31,10 +33,14 @@ export default class DomTableRef {
 
   getColumnY(columnName: string) {
     let col = this.domRefs[columnName];
-    if (!col) return null;
+    while (col == null && this.settings?.getParentColumnName && this.settings?.getParentColumnName(columnName)) {
+      columnName = this.settings?.getParentColumnName(columnName);
+      col = this.domRefs[columnName];
+    }
+    const tableRect = this.getRect();
+    if (!col) return tableRect.top + 12;
     const rect = col.getBoundingClientRect();
     const wrap = this.domWrapper.getBoundingClientRect();
-    const tableRect = this.getRect();
     let res = (rect.top + rect.bottom) / 2 - wrap.top;
     if (res < tableRect.top) res = tableRect.top;
     if (res > tableRect.bottom) res = tableRect.bottom;

packages/web/src/designer/DragColumnMemory.svelte

@@ -0,0 +1,54 @@
+<script lang="ts">
+  export let sourceDragColumn$;
+  export let targetDragColumn$;
+  export let settings;
+
+  let memory;
+</script>
+
+{#if settings?.allowCreateRefByDrag}
+  <div
+    class="wrapper"
+    draggable={!!memory}
+    title={memory ? 'Drag this column to other column for creating JOIN' : 'Drag column here for creating JOIN'}
+    on:dragstart={e => {
+      if (!settings?.allowCreateRefByDrag) return;
+      if (!memory) return;
+
+      const dragData = { ...memory };
+      sourceDragColumn$.set(dragData);
+      e.dataTransfer.setData('designer_column_drag_data', JSON.stringify(dragData));
+    }}
+    on:dragend={e => {
+      sourceDragColumn$.set(null);
+      targetDragColumn$.set(null);
+    }}
+    on:dragover={e => {
+      if ($sourceDragColumn$) {
+        e.preventDefault();
+      }
+    }}
+    on:drop={e => {
+      var data = e.dataTransfer.getData('designer_column_drag_data');
+      e.preventDefault();
+      if (!data) return;
+      memory = $sourceDragColumn$;
+      sourceDragColumn$.set(null);
+      targetDragColumn$.set(null);
+    }}
+  >
+    {#if memory}
+      {memory.columnName}
+    {:else}
+      Drag & drop column here
+    {/if}
+  </div>
+{/if}
+
+<style>
+  .wrapper {
+    border: 1px solid var(--theme-border);
+    padding: 3px;
+    color: var(--theme-font-2);
+  }
+</style>

packages/web/src/elements/MapView.svelte

@@ -1,102 +1,27 @@
-<script lang="ts" context="module">
-  export function selectionCouldBeShownOnMap(selection) {
-    if (selection.length > 0 && _.find(selection, x => isWktGeometry(x.value))) {
-      return true;
-    }
-
-    if (
-      selection.find(x => x.column.toLowerCase().includes('lat')) &&
-      (selection.find(x => x.column.toLowerCase().includes('lon')) ||
-        selection.find(x => x.column.toLowerCase().includes('lng')))
-    ) {
-      return true;
-    }
-    return false;
-  }
-</script>
-
 <script lang="ts">
   import _ from 'lodash';
   import { onMount, tick } from 'svelte';
   import 'leaflet/dist/leaflet.css';
   import leaflet from 'leaflet';
-  import wellknown from 'wellknown';
-  import { isWktGeometry, ScriptWriter, ScriptWriterJson } from 'dbgate-tools';
   import resizeObserver from '../utility/resizeObserver';
   import openNewTab from '../utility/openNewTab';
   import contextMenu from '../utility/contextMenu';
-  import { saveExportedFile, saveFileToDisk } from '../utility/exportFileTools';
-  import { getCurrentConfig } from '../stores';
+  import { saveFileToDisk } from '../utility/exportFileTools';
   import { apiCall } from '../utility/api';
 
-  export let selection;
-
   let refContainer;
   let map;
 
-  let selectionLayers = [];
-  let geoJson;
+  let layers = [];
+  export let geoJson;
 
-  function createColumnsTable(cells) {
-    if (cells.length == 0) return '';
-    return `<table>${cells.map(cell => `<tr><td>${cell.column}</td><td>${cell.value}</td></tr>`).join('\n')}</table>`;
-  }
-
-  function addSelectionToMap() {
+  function addObjectToMap() {
     if (!map) return;
-    if (!selection) return;
 
-    for (const selectionLayer of selectionLayers) {
-      selectionLayer.remove();
+    for (const layer of layers) {
+      layer.remove();
     }
-    selectionLayers = [];
-
-    const selectedRows = _.groupBy(selection || [], 'row');
-
-    const features = [];
-
-    for (const rowKey of _.keys(selectedRows)) {
-      const cells = selectedRows[rowKey];
-      const lat = cells.find(x => x.column.toLowerCase().includes('lat'));
-      const lon = cells.find(x => x.column.toLowerCase().includes('lon') || x.column.toLowerCase().includes('lng'));
-
-      const geoValues = cells.map(x => x.value).filter(isWktGeometry);
-
-      if (lat && lon) {
-        features.push({
-          type: 'Feature',
-          properties: {
-            popupContent: createColumnsTable(cells),
-          },
-          geometry: {
-            type: 'Point',
-            coordinates: [lon.value, lat.value],
-          },
-        });
-      }
-
-      if (geoValues.length > 0) {
-        // parse WKT to geoJSON array
-        features.push(
-          ...geoValues.map(wellknown).map(geometry => ({
-            type: 'Feature',
-            properties: {
-              popupContent: createColumnsTable(cells.filter(x => !isWktGeometry(x.value))),
-            },
-            geometry,
-          }))
-        );
-      }
-    }
-
-    if (features.length == 0) {
-      return;
-    }
-
-    geoJson = {
-      type: 'FeatureCollection',
-      features,
-    };
+    layers = [];
 
     const geoJsonObj = leaflet
       .geoJSON(geoJson, {
@@ -130,7 +55,7 @@
       .addTo(map);
     // geoJsonObj.bindPopup('This is the Transamerica Pyramid'); //.openPopup();
     map.fitBounds(geoJsonObj.getBounds());
-    selectionLayers.push(geoJsonObj);
+    layers.push(geoJsonObj);
   }
 
   onMount(() => {
@@ -143,20 +68,12 @@
       })
       .addTo(map);
 
-    addSelectionToMap();
-    // map.fitBounds([
-    //   [50, 15],
-    //   [50.1, 15],
-    //   [50, 15.1],
-    // ]);
-
-    // const marker = leaflet.marker([50, 15]).addTo(map);
-    // <div bind:this={refContainer} class="flex1 map-container" />
+    addObjectToMap();
   });
 
   $: {
-    selection;
-    addSelectionToMap();
+    geoJson;
+    addObjectToMap();
   }
 
   function createMenu() {
@@ -170,7 +87,7 @@
               icon: 'img map',
               tabComponent: 'MapTab',
             },
-            { editor: selection.map(x => _.omit(x, ['engine'])) }
+            { editor: geoJson }
           );
         },
       },

packages/web/src/elements/SelectionMapView.svelte

@@ -0,0 +1,117 @@
+<script lang="ts" context="module">
+  function findLatLonPaths(obj, attrTest, res = [], prefix = '') {
+    for (const key of Object.keys(obj)) {
+      if (attrTest(key, obj[key])) {
+        res.push(prefix + key);
+      }
+      if (_.isPlainObject(obj[key])) {
+        findLatLonPaths(obj[key], attrTest, res, prefix + key + '.');
+      }
+    }
+    return res;
+  }
+  export function findLatPaths(obj) {
+    return findLatLonPaths(obj, x => x.includes('lat'));
+  }
+  export function findLonPaths(obj) {
+    return findLatLonPaths(obj, x => x.includes('lon') || x.includes('lng'));
+  }
+  export function findAllObjectPaths(obj) {
+    return findLatLonPaths(obj, (_k, v) => v != null && !_.isNaN(Number(v)));
+  }
+
+  export function selectionCouldBeShownOnMap(selection) {
+    if (selection.length > 0 && _.find(selection, x => isWktGeometry(x.value))) {
+      return true;
+    }
+
+    if (
+      selection.length > 0 &&
+      _.find(selection, x => findLatPaths(x.rowData).length > 0 && findLonPaths(x.rowData).length > 0)
+    ) {
+      return true;
+    }
+    return false;
+  }
+</script>
+
+<script lang="ts">
+  import _ from 'lodash';
+  import 'leaflet/dist/leaflet.css';
+  import wellknown from 'wellknown';
+  import { isWktGeometry, stringifyCellValue } from 'dbgate-tools';
+  import MapView from './MapView.svelte';
+
+  export let selection;
+
+  export let latitudeField = '';
+  export let longitudeField = '';
+
+  let geoJson;
+
+  function createColumnsTable(cells) {
+    if (cells.length == 0) return '';
+    return `<table>${cells
+      .map(cell => `<tr><td>${cell.column}</td><td>${stringifyCellValue(cell.value)}</td></tr>`)
+      .join('\n')}</table>`;
+  }
+
+  function createGeoJson() {
+    const selectedRows = _.groupBy(selection || [], 'row');
+
+    const features = [];
+
+    for (const rowKey of _.keys(selectedRows)) {
+      const cells = selectedRows[rowKey];
+
+      const geoValues = cells.map(x => x.value).filter(isWktGeometry);
+
+      const lat = latitudeField ? Number(_.get(cells[0].rowData, latitudeField)) : NaN;
+      const lon = longitudeField ? Number(_.get(cells[0].rowData, longitudeField)) : NaN;
+
+      if (!_.isNaN(lat) && !_.isNaN(lon)) {
+        features.push({
+          type: 'Feature',
+          properties: {
+            popupContent: createColumnsTable(cells),
+          },
+          geometry: {
+            type: 'Point',
+            coordinates: [Number(lon), Number(lat)],
+          },
+        });
+      }
+
+      if (geoValues.length > 0) {
+        // parse WKT to geoJSON array
+        features.push(
+          ...geoValues.map(wellknown).map(geometry => ({
+            type: 'Feature',
+            properties: {
+              popupContent: createColumnsTable(cells.filter(x => !isWktGeometry(x.value))),
+            },
+            geometry,
+          }))
+        );
+      }
+    }
+
+    if (features.length == 0) {
+      return;
+    }
+
+    geoJson = {
+      type: 'FeatureCollection',
+      features,
+    };
+  }
+
+  $: {
+    selection;
+    latitudeField;
+    longitudeField;
+    createGeoJson();
+  }
+</script>
+
+<MapView {geoJson} />

packages/web/src/formview/ChangeSetFormer.ts

@@ -1,96 +0,0 @@
-import type { ChangeSet, ChangeSetRowDefinition } from 'dbgate-datalib';
-import {
-  changeSetContainsChanges,
-  changeSetInsertNewRow,
-  createChangeSet,
-  deleteChangeSetRows,
-  findExistingChangeSetItem,
-  getChangeSetInsertedRows,
-  TableFormViewDisplay,
-  revertChangeSetRowChanges,
-  setChangeSetValue,
-} from 'dbgate-datalib';
-import Former from './Former';
-
-export default class ChangeSetFormer extends Former {
-  public changeSet: ChangeSet;
-  public setChangeSet: Function;
-  private batchChangeSet: ChangeSet;
-  public rowDefinition: ChangeSetRowDefinition;
-  public rowStatus;
-  public rowData: {};
-
-  constructor(
-    public sourceRow: any,
-    public changeSetState,
-    public dispatchChangeSet,
-    public display: TableFormViewDisplay
-  ) {
-    super();
-    this.changeSet = changeSetState && changeSetState.value;
-    this.setChangeSet = value => dispatchChangeSet({ type: 'set', value });
-    this.batchChangeSet = null;
-    this.rowDefinition = display.getChangeSetRow(sourceRow);
-    const [matchedField, matchedChangeSetItem] = findExistingChangeSetItem(this.changeSet, this.rowDefinition);
-    this.rowData = matchedChangeSetItem ? { ...sourceRow, ...matchedChangeSetItem.fields } : sourceRow;
-    let status = 'regular';
-    if (matchedChangeSetItem && matchedField == 'updates') status = 'updated';
-    if (matchedField == 'deletes') status = 'deleted';
-    this.rowStatus = {
-      status,
-      modifiedFields:
-        matchedChangeSetItem && matchedChangeSetItem.fields ? new Set(Object.keys(matchedChangeSetItem.fields)) : null,
-    };
-  }
-
-  applyModification(changeSetReducer) {
-    if (this.batchChangeSet) {
-      this.batchChangeSet = changeSetReducer(this.batchChangeSet);
-    } else {
-      this.setChangeSet(changeSetReducer(this.changeSet));
-    }
-  }
-
-  setCellValue(uniqueName: string, value: any) {
-    const row = this.sourceRow;
-    const definition = this.display.getChangeSetField(row, uniqueName);
-    this.applyModification(chs => setChangeSetValue(chs, definition, value));
-  }
-
-  deleteRow(index: number) {
-    this.applyModification(chs => deleteChangeSetRows(chs, this.rowDefinition));
-  }
-
-  beginUpdate() {
-    this.batchChangeSet = this.changeSet;
-  }
-  endUpdate() {
-    this.setChangeSet(this.batchChangeSet);
-    this.batchChangeSet = null;
-  }
-
-  revertRowChanges() {
-    this.applyModification(chs => revertChangeSetRowChanges(chs, this.rowDefinition));
-  }
-  revertAllChanges() {
-    this.applyModification(chs => createChangeSet());
-  }
-  undo() {
-    this.dispatchChangeSet({ type: 'undo' });
-  }
-  redo() {
-    this.dispatchChangeSet({ type: 'redo' });
-  }
-  get editable() {
-    return this.display.editable;
-  }
-  get canUndo() {
-    return this.changeSetState.canUndo;
-  }
-  get canRedo() {
-    return this.changeSetState.canRedo;
-  }
-  get containsChanges() {
-    return changeSetContainsChanges(this.changeSet);
-  }
-}

packages/web/src/formview/FormView.svelte

@@ -39,8 +39,8 @@
     category: 'Data form',
     name: 'Revert row changes',
     keyText: 'CtrlOrCommand+U',
-    testEnabled: () => getCurrentDataForm()?.getFormer()?.containsChanges,
-    onClick: () => getCurrentDataForm().getFormer().revertRowChanges(),
+    testEnabled: () => getCurrentDataForm()?.getGrider()?.containsChanges,
+    onClick: () => getCurrentDataForm().getGrider().revertRowChanges(0),
   });
 
   registerCommand({
@@ -60,8 +60,8 @@
     icon: 'icon undo',
     toolbar: true,
     isRelatedToTab: true,
-    testEnabled: () => getCurrentDataForm()?.getFormer()?.canUndo,
-    onClick: () => getCurrentDataForm().getFormer().undo(),
+    testEnabled: () => getCurrentDataForm()?.getGrider()?.canUndo,
+    onClick: () => getCurrentDataForm().getGrider().undo(),
   });
 
   registerCommand({
@@ -72,8 +72,8 @@
     icon: 'icon redo',
     toolbar: true,
     isRelatedToTab: true,
-    testEnabled: () => getCurrentDataForm()?.getFormer()?.canRedo,
-    onClick: () => getCurrentDataForm().getFormer().redo(),
+    testEnabled: () => getCurrentDataForm()?.getGrider()?.canRedo,
+    onClick: () => getCurrentDataForm().getGrider().redo(),
   });
 
   registerCommand({
@@ -155,6 +155,8 @@
 </script>
 
 <script lang="ts">
+  import { getFilterValueExpression } from 'dbgate-filterparser';
+
   import { filterName } from 'dbgate-tools';
 
   import _ from 'lodash';
@@ -194,8 +196,10 @@
   export let allRowCount;
   export let rowCountBefore;
   export let isLoading;
-  export let former;
-  export let formDisplay;
+  export let grider;
+  export let display;
+  export let rowCountNotAvailable;
+  // export let formDisplay;
   export let onNavigate;
 
   let wrapperHeight = 1;
@@ -212,23 +216,35 @@
     domFocusField.focus();
   }
 
-  $: rowData = former?.rowData;
-  $: rowStatus = former?.rowStatus;
+  $: rowData = grider?.getRowData(0);
+  $: rowStatus = grider?.getRowStatus(0);
 
   $: rowCount = Math.floor((wrapperHeight - 22) / (rowHeight + 2));
 
-  $: columnChunks = _.chunk(formDisplay.columns, rowCount) as any[][];
+  $: columnChunks = _.chunk(display?.formColumns || [], rowCount) as any[][];
 
-  $: rowCountInfo = getRowCountInfo(rowCountBefore, allRowCount);
+  $: rowCountInfo = getRowCountInfo(allRowCount, display);
 
-  function getRowCountInfo(rowCountBefore, allRowCount) {
-    if (rowData == null) return 'No data';
-    if (allRowCount == null || rowCountBefore == null) return 'Loading row count...';
-    return `Row: ${(rowCountBefore + 1).toLocaleString()} / ${allRowCount.toLocaleString()}`;
+  function getRowCountInfo(allRowCount) {
+    if (rowCountNotAvailable) {
+      return `Row: ${((display.config.formViewRecordNumber || 0) + 1).toLocaleString()} / ???`;
+    }
+    if (rowData == null) {
+      if (allRowCount != null) {
+        return `Out of bounds: ${(
+          (display.config.formViewRecordNumber || 0) + 1
+        ).toLocaleString()} / ${allRowCount.toLocaleString()}`;
+      }
+      return 'No data';
+    }
+    if (allRowCount == null || display == null) return 'Loading row count...';
+    return `Row: ${(
+      (display.config.formViewRecordNumber || 0) + 1
+    ).toLocaleString()} / ${allRowCount.toLocaleString()}`;
   }
 
-  export function getFormer() {
-    return former;
+  export function getGrider() {
+    return grider;
   }
 
   // export function getFormDisplay() {
@@ -263,19 +279,41 @@
 
   export async function reconnect() {
     await apiCall('database-connections/refresh', { conid, database });
-    formDisplay.reload();
+    display.reload();
   }
 
   export async function refresh() {
-    formDisplay.reload();
+    display.reload();
   }
 
   export function filterSelectedValue() {
-    formDisplay.filterCellValue(getCellColumn(currentCell), rowData);
+    const column = getCellColumn(currentCell);
+    if (!column || !rowData) return;
+    const value = rowData[column.uniqueName];
+    const expr = getFilterValueExpression(value, column.dataType);
+    if (expr) {
+      setConfig(cfg => ({
+        ...cfg,
+        formViewRecordNumber: 0,
+        filters: {
+          ...cfg.filters,
+          [column.uniqueName]: expr,
+        },
+        addedColumns: cfg.addedColumns.includes(column.uniqueName)
+          ? cfg.addedColumns
+          : [...cfg.addedColumns, column.uniqueName],
+      }));
+      display.reload();
+    }
   }
 
   export function addToFilter() {
-    formDisplay.addFilterColumn(getCellColumn(currentCell));
+    const column = getCellColumn(currentCell);
+    if (!column) return;
+    setConfig(cfg => ({
+      ...cfg,
+      formFilterColumns: [...(cfg.formFilterColumns || []), column.uniqueName],
+    }));
   }
 
   export const activator = createActivator('FormView', false);
@@ -319,7 +357,7 @@
   function setCellValue(cell, value) {
     const column = getCellColumn(cell);
     if (!column) return;
-    former.setCellValue(column.uniqueName, value);
+    grider.setCellValue(0, column.uniqueName, value);
   }
 
   const getCellWidth = (row, col) => {
@@ -331,7 +369,7 @@
   const [inplaceEditorState, dispatchInsplaceEditor] = createReducer((state, action) => {
     switch (action.type) {
       case 'show': {
-        if (!former.editable) return {};
+        if (!grider.editable) return {};
         const column = getCellColumn(action.cell);
         if (!column) return state;
         if (column.uniquePath.length > 1) return state;
@@ -418,14 +456,16 @@
     if (event.keyCode == keycodes.numPadAdd) {
       const col = getCellColumn(currentCell);
       if (col.foreignKey) {
-        formDisplay.toggleExpandedColumn(col.uniqueName, true);
+        display.toggleExpandedColumn(col.uniqueName, true);
+        display.reload();
       }
     }
 
     if (event.keyCode == keycodes.numPadSub) {
       const col = getCellColumn(currentCell);
       if (col.foreignKey) {
-        formDisplay.toggleExpandedColumn(col.uniqueName, false);
+        display.toggleExpandedColumn(col.uniqueName, false);
+        display.reload();
       }
     }
 
@@ -448,7 +488,7 @@
     if (shouldOpenMultilineDialog(cellData)) {
       showModal(EditCellDataModal, {
         value: cellData,
-        onSave: value => former.setCellValue(column.uniqueName, value),
+        onSave: value => grider.setCellValue(0, column.uniqueName, value),
       });
       return true;
     }
@@ -476,7 +516,7 @@
       columnIndex = incrementFunc(columnIndex);
       while (
         isInRange(columnIndex) &&
-        !filterName(formDisplay.config.formColumnFilterText, formDisplay.columns[columnIndex].columnName)
+        !filterName(display.config.formColumnFilterText, display.formColumns[columnIndex].columnName)
       ) {
         columnIndex = incrementFunc(columnIndex);
       }
@@ -484,13 +524,13 @@
         columnIndex = firstInRange;
         while (
           isInRange(columnIndex) &&
-          !filterName(formDisplay.config.formColumnFilterText, formDisplay.columns[columnIndex].columnName)
+          !filterName(display.config.formColumnFilterText, display.formColumns[columnIndex].columnName)
         ) {
           columnIndex = incrementFunc(columnIndex);
         }
       }
       if (!isInRange(columnIndex)) columnIndex = lastInRange;
-      return moveCurrentCell(columnIndex % formDisplay.columns.length, Math.floor(columnIndex / rowCount) * 2);
+      return moveCurrentCell(columnIndex % display.formColumns.length, Math.floor(columnIndex / rowCount) * 2);
     };
 
     if (isCtrlOrCommandKey(event)) {
@@ -507,23 +547,23 @@
       case keycodes.rightArrow:
         return moveCurrentCell(currentCell[0], currentCell[1] + 1);
       case keycodes.upArrow:
-        if (currentCell[1] % 2 == 0 && formDisplay.config.formColumnFilterText) {
+        if (currentCell[1] % 2 == 0 && display.config.formColumnFilterText) {
           return findFilteredColumn(
             x => x - 1,
             x => x >= 0,
-            formDisplay.columns.length - 1,
+            display.formColumns.length - 1,
             0
           );
         }
 
         return moveCurrentCell(currentCell[0] - 1, currentCell[1]);
       case keycodes.downArrow:
-        if (currentCell[1] % 2 == 0 && formDisplay.config.formColumnFilterText) {
+        if (currentCell[1] % 2 == 0 && display.config.formColumnFilterText) {
           return findFilteredColumn(
             x => x + 1,
-            x => x < formDisplay.columns.length,
+            x => x < display.formColumns.length,
             0,
-            formDisplay.columns.length - 1
+            display.formColumns.length - 1
           );
         }
 
@@ -547,10 +587,10 @@
     showModal(DictionaryLookupModal, {
       conid,
       database,
-      driver: formDisplay?.driver,
+      driver: display?.driver,
       pureName: col.foreignKey.refTableName,
       schemaName: col.foreignKey.refSchemaName,
-      onConfirm: value => former.setCellValue(col.uniqueName, value),
+      onConfirm: value => grider.setCellValue(0, col.uniqueName, value),
     });
   }
 </script>
@@ -566,18 +606,19 @@
               data-row={rowIndex}
               data-col={chunkIndex * 2}
               style={rowHeight > 1 ? `height: ${rowHeight}px` : undefined}
-              class:columnFiltered={formDisplay.config.formColumnFilterText &&
-                filterName(formDisplay.config.formColumnFilterText, col.columnName)}
+              class:columnFiltered={display.config.formColumnFilterText &&
+                filterName(display.config.formColumnFilterText, col.columnName)}
               class:isSelected={currentCell[0] == rowIndex && currentCell[1] == chunkIndex * 2}
               bind:this={domCells[`${rowIndex},${chunkIndex * 2}`]}
             >
               <div class="header-cell-inner">
                 {#if col.foreignKey}
                   <FontIcon
-                    icon={plusExpandIcon(formDisplay.isExpandedColumn(col.uniqueName))}
+                    icon={plusExpandIcon(display.isExpandedColumn(col.uniqueName))}
                     on:click={e => {
                       e.stopPropagation();
-                      formDisplay.toggleExpandedColumn(col.uniqueName);
+                      display.toggleExpandedColumn(col.uniqueName);
+                      display.reload();
                     }}
                   />
                 {:else}
@@ -614,7 +655,7 @@
                   {dispatchInsplaceEditor}
                   cellValue={rowData[col.uniqueName]}
                   onSetValue={value => {
-                    former.setCellValue(col.uniqueName, value);
+                    grider.setCellValue(0, col.uniqueName, value);
                   }}
                 />
               {/if}

packages/web/src/formview/FormViewFilterColumn.svelte

@@ -8,7 +8,7 @@
   import FontIcon from '../icons/FontIcon.svelte';
 
   export let column;
-  export let formDisplay;
+  export let display;
   export let filters;
 
   export let driver;
@@ -26,7 +26,7 @@
         square
         narrow
         on:click={() => {
-          formDisplay.removeFilter(column.uniqueName);
+          display.removeFilter(column.uniqueName);
         }}
       >
         <FontIcon icon="icon close" />
@@ -35,7 +35,7 @@
     <DataFilterControl
       filterType={getFilterType(column.dataType)}
       filter={filters[column.uniqueName]}
-      setFilter={value => formDisplay.setFilter(column.uniqueName, value)}
+      setFilter={value => display.setFilter(column.uniqueName, value)}
       {driver}
       {conid}
       {database}

packages/web/src/formview/FormViewFilters.svelte

@@ -4,10 +4,10 @@
   import ManagerInnerContainer from '../elements/ManagerInnerContainer.svelte';
   import keycodes from '../utility/keycodes';
   import FormViewFilterColumn from './FormViewFilterColumn.svelte';
-  import PrimaryKeyFilterEditor from './PrimaryKeyFilterEditor.svelte';
+  // import PrimaryKeyFilterEditor from './PrimaryKeyFilterEditor.svelte';
 
   export let managerSize;
-  export let formDisplay;
+  export let display;
   export let setConfig;
 
   export let driver;
@@ -16,9 +16,9 @@
   export let schemaName;
   export let pureName;
 
-  $: baseTable = formDisplay?.baseTable;
-  $: formFilterColumns = formDisplay?.config?.formFilterColumns;
-  $: filters = formDisplay?.config?.filters;
+  $: baseTable = display?.baseTable;
+  $: formFilterColumns = display?.config?.formFilterColumns;
+  $: filters = display?.config?.filters;
 
   $: allFilterNames = _.union(_.keys(filters || {}), formFilterColumns || []);
 </script>
@@ -28,7 +28,7 @@
   <div class="flex">
     <input
       type="text"
-      value={formDisplay?.config?.formColumnFilterText || ''}
+      value={display?.config?.formColumnFilterText || ''}
       on:keydown={e => {
         if (e.keyCode == keycodes.escape) {
           setConfig(x => ({
@@ -47,23 +47,17 @@
   </div>
 </div>
 
-{#if baseTable?.primaryKey}
-  <ManagerInnerContainer width={managerSize}>
-    {#each baseTable.primaryKey.columns as col}
-      <PrimaryKeyFilterEditor {baseTable} column={col} {formDisplay} />
-    {/each}
-
-    {#each allFilterNames as uniqueName}
-      <FormViewFilterColumn
-        column={formDisplay.columns.find(x => x.uniqueName == uniqueName)}
-        {formDisplay}
-        {filters}
-        {driver}
-        {conid}
-        {database}
-        {schemaName}
-        {pureName}
-      />
-    {/each}
-  </ManagerInnerContainer>
-{/if}
+<ManagerInnerContainer width={managerSize}>
+  {#each allFilterNames as uniqueName}
+    <FormViewFilterColumn
+      column={display.formColumns.find(x => x.uniqueName == uniqueName)}
+      {display}
+      {filters}
+      {driver}
+      {conid}
+      {database}
+      {schemaName}
+      {pureName}
+    />
+  {/each}
+</ManagerInnerContainer>

packages/web/src/formview/Former.ts

@@ -1,53 +0,0 @@
-// export interface GriderRowStatus {
-//     status: 'regular' | 'updated' | 'deleted' | 'inserted';
-//     modifiedFields?: Set<string>;
-//     insertedFields?: Set<string>;
-//     deletedFields?: Set<string>;
-//   }
-
-export default abstract class Former {
-  public rowData: any;
-
-  // getRowStatus(index): GriderRowStatus {
-  //   const res: GriderRowStatus = {
-  //     status: 'regular',
-  //   };
-  //   return res;
-  // }
-  beginUpdate() {}
-  endUpdate() {}
-  setCellValue(uniqueName: string, value: any) {}
-  revertRowChanges() {}
-  revertAllChanges() {}
-  undo() {}
-  redo() {}
-  get editable() {
-    return false;
-  }
-  get canInsert() {
-    return false;
-  }
-  get allowSave() {
-    return this.containsChanges;
-  }
-  get canUndo() {
-    return false;
-  }
-  get canRedo() {
-    return false;
-  }
-  get containsChanges() {
-    return false;
-  }
-  get disableLoadNextPage() {
-    return false;
-  }
-  get errors() {
-    return null;
-  }
-  updateRow(changeObject) {
-    for (const key of Object.keys(changeObject)) {
-      this.setCellValue(key, changeObject[key]);
-    }
-  }
-}

packages/web/src/formview/JslFormView.svelte

@@ -0,0 +1,34 @@
+<script lang="ts" context="module">
+  async function loadRow(props, index) {
+    const { jslid, formatterFunction, display } = props;
+
+    const response = await apiCall('jsldata/get-rows', {
+      jslid,
+      offset: index,
+      limit: 1,
+      formatterFunction,
+      filters: display ? display.compileFilters() : null,
+    });
+
+    if (response.errorMessage) return response;
+    return response[0];
+  }
+</script>
+
+<script lang="ts">
+  import { apiCall } from '../utility/api';
+  import _ from 'lodash';
+  import LoadingFormView from './LoadingFormView.svelte';
+
+  export let display;
+
+  async function handleLoadRow() {
+    return await loadRow($$props, display.config.formViewRecordNumber || 0);
+  }
+
+  async function handleLoadRowCount() {
+    return null;
+  }
+</script>
+
+<LoadingFormView {...$$props} loadRowFunc={handleLoadRow} loadRowCountFunc={handleLoadRowCount} rowCountNotAvailable />

packages/web/src/formview/LoadingFormView.svelte

@@ -0,0 +1,85 @@
+<script lang="ts">
+  import FormView from './FormView.svelte';
+  import { apiCall } from '../utility/api';
+  import ChangeSetGrider from '../datagrid/ChangeSetGrider';
+  import _ from 'lodash';
+
+  export let changeSetState;
+  export let dispatchChangeSet;
+  export let masterLoadedTime;
+  export let conid;
+  export let database;
+  export let onReferenceSourceChanged;
+  export let display;
+
+  export let loadRowFunc;
+  export let loadRowCountFunc;
+
+  let isLoadingData = false;
+  let isLoadedData = false;
+  let rowData = null;
+  let isLoadingCount = false;
+  let isLoadedCount = false;
+  let loadedTime = new Date().getTime();
+  let allRowCount = null;
+  let errorMessage = null;
+
+  const handleLoadCurrentRow = async () => {
+    if (isLoadingData) return;
+    let newLoadedRow = false;
+    isLoadingData = true;
+    const row = await loadRowFunc(display.config.formViewRecordNumber || 0);
+    isLoadingData = false;
+    isLoadedData = true;
+    rowData = row;
+    loadedTime = new Date().getTime();
+    newLoadedRow = row;
+  };
+
+  const handleLoadRowCount = async () => {
+    isLoadingCount = true;
+    allRowCount = await loadRowCountFunc();
+    isLoadedCount = true;
+    isLoadingCount = false;
+  };
+
+  const handleNavigate = async command => {
+    display.formViewNavigate(command, allRowCount);
+  };
+
+  export function reload() {
+    isLoadingData = false;
+    isLoadedData = false;
+    isLoadingCount = false;
+    isLoadedCount = false;
+    rowData = null;
+    loadedTime = new Date().getTime();
+    allRowCount = null;
+    errorMessage = null;
+  }
+
+  $: {
+    if (masterLoadedTime && masterLoadedTime > loadedTime) {
+      display.reload();
+    }
+  }
+
+  $: {
+    if (display?.cache?.refreshTime > loadedTime) {
+      reload();
+    }
+  }
+
+  $: {
+    if (display?.isLoadedCorrectly) {
+      if (!isLoadedData && !isLoadingData) handleLoadCurrentRow();
+      if (isLoadedData && !isLoadingCount && !isLoadedCount) handleLoadRowCount();
+    }
+  }
+
+  $: grider = new ChangeSetGrider(rowData ? [rowData] : [], changeSetState, dispatchChangeSet, display);
+
+  $: if (onReferenceSourceChanged && rowData) onReferenceSourceChanged([rowData], loadedTime);
+</script>
+
+<FormView {...$$props} {grider} isLoading={isLoadingData} {allRowCount} onNavigate={handleNavigate} />

packages/web/src/formview/PrimaryKeyFilterEditor.svelte

@@ -1,51 +0,0 @@
-<script lang="ts">
-  import ColumnLabel from '../elements/ColumnLabel.svelte';
-  import InlineButton from '../buttons/InlineButton.svelte';
-
-  import FontIcon from '../icons/FontIcon.svelte';
-  import keycodes from '../utility/keycodes';
-
-  export let column;
-  export let baseTable;
-  export let formDisplay;
-
-  let domEditor;
-  $: value = formDisplay.getKeyValue(column.columnName);
-
-  const applyFilter = () => {
-    formDisplay.requestKeyValue(column.columnName, domEditor.value);
-  };
-
-  const cancelFilter = () => {
-    formDisplay.cancelRequestKey();
-    formDisplay.reload();
-  };
-
-  const handleKeyDown = ev => {
-    if (ev.keyCode == keycodes.enter) {
-      applyFilter();
-    }
-    if (ev.keyCode == keycodes.escape) {
-      cancelFilter();
-    }
-  };
-
-  $: if (domEditor) domEditor.value = value;
-</script>
-
-<div class="m-1">
-  <div class="space-between">
-    <div>
-      <FontIcon icon="img primary-key" />
-      <ColumnLabel {...baseTable.columns.find(x => x.columnName == column.columnName)} />
-    </div>
-    {#if formDisplay.config.formViewKeyRequested}
-      <InlineButton square on:click={cancelFilter}>
-        <FontIcon icon="icon delete" />
-      </InlineButton>
-    {/if}
-  </div>
-  <div class="flex">
-    <input bind:this={domEditor} type="text" on:blur={applyFilter} on:keydown={handleKeyDown} />
-  </div>
-</div>

packages/web/src/formview/SqlFormView.svelte

@@ -16,108 +16,20 @@
 </script>
 
 <script lang="ts">
-  import ChangeSetFormer from './ChangeSetFormer';
-  import FormView from './FormView.svelte';
   import { apiCall } from '../utility/api';
+  import _ from 'lodash';
+  import LoadingFormView from './LoadingFormView.svelte';
 
-  export let formDisplay;
-  export let changeSetState;
-  export let dispatchChangeSet;
-  export let masterLoadedTime;
-  export let conid;
-  export let database;
-  export let onReferenceSourceChanged;
+  export let display;
 
-  let isLoadingData = false;
-  let isLoadedData = false;
-  let rowData = null;
-  let isLoadingCount = false;
-  let isLoadedCount = false;
-  let loadedTime = new Date().getTime();
-  let allRowCount = null;
-  let rowCountBefore = null;
-  let errorMessage = null;
-
-  const handleLoadCurrentRow = async () => {
-    if (isLoadingData) return;
-    let newLoadedRow = false;
-    if (formDisplay.config.formViewKeyRequested || formDisplay.config.formViewKey) {
-      isLoadingData = true;
-      const row = await loadRow($$props, formDisplay.getCurrentRowQuery());
-      isLoadingData = false;
-      isLoadedData = true;
-      rowData = row;
-      loadedTime = new Date().getTime();
-      newLoadedRow = row;
-    }
-    if (formDisplay.config.formViewKeyRequested && newLoadedRow) {
-      formDisplay.cancelRequestKey(newLoadedRow);
-    }
-    if (!newLoadedRow && !formDisplay.config.formViewKeyRequested) {
-      await handleNavigate('first');
-    }
-  };
-
-  const handleLoadRowCount = async () => {
-    isLoadingCount = true;
-    const countRow = await loadRow($$props, formDisplay.getCountQuery());
-    const countBeforeRow = await loadRow($$props, formDisplay.getBeforeCountQuery());
-
-    isLoadedCount = true;
-    isLoadingCount = false;
-    allRowCount = countRow ? parseInt(countRow.count) : null;
-    rowCountBefore = countBeforeRow ? parseInt(countBeforeRow.count) : null;
-  };
-
-  const handleNavigate = async command => {
-    isLoadingData = true;
-    const row = await loadRow($$props, formDisplay.navigateRowQuery(command));
-    if (row) {
-      formDisplay.navigate(row);
-    }
-    isLoadingData = false;
-    isLoadedData = true;
-    isLoadedCount = false;
-    allRowCount = null;
-    rowCountBefore = null;
-    rowData = row;
-    loadedTime = new Date().getTime();
-  };
-
-  export function reload() {
-    isLoadingData = false;
-    isLoadedData = false;
-    isLoadingCount = false;
-    isLoadedCount = false;
-    rowData = null;
-    loadedTime = new Date().getTime();
-    allRowCount = null;
-    rowCountBefore = null;
-    errorMessage = null;
+  async function handleLoadRow() {
+    return await loadRow($$props, display.getPageQuery(display.config.formViewRecordNumber || 0, 1));
   }
 
-  $: {
-    if (masterLoadedTime && masterLoadedTime > loadedTime) {
-      formDisplay.reload();
-    }
+  async function handleLoadRowCount() {
+    const countRow = await loadRow($$props, display.getCountQuery());
+    return countRow ? parseInt(countRow.count) : null;
   }
-
-  $: {
-    if (formDisplay.cache.refreshTime > loadedTime) {
-      reload();
-    }
-  }
-
-  $: {
-    if (formDisplay.isLoadedCorrectly) {
-      if (!isLoadedData && !isLoadingData) handleLoadCurrentRow();
-      if (isLoadedData && !isLoadingCount && !isLoadedCount) handleLoadRowCount();
-    }
-  }
-
-  $: former = new ChangeSetFormer(rowData, changeSetState, dispatchChangeSet, formDisplay);
-
-  $: if (onReferenceSourceChanged && rowData) onReferenceSourceChanged([rowData], loadedTime);
 </script>
 
-<FormView {...$$props} {former} isLoading={isLoadingData} {allRowCount} {rowCountBefore} onNavigate={handleNavigate} />
+<LoadingFormView {...$$props} loadRowFunc={handleLoadRow} loadRowCountFunc={handleLoadRowCount} />

packages/web/src/formview/openReferenceForm.ts

@@ -1,10 +1,8 @@
+import { getFilterValueExpression } from 'dbgate-filterparser';
 import _ from 'lodash';
 import openNewTab from '../utility/openNewTab';
 
 export default function openReferenceForm(rowData, column, conid, database) {
-  const formViewKey = _.fromPairs(
-    column.foreignKey.columns.map(({ refColumnName, columnName }) => [refColumnName, rowData[columnName]])
-  );
   openNewTab(
     {
       title: column.foreignKey.refTableName,
@@ -21,7 +19,12 @@ export default function openReferenceForm(rowData, column, conid, database) {
     {
       grid: {
         isFormView: true,
-        formViewKey,
+        filters: {
+          [column.foreignKey.columns[0].refColumnName]: getFilterValueExpression(
+            rowData[column.foreignKey.columns[0].columnName],
+            'string'
+          ),
+        },
       },
     },
     {
@@ -50,6 +53,13 @@ export function openPrimaryKeyForm(rowData, baseTable, conid, database) {
     {
       grid: {
         isFormView: true,
+        filters: {
+          [baseTable.primaryKey.columns[0].columnName]: getFilterValueExpression(
+            rowData[baseTable.primaryKey.columns[0].columnName],
+            'string'
+          ),
+        },
+
         formViewKey,
       },
     },