upgraded mongodb driver

Revert "Merge branch 'update-packages'"
This reverts commit c4f322bda2, reversing changes made to f090661eb9.
2024-02-16 08:41:56 +01:00 · 2024-02-12 13:08:30 +01:00 · 2024-02-12 12:59:37 +01:00 · 2024-02-12 12:59:23 +01:00 · 2024-02-12 12:55:20 +01:00 · 2024-02-01 22:48:11 +01:00
372 changed files with 11255 additions and 3849 deletions
@@ -12,7 +12,7 @@ jobs:
    strategy:
      fail-fast: false
      matrix:
-        os: [macOS-10.15, windows-2022, ubuntu-18.04]
+        os: [macos-12, windows-2022, ubuntu-22.04]
        # os: [macOS-10.15]

    steps:
@@ -23,13 +23,16 @@ jobs:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 1
-      - name: Use Node.js 14.x
+      - name: Use Node.js 16.x
        uses: actions/setup-node@v1
        with:
-          node-version: 14.x
+          node-version: 16.x
      - name: yarn adjustPackageJson
        run: |
          yarn adjustPackageJson
+      - name: yarn set timeout
+        run: |
+          yarn config set network-timeout 100000
      - name: yarn install
        run: |
          yarn install
@@ -43,7 +46,7 @@ jobs:
        run: |
          yarn fillPackagedPlugins
      - name: Install Snapcraft
-        if: matrix.os == 'ubuntu-18.04'
+        if: matrix.os == 'ubuntu-22.04'
        uses: samuelmeuli/action-snapcraft@v1
      - name: Publish
        run: |
@@ -62,18 +65,12 @@ jobs:
          APPLE_ID: ${{ secrets.APPLE_ID }}
          APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}

-      - name: Save snap login
-        if: matrix.os == 'ubuntu-18.04'
-        run: 'echo "$SNAPCRAFT_LOGIN" > snapcraft.login'
-        shell: bash
-        env:
-          SNAPCRAFT_LOGIN: ${{secrets.SNAPCRAFT_LOGIN}}
-
      - name: publishSnap
-        if: matrix.os == 'ubuntu-18.04'
+        if: matrix.os == 'ubuntu-22.04'
        run: |
-          snapcraft login --with snapcraft.login
          snapcraft upload --release=beta app/dist/*.snap
+        env:
+          SNAPCRAFT_STORE_CREDENTIALS: ${{secrets.SNAPCRAFT_LOGIN}}

      - name: Copy artifacts
        run: |
@@ -16,8 +16,8 @@ jobs:
    strategy:
      fail-fast: false
      matrix:
-        # os: [ubuntu-18.04, windows-2016]
-        os: [macOS-10.15, windows-2022, ubuntu-18.04]
+        # os: [ubuntu-22.04, windows-2016]
+        os: [macos-12, windows-2022, ubuntu-22.04]

    steps:
      - name: Context
@@ -27,13 +27,16 @@ jobs:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 1
-      - name: Use Node.js 14.x
+      - name: Use Node.js 16.x
        uses: actions/setup-node@v1
        with:
-          node-version: 14.x
+          node-version: 16.x
      - name: yarn adjustPackageJson
        run: |
          yarn adjustPackageJson
+      - name: yarn set timeout
+        run: |
+          yarn config set network-timeout 100000
      - name: yarn install
        run: |
          # yarn --version
@@ -49,7 +52,7 @@ jobs:
        run: |
          yarn fillPackagedPlugins
      - name: Install Snapcraft
-        if: matrix.os == 'ubuntu-18.04'
+        if: matrix.os == 'ubuntu-22.04'
        uses: samuelmeuli/action-snapcraft@v1
      - name: Publish
        run: |
@@ -72,18 +75,12 @@ jobs:
        run: |
          yarn generatePadFile

-      - name: Save snap login
-        if: matrix.os == 'ubuntu-18.04'
-        run: 'echo "$SNAPCRAFT_LOGIN" > snapcraft.login'
-        shell: bash
-        env:
-          SNAPCRAFT_LOGIN: ${{secrets.SNAPCRAFT_LOGIN}}
-
      - name: publishSnap
-        if: matrix.os == 'ubuntu-18.04'
+        if: matrix.os == 'ubuntu-22.04'
        run: |
-          snapcraft login --with snapcraft.login
          snapcraft upload --release=stable app/dist/*.snap
+        env:
+          SNAPCRAFT_STORE_CREDENTIALS: ${{secrets.SNAPCRAFT_LOGIN}}

      - name: Copy artifacts
        run: |
@@ -137,12 +134,12 @@ jobs:
          mv app/dist/dbgate-pad.xml artifacts/ || true

      - name: Copy latest-linux.yml
-        if: matrix.os == 'ubuntu-18.04'
+        if: matrix.os == 'ubuntu-22.04'
        run: |
          mv app/dist/latest-linux.yml artifacts/latest-linux.yml || true

      - name: Copy latest-mac.yml
-        if: matrix.os == 'macOS-10.15'
+        if: matrix.os == 'macos-12'
        run: |
          mv app/dist/latest-mac.yml artifacts/latest-mac.yml || true

@@ -14,7 +14,7 @@ jobs:

    strategy:
      matrix:
-        os: [ubuntu-18.04]
+        os: [ubuntu-22.04]

    steps:
      - name: Context
@@ -53,10 +53,10 @@ jobs:
            type=match,pattern=\d+.\d+.\d+,suffix=-alpine,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
            type=raw,value=alpine,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}

-      - name: Use Node.js 14.x
+      - name: Use Node.js 16.x
        uses: actions/setup-node@v1
        with:
-          node-version: 14.x
+          node-version: 16.x
      - name: yarn install
        run: |
          # yarn --version
@@ -20,7 +20,7 @@ jobs:

    strategy:
      matrix:
-        os: [ubuntu-18.04]
+        os: [ubuntu-22.04]

    steps:
      - name: Context
@@ -30,10 +30,10 @@ jobs:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 1
-      - name: Use Node.js 14.x
+      - name: Use Node.js 16.x
        uses: actions/setup-node@v1
        with:
-          node-version: 14.x
+          node-version: 16.x

      - name: Configure NPM token
        env:
@@ -94,6 +94,11 @@ jobs:
        run: |
          npm publish

+      - name: Publish dbmodel
+        working-directory: packages/dbmodel
+        run: |
+          npm publish
+
      - name: Publish dbgate-plugin-csv
        working-directory: plugins/dbgate-plugin-csv
        run: |
@@ -138,3 +143,8 @@ jobs:
        working-directory: plugins/dbgate-plugin-redis
        run: |
          npm publish
+
+      - name: Publish dbgate-plugin-oracle
+        working-directory: plugins/dbgate-plugin-oracle
+        run: |
+          npm publish
@@ -8,7 +8,7 @@ on:
 jobs:
  test-runner:
    runs-on: ubuntu-latest
-    container: node:14.18
+    container: node:16.20

    steps:
      - name: Context
@@ -8,6 +8,118 @@ Builds:
 - linux - application for linux
 - win - application for Windows

+### 5.2.7
+- FIXED: fix body overflow when context menu height great than viewport #592
+- FIXED: Pass signals in entrypoint.sh #596
+- FIXED: Remove missing links to jenasoft #625
+- FIXED: add API headers on upload call #627
+- FIXED: Disabled shell scripting for NPM distribution by default
+- FIXED: Fixed data import from files #633
+- FIXED: Fixed showing GPS positions #575
+- CHANGED: Improved stability of electron client on Windows and Mac (fewer EPIPE errors)
+
+### 5.2.6
+- FIXED: DbGate creates a lot of .tmp.node files in the temp directory #561
+- FIXED: Typo in datetimeoffset dataType #556
+- FIXED: SQL export is using the wrong hour formatting #537
+- FIXED: Missing toolstrip and adds up to 200% zoom to diagram view #524
+- FIXED: MongoDB password could contain special characters #560
+
+### 5.2.5
+- ADDED: Split Windows #394
+- FIXED: Postgres index asc/desc #514
+- FIXED: Excel export not working since 5.2.3 #511
+- ADDED: Include macOS specific app icon #494
+- FIXED: Resizing window resets window contents #479
+- FIXED: Solved some minor problems with widget collapsing
+
+### 5.2.4
+- FIXED: npm version crash (#508)
+
+### 5.2.3
+- ADDED: Search entire table (multi column filter) #491
+- ADDED: OracleDB - connection to toher than default ports #496
+- CHANGED: OracleDB - status of support set to experimental
+- FIXED: OracleDB database URL - fixes: Connect to default Oracle database #489
+- ADDED: HTML, XML code highlighting for Edit cell value #485
+- FIXED: Intellisense - incorrect alias after ORDER BY clause #484
+- FIXED: Typo in SQL-Generator #481
+- ADDED: Data duplicator #480
+- FIXED: MongoDB - support for views #476
+- FIXED: "SQL:CREATE TABLE" generated SQL default value syntax errors #455
+- FIXED: Crash when right-clicking on tables #452
+- FIXED: View sort #436
+- ADDED: Arm64 version for Windows #473
+- ADDED: Sortable query results and data archive
+- CHANGED: Use transactions for saving table data
+- CHANGED: Save table structure uses transactions
+- ADDED: Table data editing - shows editing mark
+- ADDED: Editing data archive files
+- FIXED: Delete cascade options when using more than 2 tables
+- ADDED: Save to current archive commands
+- ADDED: Current archive mark is on status bar
+- FIXED: Changed package used for parsing JSONL files when browsing - fixes backend freezing
+- FIXED: SSL option for mongodb #504
+- REMOVED: Data sheet editor
+- FIXED: Creating SQLite autoincrement column
+- FIXED: Better error reporting from exports/import/dulicator
+- CHANGED: Optimalizede OracleDB analysing algorithm
+- ADDED: Mutli column filter for perspectives
+- FIXED: Fixed some scenarios using tables from different DBs
+- FIXED: Sessions with long-running queries are not killed
+
+
+### 5.2.2
+- FIXED: Optimalized load DB structure for PostgreSQL #451
+- ADDED: Auto-closing query connections after configurable (15 minutes default) no-activity interval #468
+- ADDED: Set application-name connection parameter (for PostgreSQL and MS SQL) for easier identifying of DbGate connections
+- ADDED: Filters supports binary IDs #467
+- FIXED: Ctrl+Tab works (switching tabs) #457
+- FIXED: Format code supports non-standard letters #450
+- ADDED: New logging system, log to file, ability to reduce logging #360 (using https://www.npmjs.com/package/pinomin)
+- FIXED: crash on Windows and Mac after system goes in suspend mode #458
+- ADDED: dbmodel standalone NPM package (https://www.npmjs.com/package/dbmodel) - deploy database via commandline tool
+
+
+### 5.2.1
+- FIXED: client_id param in OAuth
+- ADDED: OAuth scope parameter
+- FIXED: login page - password was not sent, when submitting by pressing ENTER
+- FIXED: Used permissions fix
+- FIXED: Export modal - fixed crash when selecting different database
+
+### 5.2.0
+- ADDED: Oracle database support #380
+- ADDED: OAuth authentification #407
+- ADDED: Active directory (Windows) authentification #261
+- ADDED: Ask database credentials when login to DB
+- ADDED: Login form instead of simple authorization (simple auth is possible with special configuration)
+- FIXED: MongoDB - connection uri regression
+- ADDED: MongoDB server summary tab
+- FIXED: Broken versioned tables in MariaDB #433
+- CHANGED: Improved editor margin #422
+- ADDED: Implemented camel case search in all search boxes
+- ADDED: MonhoDB filter empty array, not empty array
+- ADDED: Maximize button reflects window state
+- ADDED: MongoDB - database profiler
+- CHANGED: Short JSON values are shown directly in grid
+- FIXED: Fixed filtering nested fields in NDJSON viewer
+- CHANGED: Improved fuzzy search after Ctrl+P #246
+- ADDED: MongoDB: Create collection backup
+- ADDED: Single database mode
+- ADDED: Perspective designer supports joins from MongoDB nested documents and arrays
+- FIXED: Perspective designer joins on MongoDB ObjectId fields
+- ADDED: Filtering columns in designer (query designer, diagram designer, perspective designer)
+- FIXED: Clone MongoDB rows without _id attribute #404
+- CHANGED: Improved cell view with GPS latitude, longitude fields
+- ADDED: SQL: ALTER VIEW and SQL:ALTER PROCEDURE scripts
+- ADDED: Ctrl+F5 refreshes data grid also with database structure #428
+- ADDED: Perspective display modes: text, force text #439
+- FIXED: Fixed file filters #445
+- ADDED: Rename, remove connection folder, memoize opened state after app restart #425
+- FIXED: Show SQLServer alter store procedure #435
+
+
 ### 5.1.6
 - ADDED: Connection folders support #274
 - ADDED: Keyboard shortcut to hide result window and show/hide the side toolbar #406
@@ -22,6 +22,7 @@ DbGate is licensed under MIT license and is completely free.
 * MySQL
 * PostgreSQL
 * SQL Server
+* Oracle (experimental)
 * MongoDB
 * Redis
 * SQLite
@@ -66,23 +67,23 @@ DbGate is licensed under MIT license and is completely free.
 * Mongo JavaScript editor, execute Mongo script (with NodeJs syntax)
 * Redis tree view, generate script from keys, run Redis script
 * Runs as application for Windows, Linux and Mac. Or in Docker container on server and in web Browser on client.
-* Import, export from/to CSV, Excel, JSON, XML
+* Import, export from/to CSV, Excel, JSON, NDJSON, XML
 * Free table editor - quick table data editing (cleanup data after import/before export, prototype tables etc.)
-* Archives - backup your data in JSON files on local filesystem (or on DbGate server, when using web application)
+* Archives - backup your data in NDJSON files on local filesystem (or on DbGate server, when using web application)
 * Charts, export chart to HTML page
 * For detailed info, how to run DbGate in docker container, visit [docker hub](https://hub.docker.com/r/dbgate/dbgate)
 * Extensible plugin architecture
-* Perspectives - nested table view over complex relational data
+* Perspectives - nested table view over complex relational data, query designer on MongoDB databases

 ## How to contribute
 Any contributions are welcome. If you want to contribute without coding, consider following:

 * Tell your friends about DbGate or share on social networks - when more people will use DbGate, it will grow to be better
 * Write review on [Slant.co](https://www.slant.co/improve/options/41086/~dbgate-review) or [G2](https://www.g2.com/products/dbgate/reviews) 
-* Create issue, if you find problem in app, or you have idea to new feature. If issue already exists, you could leave comment on it, to prioritise most wanted issues.
+* Create issue, if you find problem in app, or you have idea to new feature. If issue already exists, you could leave comment on it, to prioritise most wanted issues
 * Create some tutorial video on [youtube](https://www.youtube.com/playlist?list=PLCo7KjCVXhr0RfUSjM9wJMsp_ShL1q61A)
 * Become a backer on [GitHub sponsors](https://github.com/sponsors/dbgate) or [Open collective](https://opencollective.com/dbgate)
-* Where a small coding is acceptable for you, you could [create plugin](https://dbgate.org/docs/plugin-development.html). Plugins for new themes can be created actually without JS coding.
+* Where a small coding is acceptable for you, you could [create plugin](https://dbgate.org/docs/plugin-development.html). Plugins for new themes can be created actually without JS coding

 Thank you!

@@ -93,7 +94,7 @@ There are many database managers now, so why DbGate?
 * Many data browsing functions based using foreign keys - master/detail, expand columns, expandable form view

 ## Design goals
-* Application simplicity - DbGate takes the best and only the best from old [DbGate](http://www.jenasoft.com/dbgate), [DatAdmin](http://www.jenasoft.com/datadmin) and [DbMouse](http://www.jenasoft.com/dbmouse) .
+* Application simplicity - DbGate takes the best and only the best from old DbGate, [DatAdmin](https://www.softpedia.com/get/Internet/Servers/Database-Utils/DatAdmin-Personal.shtml), [DbMouse](https://www.softpedia.com/get/Internet/Servers/Database-Utils/DbMouse.shtml) and [SQL Database Studio](https://en.wikipedia.org/wiki/SQL_Database_Studio)
 * Minimal dependencies
    * Frontend - Svelte
    * Backend - NodeJs, ExpressJs, database connection drivers
@@ -174,4 +175,7 @@ cd dbgate-plugin-my-new-plugin # this directory is created by wizard, edit, what
 yarn plugin # this compiles plugin and copies it into existing DbGate installation
 ```

-After restarting DbGate, you could use your new plugin from DbGate.
+After restarting DbGate, you could use your new plugin from DbGate.
+
+## Logging
+DbGate uses [pinomin logger](https://github.com/dbgate/pinomin). So by default, it produces JSON log messages into console and log files. If you want to see formatted logs, please use [pino-pretty](https://github.com/pinojs/pino-pretty) log formatter.
@@ -19,9 +19,10 @@
    "appId": "org.dbgate",
    "productName": "DbGate",
    "afterSign": "electron-builder-notarize",
+    "asarUnpack": "**/*.node",
    "mac": {
      "category": "database",
-      "icon": "icon512.png",
+      "icon": "icon512-mac.png",
      "hardenedRuntime": true,
      "entitlements": "entitlements.mac.plist",
      "entitlementsInherit": "entitlements.mac.plist",
@@ -71,7 +72,13 @@
    },
    "win": {
      "target": [
-        "nsis",
+        {
+          "target": "nsis",
+          "arch": [
+            "x64",
+            "arm64"
+          ]
+        },
        {
          "target": "zip",
          "arch": [
@@ -109,11 +116,11 @@
    "cross-env": "^6.0.3",
    "electron": "17.4.10",
    "electron-builder": "23.1.0",
-    "electron-builder-notarize": "^1.5.0"
+    "electron-builder-notarize": "^1.5.1"
  },
  "optionalDependencies": {
    "better-sqlite3": "7.6.2",
-    "oracledb": "^5.5.0",
-    "msnodesqlv8": "^2.6.0"
+    "msnodesqlv8": "^2.6.0",
+    "oracledb": "^5.5.0"
  }
 }
@@ -1,6 +1,8 @@
 const electron = require('electron');
 const os = require('os');
 const fs = require('fs');
+// const unhandled = require('electron-unhandled');
+// const { openNewGitHubIssue, debugInfo } = require('electron-util');
 const { Menu, ipcMain } = require('electron');
 const { autoUpdater } = require('electron-updater');
 const log = require('electron-log');
@@ -22,9 +24,25 @@ const configRootPath = path.join(app.getPath('userData'), 'config-root.json');
 let initialConfig = {};
 let apiLoaded = false;
 let mainModule;
+// let getLogger;
+// let loadLogsContent;

 const isMac = () => os.platform() == 'darwin';

+// unhandled({
+//   showDialog: true,
+//   reportButton: error => {
+//     openNewGitHubIssue({
+//       user: 'dbgate',
+//       repo: 'dbgate',
+//       body: `PLEASE DELETE SENSITIVE INFO BEFORE POSTING ISSUE!!!\n\n\`\`\`\n${
+//         error.stack
+//       }\n\`\`\`\n\n---\n\n${debugInfo()}\n\n\`\`\`\n${loadLogsContent ? loadLogsContent(50) : ''}\n\`\`\``,
+//     });
+//   },
+//   logger: error => (getLogger ? getLogger('electron').fatal(error) : console.error(error)),
+// });
+
 try {
  initialConfig = JSON.parse(fs.readFileSync(configRootPath, { encoding: 'utf-8' }));
 } catch (err) {
@@ -154,6 +172,10 @@ ipcMain.on('app-started', async (event, arg) => {
    mainWindow.webContents.send('run-command', runCommandOnLoad);
    runCommandOnLoad = null;
  }
+
+  if (initialConfig['winIsMaximized']) {
+    mainWindow.webContents.send('setIsMaximized', true);
+  }
 });
 ipcMain.on('window-action', async (event, arg) => {
  if (!mainWindow) {
@@ -164,11 +186,7 @@ ipcMain.on('window-action', async (event, arg) => {
      mainWindow.minimize();
      break;
    case 'maximize':
-      if (mainWindow.isMaximized()) {
-        mainWindow.unmaximize();
-      } else {
-        mainWindow.maximize();
-      }
+      mainWindow.isMaximized() ? mainWindow.unmaximize() : mainWindow.maximize();
      break;
    case 'close':
      mainWindow.close();
@@ -211,6 +229,9 @@ ipcMain.on('window-action', async (event, arg) => {
    case 'paste':
      mainWindow.webContents.paste();
      break;
+    case 'selectAll':
+      mainWindow.webContents.selectAll();
+      break;
  }
 });

@@ -308,6 +329,14 @@ function createWindow() {
      mainWindow.setIcon(path.resolve(__dirname, '../icon.png'));
    }
    // mainWindow.webContents.toggleDevTools();
+
+    mainWindow.on('maximize', () => {
+      mainWindow.webContents.send('setIsMaximized', true);
+    });
+
+    mainWindow.on('unmaximize', () => {
+      mainWindow.webContents.send('setIsMaximized', false);
+    });
  }

  if (!apiLoaded) {
@@ -327,9 +356,12 @@ function createWindow() {
    //     path.join(__dirname, process.env.DEVMODE ? '../../packages/api/src/index' : '../packages/api/dist/bundle.js')
    //   )
    // );
+    api.configureLogger();
    const main = api.getMainModule();
    main.useAllControllers(null, electron);
    mainModule = main;
+    // getLogger = api.getLogger;
+    // loadLogsContent = api.loadLogsContent;
    apiLoaded = true;
  }
  mainModule.setElectronSender(mainWindow.webContents);
@@ -21,6 +21,7 @@ module.exports = ({ editMenu }) => [
      { divider: true },
      { command: 'file.exit', hideDisabled: true },
      { command: 'app.logout', hideDisabled: true, skipInApp: true },
+      { command: 'app.disconnect', hideDisabled: true, skipInApp: true },
    ],
  },
  {
@@ -47,6 +48,7 @@ module.exports = ({ editMenu }) => [
          { command: 'edit.cut' },
          { command: 'edit.copy' },
          { command: 'edit.paste' },
+          { command: 'edit.selectAll' },
        ],
      }
    : null,
@@ -85,6 +87,9 @@ module.exports = ({ editMenu }) => [
      { command: 'sql.generator', hideDisabled: true },
      { command: 'file.import', hideDisabled: true },
      { command: 'new.modelCompare', hideDisabled: true },
+      { divider: true },
+      { command: 'folder.showLogs', hideDisabled: true },
+      { command: 'folder.showData', hideDisabled: true },
    ],
  },
  {
@@ -859,10 +859,10 @@ ejs@^3.1.7:
  dependencies:
    jake "^10.8.5"

-electron-builder-notarize@^1.5.0:
-  version "1.5.0"
-  resolved "https://registry.yarnpkg.com/electron-builder-notarize/-/electron-builder-notarize-1.5.0.tgz#ef17c3f0eac6b3908ff2bc3568aea6e4f8612b5b"
-  integrity sha512-kbVCnZX3pCKTXiPhyoMjCZYSQnwS04QmlTM2NB2D/2LCsab5UJA0Me9ZqDT3W35ENPglf1WYDKT+tx9i+xuaPA==
+electron-builder-notarize@^1.5.1:
+  version "1.5.1"
+  resolved "https://registry.yarnpkg.com/electron-builder-notarize/-/electron-builder-notarize-1.5.1.tgz#e00b868a67ef20a77f00017606626f24fdbdc445"
+  integrity sha512-xS7s9gE+1AcJIuJ4DU/LqCrmRypE1zOR/6b66egKzgP/UVh9YSa7rINos34gF/KcueNDQU39HcXcCEKiEI5wPQ==
  dependencies:
    dotenv "^8.2.0"
    electron-notarize "^1.1.1"
@@ -1,9 +1,21 @@
-FROM node:14
+FROM ubuntu:22.04

 RUN apt-get update && apt-get install -y \
+    curl \
+    gnupg \
    iputils-ping \
    iproute2 \
- && rm -rf /var/lib/apt/lists/*
+    unixodbc \
+    gcc \
+    g++ \
+    make
+
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | gpg --dearmor -o /usr/share/keyrings/nodesource-archive-keyring.gpg \
+  && echo "deb [signed-by=/usr/share/keyrings/nodesource-archive-keyring.gpg] https://deb.nodesource.com/node_16.x jammy main" | tee /etc/apt/sources.list.d/nodesource.list \
+  && echo "deb-src [signed-by=/usr/share/keyrings/nodesource-archive-keyring.gpg] https://deb.nodesource.com/node_16.x jammy main" | tee -a /etc/apt/sources.list.d/nodesource.list \
+  && apt-get update && apt-get install -y nodejs \
+  && rm -rf /var/lib/apt/lists/* \
+  && npm install -g yarn

 WORKDIR /home/dbgate-docker

@@ -8,4 +8,4 @@ then
  echo "$HOST_IP $HOST_DOMAIN" >> /etc/hosts
 fi

-node bundle.js --listen-api
+exec node bundle.js --listen-api
@@ -0,0 +1,94 @@
+const engines = require('../engines');
+const stream = require('stream');
+const { testWrapper } = require('../tools');
+const dataDuplicator = require('dbgate-api/src/shell/dataDuplicator');
+const { runCommandOnDriver } = require('dbgate-tools');
+
+describe('Data duplicator', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Insert simple data - %s',
+    testWrapper(async (conn, driver, engine) => {
+      runCommandOnDriver(conn, driver, dmp =>
+        dmp.createTable({
+          pureName: 't1',
+          columns: [
+            { columnName: 'id', dataType: 'int', autoIncrement: true, notNull: true },
+            { columnName: 'val', dataType: 'varchar(50)' },
+          ],
+          primaryKey: {
+            columns: [{ columnName: 'id' }],
+          },
+        })
+      );
+      runCommandOnDriver(conn, driver, dmp =>
+        dmp.createTable({
+          pureName: 't2',
+          columns: [
+            { columnName: 'id', dataType: 'int', autoIncrement: true, notNull: true },
+            { columnName: 'val', dataType: 'varchar(50)' },
+            { columnName: 'valfk', dataType: 'int', notNull: true },
+          ],
+          primaryKey: {
+            columns: [{ columnName: 'id' }],
+          },
+          foreignKeys: [{ refTableName: 't1', columns: [{ columnName: 'valfk', refColumnName: 'id' }] }],
+        })
+      );
+
+      const gett1 = () =>
+        stream.Readable.from([
+          { __isStreamHeader: true, __isDynamicStructure: true },
+          { id: 1, val: 'v1' },
+          { id: 2, val: 'v2' },
+          { id: 3, val: 'v3' },
+        ]);
+      const gett2 = () =>
+        stream.Readable.from([
+          { __isStreamHeader: true, __isDynamicStructure: true },
+          { id: 1, val: 'v1', valfk: 1 },
+          { id: 2, val: 'v2', valfk: 2 },
+          { id: 3, val: 'v3', valfk: 3 },
+        ]);
+
+      await dataDuplicator({
+        systemConnection: conn,
+        driver,
+        items: [
+          {
+            name: 't1',
+            operation: 'copy',
+            openStream: gett1,
+          },
+          {
+            name: 't2',
+            operation: 'copy',
+            openStream: gett2,
+          },
+        ],
+      });
+
+      await dataDuplicator({
+        systemConnection: conn,
+        driver,
+        items: [
+          {
+            name: 't1',
+            operation: 'copy',
+            openStream: gett1,
+          },
+          {
+            name: 't2',
+            operation: 'copy',
+            openStream: gett2,
+          },
+        ],
+      });
+
+      const res1 = await driver.query(conn, `select count(*) as cnt from t1`);
+      expect(res1.rows[0].cnt.toString()).toEqual('6');
+
+      const res2 = await driver.query(conn, `select count(*) as cnt from t2`);
+      expect(res2.rows[0].cnt.toString()).toEqual('6');
+    })
+  );
+});
@@ -136,8 +136,8 @@ const filterLocal = [
  '-MySQL',
  '-MariaDB',
  '-PostgreSQL',
-  'SQL Server',
-  '-SQLite',
+  '-SQL Server',
+  'SQLite',
  '-CockroachDB',
 ];

@@ -13,6 +13,8 @@
    "wait:ci": "cross-env DEVMODE=1 CITEST=1 node wait.js",

    "test:local": "cross-env DEVMODE=1 LOCALTEST=1 jest",
+    "test:local:path": "cross-env DEVMODE=1 LOCALTEST=1 jest --runTestsByPath __tests__/data-duplicator.spec.js",
+    
    "test:ci": "cross-env DEVMODE=1 CITEST=1 jest --runInBand --json --outputFile=result.json --testLocationInResults",
    
    "run:local": "docker-compose down && docker-compose up -d && yarn wait:local && yarn test:local"
@@ -104,3 +104,6 @@ magick icon.png -resize 16x16! ../app/icons/16x16.png
 magick icon.png -resize 192x192! ../packages/web/public/logo192.png
 magick icon.png -resize 512x512! ../packages/web/public/logo512.png
 magick icon.png -define icon:auto-resize="256,128,96,64,48,32,16" ../packages/web/public/favicon.ico
+
+convert icon.png -resize 800x800 -background transparent -gravity center -extent 1000x1000 iconmac.png
+magick composite  iconmac.png macbg.png -resize 600x600! ../app/icon512-mac.png
@@ -1,6 +1,6 @@
 {
  "private": true,
-  "version": "5.1.7-beta.1",
+  "version": "5.2.8-beta.7",
  "name": "dbgate-all",
  "workspaces": [
    "packages/*",
@@ -8,14 +8,18 @@
    "integration-tests"
  ],
  "scripts": {
-    "start:api": "yarn workspace dbgate-api start",
-    "start:app": "cd app && yarn start",
+    "start:api": "yarn workspace dbgate-api start | pino-pretty",
+    "start:api:json": "yarn workspace dbgate-api start",
+    "start:app": "cd app && yarn start | pino-pretty",
+    "start:app:singledb": "CONNECTIONS=con1 SERVER_con1=localhost ENGINE_con1=mysql@dbgate-plugin-mysql USER_con1=root PASSWORD_con1=Pwd2020Db SINGLE_CONNECTION=con1 SINGLE_DATABASE=Chinook yarn start:app",
    "start:api:debug": "cross-env DEBUG=* yarn workspace dbgate-api start",
    "start:app:debug": "cd app && cross-env DEBUG=* yarn start",
    "start:api:debug:ssh": "cross-env DEBUG=ssh yarn workspace dbgate-api start",
    "start:app:debug:ssh": "cd app && cross-env DEBUG=ssh yarn start",
-    "start:api:portal": "yarn workspace dbgate-api start:portal",
-    "start:api:singledb": "yarn workspace dbgate-api start:singledb",
+    "start:api:portal": "yarn workspace dbgate-api start:portal | pino-pretty",
+    "start:api:singledb": "yarn workspace dbgate-api start:singledb | pino-pretty",
+    "start:api:auth": "yarn workspace dbgate-api start:auth | pino-pretty",
+    "start:api:dblogin": "yarn workspace dbgate-api start:dblogin | pino-pretty",
    "start:web": "yarn workspace dbgate-web dev",
    "start:sqltree": "yarn workspace dbgate-sqltree start",
    "start:tools": "yarn workspace dbgate-tools start",
@@ -55,7 +59,8 @@
  },
  "dependencies": {
    "concurrently": "^5.1.0",
-    "patch-package": "^6.2.1"
+    "patch-package": "^6.2.1",
+    "pino-pretty": "^9.1.1"
  },
  "devDependencies": {
    "copyfiles": "^2.2.0",
@@ -0,0 +1 @@
+.env
@@ -0,0 +1,14 @@
+DEVMODE=1
+
+CONNECTIONS=mysql
+SINGLE_CONNECTION=mysql
+# SINGLE_DATABASE=Chinook
+
+LABEL_mysql=MySql localhost
+SERVER_mysql=localhost
+# USER_mysql=root
+PORT_mysql=3306
+# PASSWORD_mysql=Pwd2020Db
+ENGINE_mysql=mysql@dbgate-plugin-mysql
+# PASSWORD_MODE_mysql=askPassword
+PASSWORD_MODE_mysql=askUser
@@ -5,8 +5,8 @@ CONNECTIONS=mysql
 LABEL_mysql=MySql localhost
 SERVER_mysql=localhost
 USER_mysql=root
-PASSWORD_mysql=test
-PORT_mysql=3307
+PASSWORD_mysql=Pwd2020Db
+PORT_mysql=3306
 ENGINE_mysql=mysql@dbgate-plugin-mysql
 DBCONFIG_mysql=[{"name":"Chinook","connectionColor":"cyan"}]

@@ -17,6 +17,7 @@
    "dbgate"
  ],
  "dependencies": {
+    "activedirectory2": "^2.1.0",
    "async-lock": "^1.2.4",
    "axios": "^0.21.1",
    "body-parser": "^1.19.0",
@@ -25,9 +26,10 @@
    "compare-versions": "^3.6.0",
    "cors": "^2.8.5",
    "cross-env": "^6.0.3",
-    "dbgate-query-splitter": "^4.9.2",
+    "dbgate-query-splitter": "^4.9.3",
    "dbgate-sqltree": "^5.0.0-alpha.1",
    "dbgate-tools": "^5.0.0-alpha.1",
+    "dbgate-datalib": "^5.0.0-alpha.1",
    "debug": "^4.3.4",
    "diff": "^5.0.0",
    "diff2html": "^3.4.13",
@@ -35,6 +37,7 @@
    "express": "^4.17.1",
    "express-basic-auth": "^1.2.0",
    "express-fileupload": "^1.2.0",
+    "external-sorting": "^1.3.1",
    "fs-extra": "^9.1.0",
    "fs-reverse": "^0.0.3",
    "get-port": "^5.1.1",
@@ -42,12 +45,16 @@
    "is-electron": "^2.2.1",
    "js-yaml": "^4.1.0",
    "json-stable-stringify": "^1.0.1",
+    "jsonwebtoken": "^8.5.1",
    "line-reader": "^0.4.0",
    "lodash": "^4.17.21",
+    "moment": "^2.24.0",
    "ncp": "^2.0.0",
    "node-cron": "^2.0.3",
    "on-finished": "^2.4.1",
+    "pinomin": "^1.0.1",
    "portfinder": "^1.0.28",
+    "rimraf": "^3.0.0",
    "simple-encryptor": "^4.0.0",
    "ssh2": "^1.11.0",
    "tar": "^6.0.5",
@@ -57,6 +64,8 @@
    "start": "env-cmd node src/index.js --listen-api",
    "start:portal": "env-cmd -f env/portal/.env node src/index.js --listen-api",
    "start:singledb": "env-cmd -f env/singledb/.env node src/index.js --listen-api",
+    "start:auth": "env-cmd -f env/auth/.env node src/index.js --listen-api",
+    "start:dblogin": "env-cmd -f env/dblogin/.env node src/index.js --listen-api",
    "start:filedb": "env-cmd node src/index.js /home/jena/test/chinook/Chinook.db --listen-api",
    "start:singleconn": "env-cmd node src/index.js --server localhost --user root --port 3307 --engine mysql@dbgate-plugin-mysql --password test --listen-api",
    "ts": "tsc",
@@ -75,7 +84,7 @@
  },
  "optionalDependencies": {
    "better-sqlite3": "7.6.2",
-    "oracledb": "^5.5.0",
-    "msnodesqlv8": "^2.6.0"
+    "msnodesqlv8": "^2.6.0",
+    "oracledb": "^5.5.0"
  }
 }
@@ -58,7 +58,7 @@ module.exports = {

  refreshFiles_meta: true,
  async refreshFiles({ folder }) {
-    socket.emitChanged(`app-files-changed-${folder}`);
+    socket.emitChanged('app-files-changed', { app: folder });
  },

  refreshFolders_meta: true,
@@ -69,7 +69,7 @@ module.exports = {
  deleteFile_meta: true,
  async deleteFile({ folder, file, fileType }) {
    await fs.unlink(path.join(appdir(), folder, `${file}.${fileType}`));
-    socket.emitChanged(`app-files-changed-${folder}`);
+    socket.emitChanged('app-files-changed', { app: folder });
    this.emitChangedDbApp(folder);
  },

@@ -79,7 +79,7 @@ module.exports = {
      path.join(path.join(appdir(), folder), `${file}.${fileType}`),
      path.join(path.join(appdir(), folder), `${newFile}.${fileType}`)
    );
-    socket.emitChanged(`app-files-changed-${folder}`);
+    socket.emitChanged('app-files-changed', { app: folder });
    this.emitChangedDbApp(folder);
  },

@@ -95,7 +95,7 @@ module.exports = {
    if (!folder) throw new Error('Missing folder parameter');
    await fs.rmdir(path.join(appdir(), folder), { recursive: true });
    socket.emitChanged(`app-folders-changed`);
-    socket.emitChanged(`app-files-changed-${folder}`);
+    socket.emitChanged('app-files-changed', { app: folder });
    socket.emitChanged('used-apps-changed');
  },

@@ -219,7 +219,7 @@ module.exports = {

    await fs.writeFile(file, JSON.stringify(json, undefined, 2));

-    socket.emitChanged(`app-files-changed-${appFolder}`);
+    socket.emitChanged('app-files-changed', { app: appFolder });
    socket.emitChanged('used-apps-changed');
  },

@@ -271,7 +271,7 @@ module.exports = {
    const file = path.join(appdir(), appFolder, fileName);
    if (!(await fs.exists(file))) {
      await fs.writeFile(file, JSON.stringify(content, undefined, 2));
-      socket.emitChanged(`app-files-changed-${appFolder}`);
+      socket.emitChanged('app-files-changed', { app: appFolder });
      socket.emitChanged('used-apps-changed');
      return true;
    }
@@ -3,8 +3,15 @@ const readline = require('readline');
 const path = require('path');
 const { archivedir, clearArchiveLinksCache, resolveArchiveFolder } = require('../utility/directories');
 const socket = require('../utility/socket');
-const { saveFreeTableData } = require('../utility/freeTableStorage');
 const loadFilesRecursive = require('../utility/loadFilesRecursive');
+const getJslFileName = require('../utility/getJslFileName');
+const { getLogger } = require('dbgate-tools');
+const uuidv1 = require('uuid/v1');
+const dbgateApi = require('../shell');
+const jsldata = require('./jsldata');
+const platformInfo = require('../utility/platformInfo');
+
+const logger = getLogger('archive');

 module.exports = {
  folders_meta: true,
@@ -67,25 +74,28 @@ module.exports = {
        ...fileType('.matview.sql', 'matview.sql'),
      ];
    } catch (err) {
-      console.log('Error reading archive files', err.message);
+      logger.error({ err }, 'Error reading archive files');
      return [];
    }
  },

  refreshFiles_meta: true,
  async refreshFiles({ folder }) {
-    socket.emitChanged(`archive-files-changed-${folder}`);
+    socket.emitChanged('archive-files-changed', { folder });
+    return true;
  },

  refreshFolders_meta: true,
  async refreshFolders() {
    socket.emitChanged(`archive-folders-changed`);
+    return true;
  },

  deleteFile_meta: true,
  async deleteFile({ folder, file, fileType }) {
    await fs.unlink(path.join(resolveArchiveFolder(folder), `${file}.${fileType}`));
-    socket.emitChanged(`archive-files-changed-${folder}`);
+    socket.emitChanged(`archive-files-changed`, { folder });
+    return true;
  },

  renameFile_meta: true,
@@ -94,7 +104,47 @@ module.exports = {
      path.join(resolveArchiveFolder(folder), `${file}.${fileType}`),
      path.join(resolveArchiveFolder(folder), `${newFile}.${fileType}`)
    );
-    socket.emitChanged(`archive-files-changed-${folder}`);
+    socket.emitChanged(`archive-files-changed`, { folder });
+    return true;
+  },
+
+  modifyFile_meta: true,
+  async modifyFile({ folder, file, changeSet, mergedRows, mergeKey, mergeMode }) {
+    await jsldata.closeDataStore(`archive://${folder}/${file}`);
+    const changedFilePath = path.join(resolveArchiveFolder(folder), `${file}.jsonl`);
+
+    if (!fs.existsSync(changedFilePath)) {
+      if (!mergedRows) {
+        return false;
+      }
+      const fileStream = fs.createWriteStream(changedFilePath);
+      for (const row of mergedRows) {
+        await fileStream.write(JSON.stringify(row) + '\n');
+      }
+      await fileStream.close();
+
+      socket.emitChanged(`archive-files-changed`, { folder });
+      return true;
+    }
+
+    const tmpchangedFilePath = path.join(resolveArchiveFolder(folder), `${file}-${uuidv1()}.jsonl`);
+    const reader = await dbgateApi.modifyJsonLinesReader({
+      fileName: changedFilePath,
+      changeSet,
+      mergedRows,
+      mergeKey,
+      mergeMode,
+    });
+    const writer = await dbgateApi.jsonLinesWriter({ fileName: tmpchangedFilePath });
+    await dbgateApi.copyStream(reader, writer);
+    if (platformInfo.isWindows) {
+      await fs.copyFile(tmpchangedFilePath, changedFilePath);
+      await fs.unlink(tmpchangedFilePath);
+    } else {
+      await fs.unlink(changedFilePath);
+      await fs.rename(tmpchangedFilePath, changedFilePath);
+    }
+    return true;
  },

  renameFolder_meta: true,
@@ -102,6 +152,7 @@ module.exports = {
    const uniqueName = await this.getNewArchiveFolder({ database: newFolder });
    await fs.rename(path.join(archivedir(), folder), path.join(archivedir(), uniqueName));
    socket.emitChanged(`archive-folders-changed`);
+    return true;
  },

  deleteFolder_meta: true,
@@ -113,40 +164,42 @@ module.exports = {
      await fs.rmdir(path.join(archivedir(), folder), { recursive: true });
    }
    socket.emitChanged(`archive-folders-changed`);
-  },
-
-  saveFreeTable_meta: true,
-  async saveFreeTable({ folder, file, data }) {
-    await saveFreeTableData(path.join(resolveArchiveFolder(folder), `${file}.jsonl`), data);
-    socket.emitChanged(`archive-files-changed-${folder}`);
    return true;
  },

-  loadFreeTable_meta: true,
-  async loadFreeTable({ folder, file }) {
-    return new Promise((resolve, reject) => {
-      const fileStream = fs.createReadStream(path.join(resolveArchiveFolder(folder), `${file}.jsonl`));
-      const liner = readline.createInterface({
-        input: fileStream,
-      });
-      let structure = null;
-      const rows = [];
-      liner.on('line', line => {
-        const data = JSON.parse(line);
-        if (structure) rows.push(data);
-        else structure = data;
-      });
-      liner.on('close', () => {
-        resolve({ structure, rows });
-        fileStream.close();
-      });
-    });
-  },
-
  saveText_meta: true,
  async saveText({ folder, file, text }) {
    await fs.writeFile(path.join(resolveArchiveFolder(folder), `${file}.jsonl`), text);
-    socket.emitChanged(`archive-files-changed-${folder}`);
+    socket.emitChanged(`archive-files-changed`, { folder });
+    return true;
+  },
+
+  saveJslData_meta: true,
+  async saveJslData({ folder, file, jslid, changeSet }) {
+    const source = getJslFileName(jslid);
+    const target = path.join(resolveArchiveFolder(folder), `${file}.jsonl`);
+    if (changeSet) {
+      const reader = await dbgateApi.modifyJsonLinesReader({
+        fileName: source,
+        changeSet,
+      });
+      const writer = await dbgateApi.jsonLinesWriter({ fileName: target });
+      await dbgateApi.copyStream(reader, writer);
+    } else {
+      await fs.copyFile(source, target);
+      socket.emitChanged(`archive-files-changed`, { folder });
+    }
+    return true;
+  },
+
+  saveRows_meta: true,
+  async saveRows({ folder, file, rows }) {
+    const fileStream = fs.createWriteStream(path.join(resolveArchiveFolder(folder), `${file}.jsonl`));
+    for (const row of rows) {
+      await fileStream.write(JSON.stringify(row) + '\n');
+    }
+    await fileStream.close();
+    socket.emitChanged(`archive-files-changed`, { folder });
    return true;
  },

@@ -0,0 +1,150 @@
+const axios = require('axios');
+const jwt = require('jsonwebtoken');
+const getExpressPath = require('../utility/getExpressPath');
+const uuidv1 = require('uuid/v1');
+const { getLogins } = require('../utility/hasPermission');
+const { getLogger } = require('dbgate-tools');
+const AD = require('activedirectory2').promiseWrapper;
+
+const logger = getLogger('auth');
+
+const tokenSecret = uuidv1();
+
+function shouldAuthorizeApi() {
+  const logins = getLogins();
+  return !!process.env.OAUTH_AUTH || !!process.env.AD_URL || (!!logins && !process.env.BASIC_AUTH);
+}
+
+function getTokenLifetime() {
+  return process.env.TOKEN_LIFETIME || '1d';
+}
+
+function unauthorizedResponse(req, res, text) {
+  // if (req.path == getExpressPath('/config/get-settings')) {
+  //   return res.json({});
+  // }
+  // if (req.path == getExpressPath('/connections/list')) {
+  //   return res.json([]);
+  // }
+  return res.sendStatus(401).send(text);
+}
+
+function authMiddleware(req, res, next) {
+  const SKIP_AUTH_PATHS = ['/config/get', '/auth/oauth-token', '/auth/login', '/stream'];
+
+  if (!shouldAuthorizeApi()) {
+    return next();
+  }
+  let skipAuth = !!SKIP_AUTH_PATHS.find(x => req.path == getExpressPath(x));
+
+  const authHeader = req.headers.authorization;
+  if (!authHeader) {
+    if (skipAuth) {
+      return next();
+    }
+    return unauthorizedResponse(req, res, 'missing authorization header');
+  }
+  const token = authHeader.split(' ')[1];
+  try {
+    const decoded = jwt.verify(token, tokenSecret);
+    req.user = decoded;
+    return next();
+  } catch (err) {
+    if (skipAuth) {
+      return next();
+    }
+
+    logger.error({ err }, 'Sending invalid token error');
+
+    return unauthorizedResponse(req, res, 'invalid token');
+  }
+}
+
+module.exports = {
+  oauthToken_meta: true,
+  async oauthToken(params) {
+    const { redirectUri, code } = params;
+
+    const scopeParam = process.env.OAUTH_SCOPE ? `&scope=${process.env.OAUTH_SCOPE}` : '';
+    const resp = await axios.default.post(
+      `${process.env.OAUTH_TOKEN}`,
+      `grant_type=authorization_code&code=${encodeURIComponent(code)}&redirect_uri=${encodeURIComponent(
+        redirectUri
+      )}&client_id=${process.env.OAUTH_CLIENT_ID}&client_secret=${process.env.OAUTH_CLIENT_SECRET}${scopeParam}`
+    );
+
+    const { access_token, refresh_token } = resp.data;
+
+    const payload = jwt.decode(access_token);
+
+    logger.info({ payload }, 'User payload returned from OAUTH');
+
+    const login =
+      process.env.OAUTH_LOGIN_FIELD && payload && payload[process.env.OAUTH_LOGIN_FIELD]
+        ? payload[process.env.OAUTH_LOGIN_FIELD]
+        : 'oauth';
+
+    if (
+      process.env.OAUTH_ALLOWED_LOGINS &&
+      !process.env.OAUTH_ALLOWED_LOGINS.split(',').find(x => x.toLowerCase().trim() == login.toLowerCase().trim())
+    ) {
+      return { error: `Username ${login} not allowed to log in` };
+    }
+    if (access_token) {
+      return {
+        accessToken: jwt.sign({ login }, tokenSecret, { expiresIn: getTokenLifetime() }),
+      };
+    }
+
+    return { error: 'Token not found' };
+  },
+  login_meta: true,
+  async login(params) {
+    const { login, password } = params;
+
+    if (process.env.AD_URL) {
+      const adConfig = {
+        url: process.env.AD_URL,
+        baseDN: process.env.AD_BASEDN,
+        username: process.env.AD_USERNAME,
+        password: process.env.AD_PASSOWRD,
+      };
+      const ad = new AD(adConfig);
+      try {
+        const res = await ad.authenticate(login, password);
+        if (!res) {
+          return { error: 'Login failed' };
+        }
+        if (
+          process.env.AD_ALLOWED_LOGINS &&
+          !process.env.AD_ALLOWED_LOGINS.split(',').find(x => x.toLowerCase().trim() == login.toLowerCase().trim())
+        ) {
+          return { error: `Username ${login} not allowed to log in` };
+        }
+        return {
+          accessToken: jwt.sign({ login }, tokenSecret, { expiresIn: getTokenLifetime() }),
+        };
+      } catch (err) {
+        logger.error({ err }, 'Failed active directory authentization');
+        return {
+          error: err.message,
+        };
+      }
+    }
+
+    const logins = getLogins();
+    if (!logins) {
+      return { error: 'Logins not configured' };
+    }
+    const foundLogin = logins.find(x => x.login == login);
+    if (foundLogin && foundLogin.password == password) {
+      return {
+        accessToken: jwt.sign({ login }, tokenSecret, { expiresIn: getTokenLifetime() }),
+      };
+    }
+    return { error: 'Invalid credentials' };
+  },
+
+  authMiddleware,
+  shouldAuthorizeApi,
+};
@@ -2,7 +2,7 @@ const fs = require('fs-extra');
 const os = require('os');
 const path = require('path');
 const axios = require('axios');
-const { datadir } = require('../utility/directories');
+const { datadir, getLogsFilePath } = require('../utility/directories');
 const { hasPermission, getLogins } = require('../utility/hasPermission');
 const socket = require('../utility/socket');
 const _ = require('lodash');
@@ -28,18 +28,28 @@ module.exports = {
  get_meta: true,
  async get(_params, req) {
    const logins = getLogins();
-    const login = logins ? logins.find(x => x.login == (req.auth && req.auth.user)) : null;
+    const loginName =
+      req && req.user && req.user.login ? req.user.login : req && req.auth && req.auth.user ? req.auth.user : null;
+    const login = logins && loginName ? logins.find(x => x.login == loginName) : null;
    const permissions = login ? login.permissions : process.env.PERMISSIONS;

    return {
      runAsPortal: !!connections.portalConnections,
-      singleDatabase: connections.singleDatabase,
+      singleDbConnection: connections.singleDbConnection,
+      singleConnection: connections.singleConnection,
      // hideAppEditor: !!process.env.HIDE_APP_EDITOR,
      allowShellConnection: platformInfo.allowShellConnection,
      allowShellScripting: platformInfo.allowShellScripting,
      isDocker: platformInfo.isDocker,
      permissions,
      login,
+      oauth: process.env.OAUTH_AUTH,
+      oauthClient: process.env.OAUTH_CLIENT_ID,
+      oauthScope: process.env.OAUTH_SCOPE,
+      oauthLogout: process.env.OAUTH_LOGOUT,
+      isLoginForm: !!process.env.AD_URL || (!!logins && !process.env.BASIC_AUTH),
+      logsFilePath: getLogsFilePath(),
+      connectionsFilePath: path.join(datadir(), 'connections.jsonl'),
      ...currentVersion,
    };
  },
@@ -2,6 +2,7 @@ const path = require('path');
 const { fork } = require('child_process');
 const _ = require('lodash');
 const fs = require('fs-extra');
+const crypto = require('crypto');

 const { datadir, filesdir } = require('../utility/directories');
 const socket = require('../utility/socket');
@@ -11,9 +12,14 @@ const { pickSafeConnectionInfo } = require('../utility/crypting');
 const JsonLinesDatabase = require('../utility/JsonLinesDatabase');

 const processArgs = require('../utility/processArgs');
-const { safeJsonParse } = require('dbgate-tools');
+const { safeJsonParse, getLogger } = require('dbgate-tools');
 const platformInfo = require('../utility/platformInfo');
 const { connectionHasPermission, testConnectionPermission } = require('../utility/hasPermission');
+const pipeForkLogs = require('../utility/pipeForkLogs');
+
+const logger = getLogger('connections');
+
+let volatileConnections = {};

 function getNamedArgs() {
  const res = {};
@@ -49,6 +55,7 @@ function getPortalCollections() {
      server: process.env[`SERVER_${id}`],
      user: process.env[`USER_${id}`],
      password: process.env[`PASSWORD_${id}`],
+      passwordMode: process.env[`PASSWORD_MODE_${id}`],
      port: process.env[`PORT_${id}`],
      databaseUrl: process.env[`URL_${id}`],
      useDatabaseUrl: !!process.env[`URL_${id}`],
@@ -82,13 +89,13 @@ function getPortalCollections() {
      sslKeyFile: process.env[`SSL_KEY_FILE_${id}`],
      sslRejectUnauthorized: process.env[`SSL_REJECT_UNAUTHORIZED_${id}`],
    }));
-    console.log('Using connections from ENV variables:');
-    console.log(JSON.stringify(connections.map(pickSafeConnectionInfo), undefined, 2));
+
+    logger.info({ connections: connections.map(pickSafeConnectionInfo) }, 'Using connections from ENV variables');
    const noengine = connections.filter(x => !x.engine);
    if (noengine.length > 0) {
-      console.log(
-        'Warning: Invalid CONNECTIONS configutation, missing ENGINE for connection ID:',
-        noengine.map(x => x._id)
+      logger.warn(
+        { connections: noengine.map(x => x._id) },
+        'Invalid CONNECTIONS configutation, missing ENGINE for connection ID'
      );
    }
    return connections;
@@ -126,9 +133,10 @@ function getPortalCollections() {

  return null;
 }
+
 const portalConnections = getPortalCollections();

-function getSingleDatabase() {
+function getSingleDbConnection() {
  if (process.env.SINGLE_CONNECTION && process.env.SINGLE_DATABASE) {
    // @ts-ignore
    const connection = portalConnections.find(x => x._id == process.env.SINGLE_CONNECTION);
@@ -152,12 +160,31 @@ function getSingleDatabase() {
  return null;
 }

-const singleDatabase = getSingleDatabase();
+function getSingleConnection() {
+  if (getSingleDbConnection()) return null;
+  if (process.env.SINGLE_CONNECTION) {
+    // @ts-ignore
+    const connection = portalConnections.find(x => x._id == process.env.SINGLE_CONNECTION);
+    if (connection) {
+      return connection;
+    }
+  }
+  // @ts-ignore
+  const arg0 = (portalConnections || []).find(x => x._id == 'argv');
+  if (arg0) {
+    return arg0;
+  }
+  return null;
+}
+
+const singleDbConnection = getSingleDbConnection();
+const singleConnection = getSingleConnection();

 module.exports = {
  datastore: null,
  opened: [],
-  singleDatabase,
+  singleDbConnection,
+  singleConnection,
  portalConnections,

  async _init() {
@@ -179,13 +206,20 @@ module.exports = {

  test_meta: true,
  test(connection) {
-    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
-      '--is-forked-api',
-      '--start-process',
-      'connectProcess',
-      ...processArgs.getPassArgs(),
-      // ...process.argv.slice(3),
-    ]);
+    const subprocess = fork(
+      global['API_PACKAGE'] || process.argv[1],
+      [
+        '--is-forked-api',
+        '--start-process',
+        'connectProcess',
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
+      ],
+      {
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+      }
+    );
+    pipeForkLogs(subprocess);
    subprocess.send(connection);
    return new Promise(resolve => {
      subprocess.on('message', resp => {
@@ -199,6 +233,36 @@ module.exports = {
    });
  },

+  saveVolatile_meta: true,
+  async saveVolatile({ conid, user, password, test }) {
+    const old = await this.getCore({ conid });
+    const res = {
+      ...old,
+      _id: crypto.randomUUID(),
+      password,
+      passwordMode: undefined,
+      unsaved: true,
+    };
+    if (old.passwordMode == 'askUser') {
+      res.user = user;
+    }
+
+    if (test) {
+      const testRes = await this.test(res);
+      if (testRes.msgtype == 'connected') {
+        volatileConnections[res._id] = res;
+        return {
+          ...res,
+          msgtype: 'connected',
+        };
+      }
+      return testRes;
+    } else {
+      volatileConnections[res._id] = res;
+      return res;
+    }
+  },
+
  save_meta: true,
  async save(connection) {
    if (portalConnections) return;
@@ -229,6 +293,14 @@ module.exports = {
    return res;
  },

+  batchChangeFolder_meta: true,
+  async batchChangeFolder({ folder, newFolder }, req) {
+    // const updated = await this.datastore.find(x => x.parent == folder);
+    const res = await this.datastore.updateAll(x => (x.parent == folder ? { ...x, parent: newFolder } : x));
+    socket.emitChanged('connection-list-changed');
+    return res;
+  },
+
  updateDatabase_meta: true,
  async updateDatabase({ conid, database, values }, req) {
    if (portalConnections) return;
@@ -258,6 +330,10 @@ module.exports = {

  async getCore({ conid, mask = false }) {
    if (!conid) return null;
+    const volatile = volatileConnections[conid];
+    if (volatile) {
+      return volatile;
+    }
    if (portalConnections) {
      const res = portalConnections.find(x => x._id == conid) || null;
      return mask && !platformInfo.allowShellConnection ? maskConnection(res) : res;
@@ -12,6 +12,7 @@ const {
  matchPairedObjects,
  extendDatabaseInfo,
  modelCompareDbDiffOptions,
+  getLogger,
 } = require('dbgate-tools');
 const { html, parse } = require('diff2html');
 const { handleProcessCommunication } = require('../utility/processComm');
@@ -27,6 +28,10 @@ const { createTwoFilesPatch } = require('diff');
 const diff2htmlPage = require('../utility/diff2htmlPage');
 const processArgs = require('../utility/processArgs');
 const { testConnectionPermission } = require('../utility/hasPermission');
+const { MissingCredentialsError } = require('../utility/exceptions');
+const pipeForkLogs = require('../utility/pipeForkLogs');
+
+const logger = getLogger('databaseConnections');

 module.exports = {
  /** @type {import('dbgate-types').OpenedDatabaseConnection[]} */
@@ -42,24 +47,24 @@ module.exports = {
    const existing = this.opened.find(x => x.conid == conid && x.database == database);
    if (!existing) return;
    existing.structure = structure;
-    socket.emitChanged(`database-structure-changed-${conid}-${database}`);
+    socket.emitChanged('database-structure-changed', { conid, database });
  },
  handle_structureTime(conid, database, { analysedTime }) {
    const existing = this.opened.find(x => x.conid == conid && x.database == database);
    if (!existing) return;
    existing.analysedTime = analysedTime;
-    socket.emitChanged(`database-status-changed-${conid}-${database}`);
+    socket.emitChanged(`database-status-changed`, { conid, database });
  },
  handle_version(conid, database, { version }) {
    const existing = this.opened.find(x => x.conid == conid && x.database == database);
    if (!existing) return;
    existing.serverVersion = version;
-    socket.emitChanged(`database-server-version-changed-${conid}-${database}`);
+    socket.emitChanged(`database-server-version-changed`, { conid, database });
  },

  handle_error(conid, database, props) {
    const { error } = props;
-    console.log(`Error in database connection ${conid}, database ${database}: ${error}`);
+    logger.error(`Error in database connection ${conid}, database ${database}: ${error}`);
  },
  handle_response(conid, database, { msgid, ...response }) {
    const [resolve, reject] = this.requests[msgid];
@@ -72,7 +77,7 @@ module.exports = {
    if (!existing) return;
    if (existing.status && status && existing.status.counter > status.counter) return;
    existing.status = status;
-    socket.emitChanged(`database-status-changed-${conid}-${database}`);
+    socket.emitChanged(`database-status-changed`, { conid, database });
  },

  handle_ping() {},
@@ -81,13 +86,23 @@ module.exports = {
    const existing = this.opened.find(x => x.conid == conid && x.database == database);
    if (existing) return existing;
    const connection = await connections.getCore({ conid });
-    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
-      '--is-forked-api',
-      '--start-process',
-      'databaseConnectionProcess',
-      ...processArgs.getPassArgs(),
-      // ...process.argv.slice(3),
-    ]);
+    if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') {
+      throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode });
+    }
+    const subprocess = fork(
+      global['API_PACKAGE'] || process.argv[1],
+      [
+        '--is-forked-api',
+        '--start-process',
+        'databaseConnectionProcess',
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
+      ],
+      {
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+      }
+    );
+    pipeForkLogs(subprocess);
    const lastClosed = this.closed[`${conid}/${database}`];
    const newOpened = {
      conid,
@@ -125,7 +140,12 @@ module.exports = {
    const msgid = uuidv1();
    const promise = new Promise((resolve, reject) => {
      this.requests[msgid] = [resolve, reject];
-      conn.subprocess.send({ msgid, ...message });
+      try {
+        conn.subprocess.send({ msgid, ...message });
+      } catch (err) {
+        logger.error({ err }, 'Error sending request do process');
+        this.close(conn.conid, conn.database);
+      }
    });
    return promise;
  },
@@ -133,7 +153,7 @@ module.exports = {
  queryData_meta: true,
  async queryData({ conid, database, sql }, req) {
    testConnectionPermission(conid, req);
-    console.log(`Processing query, conid=${conid}, database=${database}, sql=${sql}`);
+    logger.info({ conid, database, sql }, 'Processing query');
    const opened = await this.ensureOpened(conid, database);
    // if (opened && opened.status && opened.status.name == 'error') {
    //   return opened.status;
@@ -151,11 +171,11 @@ module.exports = {
  },

  runScript_meta: true,
-  async runScript({ conid, database, sql }, req) {
+  async runScript({ conid, database, sql, useTransaction }, req) {
    testConnectionPermission(conid, req);
-    console.log(`Processing script, conid=${conid}, database=${database}, sql=${sql}`);
+    logger.info({ conid, database, sql }, 'Processing script');
    const opened = await this.ensureOpened(conid, database);
-    const res = await this.sendRequest(opened, { msgtype: 'runScript', sql });
+    const res = await this.sendRequest(opened, { msgtype: 'runScript', sql, useTransaction });
    return res;
  },

@@ -272,8 +292,19 @@ module.exports = {
    let existing = this.opened.find(x => x.conid == conid && x.database == database);

    if (existing) {
-      existing.subprocess.send({ msgtype: 'ping' });
+      try {
+        existing.subprocess.send({ msgtype: 'ping' });
+      } catch (err) {
+        logger.error({ err }, 'Error pinging DB connection');
+        this.close(conid, database);
+
+        return {
+          status: 'error',
+          message: 'Ping failed',
+        };
+      }
    } else {
+      // @ts-ignore
      existing = await this.ensureOpened(conid, database);
    }

@@ -304,7 +335,13 @@ module.exports = {
    const existing = this.opened.find(x => x.conid == conid && x.database == database);
    if (existing) {
      existing.disconnected = true;
-      if (kill) existing.subprocess.kill();
+      if (kill) {
+        try {
+          existing.subprocess.kill();
+        } catch (err) {
+          logger.error({ err }, 'Error killing subprocess');
+        }
+      }
      this.opened = this.opened.filter(x => x.conid != conid || x.database != database);
      this.closed[`${conid}/${database}`] = {
        status: {
@@ -313,7 +350,7 @@ module.exports = {
        },
        structure: existing.structure,
      };
-      socket.emitChanged(`database-status-changed-${conid}-${database}`);
+      socket.emitChanged(`database-status-changed`, { conid, database });
    }
  },

@@ -49,7 +49,7 @@ module.exports = {
  async delete({ folder, file }, req) {
    if (!hasPermission(`files/${folder}/write`, req)) return false;
    await fs.unlink(path.join(filesdir(), folder, file));
-    socket.emitChanged(`files-changed-${folder}`);
+    socket.emitChanged(`files-changed`, { folder });
    socket.emitChanged(`all-files-changed`);
    return true;
  },
@@ -58,7 +58,7 @@ module.exports = {
  async rename({ folder, file, newFile }, req) {
    if (!hasPermission(`files/${folder}/write`, req)) return false;
    await fs.rename(path.join(filesdir(), folder, file), path.join(filesdir(), folder, newFile));
-    socket.emitChanged(`files-changed-${folder}`);
+    socket.emitChanged(`files-changed`, { folder });
    socket.emitChanged(`all-files-changed`);
    return true;
  },
@@ -66,7 +66,7 @@ module.exports = {
  refresh_meta: true,
  async refresh({ folders }, req) {
    for (const folder of folders) {
-      socket.emitChanged(`files-changed-${folder}`);
+      socket.emitChanged(`files-changed`, { folder });
      socket.emitChanged(`all-files-changed`);
    }
    return true;
@@ -76,7 +76,7 @@ module.exports = {
  async copy({ folder, file, newFile }, req) {
    if (!hasPermission(`files/${folder}/write`, req)) return false;
    await fs.copyFile(path.join(filesdir(), folder, file), path.join(filesdir(), folder, newFile));
-    socket.emitChanged(`files-changed-${folder}`);
+    socket.emitChanged(`files-changed`, { folder });
    socket.emitChanged(`all-files-changed`);
    return true;
  },
@@ -112,13 +112,13 @@ module.exports = {
      if (!hasPermission(`archive/write`, req)) return false;
      const dir = resolveArchiveFolder(folder.substring('archive:'.length));
      await fs.writeFile(path.join(dir, file), serialize(format, data));
-      socket.emitChanged(`archive-files-changed-${folder.substring('archive:'.length)}`);
+      socket.emitChanged(`archive-files-changed`, { folder: folder.substring('archive:'.length) });
      return true;
    } else if (folder.startsWith('app:')) {
      if (!hasPermission(`apps/write`, req)) return false;
      const app = folder.substring('app:'.length);
      await fs.writeFile(path.join(appdir(), app, file), serialize(format, data));
-      socket.emitChanged(`app-files-changed-${app}`);
+      socket.emitChanged(`app-files-changed`, { app });
      socket.emitChanged('used-apps-changed');
      apps.emitChangedDbApp(folder);
      return true;
@@ -129,7 +129,7 @@ module.exports = {
        await fs.mkdir(dir);
      }
      await fs.writeFile(path.join(dir, file), serialize(format, data));
-      socket.emitChanged(`files-changed-${folder}`);
+      socket.emitChanged(`files-changed`, { folder });
      socket.emitChanged(`all-files-changed`);
      if (folder == 'shell') {
        scheduler.reload();
@@ -4,9 +4,9 @@ const lineReader = require('line-reader');
 const _ = require('lodash');
 const { __ } = require('lodash/fp');
 const DatastoreProxy = require('../utility/DatastoreProxy');
-const { saveFreeTableData } = require('../utility/freeTableStorage');
 const getJslFileName = require('../utility/getJslFileName');
 const JsonLinesDatastore = require('../utility/JsonLinesDatastore');
+const requirePluginFunction = require('../utility/requirePluginFunction');
 const socket = require('../utility/socket');

 function readFirstLine(file) {
@@ -99,16 +99,27 @@ module.exports = {
  //   return readerInfo;
  // },

-  async ensureDatastore(jslid) {
+  async ensureDatastore(jslid, formatterFunction) {
    let datastore = this.datastores[jslid];
-    if (!datastore) {
-      datastore = new JsonLinesDatastore(getJslFileName(jslid));
+    if (!datastore || datastore.formatterFunction != formatterFunction) {
+      if (datastore) {
+        datastore._closeReader();
+      }
+      datastore = new JsonLinesDatastore(getJslFileName(jslid), formatterFunction);
      // datastore = new DatastoreProxy(getJslFileName(jslid));
      this.datastores[jslid] = datastore;
    }
    return datastore;
  },

+  async closeDataStore(jslid) {
+    const datastore = this.datastores[jslid];
+    if (datastore) {
+      await datastore._closeReader();
+      delete this.datastores[jslid];
+    }
+  },
+
  getInfo_meta: true,
  async getInfo({ jslid }) {
    const file = getJslFileName(jslid);
@@ -131,9 +142,15 @@ module.exports = {
  },

  getRows_meta: true,
-  async getRows({ jslid, offset, limit, filters }) {
-    const datastore = await this.ensureDatastore(jslid);
-    return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters);
+  async getRows({ jslid, offset, limit, filters, sort, formatterFunction }) {
+    const datastore = await this.ensureDatastore(jslid, formatterFunction);
+    return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters, _.isEmpty(sort) ? null : sort);
+  },
+
+  exists_meta: true,
+  async exists({ jslid }) {
+    const fileName = getJslFileName(jslid);
+    return fs.existsSync(fileName);
  },

  getStats_meta: true,
@@ -150,8 +167,8 @@ module.exports = {
  },

  loadFieldValues_meta: true,
-  async loadFieldValues({ jslid, field, search }) {
-    const datastore = await this.ensureDatastore(jslid);
+  async loadFieldValues({ jslid, field, search, formatterFunction }) {
+    const datastore = await this.ensureDatastore(jslid, formatterFunction);
    const res = new Set();
    await datastore.enumRows(row => {
      if (!filterName(search, row[field])) return true;
@@ -177,15 +194,100 @@ module.exports = {
    // }
  },

-  saveFreeTable_meta: true,
-  async saveFreeTable({ jslid, data }) {
-    saveFreeTableData(getJslFileName(jslid), data);
-    return true;
-  },
-
  saveText_meta: true,
  async saveText({ jslid, text }) {
    await fs.promises.writeFile(getJslFileName(jslid), text);
    return true;
  },
+
+  saveRows_meta: true,
+  async saveRows({ jslid, rows }) {
+    const fileStream = fs.createWriteStream(getJslFileName(jslid));
+    for (const row of rows) {
+      await fileStream.write(JSON.stringify(row) + '\n');
+    }
+    await fileStream.close();
+    return true;
+  },
+
+  extractTimelineChart_meta: true,
+  async extractTimelineChart({ jslid, timestampFunction, aggregateFunction, measures }) {
+    const timestamp = requirePluginFunction(timestampFunction);
+    const aggregate = requirePluginFunction(aggregateFunction);
+    const datastore = new JsonLinesDatastore(getJslFileName(jslid));
+    let mints = null;
+    let maxts = null;
+    // pass 1 - counts stats, time range
+    await datastore.enumRows(row => {
+      const ts = timestamp(row);
+      if (!mints || ts < mints) mints = ts;
+      if (!maxts || ts > maxts) maxts = ts;
+      return true;
+    });
+    const minTime = new Date(mints).getTime();
+    const maxTime = new Date(maxts).getTime();
+    const duration = maxTime - minTime;
+    const STEPS = 100;
+    let stepCount = duration > 100 * 1000 ? STEPS : Math.round((maxTime - minTime) / 1000);
+    if (stepCount < 2) {
+      stepCount = 2;
+    }
+    const stepDuration = duration / stepCount;
+    const labels = _.range(stepCount).map(i => new Date(minTime + stepDuration / 2 + stepDuration * i));
+
+    // const datasets = measures.map(m => ({
+    //   label: m.label,
+    //   data: Array(stepCount).fill(0),
+    // }));
+
+    const mproc = measures.map(m => ({
+      ...m,
+    }));
+
+    const data = Array(stepCount)
+      .fill(0)
+      .map(() => ({}));
+
+    // pass 2 - count measures
+    await datastore.enumRows(row => {
+      const ts = timestamp(row);
+      let part = Math.round((new Date(ts).getTime() - minTime) / stepDuration);
+      if (part < 0) part = 0;
+      if (part >= stepCount) part - stepCount - 1;
+      if (data[part]) {
+        data[part] = aggregate(data[part], row, stepDuration);
+      }
+      return true;
+    });
+
+    datastore._closeReader();
+
+    // const measureByField = _.fromPairs(measures.map((m, i) => [m.field, i]));
+
+    // for (let mindex = 0; mindex < measures.length; mindex++) {
+    //   for (let stepIndex = 0; stepIndex < stepCount; stepIndex++) {
+    //     const measure = measures[mindex];
+    //     if (measure.perSecond) {
+    //       datasets[mindex].data[stepIndex] /= stepDuration / 1000;
+    //     }
+    //     if (measure.perField) {
+    //       datasets[mindex].data[stepIndex] /= datasets[measureByField[measure.perField]].data[stepIndex];
+    //     }
+    //   }
+    // }
+
+    // for (let i = 0; i < measures.length; i++) {
+    //   if (measures[i].hidden) {
+    //     datasets[i] = null;
+    //   }
+    // }
+
+    return {
+      labels,
+      datasets: mproc.map(m => ({
+        label: m.label,
+        data: data.map(d => d[m.field] || 0),
+      })),
+    };
+  },
 };
@@ -6,10 +6,17 @@ const byline = require('byline');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories');
-const { extractShellApiPlugins, extractShellApiFunctionName, jsonScriptToJavascript } = require('dbgate-tools');
+const {
+  extractShellApiPlugins,
+  extractShellApiFunctionName,
+  jsonScriptToJavascript,
+  getLogger,
+  safeJsonParse,
+} = require('dbgate-tools');
 const { handleProcessCommunication } = require('../utility/processComm');
 const processArgs = require('../utility/processArgs');
 const platformInfo = require('../utility/platformInfo');
+const logger = getLogger('runners');

 function extractPlugins(script) {
  const requireRegex = /\s*\/\/\s*@require\s+([^\s]+)\s*\n/g;
@@ -29,13 +36,14 @@ const requirePluginsTemplate = (plugins, isExport) =>

 const scriptTemplate = (script, isExport) => `
 const dbgateApi = require(${isExport ? `'dbgate-api'` : 'process.env.DBGATE_API'});
+const logger = dbgateApi.getLogger('script');
 dbgateApi.initializeApiEnvironment();
 ${requirePluginsTemplate(extractPlugins(script), isExport)}
 require=null;
 async function run() {
 ${script}
 await dbgateApi.finalizer.run();
-console.log('Finished job script');
+logger.info('Finished job script');
 }
 dbgateApi.runScript(run);
 `;
@@ -59,20 +67,23 @@ module.exports = {
  requests: {},

  dispatchMessage(runid, message) {
-    if (message) console.log('...', message.message);
-    if (_.isString(message)) {
-      socket.emit(`runner-info-${runid}`, {
-        message,
+    if (message) {
+      const json = safeJsonParse(message.message);
+
+      if (json) logger.log(json);
+      else logger.info(message.message);
+
+      const toEmit = {
        time: new Date(),
-        severity: 'info',
-      });
-    }
-    if (_.isPlainObject(message)) {
-      socket.emit(`runner-info-${runid}`, {
-        time: new Date(),
-        severity: 'info',
        ...message,
-      });
+        message: json ? json.msg : message.message,
+      };
+
+      if (json && json.level >= 50) {
+        toEmit.severity = 'error';
+      }
+
+      socket.emit(`runner-info-${runid}`, toEmit);
    }
  },

@@ -98,13 +109,15 @@ module.exports = {
    fs.writeFileSync(`${scriptFile}`, scriptText);
    fs.mkdirSync(directory);
    const pluginNames = _.union(fs.readdirSync(pluginsdir()), packagedPluginList);
-    console.log(`RUNNING SCRIPT ${scriptFile}`);
+    logger.info({ scriptFile }, 'Running script');
    // const subprocess = fork(scriptFile, ['--checkParent', '--max-old-space-size=8192'], {
    const subprocess = fork(
      scriptFile,
      [
        '--checkParent', // ...process.argv.slice(3)
        '--is-forked-api',
+        '--process-display-name',
+        'script',
        ...processArgs.getPassArgs(),
      ],
      {
@@ -117,14 +130,15 @@ module.exports = {
        },
      }
    );
-    const pipeDispatcher = severity => data =>
-      this.dispatchMessage(runid, { severity, message: data.toString().trim() });
+    const pipeDispatcher = severity => data => {
+      return this.dispatchMessage(runid, { severity, message: data.toString().trim() });
+    };

    byline(subprocess.stdout).on('data', pipeDispatcher('info'));
    byline(subprocess.stderr).on('data', pipeDispatcher('error'));
    subprocess.on('exit', code => {
      this.rejectRequest(runid, { message: 'No data retured, maybe input data source is too big' });
-      console.log('... EXIT process', code);
+      logger.info({ code, pid: subprocess.pid }, 'Exited process');
      socket.emit(`runner-done-${runid}`, code);
    });
    subprocess.on('error', error => {
@@ -4,6 +4,9 @@ const path = require('path');
 const cron = require('node-cron');
 const runners = require('./runners');
 const { hasPermission } = require('../utility/hasPermission');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('scheduler');

 const scheduleRegex = /\s*\/\/\s*@schedule\s+([^\n]+)\n/;

@@ -21,7 +24,7 @@ module.exports = {
    if (!match) return;
    const pattern = match[1];
    if (!cron.validate(pattern)) return;
-    console.log(`Schedule script ${file} with pattern ${pattern}`);
+    logger.info(`Schedule script ${file} with pattern ${pattern}`);
    const task = cron.schedule(pattern, () => runners.start({ script: text }));
    this.tasks.push(task);
  },
@@ -1,6 +1,7 @@
 const connections = require('./connections');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
+const uuidv1 = require('uuid/v1');
 const _ = require('lodash');
 const AsyncLock = require('async-lock');
 const { handleProcessCommunication } = require('../utility/processComm');
@@ -8,23 +9,29 @@ const lock = new AsyncLock();
 const config = require('./config');
 const processArgs = require('../utility/processArgs');
 const { testConnectionPermission } = require('../utility/hasPermission');
+const { MissingCredentialsError } = require('../utility/exceptions');
+const pipeForkLogs = require('../utility/pipeForkLogs');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('serverConnection');

 module.exports = {
  opened: [],
  closed: {},
  lastPinged: {},
+  requests: {},

  handle_databases(conid, { databases }) {
    const existing = this.opened.find(x => x.conid == conid);
    if (!existing) return;
    existing.databases = databases;
-    socket.emitChanged(`database-list-changed-${conid}`);
+    socket.emitChanged(`database-list-changed`, { conid });
  },
  handle_version(conid, { version }) {
    const existing = this.opened.find(x => x.conid == conid);
    if (!existing) return;
    existing.version = version;
-    socket.emitChanged(`server-version-changed-${conid}`);
+    socket.emitChanged(`server-version-changed`, { conid });
  },
  handle_status(conid, { status }) {
    const existing = this.opened.find(x => x.conid == conid);
@@ -33,19 +40,37 @@ module.exports = {
    socket.emitChanged(`server-status-changed`);
  },
  handle_ping() {},
+  handle_response(conid, { msgid, ...response }) {
+    const [resolve, reject] = this.requests[msgid];
+    resolve(response);
+    delete this.requests[msgid];
+  },

  async ensureOpened(conid) {
    const res = await lock.acquire(conid, async () => {
      const existing = this.opened.find(x => x.conid == conid);
      if (existing) return existing;
      const connection = await connections.getCore({ conid });
-      const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
-        '--is-forked-api',
-        '--start-process',
-        'serverConnectionProcess',
-        ...processArgs.getPassArgs(),
-        // ...process.argv.slice(3),
-      ]);
+      if (!connection) {
+        throw new Error(`Connection with conid="${conid}" not fund`);
+      }
+      if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') {
+        throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode });
+      }
+      const subprocess = fork(
+        global['API_PACKAGE'] || process.argv[1],
+        [
+          '--is-forked-api',
+          '--start-process',
+          'serverConnectionProcess',
+          ...processArgs.getPassArgs(),
+          // ...process.argv.slice(3),
+        ],
+        {
+          stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+        }
+      );
+      pipeForkLogs(subprocess);
      const newOpened = {
        conid,
        subprocess,
@@ -80,7 +105,13 @@ module.exports = {
    const existing = this.opened.find(x => x.conid == conid);
    if (existing) {
      existing.disconnected = true;
-      if (kill) existing.subprocess.kill();
+      if (kill) {
+        try {
+          existing.subprocess.kill();
+        } catch (err) {
+          logger.error({ err }, 'Error killing subprocess');
+        }
+      }
      this.opened = this.opened.filter(x => x.conid != conid);
      this.closed[conid] = {
        ...existing.status,
@@ -99,6 +130,7 @@ module.exports = {

  listDatabases_meta: true,
  async listDatabases({ conid }, req) {
+    if (!conid) return [];
    testConnectionPermission(conid, req);
    const opened = await this.ensureOpened(conid);
    return opened.databases;
@@ -120,16 +152,21 @@ module.exports = {
  },

  ping_meta: true,
-  async ping({ connections }) {
+  async ping({ conidArray }) {
    await Promise.all(
-      _.uniq(connections).map(async conid => {
+      _.uniq(conidArray).map(async conid => {
        const last = this.lastPinged[conid];
        if (last && new Date().getTime() - last < 30 * 1000) {
          return Promise.resolve();
        }
        this.lastPinged[conid] = new Date().getTime();
        const opened = await this.ensureOpened(conid);
-        opened.subprocess.send({ msgtype: 'ping' });
+        try {
+          opened.subprocess.send({ msgtype: 'ping' });
+        } catch (err) {
+          logger.error({ err }, 'Error pinging server connection');
+          this.close(conid);
+        }
      })
    );
    return { status: 'ok' };
@@ -161,4 +198,46 @@ module.exports = {
    opened.subprocess.send({ msgtype: 'dropDatabase', name });
    return { status: 'ok' };
  },
+
+  sendRequest(conn, message) {
+    const msgid = uuidv1();
+    const promise = new Promise((resolve, reject) => {
+      this.requests[msgid] = [resolve, reject];
+      try {
+        conn.subprocess.send({ msgid, ...message });
+      } catch (err) {
+        logger.error({ err }, 'Error sending request');
+        this.close(conn.conid);
+      }
+    });
+    return promise;
+  },
+
+  async loadDataCore(msgtype, { conid, ...args }, req) {
+    testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid);
+    const res = await this.sendRequest(opened, { msgtype, ...args });
+    if (res.errorMessage) {
+      console.error(res.errorMessage);
+
+      return {
+        errorMessage: res.errorMessage,
+      };
+    }
+    return res.result || null;
+  },
+
+  serverSummary_meta: true,
+  async serverSummary({ conid }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('serverSummary', { conid });
+  },
+
+  summaryCommand_meta: true,
+  async summaryCommand({ conid, command, row }, req) {
+    testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid);
+    if (opened.connection.isReadOnly) return false;
+    return this.loadDataCore('summaryCommand', { conid, command, row });
+  },
 };
@@ -8,6 +8,11 @@ const path = require('path');
 const { handleProcessCommunication } = require('../utility/processComm');
 const processArgs = require('../utility/processArgs');
 const { appdir } = require('../utility/directories');
+const { getLogger } = require('dbgate-tools');
+const pipeForkLogs = require('../utility/pipeForkLogs');
+const config = require('./config');
+
+const logger = getLogger('sessions');

 module.exports = {
  /** @type {import('dbgate-types').OpenedSession[]} */
@@ -82,13 +87,20 @@ module.exports = {
  async create({ conid, database }) {
    const sesid = uuidv1();
    const connection = await connections.getCore({ conid });
-    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
-      '--is-forked-api',
-      '--start-process',
-      'sessionProcess',
-      ...processArgs.getPassArgs(),
-      // ...process.argv.slice(3),
-    ]);
+    const subprocess = fork(
+      global['API_PACKAGE'] || process.argv[1],
+      [
+        '--is-forked-api',
+        '--start-process',
+        'sessionProcess',
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
+      ],
+      {
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+      }
+    );
+    pipeForkLogs(subprocess);
    const newOpened = {
      conid,
      database,
@@ -109,7 +121,12 @@ module.exports = {
      socket.emit(`session-closed-${sesid}`);
    });

-    subprocess.send({ msgtype: 'connect', ...connection, database });
+    subprocess.send({
+      msgtype: 'connect',
+      ...connection,
+      database,
+      globalSettings: await config.getSettings(),
+    });
    return _.pick(newOpened, ['conid', 'database', 'sesid']);
  },

@@ -120,7 +137,7 @@ module.exports = {
      throw new Error('Invalid session');
    }

-    console.log(`Processing query, sesid=${sesid}, sql=${sql}`);
+    logger.info({ sesid, sql }, 'Processing query');
    this.dispatchMessage(sesid, 'Query execution started');
    session.subprocess.send({ msgtype: 'executeQuery', sql });

@@ -150,6 +167,31 @@ module.exports = {
    return true;
  },

+  startProfiler_meta: true,
+  async startProfiler({ sesid }) {
+    const jslid = uuidv1();
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (!session) {
+      throw new Error('Invalid session');
+    }
+
+    logger.info({ sesid }, 'Starting profiler');
+    session.loadingReader_jslid = jslid;
+    session.subprocess.send({ msgtype: 'startProfiler', jslid });
+
+    return { state: 'ok', jslid };
+  },
+
+  stopProfiler_meta: true,
+  async stopProfiler({ sesid }) {
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (!session) {
+      throw new Error('Invalid session');
+    }
+    session.subprocess.send({ msgtype: 'stopProfiler' });
+    return { state: 'ok' };
+  },
+
  // cancel_meta: true,
  // async cancel({ sesid }) {
  //   const session = this.opened.find((x) => x.sesid == sesid);
@@ -177,7 +219,16 @@ module.exports = {
    if (!session) {
      throw new Error('Invalid session');
    }
-    session.subprocess.send({ msgtype: 'ping' });
+    try {
+      session.subprocess.send({ msgtype: 'ping' });
+    } catch (err) {
+      logger.error({ err }, 'Error pinging session');
+
+      return {
+        status: 'error',
+        message: 'Ping failed',
+      };
+    }

    return { state: 'ok' };
  },
@@ -1,6 +1,8 @@
 const path = require('path');
 const { uploadsdir } = require('../utility/directories');
 const uuidv1 = require('uuid/v1');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('uploads');

 module.exports = {
  upload_meta: {
@@ -15,7 +17,7 @@ module.exports = {
    }
    const uploadName = uuidv1();
    const filePath = path.join(uploadsdir(), uploadName);
-    console.log(`Uploading file ${data.name}, size=${data.size}`);
+    logger.info(`Uploading file ${data.name}, size=${data.size}`);

    data.mv(filePath, () => {
      res.json({
@@ -1,5 +1,96 @@
-const shell = require('./shell');
+const { setLogger, getLogger, setLoggerName } = require('dbgate-tools');
 const processArgs = require('./utility/processArgs');
+const fs = require('fs');
+const moment = require('moment');
+const path = require('path');
+const { logsdir, setLogsFilePath, getLogsFilePath } = require('./utility/directories');
+const { createLogger } = require('pinomin');
+
+if (processArgs.startProcess) {
+  setLoggerName(processArgs.startProcess.replace(/Process$/, ''));
+}
+if (processArgs.processDisplayName) {
+  setLoggerName(processArgs.processDisplayName);
+}
+
+// function loadLogsContent(maxLines) {
+//   const text = fs.readFileSync(getLogsFilePath(), { encoding: 'utf8' });
+//   if (maxLines) {
+//     const lines = text
+//       .split('\n')
+//       .map(x => x.trim())
+//       .filter(x => x);
+//     return lines.slice(-maxLines).join('\n');
+//   }
+//   return text;
+// }
+
+function configureLogger() {
+  const logsFilePath = path.join(logsdir(), `${moment().format('YYYY-MM-DD-HH-mm')}-${process.pid}.ndjson`);
+  setLogsFilePath(logsFilePath);
+  setLoggerName('main');
+
+  const logger = createLogger({
+    base: { pid: process.pid },
+    targets: [
+      {
+        type: 'console',
+        // @ts-ignore
+        level: process.env.CONSOLE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+      },
+      {
+        type: 'stream',
+        // @ts-ignore
+        level: process.env.FILE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+        stream: fs.createWriteStream(logsFilePath, { flags: 'a' }),
+      },
+    ],
+  });
+
+  // const streams = [];
+  // if (!platformInfo.isElectron) {
+  //   streams.push({
+  //     stream: process.stdout,
+  //     level: process.env.CONSOLE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  //   });
+  // }
+
+  // streams.push({
+  //   stream: fs.createWriteStream(logsFilePath),
+  //   level: process.env.FILE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  // });
+
+  // let logger = pinoms({
+  //   redact: { paths: ['hostname'], remove: true },
+  //   streams,
+  // });
+
+  // // @ts-ignore
+  // let logger = pino({
+  //   redact: { paths: ['hostname'], remove: true },
+  //   transport: {
+  //     targets: [
+  //       {
+  //         level: process.env.CONSOLE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  //         target: 'pino/file',
+  //       },
+  //       {
+  //         level: process.env.FILE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  //         target: 'pino/file',
+  //         options: { destination: logsFilePath },
+  //       },
+  //     ],
+  //   },
+  // });
+
+  setLogger(logger);
+}
+
+if (processArgs.listenApi) {
+  configureLogger();
+}
+
+const shell = require('./shell');
 const dbgateTools = require('dbgate-tools');

 global['DBGATE_TOOLS'] = dbgateTools;
@@ -8,7 +99,7 @@ if (processArgs.startProcess) {
  const proc = require('./proc');
  const module = proc[processArgs.startProcess];
  module.start();
-} 
+}

 if (processArgs.listenApi) {
  const main = require('./main');
@@ -17,5 +108,8 @@ if (processArgs.listenApi) {

 module.exports = {
  ...shell,
+  getLogger,
+  configureLogger,
+  // loadLogsContent,
  getMainModule: () => require('./main'),
 };
@@ -20,6 +20,7 @@ const jsldata = require('./controllers/jsldata');
 const config = require('./controllers/config');
 const archive = require('./controllers/archive');
 const apps = require('./controllers/apps');
+const auth = require('./controllers/auth');
 const uploads = require('./controllers/uploads');
 const plugins = require('./controllers/plugins');
 const files = require('./controllers/files');
@@ -32,6 +33,9 @@ const platformInfo = require('./utility/platformInfo');
 const getExpressPath = require('./utility/getExpressPath');
 const { getLogins } = require('./utility/hasPermission');
 const _ = require('lodash');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('main');

 function start() {
  // console.log('process.argv', process.argv);
@@ -41,7 +45,7 @@ function start() {
  const server = http.createServer(app);

  const logins = getLogins();
-  if (logins) {
+  if (logins && process.env.BASIC_AUTH) {
    app.use(
      basicAuth({
        users: _.fromPairs(logins.map(x => [x.login, x.password])),
@@ -53,6 +57,25 @@ function start() {

  app.use(cors());

+  if (platformInfo.isDocker) {
+    // server static files inside docker container
+    app.use(getExpressPath('/'), express.static('/home/dbgate-docker/public'));
+  } else if (platformInfo.isNpmDist) {
+    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../dbgate-web/public')));
+  } else if (process.env.DEVWEB) {
+    // console.log('__dirname', __dirname);
+    // console.log(path.join(__dirname, '../../web/public/build'));
+    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../web/public')));
+  } else {
+    app.get(getExpressPath('/'), (req, res) => {
+      res.send('DbGate API');
+    });
+  }
+
+  if (auth.shouldAuthorizeApi()) {
+    app.use(auth.authMiddleware);
+  }
+
  app.get(getExpressPath('/stream'), async function (req, res) {
    res.set({
      'Cache-Control': 'no-cache',
@@ -88,14 +111,10 @@ function start() {
  app.use(getExpressPath('/runners/data'), express.static(rundir()));

  if (platformInfo.isDocker) {
-    // server static files inside docker container
-    app.use(getExpressPath('/'), express.static('/home/dbgate-docker/public'));
-
    const port = process.env.PORT || 3000;
-    console.log('DbGate API listening on port (docker build)', port);
+    logger.info(`DbGate API listening on port ${port} (docker build)`);
    server.listen(port);
  } else if (platformInfo.isNpmDist) {
-    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../dbgate-web/public')));
    getPort({
      port: parseInt(
        // @ts-ignore
@@ -103,35 +122,27 @@ function start() {
      ),
    }).then(port => {
      server.listen(port, () => {
-        console.log(`DbGate API listening on port ${port} (NPM build)`);
+        logger.info(`DbGate API listening on port ${port} (NPM build)`);
      });
    });
  } else if (process.env.DEVWEB) {
-    console.log('__dirname', __dirname);
-    console.log(path.join(__dirname, '../../web/public/build'));
-    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../web/public')));
-
    const port = process.env.PORT || 3000;
-    console.log('DbGate API & web listening on port (dev web build)', port);
+    logger.info(`DbGate API & web listening on port ${port} (dev web build)`);
    server.listen(port);
  } else {
-    app.get(getExpressPath('/'), (req, res) => {
-      res.send('DbGate API');
-    });
-
    const port = process.env.PORT || 3000;
-    console.log('DbGate API listening on port (dev API build)', port);
+    logger.info(`DbGate API listening on port ${port} (dev API build)`);
    server.listen(port);
  }

  function shutdown() {
-    console.log('\nShutting down DbGate API server');
+    logger.info('\nShutting down DbGate API server');
    server.close(() => {
-      console.log('Server shut down, terminating');
+      logger.info('Server shut down, terminating');
      process.exit(0);
    });
    setTimeout(() => {
-      console.log('Server close timeout, terminating');
+      logger.info('Server close timeout, terminating');
      process.exit(0);
    }, 1000);
  }
@@ -157,6 +168,7 @@ function useAllControllers(app, electron) {
  useController(app, electron, '/scheduler', scheduler);
  useController(app, electron, '/query-history', queryHistory);
  useController(app, electron, '/apps', apps);
+  useController(app, electron, '/auth', auth);
 }

 function setElectronSender(electronSender) {
@@ -1,7 +1,7 @@
 const stableStringify = require('json-stable-stringify');
 const { splitQuery } = require('dbgate-query-splitter');
 const childProcessChecker = require('../utility/childProcessChecker');
-const { extractBoolSettingsValue, extractIntSettingsValue } = require('dbgate-tools');
+const { extractBoolSettingsValue, extractIntSettingsValue, getLogger } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
@@ -9,6 +9,8 @@ const { SqlGenerator } = require('dbgate-tools');
 const generateDeploySql = require('../shell/generateDeploySql');
 const { dumpSqlSelect } = require('dbgate-sqltree');

+const logger = getLogger('dbconnProcess');
+
 let systemConnection;
 let storedConnection;
 let afterConnectCallbacks = [];
@@ -156,12 +158,12 @@ function resolveAnalysedPromises() {
  afterAnalyseCallbacks = [];
 }

-async function handleRunScript({ msgid, sql }, skipReadonlyCheck = false) {
+async function handleRunScript({ msgid, sql, useTransaction }, skipReadonlyCheck = false) {
  await waitConnected();
  const driver = requireEngineDriver(storedConnection);
  try {
    if (!skipReadonlyCheck) ensureExecuteCustomScript(driver);
-    await driver.script(systemConnection, sql);
+    await driver.script(systemConnection, sql, { useTransaction });
    process.send({ msgtype: 'response', msgid });
  } catch (err) {
    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
@@ -269,7 +271,7 @@ async function handleSqlPreview({ msgid, objects, options }) {
    process.send({ msgtype: 'response', msgid, sql: dmp.s, isTruncated: generator.isTruncated });
    if (generator.isUnhandledException) {
      setTimeout(() => {
-        console.log('Exiting because of unhandled exception');
+        logger.error('Exiting because of unhandled exception');
        process.exit(0);
      }, 500);
    }
@@ -336,7 +338,7 @@ function start() {
  setInterval(() => {
    const time = new Date().getTime();
    if (time - lastPing > 40 * 1000) {
-      console.log('Database connection not alive, exiting');
+      logger.info('Database connection not alive, exiting');
      process.exit(0);
    }
  }, 10 * 1000);
@@ -345,9 +347,9 @@ function start() {
    if (handleProcessCommunication(message)) return;
    try {
      await handleMessage(message);
-    } catch (e) {
-      console.error('Error in DB connection', e);
-      process.send({ msgtype: 'error', error: e.message });
+    } catch (err) {
+      logger.error({ err }, 'Error in DB connection');
+      process.send({ msgtype: 'error', error: err.message });
    }
  });
 }
@@ -1,15 +1,17 @@
 const stableStringify = require('json-stable-stringify');
-const { extractBoolSettingsValue, extractIntSettingsValue } = require('dbgate-tools');
+const { extractBoolSettingsValue, extractIntSettingsValue, getLogger } = require('dbgate-tools');
 const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
+const logger = getLogger('srvconnProcess');

 let systemConnection;
 let storedConnection;
 let lastDatabases = null;
 let lastStatus = null;
 let lastPing = null;
+let afterConnectCallbacks = [];

 async function handleRefresh() {
  const driver = requireEngineDriver(storedConnection);
@@ -74,6 +76,18 @@ async function handleConnect(connection) {
    // console.error(err);
    setTimeout(() => process.exit(1), 1000);
  }
+
+  for (const [resolve] of afterConnectCallbacks) {
+    resolve();
+  }
+  afterConnectCallbacks = [];
+}
+
+function waitConnected() {
+  if (systemConnection) return Promise.resolve();
+  return new Promise((resolve, reject) => {
+    afterConnectCallbacks.push([resolve, reject]);
+  });
 }

 function handlePing() {
@@ -88,15 +102,36 @@ async function handleDatabaseOp(op, { name }) {
  } else {
    const dmp = driver.createDumper();
    dmp[op](name);
-    console.log(`RUNNING SCRIPT: ${dmp.s}`);
+    logger.info({ sql: dmp.s }, 'Running script');
    await driver.query(systemConnection, dmp.s);
  }
  await handleRefresh();
 }

+async function handleDriverDataCore(msgid, callMethod) {
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+  try {
+    const result = await callMethod(driver);
+    process.send({ msgtype: 'response', msgid, result });
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+  }
+}
+
+async function handleServerSummary({ msgid }) {
+  return handleDriverDataCore(msgid, driver => driver.serverSummary(systemConnection));
+}
+
+async function handleSummaryCommand({ msgid, command, row }) {
+  return handleDriverDataCore(msgid, driver => driver.summaryCommand(systemConnection, command, row));
+}
+
 const messageHandlers = {
  connect: handleConnect,
  ping: handlePing,
+  serverSummary: handleServerSummary,
+  summaryCommand: handleSummaryCommand,
  createDatabase: props => handleDatabaseOp('createDatabase', props),
  dropDatabase: props => handleDatabaseOp('dropDatabase', props),
 };
@@ -112,7 +147,7 @@ function start() {
  setInterval(() => {
    const time = new Date().getTime();
    if (time - lastPing > 40 * 1000) {
-      console.log('Server connection not alive, exiting');
+      logger.info('Server connection not alive, exiting');
      process.exit(0);
    }
  }, 10 * 1000);
@@ -10,12 +10,18 @@ const requireEngineDriver = require('../utility/requireEngineDriver');
 const { decryptConnection } = require('../utility/crypting');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
+const { getLogger, extractIntSettingsValue, extractBoolSettingsValue } = require('dbgate-tools');
+
+const logger = getLogger('sessionProcess');

 let systemConnection;
 let storedConnection;
 let afterConnectCallbacks = [];
 // let currentHandlers = [];
 let lastPing = null;
+let lastActivity = null;
+let currentProfiler = null;
+let executingScripts = 0;

 class TableWriter {
  constructor() {
@@ -210,7 +216,38 @@ function waitConnected() {
  });
 }

+async function handleStartProfiler({ jslid }) {
+  lastActivity = new Date().getTime();
+
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+
+  if (!allowExecuteCustomScript(driver)) {
+    process.send({ msgtype: 'done' });
+    return;
+  }
+
+  const writer = new TableWriter();
+  writer.initializeFromReader(jslid);
+
+  currentProfiler = await driver.startProfiler(systemConnection, {
+    row: data => writer.rowFromReader(data),
+  });
+  currentProfiler.writer = writer;
+}
+
+async function handleStopProfiler({ jslid }) {
+  lastActivity = new Date().getTime();
+
+  const driver = requireEngineDriver(storedConnection);
+  currentProfiler.writer.close();
+  driver.stopProfiler(systemConnection, currentProfiler);
+  currentProfiler = null;
+}
+
 async function handleExecuteQuery({ sql }) {
+  lastActivity = new Date().getTime();
+
  await waitConnected();
  const driver = requireEngineDriver(storedConnection);

@@ -227,23 +264,30 @@ async function handleExecuteQuery({ sql }) {
    //process.send({ msgtype: 'error', error: e.message });
  }

-  const resultIndexHolder = {
-    value: 0,
-  };
-  for (const sqlItem of splitQuery(sql, {
-    ...driver.getQuerySplitterOptions('stream'),
-    returnRichInfo: true,
-  })) {
-    await handleStream(driver, resultIndexHolder, sqlItem);
-    // const handler = new StreamHandler(resultIndex);
-    // const stream = await driver.stream(systemConnection, sqlItem, handler);
-    // handler.stream = stream;
-    // resultIndex = handler.resultIndex;
+  executingScripts++;
+  try {
+    const resultIndexHolder = {
+      value: 0,
+    };
+    for (const sqlItem of splitQuery(sql, {
+      ...driver.getQuerySplitterOptions('stream'),
+      returnRichInfo: true,
+    })) {
+      await handleStream(driver, resultIndexHolder, sqlItem);
+      // const handler = new StreamHandler(resultIndex);
+      // const stream = await driver.stream(systemConnection, sqlItem, handler);
+      // handler.stream = stream;
+      // resultIndex = handler.resultIndex;
+    }
+    process.send({ msgtype: 'done' });
+  } finally {
+    executingScripts--;
  }
-  process.send({ msgtype: 'done' });
 }

 async function handleExecuteReader({ jslid, sql, fileName }) {
+  lastActivity = new Date().getTime();
+
  await waitConnected();

  const driver = requireEngineDriver(storedConnection);
@@ -280,6 +324,8 @@ const messageHandlers = {
  connect: handleConnect,
  executeQuery: handleExecuteQuery,
  executeReader: handleExecuteReader,
+  startProfiler: handleStartProfiler,
+  stopProfiler: handleStopProfiler,
  ping: handlePing,
  // cancel: handleCancel,
 };
@@ -297,7 +343,25 @@ function start() {
  setInterval(() => {
    const time = new Date().getTime();
    if (time - lastPing > 25 * 1000) {
-      console.log('Session not alive, exiting');
+      logger.info('Session not alive, exiting');
+      process.exit(0);
+    }
+
+    const useSessionTimeout =
+      storedConnection && storedConnection.globalSettings
+        ? extractBoolSettingsValue(storedConnection.globalSettings, 'session.autoClose', true)
+        : false;
+    const sessionTimeout =
+      storedConnection && storedConnection.globalSettings
+        ? extractIntSettingsValue(storedConnection.globalSettings, 'session.autoCloseTimeout', 15, 1, 120)
+        : 15;
+    if (
+      useSessionTimeout &&
+      time - lastActivity > sessionTimeout * 60 * 1000 &&
+      !currentProfiler &&
+      executingScripts == 0
+    ) {
+      logger.info('Session not active, exiting');
      process.exit(0);
    }
  }, 10 * 1000);
@@ -3,6 +3,9 @@ const platformInfo = require('../utility/platformInfo');
 const childProcessChecker = require('../utility/childProcessChecker');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { SSHConnection } = require('../utility/SSHConnection');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('sshProcess');

 async function getSshConnection(connection) {
  const sshConfig = {
@@ -35,8 +38,8 @@ async function handleStart({ connection, tunnelConfig }) {
      tunnelConfig,
    });
  } catch (err) {
-    console.log('Error creating SSH tunnel connection:', err.message);
-    
+    logger.error({ err }, 'Error creating SSH tunnel connection:');
+
    process.send({
      msgtype: 'error',
      connection,
@@ -3,11 +3,14 @@ const fs = require('fs');
 const { archivedir, resolveArchiveFolder } = require('../utility/directories');
 // const socket = require('../utility/socket');
 const jsonLinesWriter = require('./jsonLinesWriter');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger();

 function archiveWriter({ folderName, fileName }) {
  const dir = resolveArchiveFolder(folderName);
  if (!fs.existsSync(dir)) {
-    console.log(`Creating directory ${dir}`);
+    logger.info(`Creating directory ${dir}`);
    fs.mkdirSync(dir);
  }
  const jsonlFile = path.join(dir, `${fileName}.jsonl`);
@@ -0,0 +1,50 @@
+const stream = require('stream');
+const path = require('path');
+const { quoteFullName, fullNameToString, getLogger } = require('dbgate-tools');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const logger = getLogger('dataDuplicator');
+const { DataDuplicator } = require('dbgate-datalib');
+const copyStream = require('./copyStream');
+const jsonLinesReader = require('./jsonLinesReader');
+const { resolveArchiveFolder } = require('../utility/directories');
+
+async function dataDuplicator({
+  connection,
+  archive,
+  items,
+  options,
+  analysedStructure = null,
+  driver,
+  systemConnection,
+}) {
+  if (!driver) driver = requireEngineDriver(connection);
+  const pool = systemConnection || (await connectUtility(driver, connection, 'write'));
+
+  logger.info(`Connected.`);
+
+  if (!analysedStructure) {
+    analysedStructure = await driver.analyseFull(pool);
+  }
+
+  const dupl = new DataDuplicator(
+    pool,
+    driver,
+    analysedStructure,
+    items.map(item => ({
+      name: item.name,
+      operation: item.operation,
+      matchColumns: item.matchColumns,
+      openStream:
+        item.openStream ||
+        (() => jsonLinesReader({ fileName: path.join(resolveArchiveFolder(archive), `${item.name}.jsonl`) })),
+    })),
+    stream,
+    copyStream,
+    options
+  );
+
+  await dupl.run();
+}
+
+module.exports = dataDuplicator;
@@ -1,5 +1,8 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('dumpDb');

 function doDump(dumper) {
  return new Promise((resolve, reject) => {
@@ -21,11 +24,11 @@ async function dumpDatabase({
  databaseName,
  schemaName,
 }) {
-  console.log(`Dumping database`);
+  logger.info(`Dumping database`);

  if (!driver) driver = requireEngineDriver(connection);
  const pool = systemConnection || (await connectUtility(driver, connection, 'read', { forceRowsAsObjects: true }));
-  console.log(`Connected.`);
+  logger.info(`Connected.`);

  const dumper = await driver.createBackupDumper(pool, {
    outputFile,
@@ -1,12 +1,15 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('execQuery');

 async function executeQuery({ connection = undefined, systemConnection = undefined, driver = undefined, sql }) {
-  console.log(`Execute query ${sql}`);
+  logger.info({ sql }, `Execute query`);

  if (!driver) driver = requireEngineDriver(connection);
  const pool = systemConnection || (await connectUtility(driver, connection, 'script'));
-  console.log(`Connected.`);
+  logger.info(`Connected.`);

  await driver.script(pool, sql);
 }
@@ -1,18 +1,26 @@
 const stream = require('stream');

-async function fakeObjectReader({ delay = 0 } = {}) {
+async function fakeObjectReader({ delay = 0, dynamicData = null } = {}) {
  const pass = new stream.PassThrough({
    objectMode: true,
  });
  function doWrite() {
-    pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }], __isStreamHeader: true });
-    pass.write({ id: 1, country: 'Czechia' });
-    pass.write({ id: 2, country: 'Austria' });
-    pass.write({ country: 'Germany', id: 3 });
-    pass.write({ country: 'Romania', id: 4 });
-    pass.write({ country: 'Great Britain', id: 5 });
-    pass.write({ country: 'Bosna, Hecegovina', id: 6 });
-    pass.end();
+    if (dynamicData) {
+      pass.write({ __isStreamHeader: true, __isDynamicStructure: true });
+      for (const item of dynamicData) {
+        pass.write(item);
+      }
+      pass.end();
+    } else {
+      pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }], __isStreamHeader: true });
+      pass.write({ id: 1, country: 'Czechia' });
+      pass.write({ id: 2, country: 'Austria' });
+      pass.write({ country: 'Germany', id: 3 });
+      pass.write({ country: 'Romania', id: 4 });
+      pass.write({ country: 'Great Britain', id: 5 });
+      pass.write({ country: 'Bosna, Hecegovina', id: 6 });
+      pass.end();
+    }
  }

  if (delay) {
@@ -0,0 +1,30 @@
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const {
+  extendDatabaseInfo,
+  databaseInfoFromYamlModel,
+  getAlterDatabaseScript,
+  DatabaseAnalyser,
+} = require('dbgate-tools');
+const importDbModel = require('../utility/importDbModel');
+const fs = require('fs');
+
+async function generateModelSql({ engine, driver, modelFolder, loadedDbModel, outputFile }) {
+  if (!driver) driver = requireEngineDriver(engine);
+
+  const dbInfo = extendDatabaseInfo(
+    loadedDbModel ? databaseInfoFromYamlModel(loadedDbModel) : await importDbModel(modelFolder)
+  );
+
+  const { sql } = getAlterDatabaseScript(
+    DatabaseAnalyser.createEmptyStructure(),
+    dbInfo,
+    {},
+    DatabaseAnalyser.createEmptyStructure(),
+    dbInfo,
+    driver
+  );
+
+  fs.writeFileSync(outputFile, sql);
+}
+
+module.exports = generateModelSql;
@@ -4,6 +4,9 @@ const connectUtility = require('../utility/connectUtility');
 const { splitQueryStream } = require('dbgate-query-splitter/lib/splitQueryStream');
 const download = require('./download');
 const stream = require('stream');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('importDb');

 class ImportStream extends stream.Transform {
  constructor(pool, driver) {
@@ -38,11 +41,11 @@ function awaitStreamEnd(stream) {
 }

 async function importDatabase({ connection = undefined, systemConnection = undefined, driver = undefined, inputFile }) {
-  console.log(`Importing database`);
+  logger.info(`Importing database`);

  if (!driver) driver = requireEngineDriver(connection);
  const pool = systemConnection || (await connectUtility(driver, connection, 'write'));
-  console.log(`Connected.`);
+  logger.info(`Connected.`);

  const downloadedFile = await download(inputFile);

@@ -23,6 +23,10 @@ const deployDb = require('./deployDb');
 const initializeApiEnvironment = require('./initializeApiEnvironment');
 const dumpDatabase = require('./dumpDatabase');
 const importDatabase = require('./importDatabase');
+const loadDatabase = require('./loadDatabase');
+const generateModelSql = require('./generateModelSql');
+const modifyJsonLinesReader = require('./modifyJsonLinesReader');
+const dataDuplicator = require('./dataDuplicator');

 const dbgateApi = {
  queryReader,
@@ -49,6 +53,10 @@ const dbgateApi = {
  initializeApiEnvironment,
  dumpDatabase,
  importDatabase,
+  loadDatabase,
+  generateModelSql,
+  modifyJsonLinesReader,
+  dataDuplicator,
 };

 requirePlugin.initializeDbgateApi(dbgateApi);
@@ -1,6 +1,9 @@
+const { getLogger } = require('dbgate-tools');
 const fs = require('fs');
 const stream = require('stream');

+const logger = getLogger('jsonArrayWriter');
+
 class StringifyStream extends stream.Transform {
  constructor() {
    super({ objectMode: true });
@@ -38,7 +41,7 @@ class StringifyStream extends stream.Transform {
 }

 async function jsonArrayWriter({ fileName, encoding = 'utf-8' }) {
-  console.log(`Writing file ${fileName}`);
+  logger.info(`Writing file ${fileName}`);
  const stringify = new StringifyStream();
  const fileStream = fs.createWriteStream(fileName, encoding);
  stringify.pipe(fileStream);
@@ -1,6 +1,8 @@
 const fs = require('fs');
 const stream = require('stream');
 const byline = require('byline');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('jsonLinesReader');

 class ParseStream extends stream.Transform {
  constructor({ limitRows }) {
@@ -31,9 +33,13 @@ class ParseStream extends stream.Transform {
 }

 async function jsonLinesReader({ fileName, encoding = 'utf-8', limitRows = undefined }) {
-  console.log(`Reading file ${fileName}`);
+  logger.info(`Reading file ${fileName}`);

-  const fileStream = fs.createReadStream(fileName, encoding);
+  const fileStream = fs.createReadStream(
+    fileName,
+    // @ts-ignore
+    encoding
+  );
  const liner = byline(fileStream);
  const parser = new ParseStream({ limitRows });
  liner.pipe(parser);
@@ -1,5 +1,7 @@
+const { getLogger } = require('dbgate-tools');
 const fs = require('fs');
 const stream = require('stream');
+const logger = getLogger('jsonLinesWriter');

 class StringifyStream extends stream.Transform {
  constructor({ header }) {
@@ -10,7 +12,9 @@ class StringifyStream extends stream.Transform {
  _transform(chunk, encoding, done) {
    let skip = false;
    if (!this.wasHeader) {
-      skip = (chunk.__isStreamHeader && !this.header) || (chunk.__isStreamHeader && chunk.__isDynamicStructure);
+      skip =
+        (chunk.__isStreamHeader && !this.header) ||
+        (chunk.__isStreamHeader && chunk.__isDynamicStructure && !chunk.__keepDynamicStreamHeader);
      this.wasHeader = true;
    }
    if (!skip) {
@@ -21,7 +25,7 @@ class StringifyStream extends stream.Transform {
 }

 async function jsonLinesWriter({ fileName, encoding = 'utf-8', header = true }) {
-  console.log(`Writing file ${fileName}`);
+  logger.info(`Writing file ${fileName}`);
  const stringify = new StringifyStream({ header });
  const fileStream = fs.createWriteStream(fileName, encoding);
  stringify.pipe(fileStream);
@@ -0,0 +1,21 @@
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const { getLogger } = require('dbgate-tools');
+const exportDbModel = require('../utility/exportDbModel');
+
+const logger = getLogger('analyseDb');
+
+async function loadDatabase({ connection = undefined, systemConnection = undefined, driver = undefined, outputDir }) {
+  logger.info(`Analysing database`);
+
+  if (!driver) driver = requireEngineDriver(connection);
+  const pool = systemConnection || (await connectUtility(driver, connection, 'read', { forceRowsAsObjects: true }));
+  logger.info(`Connected.`);
+
+  const dbInfo = await driver.analyseFull(pool);
+  logger.info(`Analyse finished`);
+
+  await exportDbModel(dbInfo, outputDir);
+}
+
+module.exports = loadDatabase;
@@ -0,0 +1,145 @@
+const fs = require('fs');
+const _ = require('lodash');
+const stream = require('stream');
+const byline = require('byline');
+const { getLogger, processJsonDataUpdateCommands, removeTablePairingId } = require('dbgate-tools');
+const logger = getLogger('modifyJsonLinesReader');
+const stableStringify = require('json-stable-stringify');
+
+class ParseStream extends stream.Transform {
+  constructor({ limitRows, changeSet, mergedRows, mergeKey, mergeMode }) {
+    super({ objectMode: true });
+    this.limitRows = limitRows;
+    this.changeSet = changeSet;
+    this.wasHeader = false;
+    this.currentRowIndex = 0;
+    if (mergeMode == 'merge') {
+      if (mergedRows && mergeKey) {
+        this.mergedRowsDict = {};
+        for (const row of mergedRows) {
+          const key = stableStringify(_.pick(row, mergeKey));
+          this.mergedRowsDict[key] = row;
+        }
+      }
+    }
+    this.mergedRowsArray = mergedRows;
+    this.mergeKey = mergeKey;
+    this.mergeMode = mergeMode;
+  }
+  _transform(chunk, encoding, done) {
+    let obj = JSON.parse(chunk);
+    if (obj.__isStreamHeader) {
+      if (this.changeSet && this.changeSet.structure) {
+        this.push({
+          ...removeTablePairingId(this.changeSet.structure),
+          __isStreamHeader: true,
+        });
+      } else {
+        this.push(obj);
+      }
+      this.wasHeader = true;
+      done();
+      return;
+    }
+
+    if (this.changeSet) {
+      if (!this.wasHeader && this.changeSet.structure) {
+        this.push({
+          ...removeTablePairingId(this.changeSet.structure),
+          __isStreamHeader: true,
+        });
+        this.wasHeader = true;
+      }
+
+      if (!this.limitRows || this.currentRowIndex < this.limitRows) {
+        if (this.changeSet.deletes.find(x => x.existingRowIndex == this.currentRowIndex)) {
+          obj = null;
+        }
+
+        const update = this.changeSet.updates.find(x => x.existingRowIndex == this.currentRowIndex);
+        if (update) {
+          if (update.document) {
+            obj = update.document;
+          } else {
+            obj = {
+              ...obj,
+              ...update.fields,
+            };
+          }
+        }
+
+        if (obj) {
+          if (this.changeSet.dataUpdateCommands) {
+            obj = processJsonDataUpdateCommands(obj, this.changeSet.dataUpdateCommands);
+          }
+          this.push(obj);
+        }
+        this.currentRowIndex += 1;
+      }
+    } else if (this.mergedRowsArray && this.mergeKey && this.mergeMode) {
+      if (this.mergeMode == 'merge') {
+        const key = stableStringify(_.pick(obj, this.mergeKey));
+        if (this.mergedRowsDict[key]) {
+          this.push({ ...obj, ...this.mergedRowsDict[key] });
+          delete this.mergedRowsDict[key];
+        } else {
+          this.push(obj);
+        }
+      } else if (this.mergeMode == 'append') {
+        this.push(obj);
+      }
+    } else {
+      this.push(obj);
+    }
+    done();
+  }
+
+  _flush(done) {
+    if (this.changeSet) {
+      for (const insert of this.changeSet.inserts) {
+        this.push({
+          ...insert.document,
+          ...insert.fields,
+        });
+      }
+    } else if (this.mergedRowsArray && this.mergeKey) {
+      if (this.mergeMode == 'merge') {
+        for (const row of this.mergedRowsArray) {
+          const key = stableStringify(_.pick(row, this.mergeKey));
+          if (this.mergedRowsDict[key]) {
+            this.push(row);
+          }
+        }
+      } else {
+        for (const row of this.mergedRowsArray) {
+          this.push(row);
+        }
+      }
+    }
+    done();
+  }
+}
+
+async function modifyJsonLinesReader({
+  fileName,
+  encoding = 'utf-8',
+  limitRows = undefined,
+  changeSet = null,
+  mergedRows = null,
+  mergeKey = null,
+  mergeMode = 'merge',
+}) {
+  logger.info(`Reading file ${fileName} with change set`);
+
+  const fileStream = fs.createReadStream(
+    fileName,
+    // @ts-ignore
+    encoding
+  );
+  const liner = byline(fileStream);
+  const parser = new ParseStream({ limitRows, changeSet, mergedRows, mergeKey, mergeMode });
+  liner.pipe(parser);
+  return parser;
+}
+
+module.exports = modifyJsonLinesReader;
@@ -1,5 +1,7 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('queryReader');

 async function queryReader({
  connection,
@@ -14,12 +16,12 @@ async function queryReader({
  // if (!sql && !json) {
  //   throw new Error('One of sql or json must be set');
  // }
-  console.log(`Reading query ${query || sql}`);
+  logger.info({ sql: query || sql }, `Reading query`);
  // else console.log(`Reading query ${JSON.stringify(json)}`);

  const driver = requireEngineDriver(connection);
  const pool = await connectUtility(driver, connection, queryType == 'json' ? 'read' : 'script');
-  console.log(`Connected.`);
+  logger.info(`Connected.`);
  const reader =
    queryType == 'json' ? await driver.readJsonQuery(pool, query) : await driver.readQuery(pool, query || sql);
  return reader;
@@ -3,6 +3,8 @@ const fs = require('fs');
 const { pluginsdir, packagedPluginsDir, getPluginBackendPath } = require('../utility/directories');
 const nativeModules = require('../nativeModules');
 const platformInfo = require('../utility/platformInfo');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('requirePlugin');

 const loadedPlugins = {};

@@ -17,7 +19,7 @@ function requirePlugin(packageName, requiredPlugin = null) {
  if (requiredPlugin == null) {
    let module;
    const modulePath = getPluginBackendPath(packageName);
-    console.log(`Loading module ${packageName} from ${modulePath}`);
+    logger.info(`Loading module ${packageName} from ${modulePath}`);
    try {
      // @ts-ignore
      module = __non_webpack_require__(modulePath);
@@ -1,5 +1,7 @@
+const { getLogger } = require('dbgate-tools');
 const childProcessChecker = require('../utility/childProcessChecker');
 const processArgs = require('../utility/processArgs');
+const logger = getLogger();

 async function runScript(func) {
  if (processArgs.checkParent) {
@@ -9,7 +11,7 @@ async function runScript(func) {
    await func();
    process.exit(0);
  } catch (err) {
-    console.log(err);
+    logger.error({ err }, `Error running script: ${err.message}`);
    process.exit(1);
  }
 }
@@ -1,8 +1,9 @@
 const fs = require('fs');
 const stream = require('stream');
 const path = require('path');
-const { driverBase } = require('dbgate-tools');
+const { driverBase, getLogger } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
+const logger = getLogger('sqlDataWriter');

 class SqlizeStream extends stream.Transform {
  constructor({ fileName, dataName }) {
@@ -40,7 +41,7 @@ class SqlizeStream extends stream.Transform {
 }

 async function sqlDataWriter({ fileName, dataName, driver, encoding = 'utf-8' }) {
-  console.log(`Writing file ${fileName}`);
+  logger.info(`Writing file ${fileName}`);
  const stringify = new SqlizeStream({ fileName, dataName });
  const fileStream = fs.createWriteStream(fileName, encoding);
  stringify.pipe(fileStream);
@@ -1,17 +1,18 @@
-const { quoteFullName, fullNameToString } = require('dbgate-tools');
+const { quoteFullName, fullNameToString, getLogger } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
+const logger = getLogger('tableReader');

 async function tableReader({ connection, pureName, schemaName }) {
  const driver = requireEngineDriver(connection);
  const pool = await connectUtility(driver, connection, 'read');
-  console.log(`Connected.`);
+  logger.info(`Connected.`);

  const fullName = { pureName, schemaName };

  if (driver.databaseEngineTypes.includes('document')) {
    // @ts-ignore
-    console.log(`Reading collection ${fullNameToString(fullName)}`);
+    logger.info(`Reading collection ${fullNameToString(fullName)}`);
    // @ts-ignore
    return await driver.readQuery(pool, JSON.stringify(fullName));
  }
@@ -20,14 +21,14 @@ async function tableReader({ connection, pureName, schemaName }) {
  const query = `select * from ${quoteFullName(driver.dialect, fullName)}`;
  if (table) {
    // @ts-ignore
-    console.log(`Reading table ${fullNameToString(table)}`);
+    logger.info(`Reading table ${fullNameToString(table)}`);
    // @ts-ignore
    return await driver.readQuery(pool, query, table);
  }
  const view = await driver.analyseSingleObject(pool, fullName, 'views');
  if (view) {
    // @ts-ignore
-    console.log(`Reading view ${fullNameToString(view)}`);
+    logger.info(`Reading view ${fullNameToString(view)}`);
    // @ts-ignore
    return await driver.readQuery(pool, query, view);
  }
@@ -1,16 +1,17 @@
-const { fullNameToString } = require('dbgate-tools');
+const { fullNameToString, getLogger } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
+const logger = getLogger('tableWriter');

 async function tableWriter({ connection, schemaName, pureName, driver, systemConnection, ...options }) {
-  console.log(`Writing table ${fullNameToString({ schemaName, pureName })}`);
+  logger.info(`Writing table ${fullNameToString({ schemaName, pureName })}`);

  if (!driver) {
    driver = requireEngineDriver(connection);
  }
  const pool = systemConnection || (await connectUtility(driver, connection, 'write'));

-  console.log(`Connected.`);
+  logger.info(`Connected.`);
  return await driver.writeTable(pool, { schemaName, pureName }, options);
 }

@@ -2,6 +2,9 @@ const { fork } = require('child_process');
 const uuidv1 = require('uuid/v1');
 const { handleProcessCommunication } = require('./processComm');
 const processArgs = require('../utility/processArgs');
+const pipeForkLogs = require('./pipeForkLogs');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('DatastoreProxy');

 class DatastoreProxy {
  constructor(file) {
@@ -30,13 +33,20 @@ class DatastoreProxy {

  async ensureSubprocess() {
    if (!this.subprocess) {
-      this.subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
-        '--is-forked-api',
-        '--start-process',
-        'jslDatastoreProcess',
-        ...processArgs.getPassArgs(),
-        // ...process.argv.slice(3),
-      ]);
+      this.subprocess = fork(
+        global['API_PACKAGE'] || process.argv[1],
+        [
+          '--is-forked-api',
+          '--start-process',
+          'jslDatastoreProcess',
+          ...processArgs.getPassArgs(),
+          // ...process.argv.slice(3),
+        ],
+        {
+          stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+        }
+      );
+      pipeForkLogs(this.subprocess);

      this.subprocess.on('message', message => {
        // @ts-ignore
@@ -60,7 +70,12 @@ class DatastoreProxy {
    const msgid = uuidv1();
    const promise = new Promise((resolve, reject) => {
      this.requests[msgid] = [resolve, reject];
-      this.subprocess.send({ msgtype: 'read', msgid, offset, limit });
+      try {
+        this.subprocess.send({ msgtype: 'read', msgid, offset, limit });
+      } catch (err) {
+        logger.error({ err }, 'Error getting rows');
+        this.subprocess = null;
+      }
    });
    return promise;
  }
@@ -69,7 +84,12 @@ class DatastoreProxy {
    const msgid = uuidv1();
    const promise = new Promise((resolve, reject) => {
      this.requests[msgid] = [resolve, reject];
-      this.subprocess.send({ msgtype: 'notify', msgid });
+      try {
+        this.subprocess.send({ msgtype: 'notify', msgid });
+      } catch (err) {
+        logger.error({ err }, 'Error notifying subprocess');
+        this.subprocess = null;
+      }
    });
    return promise;
  }
@@ -90,6 +90,12 @@ class JsonLinesDatabase {
    return obj;
  }

+  async updateAll(mapFunction) {
+    await this._ensureLoaded();
+    this.data = this.data.map(mapFunction);
+    await this._save();
+  }
+
  async patch(id, values) {
    await this._ensureLoaded();
    this.data = this.data.map(x => (x._id == id ? { ...x, ...values } : x));
@@ -1,38 +1,64 @@
-const lineReader = require('line-reader');
+const fs = require('fs');
+const os = require('os');
+const rimraf = require('rimraf');
+const path = require('path');
 const AsyncLock = require('async-lock');
 const lock = new AsyncLock();
 const stableStringify = require('json-stable-stringify');
 const { evaluateCondition } = require('dbgate-sqltree');
-
-function fetchNextLineFromReader(reader) {
-  return new Promise((resolve, reject) => {
-    if (!reader.hasNextLine()) {
-      resolve(null);
-      return;
-    }
-
-    reader.nextLine((err, line) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(line);
-      }
-    });
-  });
-}
+const requirePluginFunction = require('./requirePluginFunction');
+const esort = require('external-sorting');
+const uuidv1 = require('uuid/v1');
+const { jsldir } = require('./directories');
+const LineReader = require('./LineReader');

 class JsonLinesDatastore {
-  constructor(file) {
+  constructor(file, formatterFunction) {
    this.file = file;
+    this.formatterFunction = formatterFunction;
    this.reader = null;
    this.readedDataRowCount = 0;
    this.readedSchemaRow = false;
    // this.firstRowToBeReturned = null;
    this.notifyChangedCallback = null;
    this.currentFilter = null;
+    this.currentSort = null;
+    this.rowFormatter = requirePluginFunction(formatterFunction);
+    this.sortedFiles = {};
  }

-  _closeReader() {
+  static async sortFile(infile, outfile, sort) {
+    const tempDir = path.join(os.tmpdir(), uuidv1());
+    fs.mkdirSync(tempDir);
+
+    await esort
+      .default({
+        input: fs.createReadStream(infile),
+        output: fs.createWriteStream(outfile),
+        deserializer: JSON.parse,
+        serializer: JSON.stringify,
+        tempDir,
+        maxHeap: 100,
+        comparer: (a, b) => {
+          for (const item of sort) {
+            const { uniqueName, order } = item;
+            if (a[uniqueName] < b[uniqueName]) {
+              return order == 'ASC' ? -1 : 1;
+            }
+            if (a[uniqueName] > b[uniqueName]) {
+              return order == 'ASC' ? 1 : -1;
+            }
+          }
+          return 0;
+        },
+      })
+      .asc();
+
+    await new Promise(resolve => rimraf(tempDir, resolve));
+  }
+
+  async _closeReader() {
+    // console.log('CLOSING READER', this.reader);
    if (!this.reader) return;
    const reader = this.reader;
    this.reader = null;
@@ -40,7 +66,8 @@ class JsonLinesDatastore {
    this.readedSchemaRow = false;
    // this.firstRowToBeReturned = null;
    this.currentFilter = null;
-    reader.close(() => {});
+    this.currentSort = null;
+    await reader.close();
  }

  async notifyChanged(callback) {
@@ -53,13 +80,17 @@ class JsonLinesDatastore {
    if (call) call();
  }

-  async _openReader() {
-    return new Promise((resolve, reject) =>
-      lineReader.open(this.file, (err, reader) => {
-        if (err) reject(err);
-        resolve(reader);
-      })
-    );
+  async _openReader(fileName) {
+    // console.log('OPENING READER', fileName);
+    // console.log(fs.readFileSync(fileName, 'utf-8'));
+
+    const fileStream = fs.createReadStream(fileName);
+    return new LineReader(fileStream);
+  }
+
+  parseLine(line) {
+    const res = JSON.parse(line);
+    return this.rowFormatter ? this.rowFormatter(res) : res;
  }

  async _readLine(parse) {
@@ -69,7 +100,7 @@ class JsonLinesDatastore {
    //   return res;
    // }
    for (;;) {
-      const line = await fetchNextLineFromReader(this.reader);
+      const line = await this.reader.readLine();
      if (!line) {
        // EOF
        return null;
@@ -84,14 +115,14 @@ class JsonLinesDatastore {
        }
      }
      if (this.currentFilter) {
-        const parsedLine = JSON.parse(line);
+        const parsedLine = this.parseLine(line);
        if (evaluateCondition(this.currentFilter, parsedLine)) {
          this.readedDataRowCount += 1;
          return parse ? parsedLine : true;
        }
      } else {
        this.readedDataRowCount += 1;
-        return parse ? JSON.parse(line) : true;
+        return parse ? this.parseLine(line) : true;
      }
    }

@@ -132,14 +163,19 @@ class JsonLinesDatastore {
    // });
  }

-  async _ensureReader(offset, filter) {
-    if (this.readedDataRowCount > offset || stableStringify(filter) != stableStringify(this.currentFilter)) {
+  async _ensureReader(offset, filter, sort) {
+    if (
+      this.readedDataRowCount > offset ||
+      stableStringify(filter) != stableStringify(this.currentFilter) ||
+      stableStringify(sort) != stableStringify(this.currentSort)
+    ) {
      this._closeReader();
    }
    if (!this.reader) {
-      const reader = await this._openReader();
+      const reader = await this._openReader(sort ? this.sortedFiles[stableStringify(sort)] : this.file);
      this.reader = reader;
      this.currentFilter = filter;
+      this.currentSort = sort;
    }
    // if (!this.readedSchemaRow) {
    //   const line = await this._readLine(true); // skip structure
@@ -171,13 +207,20 @@ class JsonLinesDatastore {
    });
  }

-  async getRows(offset, limit, filter) {
+  async getRows(offset, limit, filter, sort) {
    const res = [];
+    if (sort && !this.sortedFiles[stableStringify(sort)]) {
+      const jslid = uuidv1();
+      const sortedFile = path.join(jsldir(), `${jslid}.jsonl`);
+      await JsonLinesDatastore.sortFile(this.file, sortedFile, sort);
+      this.sortedFiles[stableStringify(sort)] = sortedFile;
+    }
    await lock.acquire('reader', async () => {
-      await this._ensureReader(offset, filter);
+      await this._ensureReader(offset, filter, sort);
      // console.log(JSON.stringify(this.currentFilter, undefined, 2));
      for (let i = 0; i < limit; i += 1) {
        const line = await this._readLine(true);
+        // console.log('READED LINE', i);
        if (line == null) break;
        res.push(line);
      }
@@ -0,0 +1,88 @@
+const readline = require('readline');
+
+class Queue {
+  constructor() {
+    this.elements = {};
+    this.head = 0;
+    this.tail = 0;
+  }
+  enqueue(element) {
+    this.elements[this.tail] = element;
+    this.tail++;
+  }
+  dequeue() {
+    const item = this.elements[this.head];
+    delete this.elements[this.head];
+    this.head++;
+    return item;
+  }
+  peek() {
+    return this.elements[this.head];
+  }
+  getLength() {
+    return this.tail - this.head;
+  }
+  isEmpty() {
+    return this.getLength() === 0;
+  }
+}
+
+class LineReader {
+  constructor(input) {
+    this.input = input;
+    this.queue = new Queue();
+    this.resolve = null;
+    this.isEnded = false;
+    this.rl = readline.createInterface({
+      input,
+    });
+    this.input.pause();
+
+    this.rl.on('line', line => {
+      this.input.pause();
+      if (this.resolve) {
+        const resolve = this.resolve;
+        this.resolve = null;
+        resolve(line);
+        return;
+      }
+      this.queue.enqueue(line);
+    });
+
+    this.rl.on('close', () => {
+      if (this.resolve) {
+        const resolve = this.resolve;
+        this.resolve = null;
+        this.isEnded = true;
+        resolve(null);
+        return;
+      }
+      this.queue.enqueue(null);
+    });
+  }
+
+  readLine() {
+    if (this.isEnded) {
+      return Promise.resolve(null);
+    }
+
+    if (!this.queue.isEmpty()) {
+      const res = this.queue.dequeue();
+      if (res == null) this.isEnded = true;
+      return Promise.resolve(res);
+    }
+
+    this.input.resume();
+
+    return new Promise(resolve => {
+      this.resolve = resolve;
+    });
+  }
+
+  close() {
+    this.isEnded = true;
+    return new Promise(resolve => this.input.close(resolve));
+  }
+}
+
+module.exports = LineReader;
@@ -1,14 +1,18 @@
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('childProcessChecked');
+
 let counter = 0;

 function childProcessChecker() {
  setInterval(() => {
    try {
      process.send({ msgtype: 'ping', counter: counter++ });
-    } catch (ex) {
+    } catch (err) {
      // This will come once parent dies.
      // One way can be to check for error code ERR_IPC_CHANNEL_CLOSED
      //     and call process.exit()
-      console.log('parent died', ex.toString());
+      logger.error({ err }, 'parent died');
      process.exit(1);
    }
  }, 1000);
@@ -2,7 +2,7 @@ const fs = require('fs-extra');
 const path = require('path');
 const ageSeconds = 3600;

-async function cleanDirectory(directory) {
+async function cleanDirectory(directory, age = undefined) {
  const files = await fs.readdir(directory);
  const now = new Date().getTime();

@@ -10,7 +10,7 @@ async function cleanDirectory(directory) {
    const full = path.join(directory, file);
    const stat = await fs.stat(full);
    const mtime = stat.mtime.getTime();
-    const expirationTime = mtime + ageSeconds * 1000;
+    const expirationTime = mtime + (age || ageSeconds) * 1000;
    if (now > expirationTime) {
      if (stat.isDirectory()) {
        await fs.rmdir(full, { recursive: true });
@@ -62,14 +62,17 @@ async function connectUtility(driver, storedConnection, connectionMode, addition

    if (connection.sslCaFile) {
      connection.ssl.ca = await fs.readFile(connection.sslCaFile);
+      connection.ssl.sslCaFile = connection.sslCaFile;
    }

    if (connection.sslCertFile) {
      connection.ssl.cert = await fs.readFile(connection.sslCertFile);
+      connection.ssl.sslCertFile = connection.sslCertFile;
    }

    if (connection.sslKeyFile) {
      connection.ssl.key = await fs.readFile(connection.sslKeyFile);
+      connection.ssl.sslKeyFile = connection.sslKeyFile;
    }

    if (connection.sslCertFilePassword) {
@@ -1,20 +1,24 @@
 const os = require('os');
 const path = require('path');
 const fs = require('fs');
+const _ = require('lodash');
 const cleanDirectory = require('./cleanDirectory');
 const platformInfo = require('./platformInfo');
 const processArgs = require('./processArgs');
 const consoleObjectWriter = require('../shell/consoleObjectWriter');
+const { getLogger } = require('dbgate-tools');
+
+let logsFilePath;

 const createDirectories = {};
 const ensureDirectory = (dir, clean) => {
  if (!createDirectories[dir]) {
    if (clean && fs.existsSync(dir) && !platformInfo.isForkedApi) {
-      console.log(`Cleaning directory ${dir}`);
-      cleanDirectory(dir);
+      getLogger('directories').info(`Cleaning directory ${dir}`);
+      cleanDirectory(dir, _.isNumber(clean) ? clean : null);
    }
    if (!fs.existsSync(dir)) {
-      console.log(`Creating directory ${dir}`);
+      getLogger('directories').info(`Creating directory ${dir}`);
      fs.mkdirSync(dir);
    }
    createDirectories[dir] = true;
@@ -38,20 +42,26 @@ function datadir() {
  return dir;
 }

-const dirFunc = (dirname, clean = false) => () => {
-  const dir = path.join(datadir(), dirname);
-  ensureDirectory(dir, clean);
+const dirFunc =
+  (dirname, clean, subdirs = []) =>
+  () => {
+    const dir = path.join(datadir(), dirname);
+    ensureDirectory(dir, clean);
+    for (const subdir of subdirs) {
+      ensureDirectory(path.join(dir, subdir), false);
+    }

-  return dir;
-};
+    return dir;
+  };

 const jsldir = dirFunc('jsl', true);
 const rundir = dirFunc('run', true);
 const uploadsdir = dirFunc('uploads', true);
 const pluginsdir = dirFunc('plugins');
-const archivedir = dirFunc('archive');
+const archivedir = dirFunc('archive', false, ['default']);
 const appdir = dirFunc('apps');
 const filesdir = dirFunc('files');
+const logsdir = dirFunc('logs', 3600 * 24 * 7);

 function packagedPluginsDir() {
  // console.log('CALL DIR FROM', new Error('xxx').stack);
@@ -127,11 +137,19 @@ function migrateDataDir() {
    if (fs.existsSync(oldDir) && !fs.existsSync(newDir)) {
      fs.renameSync(oldDir, newDir);
    }
-  } catch (e) {
-    console.log('Error migrating data dir:', e.message);
+  } catch (err) {
+    getLogger('directories').error({ err }, 'Error migrating data dir');
  }
 }

+function setLogsFilePath(value) {
+  logsFilePath = value;
+}
+
+function getLogsFilePath() {
+  return logsFilePath;
+}
+
 migrateDataDir();

 module.exports = {
@@ -144,9 +162,12 @@ module.exports = {
  ensureDirectory,
  pluginsdir,
  filesdir,
+  logsdir,
  packagedPluginsDir,
  packagedPluginList,
  getPluginBackendPath,
  resolveArchiveFolder,
  clearArchiveLinksCache,
+  getLogsFilePath,
+  setLogsFilePath,
 };
@@ -0,0 +1,9 @@
+class MissingCredentialsError {
+  constructor(detail) {
+    this.detail = detail;
+  }
+}
+
+module.exports = {
+  MissingCredentialsError,
+};
@@ -1,15 +0,0 @@
-const fs = require('fs-extra');
-
-async function saveFreeTableData(file, data) {
-  const { structure, rows } = data;
-  const fileStream = fs.createWriteStream(file);
-  await fileStream.write(JSON.stringify({ __isStreamHeader: true, ...structure }) + '\n');
-  for (const row of rows) {
-    await fileStream.write(JSON.stringify(row) + '\n');
-  }
-  await fileStream.close();
-}
-
-module.exports = {
-  saveFreeTableData,
-};
@@ -0,0 +1,19 @@
+const byline = require('byline');
+const { safeJsonParse, getLogger } = require('dbgate-tools');
+const logger = getLogger();
+
+const logDispatcher = method => data => {
+  const json = safeJsonParse(data.toString());
+  if (json && json.level) {
+    logger.log(json);
+  } else {
+    logger[method](json || data.toString());
+  }
+};
+
+function pipeForkLogs(subprocess) {
+  byline(subprocess.stdout).on('data', logDispatcher('info'));
+  byline(subprocess.stderr).on('data', logDispatcher('error'));
+}
+
+module.exports = pipeForkLogs;
@@ -39,8 +39,8 @@ const platformInfo = {
  environment: process.env.NODE_ENV,
  platform,
  runningInWebpack: !!process.env.WEBPACK_DEV_SERVER_URL,
-  allowShellConnection: !processArgs.listenApiChild || !!process.env.SHELL_CONNECTION || !!isElectron(),
-  allowShellScripting: !processArgs.listenApiChild || !!process.env.SHELL_SCRIPTING || !!isElectron(),
+  allowShellConnection: (!processArgs.listenApiChild && !isNpmDist) || !!process.env.SHELL_CONNECTION || !!isElectron(),
+  allowShellScripting: (!processArgs.listenApiChild && !isNpmDist) || !!process.env.SHELL_SCRIPTING || !!isElectron(),
  defaultKeyfile: path.join(os.homedir(), '.ssh/id_rsa'),
 };

@@ -11,6 +11,7 @@ const startProcess = getNamedArg('--start-process');
 const isForkedApi = process.argv.includes('--is-forked-api');
 const pluginsDir = getNamedArg('--plugins-dir');
 const workspaceDir = getNamedArg('--workspace-dir');
+const processDisplayName = getNamedArg('--process-display-name');
 const listenApi = process.argv.includes('--listen-api');
 const listenApiChild = process.argv.includes('--listen-api-child') || listenApi;

@@ -37,4 +38,5 @@ module.exports = {
  workspaceDir,
  listenApi,
  listenApiChild,
+  processDisplayName,
 };
@@ -0,0 +1,16 @@
+const _ = require('lodash');
+const requirePlugin = require('../shell/requirePlugin');
+
+function requirePluginFunction(functionName) {
+  if (!functionName) return null;
+  if (functionName.includes('@')) {
+    const [shortName, packageName] = functionName.split('@');
+    const plugin = requirePlugin(packageName);
+    if (plugin.functions) {
+      return plugin.functions[shortName];
+    }
+  }
+  return null;
+}
+
+module.exports = requirePluginFunction;
@@ -1,4 +1,5 @@
 const _ = require('lodash');
+const stableStringify = require('json-stable-stringify');

 const sseResponses = [];
 let electronSender = null;
@@ -27,12 +28,12 @@ module.exports = {
      electronSender.send(message, data == null ? null : data);
    }
    for (const res of sseResponses) {
-      res.write(`event: ${message}\ndata: ${JSON.stringify(data == null ? null : data)}\n\n`);
+      res.write(`event: ${message}\ndata: ${stableStringify(data == null ? null : data)}\n\n`);
    }
  },
-  emitChanged(key) {
+  emitChanged(key, params = undefined) {
    // console.log('EMIT CHANGED', key);
-    this.emit('changed-cache', key);
+    this.emit('changed-cache', { key, ...params });
    // this.emit(key);
  },
 };
@@ -5,6 +5,9 @@ const AsyncLock = require('async-lock');
 const lock = new AsyncLock();
 const { fork } = require('child_process');
 const processArgs = require('../utility/processArgs');
+const { getLogger } = require('dbgate-tools');
+const pipeForkLogs = require('./pipeForkLogs');
+const logger = getLogger('sshTunnel');

 const sshTunnelCache = {};

@@ -21,18 +24,24 @@ const CONNECTION_FIELDS = [
 const TUNNEL_FIELDS = [...CONNECTION_FIELDS, 'server', 'port'];

 function callForwardProcess(connection, tunnelConfig, tunnelCacheKey) {
-  let subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
-    '--is-forked-api',
-    '--start-process',
-    'sshForwardProcess',
-    ...processArgs.getPassArgs(),
-  ]);
+  let subprocess = fork(
+    global['API_PACKAGE'] || process.argv[1],
+    ['--is-forked-api', '--start-process', 'sshForwardProcess', ...processArgs.getPassArgs()],
+    {
+      stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+    }
+  );
+  pipeForkLogs(subprocess);

-  subprocess.send({
-    msgtype: 'connect',
-    connection,
-    tunnelConfig,
-  });
+  try {
+    subprocess.send({
+      msgtype: 'connect',
+      connection,
+      tunnelConfig,
+    });
+  } catch (err) {
+    logger.error({ err }, 'Error connecting SSH');
+  }
  return new Promise((resolve, reject) => {
    subprocess.on('message', resp => {
      // @ts-ignore
@@ -45,7 +54,7 @@ function callForwardProcess(connection, tunnelConfig, tunnelCacheKey) {
      }
    });
    subprocess.on('exit', code => {
-      console.log('SSH forward process exited');
+      logger.info('SSH forward process exited');
      delete sshTunnelCache[tunnelCacheKey];
    });
  });
@@ -65,13 +74,13 @@ async function getSshTunnel(connection) {
      toHost: connection.server,
    };
    try {
-      console.log(
+      logger.info(
        `Creating SSH tunnel to ${connection.sshHost}-${connection.server}:${connection.port}, using local port ${localPort}`
      );

      const subprocess = await callForwardProcess(connection, tunnelConfig, tunnelCacheKey);

-      console.log(
+      logger.info(
        `Created SSH tunnel to ${connection.sshHost}-${connection.server}:${connection.port}, using local port ${localPort}`
      );

@@ -1,11 +1,17 @@
+const { getLogger } = require('dbgate-tools');
 const uuidv1 = require('uuid/v1');
 const { getSshTunnel } = require('./sshTunnel');
+const logger = getLogger('sshTunnelProxy');

 const dispatchedMessages = {};

 async function handleGetSshTunnelRequest({ msgid, connection }, subprocess) {
  const response = await getSshTunnel(connection);
-  subprocess.send({ msgtype: 'getsshtunnel-response', msgid, response });
+  try {
+    subprocess.send({ msgtype: 'getsshtunnel-response', msgid, response });
+  } catch (err) {
+    logger.error({ err }, 'Error sending to SSH tunnel');
+  }
 }

 function handleGetSshTunnelResponse({ msgid, response }, subprocess) {
@@ -1,7 +1,10 @@
 const _ = require('lodash');
 const express = require('express');
 const getExpressPath = require('./getExpressPath');
+const { MissingCredentialsError } = require('./exceptions');
+const { getLogger } = require('dbgate-tools');

+const logger = getLogger('useController');
 /**
 * @param {string} route
 */
@@ -9,11 +12,11 @@ module.exports = function useController(app, electron, route, controller) {
  const router = express.Router();

  if (controller._init) {
-    console.log(`Calling init controller for controller ${route}`);
+    logger.info(`Calling init controller for controller ${route}`);
    try {
      controller._init();
    } catch (err) {
-      console.log(`Error initializing controller, exiting application`, err);
+      logger.error({ err }, `Error initializing controller, exiting application`);
      process.exit(1);
    }
  }
@@ -37,6 +40,13 @@ module.exports = function useController(app, electron, route, controller) {
            if (data === undefined) return null;
            return data;
          } catch (err) {
+            if (err instanceof MissingCredentialsError) {
+              return {
+                missingCredentials: true,
+                apiErrorMessage: 'Missing credentials',
+                detail: err.detail,
+              };
+            }
            return { apiErrorMessage: err.message };
          }
        });
@@ -67,9 +77,17 @@ module.exports = function useController(app, electron, route, controller) {
        try {
          const data = await controller[key]({ ...req.body, ...req.query }, req);
          res.json(data);
-        } catch (e) {
-          console.log(e);
-          res.status(500).json({ apiErrorMessage: e.message });
+        } catch (err) {
+          logger.error({ err }, `Error when processing route ${route}/${key}`);
+          if (err instanceof MissingCredentialsError) {
+            res.json({
+              missingCredentials: true,
+              apiErrorMessage: 'Missing credentials',
+              detail: err.detail,
+            });
+          } else {
+            res.status(500).json({ apiErrorMessage: err.message });
+          }
        }
      });
    }
@@ -9,23 +9,31 @@ import {
  AllowIdentityInsert,
  Expression,
 } from 'dbgate-sqltree';
-import type { NamedObjectInfo, DatabaseInfo } from 'dbgate-types';
+import type { NamedObjectInfo, DatabaseInfo, TableInfo } from 'dbgate-types';
+import { JsonDataObjectUpdateCommand } from 'dbgate-tools';

 export interface ChangeSetItem {
  pureName: string;
  schemaName?: string;
  insertedRowIndex?: number;
+  existingRowIndex?: number;
  document?: any;
  condition?: { [column: string]: string };
  fields?: { [column: string]: string };
 }

-export interface ChangeSet {
+export interface ChangeSetItemFields {
  inserts: ChangeSetItem[];
  updates: ChangeSetItem[];
  deletes: ChangeSetItem[];
 }

+export interface ChangeSet extends ChangeSetItemFields {
+  structure?: TableInfo;
+  dataUpdateCommands?: JsonDataObjectUpdateCommand[];
+  setColumnMode?: 'fixed' | 'variable';
+}
+
 export function createChangeSet(): ChangeSet {
  return {
    inserts: [],
@@ -38,6 +46,7 @@ export interface ChangeSetRowDefinition {
  pureName: string;
  schemaName: string;
  insertedRowIndex?: number;
+  existingRowIndex?: number;
  condition?: { [column: string]: string };
 }

@@ -49,7 +58,7 @@ export interface ChangeSetFieldDefinition extends ChangeSetRowDefinition {
 export function findExistingChangeSetItem(
  changeSet: ChangeSet,
  definition: ChangeSetRowDefinition
-): [keyof ChangeSet, ChangeSetItem] {
+): [keyof ChangeSetItemFields, ChangeSetItem] {
  if (!changeSet || !definition) return ['updates', null];
  if (definition.insertedRowIndex != null) {
    return [
@@ -66,7 +75,8 @@ export function findExistingChangeSetItem(
      x =>
        x.pureName == definition.pureName &&
        x.schemaName == definition.schemaName &&
-        _.isEqual(x.condition, definition.condition)
+        ((definition.existingRowIndex != null && x.existingRowIndex == definition.existingRowIndex) ||
+          (definition.existingRowIndex == null && _.isEqual(x.condition, definition.condition)))
    );
    if (inUpdates) return ['updates', inUpdates];

@@ -74,7 +84,8 @@ export function findExistingChangeSetItem(
      x =>
        x.pureName == definition.pureName &&
        x.schemaName == definition.schemaName &&
-        _.isEqual(x.condition, definition.condition)
+        ((definition.existingRowIndex != null && x.existingRowIndex == definition.existingRowIndex) ||
+          (definition.existingRowIndex == null && _.isEqual(x.condition, definition.condition)))
    );
    if (inDeletes) return ['deletes', inDeletes];

@@ -119,6 +130,7 @@ export function setChangeSetValue(
        schemaName: definition.schemaName,
        condition: definition.condition,
        insertedRowIndex: definition.insertedRowIndex,
+        existingRowIndex: definition.existingRowIndex,
        fields: {
          [definition.uniqueName]: value,
        },
@@ -162,6 +174,7 @@ export function setChangeSetRowData(
        schemaName: definition.schemaName,
        condition: definition.condition,
        insertedRowIndex: definition.insertedRowIndex,
+        existingRowIndex: definition.existingRowIndex,
        document,
      },
    ],
@@ -395,6 +408,7 @@ export function deleteChangeSetRows(changeSet: ChangeSet, definition: ChangeSetR
          pureName: definition.pureName,
          schemaName: definition.schemaName,
          condition: definition.condition,
+          existingRowIndex: definition.existingRowIndex,
        },
      ],
    };
@@ -402,9 +416,11 @@ export function deleteChangeSetRows(changeSet: ChangeSet, definition: ChangeSetR
 }

 export function getChangeSetInsertedRows(changeSet: ChangeSet, name?: NamedObjectInfo) {
-  if (!name) return [];
+  // if (!name) return [];
  if (!changeSet) return [];
-  const rows = changeSet.inserts.filter(x => x.pureName == name.pureName && x.schemaName == name.schemaName);
+  const rows = changeSet.inserts.filter(
+    x => name == null || (x.pureName == name.pureName && x.schemaName == name.schemaName)
+  );
  const maxIndex = _.maxBy(rows, x => x.insertedRowIndex)?.insertedRowIndex;
  if (maxIndex == null) return [];
  const res = Array(maxIndex + 1).fill({});
@@ -447,5 +463,12 @@ export function changeSetInsertDocuments(changeSet: ChangeSet, documents: any[],

 export function changeSetContainsChanges(changeSet: ChangeSet) {
  if (!changeSet) return false;
-  return changeSet.deletes.length > 0 || changeSet.updates.length > 0 || changeSet.inserts.length > 0;
+  return (
+    changeSet.deletes.length > 0 ||
+    changeSet.updates.length > 0 ||
+    changeSet.inserts.length > 0 ||
+    !!changeSet.structure ||
+    !!changeSet.setColumnMode ||
+    changeSet.dataUpdateCommands?.length > 0
+  );
 }
@@ -0,0 +1,270 @@
+import { createAsyncWriteStream, getLogger, runCommandOnDriver, runQueryOnDriver } from 'dbgate-tools';
+import { DatabaseInfo, EngineDriver, ForeignKeyInfo, TableInfo } from 'dbgate-types';
+import _pick from 'lodash/pick';
+import _omit from 'lodash/omit';
+
+const logger = getLogger('dataDuplicator');
+
+export interface DataDuplicatorItem {
+  openStream: () => Promise<ReadableStream>;
+  name: string;
+  operation: 'copy' | 'lookup' | 'insertMissing';
+  matchColumns: string[];
+}
+
+export interface DataDuplicatorOptions {
+  rollbackAfterFinish?: boolean;
+  skipRowsWithUnresolvedRefs?: boolean;
+}
+
+class DuplicatorReference {
+  constructor(
+    public base: DuplicatorItemHolder,
+    public ref: DuplicatorItemHolder,
+    public isMandatory: boolean,
+    public foreignKey: ForeignKeyInfo
+  ) {}
+
+  get columnName() {
+    return this.foreignKey.columns[0].columnName;
+  }
+}
+
+class DuplicatorItemHolder {
+  references: DuplicatorReference[] = [];
+  backReferences: DuplicatorReference[] = [];
+  table: TableInfo;
+  isPlanned = false;
+  idMap = {};
+  autoColumn: string;
+  refByColumn: { [columnName: string]: DuplicatorReference } = {};
+  isReferenced: boolean;
+
+  get name() {
+    return this.item.name;
+  }
+
+  constructor(public item: DataDuplicatorItem, public duplicator: DataDuplicator) {
+    this.table = duplicator.db.tables.find(x => x.pureName.toUpperCase() == item.name.toUpperCase());
+    this.autoColumn = this.table.columns.find(x => x.autoIncrement)?.columnName;
+    if (
+      this.table.primaryKey?.columns?.length != 1 ||
+      this.table.primaryKey?.columns?.[0].columnName != this.autoColumn
+    ) {
+      this.autoColumn = null;
+    }
+  }
+
+  initializeReferences() {
+    for (const fk of this.table.foreignKeys) {
+      if (fk.columns?.length != 1) continue;
+      const refHolder = this.duplicator.itemHolders.find(y => y.name.toUpperCase() == fk.refTableName.toUpperCase());
+      if (refHolder == null) continue;
+      const isMandatory = this.table.columns.find(x => x.columnName == fk.columns[0]?.columnName)?.notNull;
+      const newref = new DuplicatorReference(this, refHolder, isMandatory, fk);
+      this.references.push(newref);
+      this.refByColumn[newref.columnName] = newref;
+
+      refHolder.isReferenced = true;
+    }
+  }
+
+  createInsertObject(chunk) {
+    const res = _omit(
+      _pick(
+        chunk,
+        this.table.columns.map(x => x.columnName)
+      ),
+      [this.autoColumn, ...this.backReferences.map(x => x.columnName)]
+    );
+
+    for (const key in res) {
+      const ref = this.refByColumn[key];
+      if (ref) {
+        // remap id
+        res[key] = ref.ref.idMap[res[key]];
+        if (ref.isMandatory && res[key] == null) {
+          // mandatory refertence not matched
+          if (this.duplicator.options.skipRowsWithUnresolvedRefs) {
+            return null;
+          }
+          throw new Error(`Unresolved reference, base=${ref.base.name}, ref=${ref.ref.name}, ${key}=${chunk[key]}`);
+        }
+      }
+    }
+
+    return res;
+  }
+
+  async runImport() {
+    const readStream = await this.item.openStream();
+    const driver = this.duplicator.driver;
+    const pool = this.duplicator.pool;
+    let inserted = 0;
+    let mapped = 0;
+    let missing = 0;
+    let skipped = 0;
+    let lastLogged = new Date();
+
+    const writeStream = createAsyncWriteStream(this.duplicator.stream, {
+      processItem: async chunk => {
+        if (chunk.__isStreamHeader) {
+          return;
+        }
+
+        const doCopy = async () => {
+          // console.log('chunk', this.name, JSON.stringify(chunk));
+          const insertedObj = this.createInsertObject(chunk);
+          // console.log('insertedObj', this.name, JSON.stringify(insertedObj));
+          if (insertedObj == null) {
+            skipped += 1;
+            return;
+          }
+          let res = await runQueryOnDriver(pool, driver, dmp => {
+            dmp.put(
+              '^insert ^into %f (%,i) ^values (%,v)',
+              this.table,
+              Object.keys(insertedObj),
+              Object.values(insertedObj)
+            );
+
+            if (
+              this.autoColumn &&
+              this.isReferenced &&
+              !this.duplicator.driver.dialect.requireStandaloneSelectForScopeIdentity
+            ) {
+              dmp.selectScopeIdentity(this.table);
+            }
+          });
+          inserted += 1;
+          if (this.autoColumn && this.isReferenced) {
+            if (this.duplicator.driver.dialect.requireStandaloneSelectForScopeIdentity) {
+              res = await runQueryOnDriver(pool, driver, dmp => dmp.selectScopeIdentity(this.table));
+            }
+            // console.log('IDRES', JSON.stringify(res));
+            const resId = Object.entries(res?.rows?.[0])?.[0]?.[1];
+            if (resId != null) {
+              this.idMap[chunk[this.autoColumn]] = resId;
+            }
+          }
+        };
+
+        switch (this.item.operation) {
+          case 'copy': {
+            await doCopy();
+            break;
+          }
+          case 'insertMissing':
+          case 'lookup': {
+            const res = await runQueryOnDriver(pool, driver, dmp =>
+              dmp.put(
+                '^select %i ^from %f ^where %i = %v',
+                this.autoColumn,
+                this.table,
+                this.item.matchColumns[0],
+                chunk[this.item.matchColumns[0]]
+              )
+            );
+            const resId = Object.entries(res?.rows?.[0])?.[0]?.[1];
+            if (resId != null) {
+              mapped += 1;
+              this.idMap[chunk[this.autoColumn]] = resId;
+            } else if (this.item.operation == 'insertMissing') {
+              await doCopy();
+            } else {
+              missing += 1;
+            }
+            break;
+          }
+        }
+
+        if (new Date().getTime() - lastLogged.getTime() > 5000) {
+          logger.info(
+            `Duplicating ${this.item.name} in progress, inserted ${inserted} rows, mapped ${mapped} rows, missing ${missing} rows, skipped ${skipped} rows`
+          );
+          lastLogged = new Date();
+        }
+        // this.idMap[oldId] = newId;
+      },
+    });
+
+    await this.duplicator.copyStream(readStream, writeStream);
+
+    // await this.duplicator.driver.writeQueryStream(this.duplicator.pool, {
+    //   mapResultId: (oldId, newId) => {
+    //     this.idMap[oldId] = newId;
+    //   },
+    // });
+
+    return { inserted, mapped, missing, skipped };
+  }
+}
+
+export class DataDuplicator {
+  itemHolders: DuplicatorItemHolder[];
+  itemPlan: DuplicatorItemHolder[] = [];
+
+  constructor(
+    public pool: any,
+    public driver: EngineDriver,
+    public db: DatabaseInfo,
+    public items: DataDuplicatorItem[],
+    public stream,
+    public copyStream: (input, output) => Promise<void>,
+    public options: DataDuplicatorOptions = {}
+  ) {
+    this.itemHolders = items.map(x => new DuplicatorItemHolder(x, this));
+    this.itemHolders.forEach(x => x.initializeReferences());
+  }
+
+  findItemToPlan(): DuplicatorItemHolder {
+    for (const item of this.itemHolders) {
+      if (item.isPlanned) continue;
+      if (item.references.every(x => x.ref.isPlanned)) {
+        return item;
+      }
+    }
+    for (const item of this.itemHolders) {
+      if (item.isPlanned) continue;
+      if (item.references.every(x => x.ref.isPlanned || !x.isMandatory)) {
+        const backReferences = item.references.filter(x => !x.ref.isPlanned);
+        item.backReferences = backReferences;
+        return item;
+      }
+    }
+    throw new Error('Cycle in mandatory references');
+  }
+
+  createPlan() {
+    while (this.itemPlan.length < this.itemHolders.length) {
+      const item = this.findItemToPlan();
+      item.isPlanned = true;
+      this.itemPlan.push(item);
+    }
+  }
+
+  async run() {
+    this.createPlan();
+
+    await runCommandOnDriver(this.pool, this.driver, dmp => dmp.beginTransaction());
+    try {
+      for (const item of this.itemPlan) {
+        const stats = await item.runImport();
+        logger.info(
+          `Duplicated ${item.name}, inserted ${stats.inserted} rows, mapped ${stats.mapped} rows, missing ${stats.missing} rows, skipped ${stats.skipped} rows`
+        );
+      }
+    } catch (err) {
+      logger.error({ err }, `Failed duplicator job, rollbacking. ${err.message}`);
+      await runCommandOnDriver(this.pool, this.driver, dmp => dmp.rollbackTransaction());
+      return;
+    }
+    if (this.options.rollbackAfterFinish) {
+      logger.info('Rollbacking transaction, nothing was changed');
+      await runCommandOnDriver(this.pool, this.driver, dmp => dmp.rollbackTransaction());
+    } else {
+      logger.info('Committing duplicator transaction');
+      await runCommandOnDriver(this.pool, this.driver, dmp => dmp.commitTransaction());
+    }
+  }
+}
@@ -1,120 +0,0 @@
-import _ from 'lodash';
-import { GridConfig, GridCache, GridConfigColumns, createGridCache, GroupFunc } from './GridConfig';
-import type { TableInfo, EngineDriver, DatabaseInfo, SqlDialect } from 'dbgate-types';
-import { getFilterValueExpression } from 'dbgate-filterparser';
-import { ChangeCacheFunc, ChangeConfigFunc, DisplayColumn } from './GridDisplay';
-
-export class FormViewDisplay {
-  isLoadedCorrectly = true;
-  columns: DisplayColumn[];
-  public baseTable: TableInfo;
-  dialect: SqlDialect;
-
-  constructor(
-    public config: GridConfig,
-    protected setConfig: ChangeConfigFunc,
-    public cache: GridCache,
-    protected setCache: ChangeCacheFunc,
-    public driver?: EngineDriver,
-    public dbinfo: DatabaseInfo = null,
-    public serverVersion = null
-  ) {
-    this.dialect = (driver?.dialectByVersion && driver?.dialectByVersion(serverVersion)) || driver?.dialect;
-  }
-
-  addFilterColumn(column) {
-    if (!column) return;
-    this.setConfig(cfg => ({
-      ...cfg,
-      formFilterColumns: [...(cfg.formFilterColumns || []), column.uniqueName],
-    }));
-  }
-
-  filterCellValue(column, rowData) {
-    if (!column || !rowData) return;
-    const value = rowData[column.uniqueName];
-    const expr = getFilterValueExpression(value, column.dataType);
-    if (expr) {
-      this.setConfig(cfg => ({
-        ...cfg,
-        filters: {
-          ...cfg.filters,
-          [column.uniqueName]: expr,
-        },
-        addedColumns: cfg.addedColumns.includes(column.uniqueName)
-          ? cfg.addedColumns
-          : [...cfg.addedColumns, column.uniqueName],
-      }));
-      this.reload();
-    }
-  }
-
-  setFilter(uniqueName, value) {
-    this.setConfig(cfg => ({
-      ...cfg,
-      filters: {
-        ...cfg.filters,
-        [uniqueName]: value,
-      },
-    }));
-    this.reload();
-  }
-
-  removeFilter(uniqueName) {
-    const reloadRequired = !!this.config.filters[uniqueName];
-    this.setConfig(cfg => ({
-      ...cfg,
-      formFilterColumns: (cfg.formFilterColumns || []).filter(x => x != uniqueName),
-      filters: _.omit(cfg.filters || [], uniqueName),
-    }));
-    if (reloadRequired) this.reload();
-  }
-
-  reload() {
-    this.setCache(cache => ({
-      // ...cache,
-      ...createGridCache(),
-      refreshTime: new Date().getTime(),
-    }));
-  }
-
-  getKeyValue(columnName) {
-    const { formViewKey, formViewKeyRequested } = this.config;
-    if (formViewKeyRequested && formViewKeyRequested[columnName]) return formViewKeyRequested[columnName];
-    if (formViewKey && formViewKey[columnName]) return formViewKey[columnName];
-    return null;
-  }
-
-  requestKeyValue(columnName, value) {
-    if (this.getKeyValue(columnName) == value) return;
-
-    this.setConfig(cfg => ({
-      ...cfg,
-      formViewKeyRequested: {
-        ...cfg.formViewKey,
-        ...cfg.formViewKeyRequested,
-        [columnName]: value,
-      },
-    }));
-    this.reload();
-  }
-
-  extractKey(row) {
-    if (!row || !this.baseTable || !this.baseTable.primaryKey) {
-      return null;
-    }
-    const formViewKey = _.pick(
-      row,
-      this.baseTable.primaryKey.columns.map(x => x.columnName)
-    );
-    return formViewKey;
-  }
-
-  cancelRequestKey(rowData) {
-    this.setConfig(cfg => ({
-      ...cfg,
-      formViewKeyRequested: null,
-      formViewKey: rowData ? this.extractKey(rowData) : cfg.formViewKey,
-    }));
-  }
-}
@@ -27,10 +27,10 @@ export interface GridConfig extends GridConfigColumns {
  childConfig?: GridConfig;
  reference?: GridReferenceDefinition;
  isFormView?: boolean;
-  formViewKey?: { [uniqueName: string]: string };
-  formViewKeyRequested?: { [uniqueName: string]: string };
+  formViewRecordNumber?: number;
  formFilterColumns: string[];
  formColumnFilterText?: string;
+  multiColumnFilter?: string;
 }

 export interface GridCache {
@@ -14,7 +14,7 @@ import type {
 import { parseFilter, getFilterType } from 'dbgate-filterparser';
 import { filterName } from 'dbgate-tools';
 import { ChangeSetFieldDefinition, ChangeSetRowDefinition } from './ChangeSet';
-import { Expression, Select, treeToSql, dumpSqlSelect, Condition } from 'dbgate-sqltree';
+import { Expression, Select, treeToSql, dumpSqlSelect, Condition, CompoudCondition } from 'dbgate-sqltree';
 import { isTypeLogical } from 'dbgate-tools';

 export interface DisplayColumn {
@@ -70,6 +70,7 @@ export abstract class GridDisplay {
  }
  dialect: SqlDialect;
  columns: DisplayColumn[];
+  formColumns: DisplayColumn[] = [];
  baseTable?: TableInfo;
  baseView?: ViewInfo;
  baseCollection?: CollectionInfo;
@@ -83,6 +84,7 @@ export abstract class GridDisplay {
    return this.baseTable || this.baseView;
  }
  changeSetKeyFields: string[] = null;
+  editableStructure: TableInfo = null;
  sortable = false;
  groupable = false;
  filterable = false;
@@ -211,6 +213,32 @@ export abstract class GridDisplay {
      }
    }

+    if (this.baseTableOrView && this.config.multiColumnFilter) {
+      try {
+        const condition = parseFilter(this.config.multiColumnFilter, 'string');
+        if (condition) {
+          const orCondition: CompoudCondition = {
+            conditionType: 'or',
+            conditions: [],
+          };
+          for (const column of this.baseTableOrView.columns) {
+            orCondition.conditions.push(
+              _.cloneDeepWith(condition, (expr: Expression) => {
+                if (expr.exprType == 'placeholder') {
+                  return this.createColumnExpression(column, { alias: 'basetbl' });
+                }
+              })
+            );
+          }
+          if (orCondition.conditions.length > 0) {
+            conditions.push(orCondition);
+          }
+        }
+      } catch (err) {
+        console.warn(err.message);
+      }
+    }
+
    if (conditions.length > 0) {
      select.where = {
        conditionType: 'and',
@@ -329,6 +357,16 @@ export abstract class GridDisplay {
        ...cfg.filters,
        [uniqueName]: value,
      },
+      formViewRecordNumber: 0,
+    }));
+    this.reload();
+  }
+
+  setMutliColumnFilter(value) {
+    this.setConfig(cfg => ({
+      ...cfg,
+      multiColumnFilter: value,
+      formViewRecordNumber: 0,
    }));
    this.reload();
  }
@@ -351,6 +389,7 @@ export abstract class GridDisplay {
    this.setConfig(cfg => ({
      ...cfg,
      filters: _.omit(cfg.filters, [uniqueName]),
+      formFilterColumns: (cfg.formFilterColumns || []).filter(x => x != uniqueName),
    }));
    this.reload();
  }
@@ -457,30 +496,39 @@ export abstract class GridDisplay {
    return _.pick(row, this.changeSetKeyFields);
  }

-  getChangeSetField(row, uniqueName, insertedRowIndex): ChangeSetFieldDefinition {
+  getChangeSetField(
+    row,
+    uniqueName,
+    insertedRowIndex,
+    existingRowIndex = null,
+    baseNameOmitable = false
+  ): ChangeSetFieldDefinition {
    const col = this.columns.find(x => x.uniqueName == uniqueName);
    if (!col) return null;
    const baseObj = this.baseTableOrSimilar;
-    if (!baseObj) return null;
-    if (baseObj.pureName != col.pureName || baseObj.schemaName != col.schemaName) {
-      return null;
+    if (!baseNameOmitable) {
+      if (!baseObj) return null;
+      if (baseObj.pureName != col.pureName || baseObj.schemaName != col.schemaName) {
+        return null;
+      }
    }

    return {
-      ...this.getChangeSetRow(row, insertedRowIndex),
+      ...this.getChangeSetRow(row, insertedRowIndex, existingRowIndex, baseNameOmitable),
      uniqueName: uniqueName,
      columnName: col.columnName,
    };
  }

-  getChangeSetRow(row, insertedRowIndex): ChangeSetRowDefinition {
+  getChangeSetRow(row, insertedRowIndex, existingRowIndex, baseNameOmitable = false): ChangeSetRowDefinition {
    const baseObj = this.baseTableOrSimilar;
-    if (!baseObj) return null;
+    if (!baseNameOmitable && !baseObj) return null;
    return {
-      pureName: baseObj.pureName,
-      schemaName: baseObj.schemaName,
+      pureName: baseObj?.pureName,
+      schemaName: baseObj?.schemaName,
      insertedRowIndex,
-      condition: insertedRowIndex == null ? this.getChangeSetCondition(row) : null,
+      existingRowIndex,
+      condition: insertedRowIndex == null && existingRowIndex == null ? this.getChangeSetCondition(row) : null,
    };
  }

@@ -516,13 +564,15 @@ export abstract class GridDisplay {
        alias: 'basetbl',
      },
      columns: columns.map(col => this.createColumnExpression(col, { alias: 'basetbl' })),
-      orderBy: [
-        {
-          exprType: 'column',
-          columnName: orderColumnName,
-          direction: 'ASC',
-        },
-      ],
+      orderBy: this.driver?.requiresDefaultSortCriteria
+        ? [
+            {
+              exprType: 'column',
+              columnName: orderColumnName,
+              direction: 'ASC',
+            },
+          ]
+        : null,
    };
    const displayedColumnInfo = _.keyBy(
      this.columns.map(col => ({ ...col, sourceAlias: 'basetbl' })),
@@ -688,7 +738,7 @@ export abstract class GridDisplay {
    // return sql;
  }

-  compileFilters(): Condition {
+  compileJslFilters(): Condition {
    const filters = this.config && this.config.filters;
    if (!filters) return null;
    const conditions = [];
@@ -711,6 +761,17 @@ export abstract class GridDisplay {
        // filter parse error - ignore filter
      }
    }
+
+    if (this.config.multiColumnFilter) {
+      const placeholderCondition = parseFilter(this.config.multiColumnFilter, 'string');
+      if (placeholderCondition) {
+        conditions.push({
+          conditionType: 'anyColumnPass',
+          placeholderCondition,
+        });
+      }
+    }
+
    if (conditions.length == 0) return null;
    return {
      conditionType: 'and',
@@ -718,22 +779,11 @@ export abstract class GridDisplay {
    };
  }

-  switchToFormView(rowData) {
-    if (!this.baseTable) return;
-    const { primaryKey } = this.baseTable;
-    if (!primaryKey) return;
-    const { columns } = primaryKey;
-
+  switchToFormView(rowIndex) {
    this.setConfig(cfg => ({
      ...cfg,
      isFormView: true,
-      formViewKey: rowData
-        ? _.pick(
-            rowData,
-            columns.map(x => x.columnName)
-          )
-        : null,
-      formViewKeyRequested: null,
+      formViewRecordNumber: rowIndex,
    }));
  }

@@ -743,6 +793,36 @@ export abstract class GridDisplay {
      isJsonView: true,
    }));
  }
+
+  formViewNavigate(command, allRowCount) {
+    switch (command) {
+      case 'begin':
+        this.setConfig(cfg => ({
+          ...cfg,
+          formViewRecordNumber: 0,
+        }));
+        break;
+      case 'previous':
+        this.setConfig(cfg => ({
+          ...cfg,
+          formViewRecordNumber: Math.max((cfg.formViewRecordNumber || 0) - 1, 0),
+        }));
+        break;
+      case 'next':
+        this.setConfig(cfg => ({
+          ...cfg,
+          formViewRecordNumber: Math.max((cfg.formViewRecordNumber || 0) + 1, 0),
+        }));
+        break;
+      case 'end':
+        this.setConfig(cfg => ({
+          ...cfg,
+          formViewRecordNumber: Math.max(allRowCount - 1, 0),
+        }));
+        break;
+    }
+    this.reload();
+  }
 }

 export function reloadDataCacheFunc(cache: GridCache): GridCache {
@@ -13,14 +13,18 @@ export class JslGridDisplay extends GridDisplay {
    setCache: ChangeCacheFunc,
    rows: any,
    isDynamicStructure: boolean,
-    supportsReload: boolean
+    supportsReload: boolean,
+    editable: boolean = false
  ) {
    super(config, setConfig, cache, setCache, null);

    this.filterable = true;
+    this.sortable = true;
    this.supportsReload = supportsReload;
    this.isDynamicStructure = isDynamicStructure;
    this.filterTypeOverride = 'eval';
+    this.editable = editable;
+    this.editableStructure = editable ? structure : null;

    if (structure?.columns) {
      this.columns = _.uniqBy(
@@ -48,5 +52,7 @@ export class JslGridDisplay extends GridDisplay {
    }

    if (!this.columns) this.columns = [];
+
+    this.formColumns = this.columns;
  }
 }
@@ -11,7 +11,7 @@ export interface MacroDefinition {
  name: string;
  group: string;
  description?: string;
-  type: 'transformValue';
+  type: 'transformValue' | 'transformRow';
  code: string;
  args?: MacroArgument[];
 }
@@ -5,6 +5,7 @@ import _difference from 'lodash/difference';
 import debug from 'debug';
 import stableStringify from 'json-stable-stringify';
 import { PerspectiveDataPattern } from './PerspectiveDataPattern';
+import { perspectiveValueMatcher } from './perspectiveTools';

 const dbg = debug('dbgate:PerspectiveCache');

@@ -17,7 +18,9 @@ export class PerspectiveBindingGroup {
  bindingValues: any[];

  matchRow(row) {
-    return this.table.bindingColumns.every((column, index) => row[column] == this.bindingValues[index]);
+    return this.table.bindingColumns.every((column, index) =>
+    perspectiveValueMatcher(row[column], this.bindingValues[index])
+    );
  }
 }

@@ -69,7 +72,11 @@ export class PerspectiveCacheTable {
  }

  storeGroupSize(props: PerspectiveDataLoadProps, bindingValues: any[], count: number) {
-    const originalBindingValue = props.bindingValues.find(v => _zip(v, bindingValues).every(([x, y]) => x == y));
+    const originalBindingValue = props.bindingValues.find(v =>
+      _zip(v, bindingValues).every(([x, y]) => perspectiveValueMatcher(x, y))
+    );
+    // console.log('storeGroupSize NEW', bindingValues);
+    // console.log('storeGroupSize ORIGINAL', originalBindingValue);
    if (originalBindingValue) {
      const key = stableStringify(originalBindingValue);
      // console.log('SET SIZE', originalBindingValue, bindingValues, key, count);
@@ -80,6 +80,8 @@ export interface PerspectiveNodeConfig {
  isAutoGenerated?: true | undefined;
  isNodeChecked?: boolean;

+  multiColumnFilter?: string;
+
  position?: {
    x: number;
    y: number;
@@ -51,7 +51,7 @@ function addObjectToColumns(columns: PerspectiveDataPatternColumn[], row) {
      if (!column.types.includes(type)) {
        column.types.push(type);
      }
-      if (_isPlainObject(value)) {
+      if (_isPlainObject(value) && type != 'oid') {
        addObjectToColumns(column.columns, value);
      }
      if (_isArray(value)) {
@@ -47,6 +47,7 @@ export class PerspectiveDataProvider {

  async loadDataNested(props: PerspectiveDataLoadProps): Promise<{ rows: any[]; incomplete: boolean }> {
    const tableCache = this.cache.getTableCache(props);
+    // console.log('loadDataNested', props);

    const uncached = tableCache.getUncachedBindingGroups(props);
    if (uncached.length > 0) {
@@ -54,7 +55,7 @@ export class PerspectiveDataProvider {
        ...props,
        bindingValues: uncached,
      });
-      // console.log('COUNTS', counts);
+      // console.log('loadDataNested COUNTS', counts);
      for (const resetItem of uncached) {
        tableCache.storeGroupSize(props, resetItem, 0);
      }
@@ -196,6 +197,14 @@ export class PerspectiveDataProvider {
      },
    });

+    if (!nextRows) {
+      // return tableCache.getRowsResult(props);
+      return {
+        rows: [],
+        incomplete: false,
+      };
+    }
+
    if (nextRows.errorMessage) {
      throw new Error(nextRows.errorMessage);
    }
@@ -22,6 +22,7 @@ import {
  PerspectiveReferenceConfig,
 } from './PerspectiveConfig';
 import _isEqual from 'lodash/isEqual';
+import _isArray from 'lodash/isArray';
 import _cloneDeep from 'lodash/cloneDeep';
 import _compact from 'lodash/compact';
 import _uniq from 'lodash/uniq';
@@ -34,10 +35,15 @@ import { PerspectiveDataLoadProps, PerspectiveDataProvider } from './Perspective
 import stableStringify from 'json-stable-stringify';
 import { getFilterType, parseFilter } from 'dbgate-filterparser';
 import { FilterType } from 'dbgate-filterparser/lib/types';
-import { Condition, Expression, Select } from 'dbgate-sqltree';
+import { CompoudCondition, Condition, Expression, Select } from 'dbgate-sqltree';
 // import { getPerspectiveDefaultColumns } from './getPerspectiveDefaultColumns';
 import uuidv1 from 'uuid/v1';
 import { PerspectiveDataPatternColumn } from './PerspectiveDataPattern';
+import {
+  getPerspectiveMostNestedChildColumnName,
+  getPerspectiveParentColumnName,
+  perspectiveValueMatcher,
+} from './perspectiveTools';

 export interface PerspectiveDataLoadPropsWithNode {
  props: PerspectiveDataLoadProps;
@@ -72,7 +78,7 @@ export abstract class PerspectiveTreeNode {
    public setConfig: ChangePerspectiveConfigFunc,
    public parentNode: PerspectiveTreeNode,
    public dataProvider: PerspectiveDataProvider,
-    public databaseConfig: PerspectiveDatabaseConfig,
+    public defaultDatabaseConfig: PerspectiveDatabaseConfig,
    public designerId: string
  ) {
    this.nodeConfig = config.nodes.find(x => x.designerId == designerId);
@@ -120,6 +126,12 @@ export abstract class PerspectiveTreeNode {
  get engineType(): PerspectiveDatabaseEngineType {
    return null;
  }
+  get databaseConfig(): PerspectiveDatabaseConfig {
+    const res = { ...this.defaultDatabaseConfig };
+    if (this.nodeConfig?.conid) res.conid = this.nodeConfig?.conid;
+    if (this.nodeConfig?.database) res.database = this.nodeConfig?.database;
+    return res;
+  }
  abstract getNodeLoadProps(parentRows: any[]): PerspectiveDataLoadProps;
  get isRoot() {
    return this.parentNode == null;
@@ -137,6 +149,16 @@ export abstract class PerspectiveTreeNode {
  get generatesDataGridColumn() {
    return this.isCheckedColumn;
  }
+  get validParentDesignerId() {
+    if (this.designerId) return this.designerId;
+    return this.parentNode?.validParentDesignerId;
+  }
+  get preloadedLevelData() {
+    return false;
+  }
+  get findByDesignerIdWithoutDesignerId() {
+    return false;
+  }
  matchChildRow(parentRow: any, childRow: any): boolean {
    return true;
  }
@@ -210,7 +232,7 @@ export abstract class PerspectiveTreeNode {

  get hasUncheckedNodeInPath() {
    if (!this.parentNode) return false;
-    if (!this.isCheckedNode) return true;
+    if (this.designerId && !this.isCheckedNode) return true;
    return this.parentNode.hasUncheckedNodeInPath;
  }

@@ -319,10 +341,66 @@ export abstract class PerspectiveTreeNode {
    );
  }

+  getMutliColumnSqlCondition(source): Condition {
+    if (!this.nodeConfig?.multiColumnFilter) return null;
+    const base = this.getBaseTableFromThis() as TableInfo | ViewInfo;
+    if (!base) return null;
+    try {
+      const condition = parseFilter(this.nodeConfig?.multiColumnFilter, 'string');
+      if (condition) {
+        const orCondition: CompoudCondition = {
+          conditionType: 'or',
+          conditions: [],
+        };
+        for (const column of base.columns || []) {
+          orCondition.conditions.push(
+            _cloneDeepWith(condition, (expr: Expression) => {
+              if (expr.exprType == 'placeholder') {
+                return {
+                  exprType: 'column',
+                  alias: source,
+                  columnName: column.columnName,
+                };
+              }
+            })
+          );
+        }
+        if (orCondition.conditions.length > 0) {
+          return orCondition;
+        }
+      }
+    } catch (err) {
+      console.warn(err.message);
+    }
+    return null;
+  }
+
+  getMutliColumnMongoCondition(): {} {
+    if (!this.nodeConfig?.multiColumnFilter) return null;
+    const pattern = this.dataProvider?.dataPatterns?.[this.designerId];
+    if (!pattern) return null;
+
+    const condition = parseFilter(this.nodeConfig?.multiColumnFilter, 'mongo');
+    if (!condition) return null;
+    const res = pattern.columns.map(col => {
+      return _cloneDeepWith(condition, expr => {
+        if (expr.__placeholder__) {
+          return {
+            [col.name]: expr.__placeholder__,
+          };
+        }
+      });
+    });
+    return {
+      $or: res,
+    };
+  }
+
  getChildrenSqlCondition(source = null): Condition {
    const conditions = _compact([
      ...this.childNodes.map(x => x.parseFilterCondition(source)),
      ...this.buildParentFilterConditions(),
+      this.getMutliColumnSqlCondition(source),
    ]);
    if (conditions.length == 0) {
      return null;
@@ -337,7 +415,10 @@ export abstract class PerspectiveTreeNode {
  }

  getChildrenMongoCondition(source = null): {} {
-    const conditions = _compact([...this.childNodes.map(x => x.parseFilterCondition(source))]);
+    const conditions = _compact([
+      ...this.childNodes.map(x => x.parseFilterCondition(source)),
+      this.getMutliColumnMongoCondition(),
+    ]);
    if (conditions.length == 0) {
      return null;
    }
@@ -380,7 +461,7 @@ export abstract class PerspectiveTreeNode {
    }
    return res;
  }
-  getBaseTableFromThis() {
+  getBaseTableFromThis(): TableInfo | ViewInfo | CollectionInfo {
    return null;
  }

@@ -404,7 +485,7 @@ export abstract class PerspectiveTreeNode {
  // }

  findNodeByDesignerId(designerId: string): PerspectiveTreeNode {
-    if (!this.designerId) {
+    if (!this.designerId && !this.findByDesignerIdWithoutDesignerId) {
      return null;
    }
    if (!designerId) {
@@ -518,11 +599,11 @@ export class PerspectiveTableColumnNode extends PerspectiveTreeNode {
    config: PerspectiveConfig,
    setConfig: ChangePerspectiveConfigFunc,
    dataProvider: PerspectiveDataProvider,
-    databaseConfig: PerspectiveDatabaseConfig,
+    defaultDatabaseConfig: PerspectiveDatabaseConfig,
    parentNode: PerspectiveTreeNode,
    designerId: string
  ) {
-    super(dbs, config, setConfig, parentNode, dataProvider, databaseConfig, designerId);
+    super(dbs, config, setConfig, parentNode, dataProvider, defaultDatabaseConfig, designerId);

    this.isTable = !!this.db?.tables?.find(x => x.schemaName == table.schemaName && x.pureName == table.pureName);
    this.isView = !!this.db?.views?.find(x => x.schemaName == table.schemaName && x.pureName == table.pureName);
@@ -674,7 +755,7 @@ export class PerspectiveTableColumnNode extends PerspectiveTreeNode {
      this.config,
      this.setConfig,
      this.dataProvider,
-      this.databaseConfig,
+      this.defaultDatabaseConfig,
      this
    );
  }
@@ -752,21 +833,28 @@ export class PerspectivePatternColumnNode extends PerspectiveTreeNode {
    config: PerspectiveConfig,
    setConfig: ChangePerspectiveConfigFunc,
    dataProvider: PerspectiveDataProvider,
-    databaseConfig: PerspectiveDatabaseConfig,
+    defaultDatabaseConfig: PerspectiveDatabaseConfig,
    parentNode: PerspectiveTreeNode,
    designerId: string
  ) {
-    super(dbs, config, setConfig, parentNode, dataProvider, databaseConfig, designerId);
+    super(dbs, config, setConfig, parentNode, dataProvider, defaultDatabaseConfig, designerId);
    this.parentNodeConfig = this.tableNodeOrParent?.nodeConfig;
+    // console.log('PATTERN COLUMN', column);
  }

  get isChildColumn() {
    return this.parentNode instanceof PerspectivePatternColumnNode;
  }

+  get findByDesignerIdWithoutDesignerId() {
+    return this.isExpandable;
+  }
+
  // matchChildRow(parentRow: any, childRow: any): boolean {
-  //   if (!this.foreignKey) return false;
-  //   return parentRow[this.foreignKey.columns[0].columnName] == childRow[this.foreignKey.columns[0].refColumnName];
+  //   console.log('MATCH PATTENR ROW', parentRow, childRow);
+  //   return false;
+  //   // if (!this.foreignKey) return false;
+  //   // return parentRow[this.foreignKey.columns[0].columnName] == childRow[this.foreignKey.columns[0].refColumnName];
  // }

  // getChildMatchColumns() {
@@ -808,12 +896,19 @@ export class PerspectivePatternColumnNode extends PerspectiveTreeNode {
  // }

  getNodeLoadProps(parentRows: any[]): PerspectiveDataLoadProps {
+    // console.log('GETTING PATTERN', parentRows);
    return null;
  }

-  get generatesHiearchicGridColumn() {
+  get generatesHiearchicGridColumn(): boolean {
+    // return true;
    // console.log('generatesHiearchicGridColumn', this.parentTableNode?.nodeConfig?.checkedColumns, this.codeName + '::');
-    return !!this.tableNodeOrParent?.nodeConfig?.checkedColumns?.find(x => x.startsWith(this.codeName + '::'));
+    if (this.tableNodeOrParent?.nodeConfig?.checkedColumns?.find(x => x.startsWith(this.codeName + '::'))) {
+      return true;
+    }
+    // return false;
+
+    return this.hasCheckedJoinChild();
  }

  // get generatesHiearchicGridColumn() {
@@ -859,7 +954,11 @@ export class PerspectivePatternColumnNode extends PerspectiveTreeNode {
    return 'mongo';
  }

-  generateChildNodes(): PerspectiveTreeNode[] {
+  get preloadedLevelData() {
+    return true;
+  }
+
+  generatePatternChildNodes(): PerspectivePatternColumnNode[] {
    return this.column.columns.map(
      column =>
        new PerspectivePatternColumnNode(
@@ -870,12 +969,97 @@ export class PerspectivePatternColumnNode extends PerspectiveTreeNode {
          this.config,
          this.setConfig,
          this.dataProvider,
-          this.databaseConfig,
+          this.defaultDatabaseConfig,
          this,
          null
        )
    );
-    return [];
+  }
+
+  hasCheckedJoinChild() {
+    for (const node of this.childNodes) {
+      if (node instanceof PerspectivePatternColumnNode) {
+        if (node.hasCheckedJoinChild()) return true;
+      }
+      if (node.isCheckedNode) return true;
+    }
+    return false;
+  }
+
+  generateChildNodes(): PerspectiveTreeNode[] {
+    const patternChildren = this.generatePatternChildNodes();
+
+    const customs = [];
+    // console.log('GETTING CHILDREN', this.config.nodes, this.config.references);
+    for (const node of this.config.nodes) {
+      for (const ref of this.config.references) {
+        const validDesignerId = this.validParentDesignerId;
+        if (
+          (ref.sourceId == validDesignerId && ref.targetId == node.designerId) ||
+          (ref.targetId == validDesignerId && ref.sourceId == node.designerId)
+        ) {
+          // console.log('TESTING REF', ref, this.codeName);
+          if (ref.columns.length != 1) continue;
+          // console.log('CP1');
+          if (
+            ref.sourceId == validDesignerId &&
+            this.codeName == getPerspectiveParentColumnName(ref.columns[0].source)
+          ) {
+            if (ref.columns[0].target.includes('::')) continue;
+          } else if (
+            ref.targetId == validDesignerId &&
+            this.codeName == getPerspectiveParentColumnName(ref.columns[0].target)
+          ) {
+            if (ref.columns[0].source.includes('::')) continue;
+          } else {
+            continue;
+          }
+          // console.log('CP2');
+
+          const newConfig: PerspectiveDatabaseConfig = { ...this.defaultDatabaseConfig };
+          if (node.conid) newConfig.conid = node.conid;
+          if (node.database) newConfig.database = node.database;
+          const db = this.dbs?.[newConfig.conid]?.[newConfig.database];
+          const table = db?.tables?.find(x => x.pureName == node.pureName && x.schemaName == node.schemaName);
+          const view = db?.views?.find(x => x.pureName == node.pureName && x.schemaName == node.schemaName);
+          const collection = db?.collections?.find(x => x.pureName == node.pureName && x.schemaName == node.schemaName);
+
+          const join: PerspectiveCustomJoinConfig = {
+            refNodeDesignerId: node.designerId,
+            referenceDesignerId: ref.designerId,
+            baseDesignerId: validDesignerId,
+            joinName: node.alias,
+            refTableName: node.pureName,
+            refSchemaName: node.schemaName,
+            conid: node.conid,
+            database: node.database,
+            columns:
+              ref.sourceId == validDesignerId
+                ? ref.columns.map(col => ({ baseColumnName: col.source, refColumnName: col.target }))
+                : ref.columns.map(col => ({ baseColumnName: col.target, refColumnName: col.source })),
+          };
+
+          if (table || view || collection) {
+            customs.push(
+              new PerspectiveCustomJoinTreeNode(
+                join,
+                table || view || collection,
+                this.dbs,
+                this.config,
+                this.setConfig,
+                this.dataProvider,
+                this.defaultDatabaseConfig,
+                this,
+                node.designerId
+              )
+            );
+          }
+        }
+      }
+    }
+
+    return [...patternChildren, ...customs];
+    // return [];
    // if (!this.foreignKey) return [];
    // const tbl = this?.db?.tables?.find(
    //   x => x.pureName == this.foreignKey?.refTableName && x.schemaName == this.foreignKey?.refSchemaName
@@ -954,11 +1138,11 @@ export class PerspectiveTableNode extends PerspectiveTreeNode {
    config: PerspectiveConfig,
    setConfig: ChangePerspectiveConfigFunc,
    public dataProvider: PerspectiveDataProvider,
-    databaseConfig: PerspectiveDatabaseConfig,
+    defaultDatabaseConfig: PerspectiveDatabaseConfig,
    parentNode: PerspectiveTreeNode,
    designerId: string
  ) {
-    super(dbs, config, setConfig, parentNode, dataProvider, databaseConfig, designerId);
+    super(dbs, config, setConfig, parentNode, dataProvider, defaultDatabaseConfig, designerId);
  }

  get engineType(): PerspectiveDatabaseEngineType {
@@ -999,7 +1183,7 @@ export class PerspectiveTableNode extends PerspectiveTreeNode {
      this.config,
      this.setConfig,
      this.dataProvider,
-      this.databaseConfig,
+      this.defaultDatabaseConfig,
      this
    );
  }
@@ -1041,12 +1225,12 @@ export class PerspectiveTableReferenceNode extends PerspectiveTableNode {
    config: PerspectiveConfig,
    setConfig: ChangePerspectiveConfigFunc,
    public dataProvider: PerspectiveDataProvider,
-    databaseConfig: PerspectiveDatabaseConfig,
+    defaultDatabaseConfig: PerspectiveDatabaseConfig,
    public isMultiple: boolean,
    parentNode: PerspectiveTreeNode,
    designerId: string
  ) {
-    super(table, dbs, config, setConfig, dataProvider, databaseConfig, parentNode, designerId);
+    super(table, dbs, config, setConfig, dataProvider, defaultDatabaseConfig, parentNode, designerId);
  }

  matchChildRow(parentRow: any, childRow: any): boolean {
@@ -1145,16 +1329,22 @@ export class PerspectiveCustomJoinTreeNode extends PerspectiveTableNode {
    config: PerspectiveConfig,
    setConfig: ChangePerspectiveConfigFunc,
    public dataProvider: PerspectiveDataProvider,
-    databaseConfig: PerspectiveDatabaseConfig,
+    defaultDatabaseConfig: PerspectiveDatabaseConfig,
    parentNode: PerspectiveTreeNode,
    designerId: string
  ) {
-    super(table, dbs, config, setConfig, dataProvider, databaseConfig, parentNode, designerId);
+    super(table, dbs, config, setConfig, dataProvider, defaultDatabaseConfig, parentNode, designerId);
  }

  matchChildRow(parentRow: any, childRow: any): boolean {
+    // console.log('MATCH ROW', parentRow, childRow);
    for (const column of this.customJoin.columns) {
-      if (parentRow[column.baseColumnName] != childRow[column.refColumnName]) {
+      if (
+        !perspectiveValueMatcher(
+          parentRow[getPerspectiveMostNestedChildColumnName(column.baseColumnName)],
+          childRow[column.refColumnName]
+        )
+      ) {
        return false;
      }
    }
@@ -1171,17 +1361,68 @@ export class PerspectiveCustomJoinTreeNode extends PerspectiveTableNode {

  getNodeLoadProps(parentRows: any[]): PerspectiveDataLoadProps {
    // console.log('CUSTOM JOIN', this.customJoin);
+    // console.log('PARENT ROWS', parentRows);
+
    // console.log('this.getDataLoadColumns()', this.getDataLoadColumns());
    const isMongo = isCollectionInfo(this.table);

+    // const bindingValues = [];
+
+    // for (const row of parentRows) {
+    //   const rowBindingValueArrays = [];
+    //   for (const col of this.customJoin.columns) {
+    //     const path = col.baseColumnName.split('::');
+    //     const values = [];
+
+    //     function processSubpath(parent, subpath) {
+    //       if (subpath.length == 0) {
+    //         values.push(parent);
+    //         return;
+    //       }
+    //       if (parent == null) {
+    //         return;
+    //       }
+
+    //       const obj = parent[subpath[0]];
+    //       if (_isArray(obj)) {
+    //         for (const elem of obj) {
+    //           processSubpath(elem, subpath.slice(1));
+    //         }
+    //       } else {
+    //         processSubpath(obj, subpath.slice(1));
+    //       }
+    //     }
+
+    //     processSubpath(row, path);
+
+    //     rowBindingValueArrays.push(values);
+    //   }
+
+    //   const valueCount = Math.max(...rowBindingValueArrays.map(x => x.length));
+
+    //   for (let i = 0; i < valueCount; i += 1) {
+    //     const value = Array(this.customJoin.columns.length);
+    //     for (let col = 0; col < this.customJoin.columns.length; col++) {
+    //       value[col] = rowBindingValueArrays[col][i % rowBindingValueArrays[col].length];
+    //     }
+    //     bindingValues.push(value);
+    //   }
+    // }
+    const bindingValues = parentRows.map(row =>
+      this.customJoin.columns.map(x => row[getPerspectiveMostNestedChildColumnName(x.baseColumnName)])
+    );
+
+    // console.log('bindingValues', bindingValues);
+    // console.log(
+    //   'bindingValues UNIQ',
+    //   _uniqBy(bindingValues, x => JSON.stringify(x))
+    // );
+
    return {
      schemaName: this.table.schemaName,
      pureName: this.table.pureName,
      bindingColumns: this.getParentMatchColumns(),
-      bindingValues: _uniqBy(
-        parentRows.map(row => this.customJoin.columns.map(x => row[x.baseColumnName])),
-        stableStringify
-      ),
+      bindingValues: _uniqBy(bindingValues, x => JSON.stringify(x)),
      dataColumns: this.getDataLoadColumns(),
      allColumns: isMongo,
      databaseConfig: this.databaseConfig,
@@ -1298,7 +1539,7 @@ export function getTableChildPerspectiveNodes(
  config: PerspectiveConfig,
  setConfig: ChangePerspectiveConfigFunc,
  dataProvider: PerspectiveDataProvider,
-  databaseConfig: PerspectiveDatabaseConfig,
+  defaultDatabaseConfig: PerspectiveDatabaseConfig,
  parentNode: PerspectiveTreeNode
 ) {
  if (!table) return [];
@@ -1320,7 +1561,7 @@ export function getTableChildPerspectiveNodes(
              config,
              setConfig,
              dataProvider,
-              databaseConfig,
+              defaultDatabaseConfig,
              parentNode,
              designerId
            )
@@ -1331,7 +1572,7 @@ export function getTableChildPerspectiveNodes(
              config,
              setConfig,
              dataProvider,
-              databaseConfig,
+              defaultDatabaseConfig,
              parentNode,
              designerId
            )
@@ -1350,7 +1591,7 @@ export function getTableChildPerspectiveNodes(
            config,
            setConfig,
            dataProvider,
-            databaseConfig,
+            defaultDatabaseConfig,
            parentNode,
            designerId
          )
@@ -1389,7 +1630,7 @@ export function getTableChildPerspectiveNodes(
                config,
                setConfig,
                dataProvider,
-                databaseConfig,
+                defaultDatabaseConfig,
                isMultiple,
                parentNode,
                designerId
@@ -1412,7 +1653,10 @@ export function getTableChildPerspectiveNodes(
        (ref.sourceId == parentNode.designerId && ref.targetId == node.designerId) ||
        (ref.targetId == parentNode.designerId && ref.sourceId == node.designerId)
      ) {
-        const newConfig = { ...databaseConfig };
+        if (ref.columns.find(x => x.source.includes('::') || x.target.includes('::'))) {
+          continue;
+        }
+        const newConfig: PerspectiveDatabaseConfig = { ...defaultDatabaseConfig };
        if (node.conid) newConfig.conid = node.conid;
        if (node.database) newConfig.database = node.database;
        const db = dbs?.[newConfig.conid]?.[newConfig.database];
@@ -1444,7 +1688,7 @@ export function getTableChildPerspectiveNodes(
              config,
              setConfig,
              dataProvider,
-              newConfig,
+              defaultDatabaseConfig,
              parentNode,
              node.designerId
            )
@@ -1,272 +0,0 @@
-import { FormViewDisplay } from './FormViewDisplay';
-import _ from 'lodash';
-import { ChangeCacheFunc, DisplayColumn, ChangeConfigFunc } from './GridDisplay';
-import type { EngineDriver, NamedObjectInfo, DatabaseInfo } from 'dbgate-types';
-import { GridConfig, GridCache } from './GridConfig';
-import { mergeConditions, Condition, OrderByExpression } from 'dbgate-sqltree';
-import { TableGridDisplay } from './TableGridDisplay';
-import stableStringify from 'json-stable-stringify';
-import { ChangeSetFieldDefinition, ChangeSetRowDefinition } from './ChangeSet';
-import { DictionaryDescriptionFunc } from '.';
-
-export class TableFormViewDisplay extends FormViewDisplay {
-  // use utility functions from GridDisplay and publish result in FromViewDisplay interface
-  private gridDisplay: TableGridDisplay;
-
-  constructor(
-    public tableName: NamedObjectInfo,
-    driver: EngineDriver,
-    config: GridConfig,
-    setConfig: ChangeConfigFunc,
-    cache: GridCache,
-    setCache: ChangeCacheFunc,
-    dbinfo: DatabaseInfo,
-    displayOptions,
-    serverVersion,
-    getDictionaryDescription: DictionaryDescriptionFunc = null,
-    isReadOnly = false
-  ) {
-    super(config, setConfig, cache, setCache, driver, dbinfo, serverVersion);
-    this.gridDisplay = new TableGridDisplay(
-      tableName,
-      driver,
-      config,
-      setConfig,
-      cache,
-      setCache,
-      dbinfo,
-      displayOptions,
-      serverVersion,
-      getDictionaryDescription,
-      isReadOnly
-    );
-    this.gridDisplay.addAllExpandedColumnsToSelected = true;
-
-    this.isLoadedCorrectly = this.gridDisplay.isLoadedCorrectly && !!this.driver;
-    this.columns = [];
-    this.addDisplayColumns(this.gridDisplay.columns);
-    this.baseTable = this.gridDisplay.baseTable;
-    this.gridDisplay.hintBaseColumns = this.columns;
-  }
-
-  addDisplayColumns(columns: DisplayColumn[]) {
-    for (const col of columns) {
-      this.columns.push(col);
-      if (this.gridDisplay.isExpandedColumn(col.uniqueName)) {
-        const table = this.gridDisplay.getFkTarget(col);
-        if (table) {
-          const subcolumns = this.gridDisplay.getDisplayColumns(table, col.uniquePath);
-          this.addDisplayColumns(subcolumns);
-        }
-      }
-    }
-  }
-
-  getPrimaryKeyEqualCondition(row = null): Condition {
-    if (!row) row = this.config.formViewKeyRequested || this.config.formViewKey;
-    if (!row) return null;
-    const { primaryKey } = this.gridDisplay.baseTable;
-    if (!primaryKey) return null;
-    return {
-      conditionType: 'and',
-      conditions: primaryKey.columns.map(({ columnName }) => ({
-        conditionType: 'binary',
-        operator: '=',
-        left: {
-          exprType: 'column',
-          columnName,
-          source: {
-            alias: 'basetbl',
-          },
-        },
-        right: {
-          exprType: 'value',
-          value: row[columnName],
-        },
-      })),
-    };
-  }
-
-  getPrimaryKeyOperatorCondition(operator): Condition {
-    if (!this.config.formViewKey) return null;
-    const conditions = [];
-
-    const { primaryKey } = this.gridDisplay.baseTable;
-    if (!primaryKey) return null;
-    for (let index = 0; index < primaryKey.columns.length; index++) {
-      conditions.push({
-        conditionType: 'and',
-        conditions: [
-          ...primaryKey.columns.slice(0, index).map(({ columnName }) => ({
-            conditionType: 'binary',
-            operator: '=',
-            left: {
-              exprType: 'column',
-              columnName,
-              source: {
-                alias: 'basetbl',
-              },
-            },
-            right: {
-              exprType: 'value',
-              value: this.config.formViewKey[columnName],
-            },
-          })),
-          ...primaryKey.columns.slice(index).map(({ columnName }) => ({
-            conditionType: 'binary',
-            operator: operator,
-            left: {
-              exprType: 'column',
-              columnName,
-              source: {
-                alias: 'basetbl',
-              },
-            },
-            right: {
-              exprType: 'value',
-              value: this.config.formViewKey[columnName],
-            },
-          })),
-        ],
-      });
-    }
-
-    if (conditions.length == 1) {
-      return conditions[0];
-    }
-
-    return {
-      conditionType: 'or',
-      conditions,
-    };
-  }
-
-  getSelect() {
-    if (!this.driver) return null;
-    const select = this.gridDisplay.createSelect();
-    if (!select) return null;
-    select.topRecords = 1;
-    return select;
-  }
-
-  getCurrentRowQuery() {
-    const select = this.getSelect();
-    if (!select) return null;
-
-    select.where = mergeConditions(select.where, this.getPrimaryKeyEqualCondition());
-    return select;
-  }
-
-  getCountSelect() {
-    const select = this.getSelect();
-    if (!select) return null;
-    select.orderBy = null;
-    select.columns = [
-      {
-        exprType: 'raw',
-        sql: 'COUNT(*)',
-        alias: 'count',
-      },
-    ];
-    select.topRecords = null;
-    return select;
-  }
-
-  getCountQuery() {
-    if (!this.driver) return null;
-    const select = this.getCountSelect();
-    if (!select) return null;
-    return select;
-  }
-
-  getBeforeCountQuery() {
-    if (!this.driver) return null;
-    const select = this.getCountSelect();
-    if (!select) return null;
-    select.where = mergeConditions(select.where, this.getPrimaryKeyOperatorCondition('<'));
-    return select;
-  }
-
-  navigate(row) {
-    const formViewKey = this.extractKey(row);
-    this.setConfig(cfg => ({
-      ...cfg,
-      formViewKey,
-    }));
-  }
-
-  isLoadedCurrentRow(row) {
-    if (!row) return false;
-    const formViewKey = this.extractKey(row);
-    return stableStringify(formViewKey) == stableStringify(this.config.formViewKey);
-  }
-
-  navigateRowQuery(commmand: 'begin' | 'previous' | 'next' | 'end') {
-    if (!this.driver) return null;
-    const select = this.gridDisplay.createSelect();
-    if (!select) return null;
-    const { primaryKey } = this.gridDisplay.baseTable;
-
-    function getOrderBy(direction): OrderByExpression[] {
-      return primaryKey.columns.map(({ columnName }) => ({
-        exprType: 'column',
-        columnName,
-        direction,
-      }));
-    }
-
-    select.topRecords = 1;
-    switch (commmand) {
-      case 'begin':
-        select.orderBy = getOrderBy('ASC');
-        break;
-      case 'end':
-        select.orderBy = getOrderBy('DESC');
-        break;
-      case 'previous':
-        select.orderBy = getOrderBy('DESC');
-        select.where = mergeConditions(select.where, this.getPrimaryKeyOperatorCondition('<'));
-        break;
-      case 'next':
-        select.orderBy = getOrderBy('ASC');
-        select.where = mergeConditions(select.where, this.getPrimaryKeyOperatorCondition('>'));
-        break;
-    }
-
-    return select;
-  }
-
-  getChangeSetRow(row): ChangeSetRowDefinition {
-    if (!this.baseTable) return null;
-    return {
-      pureName: this.baseTable.pureName,
-      schemaName: this.baseTable.schemaName,
-      condition: this.extractKey(row),
-    };
-  }
-
-  getChangeSetField(row, uniqueName): ChangeSetFieldDefinition {
-    const col = this.columns.find(x => x.uniqueName == uniqueName);
-    if (!col) return null;
-    if (!this.baseTable) return null;
-    if (this.baseTable.pureName != col.pureName || this.baseTable.schemaName != col.schemaName) return null;
-    return {
-      ...this.getChangeSetRow(row),
-      uniqueName: uniqueName,
-      columnName: col.columnName,
-    };
-  }
-
-  toggleExpandedColumn(uniqueName: string, value?: boolean) {
-    this.gridDisplay.toggleExpandedColumn(uniqueName, value);
-    this.gridDisplay.reload();
-  }
-
-  isExpandedColumn(uniqueName: string) {
-    return this.gridDisplay.isExpandedColumn(uniqueName);
-  }
-
-  get editable() {
-    return this.gridDisplay.editable;
-  }
-}
@@ -51,6 +51,7 @@ export class TableGridDisplay extends GridDisplay {
    }

    this.columns = this.getDisplayColumns(this.table, []);
+    this.addFormDisplayColumns(this.getDisplayColumns(this.table, []));
    this.filterable = true;
    this.sortable = true;
    this.groupable = true;
@@ -62,6 +63,24 @@ export class TableGridDisplay extends GridDisplay {
        ? this.table.primaryKey.columns.map(x => x.columnName)
        : this.table.columns.map(x => x.columnName);
    }
+    
+    if (this.config.isFormView) {
+      this.addAllExpandedColumnsToSelected = true;
+      this.hintBaseColumns = this.formColumns;
+    }
+  }
+
+  addFormDisplayColumns(columns) {
+    for (const col of columns) {
+      this.formColumns.push(col);
+      if (this.isExpandedColumn(col.uniqueName)) {
+        const table = this.getFkTarget(col);
+        if (table) {
+          const subcolumns = this.getDisplayColumns(table, col.uniquePath);
+          this.addFormDisplayColumns(subcolumns);
+        }
+      }
+    }
  }

  findTable({ schemaName = undefined, pureName }) {
@@ -15,6 +15,7 @@ export class ViewGridDisplay extends GridDisplay {
  ) {
    super(config, setConfig, cache, setCache, driver, serverVersion);
    this.columns = this.getDisplayColumns(view);
+    this.formColumns = this.columns;
    this.filterable = true;
    this.sortable = true;
    this.groupable = false;
@@ -55,7 +55,7 @@ function processDependencies(
              schemaName: fk.schemaName,
            },
            alias: 't0',
-            relations: subFkPath.map((fkItem, fkIndex) => ({
+            relations: [...subFkPath].reverse().map((fkItem, fkIndex) => ({
              joinType: 'INNER JOIN',
              alias: `t${fkIndex + 1}`,
              name: {
@@ -123,7 +123,16 @@ export function getDeleteCascades(changeSet: ChangeSet, dbinfo: DatabaseInfo): C
    const table = dbinfo.tables.find(x => x.pureName == baseCmd.pureName && x.schemaName == baseCmd.schemaName);
    if (!table.primaryKey) continue;

-    processDependencies(changeSet, result, allForeignKeys, [], table, baseCmd, dbinfo, [table.pureName]);
+    const itemResult: ChangeSetDeleteCascade[] = [];
+    processDependencies(changeSet, itemResult, allForeignKeys, [], table, baseCmd, dbinfo, [table.pureName]);
+    for (const item of itemResult) {
+      const existing = result.find(x => x.title == item.title);
+      if (existing) {
+        existing.commands.push(...item.commands);
+      } else {
+        result.push(item);
+      }
+    }

    // let resItem = result.find(x => x.title == baseCmd.pureName);
    // if (!resItem) {
@@ -6,12 +6,8 @@ export * from './TableGridDisplay';
 export * from './ViewGridDisplay';
 export * from './JslGridDisplay';
 export * from './ChangeSet';
-export * from './FreeTableGridDisplay';
-export * from './FreeTableModel';
 export * from './MacroDefinition';
 export * from './runMacro';
-export * from './FormViewDisplay';
-export * from './TableFormViewDisplay';
 export * from './CollectionGridDisplay';
 export * from './deleteCascade';
 export * from './PerspectiveDisplay';
@@ -21,3 +17,7 @@ export * from './PerspectiveConfig';
 export * from './processPerspectiveDefaultColunns';
 export * from './PerspectiveDataPattern';
 export * from './PerspectiveDataLoader';
+export * from './perspectiveTools';
+export * from './DataDuplicator';
+export * from './FreeTableGridDisplay';
+export * from './FreeTableModel';
@@ -0,0 +1,22 @@
+export function getPerspectiveParentColumnName(columnName: string) {
+  const path = columnName.split('::');
+  if (path.length >= 2) return path.slice(0, -1).join('::');
+  return null;
+}
+
+export function getPerspectiveMostNestedChildColumnName(columnName: string) {
+  const path = columnName.split('::');
+  return path[path.length - 1];
+}
+
+// export function perspectiveValueMatcher(value1, value2): boolean {
+//   if (value1?.$oid && value2?.$oid) return value1.$oid == value2.$oid;
+//   if (Array.isArray(value1)) return !!value1.find(x => perspectiveValueMatcher(x, value2));
+//   if (Array.isArray(value2)) return !!value2.find(x => perspectiveValueMatcher(value1, x));
+//   return value1 == value2;
+// }
+
+export function perspectiveValueMatcher(value1, value2): boolean {
+  if (value1?.$oid && value2?.$oid) return value1.$oid == value2.$oid;
+  return value1 == value2;
+}
@@ -1,10 +1,9 @@
-import { FreeTableModel } from './FreeTableModel';
 import _ from 'lodash';
 import uuidv1 from 'uuid/v1';
 import uuidv4 from 'uuid/v4';
 import moment from 'moment';
 import { MacroDefinition, MacroSelectedCell } from './MacroDefinition';
-import { ChangeSet, setChangeSetValue } from './ChangeSet';
+import { ChangeSet, setChangeSetValue, setChangeSetRowData } from './ChangeSet';
 import { GridDisplay } from './GridDisplay';

 const getMacroFunction = {
@@ -13,13 +12,8 @@ const getMacroFunction = {
    ${code}
 }
 `,
-  transformRows: code => `
-(rows, args, modules, selectedCells, cols, columns) => {
-  ${code}
-}
-`,
-  transformData: code => `
-(rows, args, modules, selectedCells, cols, columns) => {
+  transformRow: code => `
+(row, args, modules, rowIndex, columns) => {
  ${code}
 }
 `,
@@ -32,160 +26,6 @@ const modules = {
  moment,
 };

-function runTramsformValue(
-  func,
-  macroArgs: {},
-  data: FreeTableModel,
-  preview: boolean,
-  selectedCells: MacroSelectedCell[],
-  errors: string[] = []
-) {
-  const selectedRows = _.groupBy(selectedCells, 'row');
-  const rows = data.rows.map((row, rowIndex) => {
-    const selectedRow = selectedRows[rowIndex];
-    if (selectedRow) {
-      const modifiedFields = [];
-      let res = null;
-      for (const cell of selectedRow) {
-        const { column } = cell;
-        const oldValue = row[column];
-        let newValue = oldValue;
-        try {
-          newValue = func(oldValue, macroArgs, modules, rowIndex, row, column);
-        } catch (err) {
-          errors.push(`Error processing column ${column} on row ${rowIndex}: ${err.message}`);
-        }
-        if (newValue != oldValue) {
-          if (res == null) {
-            res = { ...row };
-          }
-          res[column] = newValue;
-          if (preview) modifiedFields.push(column);
-        }
-      }
-      if (res) {
-        if (modifiedFields.length > 0) {
-          return {
-            ...res,
-            __modifiedFields: new Set(modifiedFields),
-          };
-        }
-        return res;
-      }
-      return row;
-    } else {
-      return row;
-    }
-  });
-
-  return {
-    structure: data.structure,
-    rows,
-  };
-}
-
-function removePreviewRowFlags(rows) {
-  rows = rows.filter(row => row.__rowStatus != 'deleted');
-  rows = rows.map(row => {
-    if (row.__rowStatus || row.__modifiedFields || row.__insertedFields || row.__deletedFields)
-      return _.omit(row, ['__rowStatus', '__modifiedFields', '__insertedFields', '__deletedFields']);
-    return row;
-  });
-  return rows;
-}
-
-function runTramsformRows(
-  func,
-  macroArgs: {},
-  data: FreeTableModel,
-  preview: boolean,
-  selectedCells: MacroSelectedCell[],
-  errors: string[] = []
-) {
-  let rows = data.rows;
-  try {
-    rows = func(
-      data.rows,
-      macroArgs,
-      modules,
-      selectedCells,
-      data.structure.columns.map(x => x.columnName),
-      data.structure.columns
-    );
-    if (!preview) {
-      rows = removePreviewRowFlags(rows);
-    }
-  } catch (err) {
-    errors.push(`Error processing rows: ${err.message}`);
-  }
-  return {
-    structure: data.structure,
-    rows,
-  };
-}
-
-function runTramsformData(
-  func,
-  macroArgs: {},
-  data: FreeTableModel,
-  preview: boolean,
-  selectedCells: MacroSelectedCell[],
-  errors: string[] = []
-) {
-  try {
-    let { rows, columns, cols } = func(
-      data.rows,
-      macroArgs,
-      modules,
-      selectedCells,
-      data.structure.columns.map(x => x.columnName),
-      data.structure.columns
-    );
-    if (cols && !columns) {
-      columns = cols.map(columnName => ({ columnName }));
-    }
-    columns = _.uniqBy(columns, 'columnName');
-    if (!preview) {
-      rows = removePreviewRowFlags(rows);
-    }
-    return {
-      structure: { columns },
-      rows,
-    };
-  } catch (err) {
-    errors.push(`Error processing data: ${err.message}`);
-  }
-  return data;
-}
-
-export function runMacro(
-  macro: MacroDefinition,
-  macroArgs: {},
-  data: FreeTableModel,
-  preview: boolean,
-  selectedCells: MacroSelectedCell[],
-  errors: string[] = []
-): FreeTableModel {
-  let func;
-  try {
-    func = eval(getMacroFunction[macro.type](macro.code));
-  } catch (err) {
-    errors.push(`Error compiling macro ${macro.name}: ${err.message}`);
-    return data;
-  }
-  if (macro.type == 'transformValue') {
-    return runTramsformValue(func, macroArgs, data, preview, selectedCells, errors);
-  }
-  if (macro.type == 'transformRows') {
-    return runTramsformRows(func, macroArgs, data, preview, selectedCells, errors);
-  }
-  if (macro.type == 'transformData') {
-    // @ts-ignore
-    return runTramsformData(func, macroArgs, data, preview, selectedCells, errors);
-  }
-  return data;
-}
-
 export function compileMacroFunction(macro: MacroDefinition, errors = []) {
  if (!macro) return null;
  let func;
@@ -198,7 +38,7 @@ export function compileMacroFunction(macro: MacroDefinition, errors = []) {
  }
 }

-export function runMacroOnValue(compiledFunc, macroArgs, value, rowIndex, row, column, errors = []) {
+export function runMacroOnValue(compiledFunc, macroArgs, value, rowIndex: number, row, column: string, errors = []) {
  if (!compiledFunc) return value;
  try {
    const res = compiledFunc(value, macroArgs, modules, rowIndex, row, column);
@@ -209,31 +49,62 @@ export function runMacroOnValue(compiledFunc, macroArgs, value, rowIndex, row, c
  }
 }

+export function runMacroOnRow(compiledFunc, macroArgs, rowIndex: number, row: any, columns: string[], errors = []) {
+  if (!compiledFunc) return row;
+  try {
+    const res = compiledFunc(row, macroArgs, modules, rowIndex, columns);
+    return res;
+  } catch (err) {
+    errors.push(`Error processing row ${rowIndex}: ${err.message}`);
+    return row;
+  }
+}
+
 export function runMacroOnChangeSet(
  macro: MacroDefinition,
  macroArgs: {},
  selectedCells: MacroSelectedCell[],
  changeSet: ChangeSet,
-  display: GridDisplay
+  display: GridDisplay,
+  useRowIndexInsteaOfCondition: boolean
 ): ChangeSet {
  const errors = [];
  const compiledMacroFunc = compileMacroFunction(macro, errors);
  if (!compiledMacroFunc) return null;

-  let res = changeSet;
-  for (const cell of selectedCells) {
-    const definition = display.getChangeSetField(cell.rowData, cell.column, undefined);
-    const macroResult = runMacroOnValue(
-      compiledMacroFunc,
-      macroArgs,
-      cell.value,
-      cell.row,
-      cell.rowData,
-      cell.column,
-      errors
-    );
-    res = setChangeSetValue(res, definition, macroResult);
+  if (macro.type == 'transformValue') {
+    let res = changeSet;
+    for (const cell of selectedCells) {
+      const definition = display.getChangeSetField(
+        cell.rowData,
+        cell.column,
+        undefined,
+        useRowIndexInsteaOfCondition ? cell.row : undefined,
+        useRowIndexInsteaOfCondition
+      );
+      const macroResult = runMacroOnValue(
+        compiledMacroFunc,
+        macroArgs,
+        cell.value,
+        cell.row,
+        cell.rowData,
+        cell.column,
+        errors
+      );
+      res = setChangeSetValue(res, definition, macroResult);
+    }
+    return res;
+  }
+  if (macro.type == 'transformRow') {
+    let res = changeSet;
+    const rowIndexes = _.uniq(selectedCells.map(x => x.row));
+    for (const index of rowIndexes) {
+      const rowData = selectedCells.find(x => x.row == index)?.rowData;
+      const columns = _.uniq(selectedCells.map(x => x.column));
+      const definition = display.getChangeSetRow(rowData, null, index, true);
+      const newRow = runMacroOnRow(compiledMacroFunc, macroArgs, index, rowData, columns);
+      res = setChangeSetRowData(res, definition, newRow);
+    }
+    return res;
  }
-
-  return res;
 }
--- a/Show More
+++ b/Show More