v5.5.7-beta.36

This reverts commit 57c62fbe27.

.github/workflows/build-app-beta.yaml

@@ -30,6 +30,9 @@ jobs:
       - name: yarn adjustPackageJson
         run: |
           yarn adjustPackageJson
+      - name: yarn adjustAppPackageJson
+        run: |
+          yarn adjustAppPackageJson
       - name: setUpdaterChannel beta
         run: |
           node setUpdaterChannel beta
@@ -47,9 +50,6 @@ jobs:
             yarn printSecrets
         env:
           GIST_UPLOAD_SECRET : ${{secrets.GIST_UPLOAD_SECRET}}
-      - name: fillNativeModulesElectron
-        run: |
-          yarn fillNativeModulesElectron
       - name: fillPackagedPlugins
         run: |
           yarn fillPackagedPlugins

.github/workflows/build-app-pro-beta.yaml

@@ -55,6 +55,9 @@ jobs:
           cd ..
           cd dbgate-merged
           yarn adjustPackageJson
+      - name: yarn adjustAppPackageJson
+        run: |
+          yarn adjustAppPackageJson
       - name: adjustPackageJsonPremium
         run: |
           cd ..
@@ -87,11 +90,6 @@ jobs:
             yarn printSecrets
         env:
           GIST_UPLOAD_SECRET : ${{secrets.GIST_UPLOAD_SECRET}}
-      - name: fillNativeModulesElectron
-        run: |
-          cd ..
-          cd dbgate-merged
-          yarn fillNativeModulesElectron
       - name: fillPackagedPlugins
         run: |
           cd ..

.github/workflows/build-app-pro.yaml

@@ -56,6 +56,9 @@ jobs:
           cd ..
           cd dbgate-merged
           yarn adjustPackageJson
+      - name: yarn adjustAppPackageJson
+        run: |
+          yarn adjustAppPackageJson
       - name: yarn adjustPackageJsonPremium
         run: |
           cd ..
@@ -88,11 +91,6 @@ jobs:
           yarn printSecrets
         env:
           GIST_UPLOAD_SECRET : ${{secrets.GIST_UPLOAD_SECRET}}
-      - name: fillNativeModulesElectron
-        run: |
-          cd ..
-          cd dbgate-merged
-          yarn fillNativeModulesElectron
       - name: fillPackagedPlugins
         run: |
           cd ..

.github/workflows/build-app.yaml

@@ -34,6 +34,9 @@ jobs:
       - name: yarn adjustPackageJson
         run: |
           yarn adjustPackageJson
+      - name: yarn adjustAppPackageJson
+        run: |
+          yarn adjustAppPackageJson
       - name: yarn set timeout
         run: |
           yarn config set network-timeout 100000
@@ -50,9 +53,6 @@ jobs:
             yarn printSecrets
         env:
           GIST_UPLOAD_SECRET : ${{secrets.GIST_UPLOAD_SECRET}}
-      - name: fillNativeModulesElectron
-        run: |
-          yarn fillNativeModulesElectron
       - name: fillPackagedPlugins
         run: |
           yarn fillPackagedPlugins

.gitignore

@@ -28,8 +28,6 @@ docker/plugins
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
-app/src/nativeModulesContent.js
-packages/api/src/nativeModulesContent.js
 packages/api/src/packagedPluginsContent.js
 .VSCodeCounter
 

README.md

@@ -26,7 +26,7 @@ DbGate is licensed under GPL-3.0 license and is free to use for any purpose.
 * MongoDB
 * Redis
 * SQLite
-* Amazon Redshift
+* Amazon Redshift (Premium)
 * CockroachDB
 * MariaDB
 * CosmosDB (Premium)

adjustAppPackageJson.js

@@ -0,0 +1,9 @@
+const fs = require('fs');
+
+function adjustRootPackageJson(file) {
+  const json = JSON.parse(fs.readFileSync(file, { encoding: 'utf-8' }));
+  json.workspaces.push('app');
+  fs.writeFileSync(file, JSON.stringify(json, null, 2), 'utf-8');
+}
+
+adjustRootPackageJson('package.json');

adjustPackageJson.js

@@ -1,10 +1,34 @@
 const fs = require('fs');
+const path = require('path');
 
 function adjustFile(file) {
   const json = JSON.parse(fs.readFileSync(file, { encoding: 'utf-8' }));
+
+  for (const packageName of fs.readdirSync('plugins')) {
+    if (!packageName.startsWith('dbgate-plugin-')) continue;
+    const pluginJson = JSON.parse(
+      fs.readFileSync(path.join('plugins', packageName, 'package.json'), { encoding: 'utf-8' })
+    );
+    for (const depkey of ['dependencies', 'optionalDependencies']) {
+      for (const dependency of Object.keys(pluginJson[depkey] || {})) {
+        if (!json[depkey]) {
+          json[depkey] = {};
+        }
+        if (json[depkey][dependency]) {
+          if (json[depkey][dependency] != pluginJson[depkey][dependency]) {
+            console.log(`Dependency ${dependency} in ${packageName} is different from ${file}`);
+          }
+          continue;
+        }
+        json[depkey][dependency] = pluginJson[depkey][dependency];
+      }
+    }
+  }
+
   if (process.platform != 'win32') {
     delete json.optionalDependencies.msnodesqlv8;
   }
+
   fs.writeFileSync(file, JSON.stringify(json, null, 2), 'utf-8');
 }
 

app/package.json

@@ -130,10 +130,5 @@
     "electron": "30.0.2",
     "electron-builder": "23.1.0",
     "electron-builder-notarize": "^1.5.2"
-  },
-  "optionalDependencies": {
-    "better-sqlite3": "9.6.0",
-    "msnodesqlv8": "^4.2.1",
-    "oracledb": "^6.6.0"
   }
 }

app/src/electron.js

@@ -430,7 +430,6 @@ function createWindow() {
     );
 
     global.API_PACKAGE = apiPackage;
-    global.NATIVE_MODULES = path.join(__dirname, 'nativeModules');
 
     // console.log('global.API_PACKAGE', global.API_PACKAGE);
     const api = require(apiPackage);

app/src/mainMenuDefinition.js

@@ -9,9 +9,9 @@ module.exports = ({ editMenu }) => [
       { command: 'new.queryDesign', hideDisabled: true },
       { command: 'new.diagram', hideDisabled: true },
       { command: 'new.perspective', hideDisabled: true },
-      { command: 'new.freetable', hideDisabled: true },
       { command: 'new.shell', hideDisabled: true },
       { command: 'new.jsonl', hideDisabled: true },
+      { command: 'new.modelTransform', hideDisabled: true },
       { divider: true },
       { command: 'file.open', hideDisabled: true },
       { command: 'file.openArchive', hideDisabled: true },

app/src/nativeModules.js

@@ -1,3 +0,0 @@
-const content = require('./nativeModulesContent');
-
-module.exports = content;

fillNativeModules.js

@@ -1,24 +0,0 @@
-const fs = require('fs');
-
-let fillContent = '';
-
-if (process.platform == 'win32') {
-  fillContent += `content.msnodesqlv8 = () => require('msnodesqlv8');\n`;
-}
-fillContent += `content['better-sqlite3'] = () => require('better-sqlite3');\n`;
-fillContent += `content['oracledb'] = () => require('oracledb');\n`;
-
-const getContent = empty => `
-// this file is generated automatically by script fillNativeModules.js, do not edit it manually
-const content = {};
-
-${empty ? '' : fillContent}
-
-module.exports = content;
-`;
-
-fs.writeFileSync(
-  'packages/api/src/nativeModulesContent.js',
-  getContent(process.argv.includes('--electron') ? true : false)
-);
-fs.writeFileSync('app/src/nativeModulesContent.js', getContent(false));

integration-tests/__tests__/alter-database.spec.js

@@ -5,10 +5,12 @@ const { testWrapper } = require('../tools');
 const engines = require('../engines');
 const { getAlterDatabaseScript, extendDatabaseInfo, generateDbPairingId } = require('dbgate-tools');
 
-function flatSource() {
+const initSql = ['CREATE TABLE t1 (id int primary key)', 'CREATE TABLE t2 (id int primary key)'];
+
+function flatSource(engineCond = x => !x.skipReferences) {
   return _.flatten(
     engines
-      .filter(x => !x.skipReferences)
+      .filter(engineCond)
       .map(engine => (engine.objects || []).map(object => [engine.label, object.type, object, engine]))
   );
 }
@@ -66,5 +68,24 @@ describe('Alter database', () => {
       expect(db[type].length).toEqual(0);
     })
   );
-});
 
+  test.each(flatSource(x => x.supportRenameSqlObject))(
+    'Rename object - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
+
+      await driver.query(conn, object.create1, { discardResult: true });
+
+      const structure = extendDatabaseInfo(await driver.analyseFull(conn));
+
+      const dmp = driver.createDumper();
+      dmp.renameSqlObject(structure[type][0], 'renamed1');
+
+      await driver.query(conn, dmp.s);
+
+      const structure2 = await driver.analyseFull(conn);
+      expect(structure2[type].length).toEqual(1);
+      expect(structure2[type][0].pureName).toEqual('renamed1');
+    })
+  );
+});

integration-tests/__tests__/data-duplicator.spec.js

@@ -91,4 +91,68 @@ describe('Data duplicator', () => {
       expect(res2.rows[0].cnt.toString()).toEqual('6');
     })
   );
+
+  test.each(engines.filter(x => !x.skipDataDuplicator).map(engine => [engine.label, engine]))(
+    'Skip nullable weak refs - %s',
+    testWrapper(async (conn, driver, engine) => {
+      runCommandOnDriver(conn, driver, dmp =>
+        dmp.createTable({
+          pureName: 't1',
+          columns: [
+            { columnName: 'id', dataType: 'int', notNull: true },
+            { columnName: 'val', dataType: 'varchar(50)' },
+          ],
+          primaryKey: {
+            columns: [{ columnName: 'id' }],
+          },
+        })
+      );
+      runCommandOnDriver(conn, driver, dmp =>
+        dmp.createTable({
+          pureName: 't2',
+          columns: [
+            { columnName: 'id', dataType: 'int', autoIncrement: true, notNull: true },
+            { columnName: 'val', dataType: 'varchar(50)' },
+            { columnName: 'valfk', dataType: 'int', notNull: false },
+          ],
+          primaryKey: {
+            columns: [{ columnName: 'id' }],
+          },
+          foreignKeys: [{ refTableName: 't1', columns: [{ columnName: 'valfk', refColumnName: 'id' }] }],
+        })
+      );
+      runCommandOnDriver(conn, driver, dmp => dmp.put("insert into ~t1 (~id, ~val) values (1, 'first')"));
+
+      const gett2 = () =>
+        stream.Readable.from([
+          { __isStreamHeader: true, __isDynamicStructure: true },
+          { id: 1, val: 'v1', valfk: 1 },
+          { id: 2, val: 'v2', valfk: 2 },
+        ]);
+
+      await dataDuplicator({
+        systemConnection: conn,
+        driver,
+        items: [
+          {
+            name: 't2',
+            operation: 'copy',
+            openStream: gett2,
+          },
+        ],
+        options: {
+          setNullForUnresolvedNullableRefs: true,
+        },
+      });
+
+      const res1 = await driver.query(conn, `select count(*) as cnt from t1`);
+      expect(res1.rows[0].cnt.toString()).toEqual('1');
+
+      const res2 = await driver.query(conn, `select count(*) as cnt from t2`);
+      expect(res2.rows[0].cnt.toString()).toEqual('2');
+
+      const res3 = await driver.query(conn, `select count(*) as cnt from t2 where valfk is not null`);
+      expect(res3.rows[0].cnt.toString()).toEqual('1');
+    })
+  );
 });

integration-tests/__tests__/deploy-database.spec.js

@@ -8,9 +8,13 @@ const { databaseInfoFromYamlModel } = require('dbgate-tools');
 const generateDeploySql = require('dbgate-api/src/shell/generateDeploySql');
 const connectUtility = require('dbgate-api/src/utility/connectUtility');
 
-function checkStructure(structure, model, { checkRenameDeletedObjects = false, disallowExtraObjects = false }) {
+function checkStructure(
+  engine,
+  structure,
+  model,
+  { checkRenameDeletedObjects = false, disallowExtraObjects = false } = {}
+) {
   const expected = databaseInfoFromYamlModel(model);
-  expect(structure.tables.length).toEqual(expected.tables.length);
 
   for (const expectedTable of expected.tables) {
     const realTable = structure.tables.find(x => x.pureName == expectedTable.pureName);
@@ -18,7 +22,9 @@ function checkStructure(structure, model, { checkRenameDeletedObjects = false, d
     for (const column of expectedTable.columns) {
       const realColumn = realTable.columns.find(x => x.columnName == column.columnName);
       expect(realColumn).toBeTruthy();
-      expect(realColumn.notNull).toEqual(column.notNull);
+      if (!engine.skipNullability) {
+        expect(realColumn.notNull).toEqual(column.notNull);
+      }
     }
 
     for (const realColumn of realTable.columns) {
@@ -47,21 +53,46 @@ function checkStructure(structure, model, { checkRenameDeletedObjects = false, d
       }
     }
   }
+
+  for (const expectedView of expected.views) {
+    const realView = structure.views.find(x => x.pureName == expectedView.pureName);
+    expect(realView).toBeTruthy();
+  }
+
+  for (const realView of structure.views) {
+    const expectedView = expected.views.find(x => x.pureName == realView.pureName);
+    if (!expectedView) {
+      if (disallowExtraObjects) {
+        expect(realView).toBeFalsy();
+      }
+    }
+  }
 }
 
-async function testDatabaseDeploy(conn, driver, dbModelsYaml, options) {
-  const { testEmptyLastScript, checkDeletedObjects, finalCheckAgainstModel, finalCheckAgainstFirstModel } =
-    options || {};
+async function testDatabaseDeploy(engine, conn, driver, dbModelsYaml, options) {
+  const { testEmptyLastScript, finalCheckAgainstModel, markDeleted, allowDropStatements } = options || {};
   let index = 0;
+  const dbdiffOptionsExtra = markDeleted
+    ? {
+        deletedTablePrefix: '_deleted_',
+        deletedColumnPrefix: '_deleted_',
+        deletedSqlObjectPrefix: '_deleted_',
+      }
+    : {};
+  dbdiffOptionsExtra.schemaMode = 'ignore';
+
   for (const loadedDbModel of dbModelsYaml) {
     const { sql, isEmpty } = await generateDeploySql({
       systemConnection: conn.isPreparedOnly ? undefined : conn,
       connection: conn.isPreparedOnly ? conn : undefined,
       driver,
       loadedDbModel,
+      dbdiffOptionsExtra,
     });
     console.debug('Generated deploy script:', sql);
-    expect(sql.toUpperCase().includes('DROP ')).toBeFalsy();
+    if (!allowDropStatements) {
+      expect(sql.toUpperCase().includes('DROP ')).toBeFalsy();
+    }
 
     console.log('dbModelsYaml.length', dbModelsYaml.length, index);
     if (testEmptyLastScript && index == dbModelsYaml.length - 1) {
@@ -73,6 +104,7 @@ async function testDatabaseDeploy(conn, driver, dbModelsYaml, options) {
       connection: conn.isPreparedOnly ? conn : undefined,
       driver,
       loadedDbModel,
+      dbdiffOptionsExtra,
     });
 
     index++;
@@ -81,18 +113,14 @@ async function testDatabaseDeploy(conn, driver, dbModelsYaml, options) {
   const dbhan = conn.isPreparedOnly ? await connectUtility(driver, conn, 'read') : conn;
   const structure = await driver.analyseFull(dbhan);
   if (conn.isPreparedOnly) await driver.close(dbhan);
-  checkStructure(
-    structure,
-    finalCheckAgainstFirstModel ? dbModelsYaml[0] : finalCheckAgainstModel ?? dbModelsYaml[dbModelsYaml.length - 1],
-    options
-  );
+  checkStructure(engine, structure, finalCheckAgainstModel ?? dbModelsYaml[dbModelsYaml.length - 1], options);
 }
 
 describe('Deploy database', () => {
   test.each(engines.map(engine => [engine.label, engine]))(
     'Deploy database simple - %s',
     testWrapper(async (conn, driver, engine) => {
-      await testDatabaseDeploy(conn, driver, [
+      await testDatabaseDeploy(engine, conn, driver, [
         [
           {
             name: 't1.table.yaml',
@@ -110,7 +138,7 @@ describe('Deploy database', () => {
   test.each(engines.map(engine => [engine.label, engine]))(
     'Deploy database simple - %s - not connected',
     testWrapperPrepareOnly(async (conn, driver, engine) => {
-      await testDatabaseDeploy(conn, driver, [
+      await testDatabaseDeploy(engine, conn, driver, [
         [
           {
             name: 't1.table.yaml',
@@ -129,6 +157,7 @@ describe('Deploy database', () => {
     'Deploy database simple twice - %s',
     testWrapper(async (conn, driver, engine) => {
       await testDatabaseDeploy(
+        engine,
         conn,
         driver,
         [
@@ -161,7 +190,7 @@ describe('Deploy database', () => {
   test.each(engines.map(engine => [engine.label, engine]))(
     'Add column - %s',
     testWrapper(async (conn, driver, engine) => {
-      await testDatabaseDeploy(conn, driver, [
+      await testDatabaseDeploy(engine, conn, driver, [
         [
           {
             name: 't1.table.yaml',
@@ -193,6 +222,7 @@ describe('Deploy database', () => {
     'Dont drop column - %s',
     testWrapper(async (conn, driver, engine) => {
       await testDatabaseDeploy(
+        engine,
         conn,
         driver,
         [
@@ -229,6 +259,7 @@ describe('Deploy database', () => {
     'Foreign keys - %s',
     testWrapper(async (conn, driver, engine) => {
       await testDatabaseDeploy(
+        engine,
         conn,
         driver,
         [
@@ -283,7 +314,7 @@ describe('Deploy database', () => {
   test.each(engines.filter(x => !x.skipDataModifications).map(engine => [engine.label, engine]))(
     'Deploy preloaded data - %s',
     testWrapper(async (conn, driver, engine) => {
-      await testDatabaseDeploy(conn, driver, [
+      await testDatabaseDeploy(engine, conn, driver, [
         [
           {
             name: 't1.table.yaml',
@@ -312,7 +343,7 @@ describe('Deploy database', () => {
   test.each(engines.filter(x => !x.skipDataModifications).map(engine => [engine.label, engine]))(
     'Deploy preloaded data - update - %s',
     testWrapper(async (conn, driver, engine) => {
-      await testDatabaseDeploy(conn, driver, [
+      await testDatabaseDeploy(engine, conn, driver, [
         [
           {
             name: 't1.table.yaml',
@@ -359,7 +390,7 @@ describe('Deploy database', () => {
   test.each(engines.enginesPostgre.map(engine => [engine.label, engine]))(
     'Current timestamp default value - %s',
     testWrapper(async (conn, driver, engine) => {
-      await testDatabaseDeploy(conn, driver, [
+      await testDatabaseDeploy(engine, conn, driver, [
         [
           {
             name: 't1.table.yaml',
@@ -385,66 +416,274 @@ describe('Deploy database', () => {
     })
   );
 
+  const T1 = {
+    name: 't1.table.yaml',
+    json: {
+      name: 't1',
+      columns: [
+        { name: 'id', type: 'int' },
+        { name: 'val', type: 'int' },
+      ],
+      primaryKey: ['id'],
+    },
+  };
+
+  const T1_DELETED = {
+    name: '_deleted_t1.table.yaml',
+    json: {
+      name: '_deleted_t1',
+      columns: [
+        { name: 'id', type: 'int' },
+        { name: 'val', type: 'int' },
+      ],
+      primaryKey: ['id'],
+    },
+  };
+
+  const T1_NO_VAL = {
+    name: 't1.table.yaml',
+    json: {
+      name: 't1',
+      columns: [{ name: 'id', type: 'int' }],
+      primaryKey: ['id'],
+    },
+  };
+
+  const T1_DELETED_VAL = {
+    name: 't1.table.yaml',
+    json: {
+      name: 't1',
+      columns: [
+        { name: 'id', type: 'int' },
+        { name: '_deleted_val', type: 'int' },
+      ],
+      primaryKey: ['id'],
+    },
+  };
+
+  const V1 = {
+    name: 'v1.view.sql',
+    text: 'create view v1 as select * from t1',
+  };
+
+  const V1_VARIANT2 = {
+    name: 'v1.view.sql',
+    text: 'create view v1 as select 1 as c1',
+  };
+
+  const V1_DELETED = {
+    name: '_deleted_v1.view.sql',
+    text: 'create view _deleted_v1 as select * from t1',
+  };
+
   test.each(engines.map(engine => [engine.label, engine]))(
     'Dont remove column - %s',
     testWrapper(async (conn, driver, engine) => {
-      await testDatabaseDeploy(
-        conn,
-        driver,
-        [
-          [
-            {
-              name: 't1.table.yaml',
-              json: {
-                name: 't1',
-                columns: [
-                  { name: 'id', type: 'int' },
-                  { name: 'val', type: 'int' },
-                ],
-                primaryKey: ['id'],
-              },
-            },
-          ],
-          [
-            {
-              name: 't1.table.yaml',
-              json: {
-                name: 't1',
-                columns: [{ name: 'id', type: 'int' }],
-                primaryKey: ['id'],
-              },
-            },
-          ],
-        ],
-        { finalCheckAgainstFirstModel: true, disallowExtraObjects: true }
-      );
+      await testDatabaseDeploy(engine, conn, driver, [[T1], [T1_NO_VAL]], {
+        finalCheckAgainstModel: [T1],
+        disallowExtraObjects: true,
+      });
     })
   );
 
   test.each(engines.map(engine => [engine.label, engine]))(
     'Dont remove table - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(engine, conn, driver, [[T1], []], {
+        finalCheckAgainstModel: [T1],
+        disallowExtraObjects: true,
+      });
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Mark table removed - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(engine, conn, driver, [[T1], [], []], {
+        markDeleted: true,
+        disallowExtraObjects: true,
+        finalCheckAgainstModel: [T1_DELETED],
+      });
+    })
+  );
+
+  test.each(engines.filter(engine => engine.supportRenameSqlObject).map(engine => [engine.label, engine]))(
+    'Mark view removed - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(engine, conn, driver, [[T1, V1], [T1], [T1]], {
+        markDeleted: true,
+        disallowExtraObjects: true,
+        finalCheckAgainstModel: [T1, V1_DELETED],
+      });
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Mark column removed - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(engine, conn, driver, [[T1], [T1_NO_VAL]], {
+        markDeleted: true,
+        disallowExtraObjects: true,
+        finalCheckAgainstModel: [T1_DELETED_VAL],
+      });
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Undelete table - %s',
     testWrapper(async (conn, driver, engine) => {
       await testDatabaseDeploy(
+        engine,
         conn,
         driver,
         [
-          [
-            {
-              name: 't1.table.yaml',
-              json: {
-                name: 't1',
-                columns: [
-                  { name: 'id', type: 'int' },
-                  { name: 'val', type: 'int' },
-                ],
-                primaryKey: ['id'],
-              },
-            },
-          ],
+          [T1],
+          // delete table
           [],
+          // undelete table
+          [T1],
         ],
-        { finalCheckAgainstFirstModel: true, disallowExtraObjects: true }
+        {
+          markDeleted: true,
+          disallowExtraObjects: true,
+        }
       );
     })
   );
+
+  test.each(engines.filter(engine => engine.supportRenameSqlObject).map(engine => [engine.label, engine]))(
+    'Undelete view - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(engine, conn, driver, [[T1, V1], [T1], [T1, V1]], {
+        markDeleted: true,
+        disallowExtraObjects: true,
+        allowDropStatements: true,
+      });
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Undelete column - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(engine, conn, driver, [[T1], [T1_NO_VAL], [T1]], {
+        markDeleted: true,
+        disallowExtraObjects: true,
+      });
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'View redeploy - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(
+        engine,
+        conn,
+        driver,
+        [
+          [T1, V1],
+          [T1, V1],
+          [T1, V1],
+        ],
+        {
+          markDeleted: true,
+          disallowExtraObjects: true,
+          allowDropStatements: true,
+        }
+      );
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Change view - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(
+        engine,
+        conn,
+        driver,
+        [
+          [T1, V1],
+          [T1, V1_VARIANT2],
+        ],
+        {
+          markDeleted: true,
+          disallowExtraObjects: true,
+          allowDropStatements: true,
+        }
+      );
+    })
+  );
+
+  test.each(engines.filter(x => !x.skipDataModifications).map(engine => [engine.label, engine]))(
+    'Script drived deploy - basic predeploy - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(engine, conn, driver, [
+        [
+          {
+            name: '1.predeploy.sql',
+            text: 'create table t1 (id int primary key); insert into t1 (id) values (1);',
+          },
+        ],
+      ]);
+
+      const res1 = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM t1');
+      expect(res1.rows[0].cnt == 1).toBeTruthy();
+
+      const res2 = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM dbgate_deploy_journal');
+      expect(res2.rows[0].cnt == 1).toBeTruthy();
+    })
+  );
+
+  test.each(engines.filter(x => !x.skipDataModifications).map(engine => [engine.label, engine]))(
+    'Script drived deploy - install+uninstall - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(engine, conn, driver, [
+        [
+          {
+            name: 't1.uninstall.sql',
+            text: 'drop table t1',
+          },
+          {
+            name: 't1.install.sql',
+            text: 'create table t1 (id int primary key); insert into t1 (id) values (1)',
+          },
+          {
+            name: 't2.once.sql',
+            text: 'create table t2 (id int primary key); insert into t2 (id) values (1)',
+          },
+        ],
+        [
+          {
+            name: 't1.uninstall.sql',
+            text: 'drop table t1',
+          },
+          {
+            name: 't1.install.sql',
+            text: 'create table t1 (id int primary key, val int); insert into t1 (id, val) values (1, 11)',
+          },
+          {
+            name: 't2.once.sql',
+            text: 'insert into t2 (id) values (2)',
+          },
+        ],
+      ]);
+
+      const res1 = await driver.query(conn, 'SELECT val from t1 where id = 1');
+      expect(res1.rows[0].val == 11).toBeTruthy();
+
+      const res2 = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM t2');
+      expect(res2.rows[0].cnt == 1).toBeTruthy();
+
+      const res3 = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM dbgate_deploy_journal');
+      expect(res3.rows[0].cnt == 3).toBeTruthy();
+
+      const res4 = await driver.query(conn, "SELECT run_count from dbgate_deploy_journal where name = 't2.once.sql'");
+      expect(res4.rows[0].run_count == 1).toBeTruthy();
+
+      const res5 = await driver.query(
+        conn,
+        "SELECT run_count from dbgate_deploy_journal where name = 't1.install.sql'"
+      );
+      expect(res5.rows[0].run_count == 2).toBeTruthy();
+    })
+  );
 });

integration-tests/engines.js

@@ -96,6 +96,7 @@ const engines = [
       },
     ],
     supportSchemas: true,
+    supportRenameSqlObject: true,
     defaultSchemaName: 'public',
     dumpFile: 'data/chinook-postgre.sql',
     dumpChecks: [
@@ -129,6 +130,7 @@ const engines = [
       },
     ],
     supportSchemas: true,
+    supportRenameSqlObject: true,
     defaultSchemaName: 'dbo',
     // skipSeparateSchemas: true,
   },

package.json

@@ -1,6 +1,6 @@
 {
   "private": true,
-  "version": "5.5.7-beta.14",
+  "version": "5.5.7-beta.36",
   "name": "dbgate-all",
   "workspaces": [
     "packages/*",
@@ -33,7 +33,7 @@
     "build:filterparser": "yarn workspace dbgate-filterparser build",
     "build:tools": "yarn workspace dbgate-tools build",
     "build:lib": "yarn build:sqltree && yarn build:tools && yarn build:filterparser && yarn build:datalib",
-    "build:app": "yarn plugins:copydist && cd app && yarn install && yarn build",
+    "build:app": "yarn plugins:copydist && cd app && yarn build",
     "build:api": "yarn workspace dbgate-api build",
     "build:web": "yarn workspace dbgate-web build",
     "build:plugins:frontend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:frontend",
@@ -47,8 +47,7 @@
     "printSecrets": "node printSecrets",
     "generatePadFile": "node generatePadFile",
     "adjustPackageJson": "node adjustPackageJson",
-    "fillNativeModules": "node fillNativeModules",
-    "fillNativeModulesElectron": "node fillNativeModules --electron",
+    "adjustAppPackageJson": "node adjustAppPackageJson",
     "fillPackagedPlugins": "node fillPackagedPlugins",
     "resetPackagedPlugins": "node resetPackagedPlugins",
     "prettier": "prettier --write packages/api/src && prettier --write packages/datalib/src && prettier --write packages/filterparser/src && prettier --write packages/sqltree/src && prettier --write packages/tools/src && prettier --write packages/types && prettier --write packages/web/src && prettier --write app/src",
@@ -62,7 +61,7 @@
     "ts:api": "yarn workspace dbgate-api ts",
     "ts:web": "yarn workspace dbgate-web ts",
     "ts": "yarn ts:api && yarn ts:web",
-    "postinstall": "yarn resetPackagedPlugins && yarn build:lib && patch-package && yarn fillNativeModules && yarn build:plugins:frontend",
+    "postinstall": "yarn resetPackagedPlugins && yarn build:lib && patch-package && yarn build:plugins:frontend",
     "dbgate-serve": "node packages/dbgate/bin/dbgate-serve.js"
   },
   "dependencies": {

packages/api/package.json

@@ -19,7 +19,7 @@
   "dependencies": {
     "@aws-sdk/rds-signer": "^3.665.0",
     "activedirectory2": "^2.1.0",
-    "async-lock": "^1.2.4",
+    "async-lock": "^1.2.6",
     "axios": "^0.21.1",
     "body-parser": "^1.19.0",
     "bufferutil": "^4.0.1",
@@ -85,10 +85,5 @@
     "typescript": "^4.4.3",
     "webpack": "^5.91.0",
     "webpack-cli": "^5.1.4"
-  },
-  "optionalDependencies": {
-    "better-sqlite3": "9.6.0",
-    "msnodesqlv8": "^4.2.1",
-    "oracledb": "^6.6.0"
   }
 }

packages/api/src/controllers/config.js

@@ -69,7 +69,7 @@ module.exports = {
       !adminConfig?.adminPasswordState
     );
 
-    return {
+    const configResult = {
       runAsPortal: !!connections.portalConnections,
       singleDbConnection: connections.singleDbConnection,
       singleConnection: singleConnection,
@@ -95,13 +95,15 @@ module.exports = {
         !process.env.BASIC_AUTH
       ),
       isAdminPasswordMissing,
-      isInvalidToken: req.isInvalidToken,
+      isInvalidToken: req?.isInvalidToken,
       adminPasswordState: adminConfig?.adminPasswordState,
       storageDatabase: process.env.STORAGE_DATABASE,
       logsFilePath: getLogsFilePath(),
       connectionsFilePath: path.join(datadir(), 'connections.jsonl'),
       ...currentVersion,
     };
+
+    return configResult;
   },
 
   logout_meta: {

packages/api/src/controllers/connections.js

@@ -368,6 +368,11 @@ module.exports = {
 
   get_meta: true,
   async get({ conid }, req) {
+    if (conid == '__model') {
+      return {
+        _id: '__model',
+      };
+    }
     testConnectionPermission(conid, req);
     return this.getCore({ conid, mask: true });
   },

packages/api/src/controllers/databaseConnections.js

@@ -13,6 +13,7 @@ const {
   modelCompareDbDiffOptions,
   getLogger,
   extractErrorLogData,
+  filterStructureBySchema,
 } = require('dbgate-tools');
 const { html, parse } = require('diff2html');
 const { handleProcessCommunication } = require('../utility/processComm');
@@ -31,6 +32,8 @@ const { testConnectionPermission } = require('../utility/hasPermission');
 const { MissingCredentialsError } = require('../utility/exceptions');
 const pipeForkLogs = require('../utility/pipeForkLogs');
 const crypto = require('crypto');
+const loadModelTransform = require('../utility/loadModelTransform');
+const exportDbModelSql = require('../utility/exportDbModelSql');
 
 const logger = getLogger('databaseConnections');
 
@@ -349,6 +352,11 @@ module.exports = {
 
   syncModel_meta: true,
   async syncModel({ conid, database, isFullRefresh }, req) {
+    if (conid == '__model') {
+      socket.emitChanged('database-structure-changed', { conid, database });
+      return { status: 'ok' };
+    }
+
     testConnectionPermission(conid, req);
     const conn = await this.ensureOpened(conid, database);
     conn.subprocess.send({ msgtype: 'syncModel', isFullRefresh });
@@ -392,11 +400,12 @@ module.exports = {
   },
 
   structure_meta: true,
-  async structure({ conid, database }, req) {
+  async structure({ conid, database, modelTransFile = null }, req) {
     testConnectionPermission(conid, req);
     if (conid == '__model') {
       const model = await importDbModel(database);
-      return model;
+      const trans = await loadModelTransform(modelTransFile);
+      return trans ? trans(model) : model;
     }
 
     const opened = await this.ensureOpened(conid, database);
@@ -432,14 +441,35 @@ module.exports = {
   },
 
   exportModel_meta: true,
-  async exportModel({ conid, database }, req) {
+  async exportModel({ conid, database, outputFolder, schema }, req) {
     testConnectionPermission(conid, req);
-    const archiveFolder = await archive.getNewArchiveFolder({ database });
-    await fs.mkdir(path.join(archivedir(), archiveFolder));
+
+    const realFolder = outputFolder.startsWith('archive:')
+      ? resolveArchiveFolder(outputFolder.substring('archive:'.length))
+      : outputFolder;
+
     const model = await this.structure({ conid, database });
-    await exportDbModel(model, path.join(archivedir(), archiveFolder));
-    socket.emitChanged(`archive-folders-changed`);
-    return { archiveFolder };
+    const filteredModel = schema ? filterStructureBySchema(model, schema) : model;
+    await exportDbModel(extendDatabaseInfo(filteredModel), realFolder);
+
+    if (outputFolder.startsWith('archive:')) {
+      socket.emitChanged(`archive-files-changed`, { folder: outputFolder.substring('archive:'.length) });
+    }
+    return { status: 'ok' };
+  },
+
+  exportModelSql_meta: true,
+  async exportModelSql({ conid, database, outputFolder, outputFile, schema }, req) {
+    testConnectionPermission(conid, req);
+
+    const connection = await connections.getCore({ conid });
+    const driver = requireEngineDriver(connection);
+
+    const model = await this.structure({ conid, database });
+    const filteredModel = schema ? filterStructureBySchema(model, schema) : model;
+    await exportDbModelSql(extendDatabaseInfo(filteredModel), driver, outputFolder, outputFile);
+
+    return { status: 'ok' };
   },
 
   generateDeploySql_meta: true,

packages/api/src/controllers/runners.js

@@ -12,6 +12,7 @@ const {
   jsonScriptToJavascript,
   getLogger,
   safeJsonParse,
+  pinoLogRecordToMessageRecord,
 } = require('dbgate-tools');
 const { handleProcessCommunication } = require('../utility/processComm');
 const processArgs = require('../utility/processArgs');
@@ -68,18 +69,20 @@ module.exports = {
 
   dispatchMessage(runid, message) {
     if (message) {
-      const json = safeJsonParse(message.message);
+      if (_.isPlainObject(message)) logger.log(message);
+      else logger.info(message);
 
-      if (json) logger.log(json);
-      else logger.info(message.message);
+      const toEmit = _.isPlainObject(message)
+        ? {
+            time: new Date(),
+            ...message,
+          }
+        : {
+            message,
+            time: new Date(),
+          };
 
-      const toEmit = {
-        time: new Date(),
-        ...message,
-        message: json ? json.msg : message.message,
-      };
-
-      if (json && json.level >= 50) {
+      if (toEmit.level >= 50) {
         toEmit.severity = 'error';
       }
 
@@ -131,7 +134,16 @@ module.exports = {
       }
     );
     const pipeDispatcher = severity => data => {
-      return this.dispatchMessage(runid, { severity, message: data.toString().trim() });
+      const json = safeJsonParse(data, null);
+
+      if (json) {
+        return this.dispatchMessage(runid, pinoLogRecordToMessageRecord(json));
+      } else {
+        return this.dispatchMessage(runid, {
+          message: json == null ? data.toString().trim() : null,
+          severity,
+        });
+      }
     };
 
     byline(subprocess.stdout).on('data', pipeDispatcher('info'));
@@ -165,7 +177,7 @@ module.exports = {
 
   start_meta: true,
   async start({ script }) {
-    const runid = crypto.randomUUID()
+    const runid = crypto.randomUUID();
 
     if (script.type == 'json') {
       const js = jsonScriptToJavascript(script);

packages/api/src/controllers/serverConnections.js

@@ -134,6 +134,7 @@ module.exports = {
   listDatabases_meta: true,
   async listDatabases({ conid }, req) {
     if (!conid) return [];
+    if (conid == '__model') return [];
     testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid);
     return opened.databases;
@@ -172,7 +173,7 @@ module.exports = {
         }
       })
     );
-    socket.setStreamIdFilter(strmid, { conid: conidArray });
+    socket.setStreamIdFilter(strmid, { conid: [...(conidArray ?? []), '__model'] });
     return { status: 'ok' };
   },
 

packages/api/src/nativeModules.js

@@ -1,13 +0,0 @@
-const argIndex = process.argv.indexOf('--native-modules');
-const redirectFile = global['NATIVE_MODULES'] || (argIndex > 0 ? process.argv[argIndex + 1] : null);
-
-function requireDynamic(file) {
-  try {
-    // @ts-ignore
-    return __non_webpack_require__(redirectFile);
-  } catch (err) {
-    return require(redirectFile);
-  }
-}
-
-module.exports = redirectFile ? requireDynamic(redirectFile) : require('./nativeModulesContent');

packages/api/src/shell/autoIndexForeignKeysTransform.js

@@ -8,7 +8,7 @@ const autoIndexForeignKeysTransform = () => database => {
           ...(table.indexes || []),
           ...table.foreignKeys.map(fk => ({
             constraintName: `IX_${fk.constraintName}`,
-            columns: fk.columns,
+            columns: fk.columns.map(x => ({ columnName: x.columnName })),
           })),
         ],
       };

packages/api/src/shell/dataDuplicator.js

@@ -12,6 +12,7 @@ const { resolveArchiveFolder } = require('../utility/directories');
 async function dataDuplicator({
   connection,
   archive,
+  folder,
   items,
   options,
   analysedStructure = null,
@@ -19,7 +20,7 @@ async function dataDuplicator({
   systemConnection,
 }) {
   if (!driver) driver = requireEngineDriver(connection);
-  
+
   const dbhan = systemConnection || (await connectUtility(driver, connection, 'write'));
 
   try {
@@ -29,6 +30,12 @@ async function dataDuplicator({
       analysedStructure = await driver.analyseFull(dbhan);
     }
 
+    const sourceDir = archive
+      ? resolveArchiveFolder(archive)
+      : folder?.startsWith('archive:')
+      ? resolveArchiveFolder(folder.substring('archive:'.length))
+      : folder;
+
     const dupl = new DataDuplicator(
       dbhan,
       driver,
@@ -38,8 +45,7 @@ async function dataDuplicator({
         operation: item.operation,
         matchColumns: item.matchColumns,
         openStream:
-          item.openStream ||
-          (() => jsonLinesReader({ fileName: path.join(resolveArchiveFolder(archive), `${item.name}.jsonl`) })),
+          item.openStream || (() => jsonLinesReader({ fileName: path.join(sourceDir, `${item.name}.jsonl`) })),
       })),
       stream,
       copyStream,

packages/api/src/shell/deployDb.js

@@ -1,5 +1,10 @@
 const generateDeploySql = require('./generateDeploySql');
 const executeQuery = require('./executeQuery');
+const { ScriptDrivedDeployer } = require('dbgate-datalib');
+const connectUtility = require('../utility/connectUtility');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const loadModelFolder = require('../utility/loadModelFolder');
+const crypto = require('crypto');
 
 async function deployDb({
   connection,
@@ -9,18 +14,43 @@ async function deployDb({
   modelFolder,
   loadedDbModel,
   modelTransforms,
+  dbdiffOptionsExtra,
+  ignoreNameRegex = '',
+  targetSchema = null,
 }) {
-  const { sql } = await generateDeploySql({
-    connection,
-    systemConnection,
-    driver,
-    analysedStructure,
-    modelFolder,
-    loadedDbModel,
-    modelTransforms,
-  });
-  // console.log('RUNNING DEPLOY SCRIPT:', sql);
-  await executeQuery({ connection, systemConnection, driver, sql, logScriptItems: true });
+  if (!driver) driver = requireEngineDriver(connection);
+  const dbhan = systemConnection || (await connectUtility(driver, connection, 'read'));
+
+  try {
+    const scriptDeployer = new ScriptDrivedDeployer(
+      dbhan,
+      driver,
+      Array.isArray(loadedDbModel) ? loadedDbModel : modelFolder ? await loadModelFolder(modelFolder) : [],
+      crypto
+    );
+    await scriptDeployer.runPre();
+
+    const { sql } = await generateDeploySql({
+      connection,
+      systemConnection: dbhan,
+      driver,
+      analysedStructure,
+      modelFolder,
+      loadedDbModel,
+      modelTransforms,
+      dbdiffOptionsExtra,
+      ignoreNameRegex,
+      targetSchema,
+    });
+    // console.log('RUNNING DEPLOY SCRIPT:', sql);
+    await executeQuery({ connection, systemConnection: dbhan, driver, sql, logScriptItems: true });
+
+    await scriptDeployer.runPost();
+  } finally {
+    if (!systemConnection) {
+      await driver.close(dbhan);
+    }
+  }
 }
 
 module.exports = deployDb;

packages/api/src/shell/dropAllDbObjects.js

@@ -0,0 +1,42 @@
+const executeQuery = require('./executeQuery');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const { getLogger, extendDatabaseInfo } = require('dbgate-tools');
+
+const logger = getLogger('dropAllDbObjects');
+
+async function dropAllDbObjects({ connection, systemConnection, driver, analysedStructure }) {
+  if (!driver) driver = requireEngineDriver(connection);
+
+  const dbhan = systemConnection || (await connectUtility(driver, connection, 'write'));
+
+  logger.info(`Connected.`);
+
+  if (!analysedStructure) {
+    analysedStructure = await driver.analyseFull(dbhan);
+  }
+
+  analysedStructure = extendDatabaseInfo(analysedStructure);
+
+  const dmp = driver.createDumper();
+
+  for (const table of analysedStructure.tables) {
+    for (const fk of table.foreignKeys) {
+      dmp.dropForeignKey(fk);
+    }
+  }
+  for (const table of analysedStructure.tables) {
+    dmp.dropTable(table);
+  }
+  for (const field of Object.keys(analysedStructure)) {
+    if (dmp.getSqlObjectSqlName(field)) {
+      for (const obj of analysedStructure[field]) {
+        dmp.dropSqlObject(obj);
+      }
+    }
+  }
+
+  await executeQuery({ connection, systemConnection, driver, sql: dmp.s, logScriptItems: true });
+}
+
+module.exports = dropAllDbObjects;

packages/api/src/shell/executeQuery.js

@@ -1,3 +1,4 @@
+const fs = require('fs-extra');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { getLogger, getLimitedQuery } = require('dbgate-tools');
@@ -9,6 +10,7 @@ async function executeQuery({
   systemConnection = undefined,
   driver = undefined,
   sql,
+  sqlFile = undefined,
   logScriptItems = false,
 }) {
   if (!logScriptItems) {
@@ -18,6 +20,11 @@ async function executeQuery({
   if (!driver) driver = requireEngineDriver(connection);
   const dbhan = systemConnection || (await connectUtility(driver, connection, 'script'));
 
+  if (sqlFile) {
+    logger.debug(`Loading SQL file ${sqlFile}`);
+    sql = await fs.readFile(sqlFile, { encoding: 'utf-8' });
+  }
+
   try {
     logger.info(`Connected.`);
 

packages/api/src/shell/generateDeploySql.js

@@ -6,6 +6,10 @@ const {
   extendDatabaseInfo,
   modelCompareDbDiffOptions,
   enrichWithPreloadedRows,
+  skipNamesInStructureByRegex,
+  replaceSchemaInStructure,
+  filterStructureBySchema,
+  skipDbGateInternalObjects,
 } = require('dbgate-tools');
 const importDbModel = require('../utility/importDbModel');
 const requireEngineDriver = require('../utility/requireEngineDriver');
@@ -19,6 +23,9 @@ async function generateDeploySql({
   modelFolder = undefined,
   loadedDbModel = undefined,
   modelTransforms = undefined,
+  dbdiffOptionsExtra = {},
+  ignoreNameRegex = '',
+  targetSchema = null,
 }) {
   if (!driver) driver = requireEngineDriver(connection);
 
@@ -29,6 +36,11 @@ async function generateDeploySql({
       analysedStructure = await driver.analyseFull(dbhan);
     }
 
+    if (ignoreNameRegex) {
+      analysedStructure = skipNamesInStructureByRegex(analysedStructure, new RegExp(ignoreNameRegex, 'i'));
+    }
+    analysedStructure = skipDbGateInternalObjects(analysedStructure);
+
     let deployedModelSource = loadedDbModel
       ? databaseInfoFromYamlModel(loadedDbModel)
       : await importDbModel(modelFolder);
@@ -37,6 +49,11 @@ async function generateDeploySql({
       deployedModelSource = transform(deployedModelSource);
     }
 
+    if (targetSchema) {
+      deployedModelSource = replaceSchemaInStructure(deployedModelSource, targetSchema);
+      analysedStructure = filterStructureBySchema(analysedStructure, targetSchema);
+    }
+
     const deployedModel = generateDbPairingId(extendDatabaseInfo(deployedModelSource));
     const currentModel = generateDbPairingId(extendDatabaseInfo(analysedStructure));
     const opts = {
@@ -48,6 +65,8 @@ async function generateDeploySql({
       noDropSqlObject: true,
       noRenameTable: true,
       noRenameColumn: true,
+
+      ...dbdiffOptionsExtra,
     };
     const currentModelPaired = matchPairedObjects(deployedModel, currentModel, opts);
     const currentModelPairedPreloaded = await enrichWithPreloadedRows(deployedModel, currentModelPaired, dbhan, driver);

packages/api/src/shell/index.js

@@ -33,6 +33,8 @@ const jsonReader = require('./jsonReader');
 const dataTypeMapperTransform = require('./dataTypeMapperTransform');
 const sqlTextReplacementTransform = require('./sqlTextReplacementTransform');
 const autoIndexForeignKeysTransform = require('./autoIndexForeignKeysTransform');
+const generateDeploySql = require('./generateDeploySql');
+const dropAllDbObjects = require('./dropAllDbObjects');
 
 const dbgateApi = {
   queryReader,
@@ -69,6 +71,8 @@ const dbgateApi = {
   dataTypeMapperTransform,
   sqlTextReplacementTransform,
   autoIndexForeignKeysTransform,
+  generateDeploySql,
+  dropAllDbObjects,
 };
 
 requirePlugin.initializeDbgateApi(dbgateApi);

packages/api/src/shell/requirePlugin.js

@@ -1,7 +1,6 @@
 const path = require('path');
 const fs = require('fs');
 const { pluginsdir, packagedPluginsDir, getPluginBackendPath } = require('../utility/directories');
-const nativeModules = require('../nativeModules');
 const platformInfo = require('../utility/platformInfo');
 const authProxy = require('../utility/authProxy');
 const { getLogger } = require('dbgate-tools');
@@ -11,7 +10,6 @@ const loadedPlugins = {};
 
 const dbgateEnv = {
   dbgateApi: null,
-  nativeModules,
   platformInfo,
   authProxy,
 };

packages/api/src/shell/sqlTextReplacementTransform.js

@@ -7,6 +7,7 @@ function replaceInText(text, replacements) {
 }
 
 function replaceInCollection(collection, replacements) {
+  if (!collection) return collection;
   return collection.map(item => {
     if (item.createSql) {
       return {
@@ -22,6 +23,9 @@ const sqlTextReplacementTransform = replacements => database => {
   return {
     ...database,
     views: replaceInCollection(database.views, replacements),
+    matviews: replaceInCollection(database.matviews, replacements),
+    procedures: replaceInCollection(database.procedures, replacements),
+    functions: replaceInCollection(database.functions, replacements),
   };
 };
 

packages/api/src/utility/exportDbModelSql.js

@@ -0,0 +1,80 @@
+const fs = require('fs-extra');
+const path = require('path');
+const { getSchemasUsedByStructure } = require('dbgate-tools');
+
+async function exportDbModelSql(dbModel, driver, outputDir, outputFile) {
+  const { tables, views, procedures, functions, triggers, matviews } = dbModel;
+
+  const usedSchemas = getSchemasUsedByStructure(dbModel);
+  const useSchemaDir = usedSchemas.length > 1;
+
+  const createdDirs = new Set();
+  async function ensureDir(dir) {
+    if (!createdDirs.has(dir)) {
+      await fs.mkdir(dir, { recursive: true });
+      createdDirs.add(dir);
+    }
+  }
+
+  async function writeLists(writeList) {
+    await writeList(views, 'views');
+    await writeList(procedures, 'procedures');
+    await writeList(functions, 'functions');
+    await writeList(triggers, 'triggers');
+    await writeList(matviews, 'matviews');
+  }
+
+  if (outputFile) {
+    const dmp = driver.createDumper();
+    for (const table of tables || []) {
+      dmp.createTable({
+        ...table,
+        foreignKeys: [],
+        dependencies: [],
+      });
+    }
+    for (const table of tables || []) {
+      for (const fk of table.foreignKeys || []) {
+        dmp.createForeignKey(fk);
+      }
+    }
+    writeLists((list, folder) => {
+      for (const obj of list || []) {
+        dmp.createSqlObject(obj);
+      }
+    });
+
+    const script = dmp.s;
+    await fs.writeFile(outputFile, script);
+  }
+
+  if (outputDir) {
+    for (const table of tables || []) {
+      const tablesDir = useSchemaDir
+        ? path.join(outputDir, table.schemaName ?? 'default', 'tables')
+        : path.join(outputDir, 'tables');
+      await ensureDir(tablesDir);
+      const dmp = driver.createDumper();
+      dmp.createTable({
+        ...table,
+        foreignKeys: [],
+        dependencies: [],
+      });
+      await fs.writeFile(path.join(tablesDir, `${table.pureName}.sql`), dmp.s);
+    }
+
+    await writeLists(async (list, folder) => {
+      for (const obj of list || []) {
+        const objdir = useSchemaDir
+          ? path.join(outputDir, obj.schemaName ?? 'default', folder)
+          : path.join(outputDir, folder);
+        await ensureDir(objdir);
+        const dmp = driver.createDumper();
+        dmp.createSqlObject(obj);
+        await fs.writeFile(path.join(objdir, `${obj.pureName}.sql`), dmp.s);
+      }
+    });
+  }
+}
+
+module.exports = exportDbModelSql;

packages/api/src/utility/importDbModel.js

@@ -1,28 +1,8 @@
-const fs = require('fs-extra');
-const path = require('path');
-const yaml = require('js-yaml');
 const { databaseInfoFromYamlModel, DatabaseAnalyser } = require('dbgate-tools');
-const { startsWith } = require('lodash');
-const { archivedir, resolveArchiveFolder } = require('./directories');
-const loadFilesRecursive = require('./loadFilesRecursive');
+const loadModelFolder = require('./loadModelFolder');
 
 async function importDbModel(inputDir) {
-  const files = [];
-
-  const dir = inputDir.startsWith('archive:') ? resolveArchiveFolder(inputDir.substring('archive:'.length)) : inputDir;
-
-  for (const name of await loadFilesRecursive(dir)) {
-    if (name.endsWith('.table.yaml') || name.endsWith('.sql')) {
-      const text = await fs.readFile(path.join(dir, name), { encoding: 'utf-8' });
-
-      files.push({
-        name: path.parse(name).base,
-        text,
-        json: name.endsWith('.yaml') ? yaml.load(text) : null,
-      });
-    }
-  }
-
+  const files = await loadModelFolder(inputDir);
   return databaseInfoFromYamlModel(files);
 }
 

packages/api/src/utility/loadModelFolder.js

@@ -0,0 +1,29 @@
+const fs = require('fs-extra');
+const path = require('path');
+const yaml = require('js-yaml');
+const { resolveArchiveFolder } = require('./directories');
+const loadFilesRecursive = require('./loadFilesRecursive');
+
+async function loadModelFolder(inputDir) {
+  const files = [];
+
+  const dir = inputDir.startsWith('archive:')
+    ? resolveArchiveFolder(inputDir.substring('archive:'.length))
+    : path.resolve(inputDir);
+
+  for (const name of await loadFilesRecursive(dir)) {
+    if (name.endsWith('.table.yaml') || name.endsWith('.sql')) {
+      const text = await fs.readFile(path.join(dir, name), { encoding: 'utf-8' });
+
+      files.push({
+        name: path.parse(name).base,
+        text,
+        json: name.endsWith('.yaml') ? yaml.load(text) : null,
+      });
+    }
+  }
+
+  return files;
+}
+
+module.exports = loadModelFolder;

packages/api/src/utility/loadModelTransform.js

@@ -0,0 +1,36 @@
+const { filesdir } = require('./directories');
+const path = require('path');
+const fs = require('fs-extra');
+const _ = require('lodash');
+const dbgateApi = require('../shell');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
+const logger = getLogger('loadModelTransform');
+
+function modelTransformFromJson(json) {
+  if (!dbgateApi[json.transform]) return null;
+  const creator = dbgateApi[json.transform];
+  return creator(...json.arguments);
+}
+
+async function loadModelTransform(file) {
+  if (!file) return null;
+  try {
+    const dir = filesdir();
+    const fullPath = path.join(dir, 'modtrans', file);
+    const text = await fs.readFile(fullPath, { encoding: 'utf-8' });
+    const json = JSON.parse(text);
+    if (_.isArray(json)) {
+      const array = _.compact(json.map(x => modelTransformFromJson(x)));
+      return array.length ? structure => array.reduce((acc, val) => val(acc), structure) : null;
+    }
+    if (_.isPlainObject(json)) {
+      return modelTransformFromJson(json);
+    }
+    return null;
+  } catch (err) {
+    logger.error(extractErrorLogData(err), `Error loading model transform ${file}`);
+    return null;
+  }
+}
+
+module.exports = loadModelTransform;

packages/api/src/utility/platformInfo.js

@@ -43,9 +43,17 @@ const platformInfo = {
   platform,
   runningInWebpack: !!process.env.WEBPACK_DEV_SERVER_URL,
   allowShellConnection:
-    (!processArgs.listenApiChild && !isNpmDist) || !!process.env.SHELL_CONNECTION || !!isElectron() || !!isDbModel,
+    (!processArgs.listenApiChild && !isNpmDist) ||
+    !!process.env.SHELL_CONNECTION ||
+    !!isElectron() ||
+    !!isDbModel ||
+    isDevMode,
   allowShellScripting:
-    (!processArgs.listenApiChild && !isNpmDist) || !!process.env.SHELL_SCRIPTING || !!isElectron() || !!isDbModel,
+    (!processArgs.listenApiChild && !isNpmDist) ||
+    !!process.env.SHELL_SCRIPTING ||
+    !!isElectron() ||
+    !!isDbModel ||
+    isDevMode,
   allowConnectionFromEnvVariables: !!isDbModel,
   defaultKeyfile: path.join(os.homedir(), '.ssh/id_rsa'),
   isAwsUbuntuLayout,

packages/api/src/utility/processArgs.js

@@ -17,9 +17,6 @@ const listenApiChild = process.argv.includes('--listen-api-child') || listenApi;
 
 function getPassArgs() {
   const res = [];
-  if (global['NATIVE_MODULES']) {
-    res.push('--native-modules', global['NATIVE_MODULES']);
-  }
   if (global['PLUGINS_DIR']) {
     res.push('--plugins-dir', global['PLUGINS_DIR']);
   }

packages/datalib/src/DataDuplicator.ts

@@ -21,6 +21,7 @@ export interface DataDuplicatorItem {
 export interface DataDuplicatorOptions {
   rollbackAfterFinish?: boolean;
   skipRowsWithUnresolvedRefs?: boolean;
+  setNullForUnresolvedNullableRefs?: boolean;
 }
 
 class DuplicatorReference {
@@ -36,9 +37,19 @@ class DuplicatorReference {
   }
 }
 
+class DuplicatorWeakReference {
+  constructor(public base: DuplicatorItemHolder, public ref: TableInfo, public foreignKey: ForeignKeyInfo) {}
+
+  get columnName() {
+    return this.foreignKey.columns[0].columnName;
+  }
+}
+
 class DuplicatorItemHolder {
   references: DuplicatorReference[] = [];
   backReferences: DuplicatorReference[] = [];
+  // not mandatory references to entities out of the model
+  weakReferences: DuplicatorWeakReference[] = [];
   table: TableInfo;
   isPlanned = false;
   idMap = {};
@@ -65,23 +76,33 @@ class DuplicatorItemHolder {
     for (const fk of this.table.foreignKeys) {
       if (fk.columns?.length != 1) continue;
       const refHolder = this.duplicator.itemHolders.find(y => y.name.toUpperCase() == fk.refTableName.toUpperCase());
-      if (refHolder == null) continue;
       const isMandatory = this.table.columns.find(x => x.columnName == fk.columns[0]?.columnName)?.notNull;
-      const newref = new DuplicatorReference(this, refHolder, isMandatory, fk);
-      this.references.push(newref);
-      this.refByColumn[newref.columnName] = newref;
+      if (refHolder == null) {
+        if (!isMandatory) {
+          const weakref = new DuplicatorWeakReference(
+            this,
+            this.duplicator.db.tables.find(x => x.pureName == fk.refTableName),
+            fk
+          );
+          this.weakReferences.push(weakref);
+        }
+      } else {
+        const newref = new DuplicatorReference(this, refHolder, isMandatory, fk);
+        this.references.push(newref);
+        this.refByColumn[newref.columnName] = newref;
 
-      refHolder.isReferenced = true;
+        refHolder.isReferenced = true;
+      }
     }
   }
 
-  createInsertObject(chunk) {
+  createInsertObject(chunk, weakrefcols: string[]) {
     const res = _omit(
       _pick(
         chunk,
         this.table.columns.map(x => x.columnName)
       ),
-      [this.autoColumn, ...this.backReferences.map(x => x.columnName)]
+      [this.autoColumn, ...this.backReferences.map(x => x.columnName), ...weakrefcols]
     );
 
     for (const key in res) {
@@ -102,6 +123,28 @@ class DuplicatorItemHolder {
     return res;
   }
 
+  // returns list of columns that are weak references and are not resolved
+  async getMissingWeakRefsForRow(row): Promise<string[]> {
+    if (!this.duplicator.options.setNullForUnresolvedNullableRefs || !this.weakReferences?.length) {
+      return [];
+    }
+
+    const qres = await runQueryOnDriver(this.duplicator.pool, this.duplicator.driver, dmp => {
+      dmp.put('^select ');
+      dmp.putCollection(',', this.weakReferences, weakref => {
+        dmp.put(
+          '(^case ^when ^exists (^select * ^from %f where %i = %v) ^then 1 ^else 0 ^end) as %i',
+          weakref.ref,
+          weakref.foreignKey.columns[0].refColumnName,
+          row[weakref.foreignKey.columns[0].columnName],
+          weakref.foreignKey.columns[0].columnName
+        );
+      });
+    });
+    const qrow = qres.rows[0];
+    return this.weakReferences.filter(x => qrow[x.columnName] == 0).map(x => x.columnName);
+  }
+
   async runImport() {
     const readStream = await this.item.openStream();
     const driver = this.duplicator.driver;
@@ -112,6 +155,8 @@ class DuplicatorItemHolder {
     let skipped = 0;
     let lastLogged = new Date();
 
+    const existingWeakRefs = {};
+
     const writeStream = createAsyncWriteStream(this.duplicator.stream, {
       processItem: async chunk => {
         if (chunk.__isStreamHeader) {
@@ -120,7 +165,8 @@ class DuplicatorItemHolder {
 
         const doCopy = async () => {
           // console.log('chunk', this.name, JSON.stringify(chunk));
-          const insertedObj = this.createInsertObject(chunk);
+          const weakrefcols = await this.getMissingWeakRefsForRow(chunk);
+          const insertedObj = this.createInsertObject(chunk, weakrefcols);
           // console.log('insertedObj', this.name, JSON.stringify(insertedObj));
           if (insertedObj == null) {
             skipped += 1;

packages/datalib/src/ScriptDrivedDeployer.ts

@@ -0,0 +1,175 @@
+import { DatabaseModelFile, extractErrorLogData, getLogger, runCommandOnDriver, runQueryOnDriver } from 'dbgate-tools';
+import { EngineDriver } from 'dbgate-types';
+import _sortBy from 'lodash/sortBy';
+
+const logger = getLogger('ScriptDrivedDeployer');
+
+interface DeployScriptJournalItem {
+  id: number;
+  name: string;
+  category: string;
+  first_run_date: string;
+  last_run_date: string;
+  script_hash: string;
+}
+
+export class ScriptDrivedDeployer {
+  predeploy: DatabaseModelFile[] = [];
+  uninstall: DatabaseModelFile[] = [];
+  install: DatabaseModelFile[] = [];
+  once: DatabaseModelFile[] = [];
+  postdeploy: DatabaseModelFile[] = [];
+  isEmpty = false;
+
+  journalItems: DeployScriptJournalItem[] = [];
+
+  constructor(public dbhan: any, public driver: EngineDriver, public files: DatabaseModelFile[], public crypto: any) {
+    this.predeploy = files.filter(x => x.name.endsWith('.predeploy.sql'));
+    this.uninstall = files.filter(x => x.name.endsWith('.uninstall.sql'));
+    this.install = files.filter(x => x.name.endsWith('.install.sql'));
+    this.once = files.filter(x => x.name.endsWith('.once.sql'));
+    this.postdeploy = files.filter(x => x.name.endsWith('.postdeploy.sql'));
+    this.isEmpty =
+      this.predeploy.length === 0 &&
+      this.uninstall.length === 0 &&
+      this.install.length === 0 &&
+      this.once.length === 0 &&
+      this.postdeploy.length === 0;
+  }
+
+  async loadJournalItems() {
+    try {
+      const { rows } = await runQueryOnDriver(this.dbhan, this.driver, dmp =>
+        dmp.put('select * from ~dbgate_deploy_journal')
+      );
+      this.journalItems = rows;
+      logger.debug(`Loaded ${rows.length} items from DbGate deploy journal`);
+    } catch (err) {
+      logger.warn(
+        extractErrorLogData(err),
+        'Error loading DbGate deploy journal, creating table dbgate_deploy_journal'
+      );
+      const dmp = this.driver.createDumper();
+      dmp.createTable({
+        pureName: 'dbgate_deploy_journal',
+        columns: [
+          { columnName: 'id', dataType: 'int', autoIncrement: true, notNull: true, pureName: 'dbgate_deploy_journal' },
+          { columnName: 'name', dataType: 'varchar(100)', notNull: true, pureName: 'dbgate_deploy_journal' },
+          { columnName: 'category', dataType: 'varchar(100)', notNull: true, pureName: 'dbgate_deploy_journal' },
+          { columnName: 'script_hash', dataType: 'varchar(100)', notNull: true, pureName: 'dbgate_deploy_journal' },
+          { columnName: 'first_run_date', dataType: 'varchar(100)', notNull: true, pureName: 'dbgate_deploy_journal' },
+          { columnName: 'last_run_date', dataType: 'varchar(100)', notNull: true, pureName: 'dbgate_deploy_journal' },
+          { columnName: 'run_count', dataType: 'int', notNull: true, pureName: 'dbgate_deploy_journal' },
+        ],
+        foreignKeys: [],
+        primaryKey: {
+          columns: [{ columnName: 'id' }],
+          constraintType: 'primaryKey',
+          pureName: 'dbgate_deploy_journal',
+        },
+      });
+      await this.driver.query(this.dbhan, dmp.s, { discardResult: true });
+    }
+  }
+
+  async runPre() {
+    // don't create journal table if no scripts are present
+    if (this.isEmpty) return;
+    await this.loadJournalItems();
+    await this.runFiles(this.predeploy, 'predeploy');
+  }
+
+  async runPost() {
+    await this.runFiles(this.install, 'install');
+    await this.runFiles(this.once, 'once');
+    await this.runFiles(this.postdeploy, 'postdeploy');
+  }
+
+  async run() {
+    await this.runPre();
+    await this.runPost();
+  }
+
+  async runFiles(files: DatabaseModelFile[], category: string) {
+    for (const file of _sortBy(files, x => x.name)) {
+      await this.runFile(file, category);
+    }
+  }
+
+  async saveToJournal(file: DatabaseModelFile, category: string, hash: string) {
+    const existing = this.journalItems.find(x => x.name == file.name);
+    if (existing) {
+      await runCommandOnDriver(this.dbhan, this.driver, dmp => {
+        dmp.put(
+          'update ~dbgate_deploy_journal set ~last_run_date = %v, ~script_hash = %v, ~run_count = ~run_count + 1 where ~id = %v',
+          new Date().toISOString(),
+          hash,
+          existing.id
+        );
+      });
+    } else {
+      await runCommandOnDriver(this.dbhan, this.driver, dmp => {
+        dmp.put(
+          'insert into ~dbgate_deploy_journal (~name, ~category, ~first_run_date, ~last_run_date, ~script_hash, ~run_count) values (%v, %v, %v, %v, %v, 1)',
+          file.name,
+          category,
+          new Date().toISOString(),
+          new Date().toISOString(),
+          hash
+        );
+      });
+    }
+  }
+
+  async runFileCore(file: DatabaseModelFile, category: string, hash: string) {
+    if (this.driver.supportsTransactions) {
+      runCommandOnDriver(this.dbhan, this.driver, dmp => dmp.beginTransaction());
+    }
+
+    logger.debug(`Running ${category} script ${file.name}`);
+    try {
+      await this.driver.script(this.dbhan, file.text);
+      await this.saveToJournal(file, category, hash);
+    } catch (err) {
+      logger.error(extractErrorLogData(err), `Error running ${category} script ${file.name}`);
+      if (this.driver.supportsTransactions) {
+        runCommandOnDriver(this.dbhan, this.driver, dmp => dmp.rollbackTransaction());
+        return;
+      }
+    }
+
+    if (this.driver.supportsTransactions) {
+      runCommandOnDriver(this.dbhan, this.driver, dmp => dmp.commitTransaction());
+    }
+  }
+
+  async runFile(file: DatabaseModelFile, category: string) {
+    const hash = this.crypto.createHash('md5').update(file.text.trim()).digest('hex');
+    const journalItem = this.journalItems.find(x => x.name == file.name);
+    const isEqual = journalItem && journalItem.script_hash == hash;
+
+    switch (category) {
+      case 'predeploy':
+      case 'postdeploy':
+        await this.runFileCore(file, category, hash);
+        break;
+      case 'once':
+        if (journalItem) return;
+        await this.runFileCore(file, category, hash);
+        break;
+      case 'install':
+        if (isEqual) return;
+        const uninstallFile = this.uninstall.find(x => x.name == file.name.replace('.install.sql', '.uninstall.sql'));
+        if (uninstallFile && journalItem) {
+          // file was previously installed, uninstall first
+          await this.runFileCore(
+            uninstallFile,
+            'uninstall',
+            this.crypto.createHash('md5').update(uninstallFile.text.trim()).digest('hex')
+          );
+        }
+        await this.runFileCore(file, category, hash);
+        break;
+    }
+  }
+}

packages/datalib/src/index.ts

@@ -22,3 +22,4 @@ export * from './DataDuplicator';
 export * from './FreeTableGridDisplay';
 export * from './FreeTableModel';
 export * from './CustomGridDisplay';
+export * from './ScriptDrivedDeployer';

packages/filterparser/src/parseFilter.ts

@@ -485,6 +485,7 @@ const createParser = (filterBehaviour: FilterBehaviour) => {
   if (filterBehaviour.supportDatetimeSymbols) {
     allowedElements.push(
       'today',
+      'yesterday',
       'tomorrow',
       'lastWeek',
       'thisWeek',

packages/tools/src/DatabaseAnalyser.ts

@@ -70,7 +70,7 @@ export class DatabaseAnalyser {
   }
 
   async fullAnalysis() {
-    logger.info(`Performing full analysis, DB=${dbNameLogCategory(this.dbhan.database)}, engine=${this.driver.engine}`);
+    logger.debug(`Performing full analysis, DB=${dbNameLogCategory(this.dbhan.database)}, engine=${this.driver.engine}`);
     const res = this.addEngineField(await this._runAnalysis());
     // console.log('FULL ANALYSIS', res);
     return res;

packages/tools/src/ScriptWriter.ts

@@ -58,7 +58,7 @@ export class ScriptWriter {
   }
 
   dataDuplicator(options) {
-    this._put(`await dbgateApi.dataDuplicator(${JSON.stringify(options)});`);
+    this._put(`await dbgateApi.dataDuplicator(${JSON.stringify(options, null, 2)});`);
   }
 
   comment(s) {

packages/tools/src/SqlDumper.ts

@@ -515,6 +515,9 @@ export class SqlDumper implements AlterProcessor {
       this.put('%i %k', col.columnName, col.isDescending == true ? 'DESC' : 'ASC');
     });
     this.put('&<&n)');
+    if (ix.filterDefinition && this.dialect.filteredIndexes) {
+      this.put('&n^where %s', ix.filterDefinition);
+    }
     this.endCommand();
   }
 
@@ -588,6 +591,8 @@ export class SqlDumper implements AlterProcessor {
 
   renameTable(obj: TableInfo, newname: string) {}
 
+  renameSqlObject(obj: SqlObjectInfo, newname: string) {}
+
   beginTransaction() {
     this.putCmd('^begin ^transaction');
   }

packages/tools/src/alterPlan.ts

@@ -1,5 +1,5 @@
 import _ from 'lodash';
-import { DbDiffOptions, generateTablePairingId } from './diffTools';
+import { DbDiffOptions, generateTablePairingId, hasDeletedPrefix } from './diffTools';
 import {
   AlterProcessor,
   ColumnInfo,
@@ -39,6 +39,12 @@ interface AlterOperation_RenameTable {
   newName: string;
 }
 
+interface AlterOperation_RenameSqlObject {
+  operationType: 'renameSqlObject';
+  object: SqlObjectInfo;
+  newName: string;
+}
+
 interface AlterOperation_CreateColumn {
   operationType: 'createColumn';
   newObject: ColumnInfo;
@@ -75,6 +81,7 @@ interface AlterOperation_ChangeConstraint {
 interface AlterOperation_DropConstraint {
   operationType: 'dropConstraint';
   oldObject: ConstraintInfo;
+  isRecreate?: boolean;
 }
 
 interface AlterOperation_RenameConstraint {
@@ -104,7 +111,7 @@ interface AlterOperation_SetTableOption {
   optionValue: string;
 }
 
-type AlterOperation =
+export type AlterOperation =
   | AlterOperation_CreateColumn
   | AlterOperation_ChangeColumn
   | AlterOperation_DropColumn
@@ -120,7 +127,8 @@ type AlterOperation =
   | AlterOperation_DropSqlObject
   | AlterOperation_RecreateTable
   | AlterOperation_FillPreloadedRows
-  | AlterOperation_SetTableOption;
+  | AlterOperation_SetTableOption
+  | AlterOperation_RenameSqlObject;
 
 export class AlterPlan {
   recreates = {
@@ -217,6 +225,14 @@ export class AlterPlan {
     });
   }
 
+  renameSqlObject(table: TableInfo, newName: string) {
+    this.operations.push({
+      operationType: 'renameSqlObject',
+      object: table,
+      newName,
+    });
+  }
+
   renameColumn(column: ColumnInfo, newName: string) {
     this.operations.push({
       operationType: 'renameColumn',
@@ -322,15 +338,16 @@ export class AlterPlan {
         if (op.operationType == testedOperationType) {
           const constraints = this._getDependendColumnConstraints(testedObject as ColumnInfo, testedDependencies);
 
-          if (constraints.length > 0 && this.opts.noDropConstraint) {
-            return [];
-          }
+          // if (constraints.length > 0 && this.opts.noDropConstraint) {
+          //   return [];
+          // }
 
           const res: AlterOperation[] = [
             ...constraints.map(oldObject => {
               const opRes: AlterOperation = {
                 operationType: 'dropConstraint',
                 oldObject,
+                isRecreate: true,
               };
               return opRes;
             }),
@@ -367,15 +384,16 @@ export class AlterPlan {
       }
 
       if (op.operationType == 'changeConstraint') {
-        if (this.opts.noDropConstraint) {
-          // skip constraint recreate
-          return [];
-        }
+        // if (this.opts.noDropConstraint) {
+        //   // skip constraint recreate
+        //   return [];
+        // }
 
         this.recreates.constraints += 1;
         const opDrop: AlterOperation = {
           operationType: 'dropConstraint',
           oldObject: op.oldObject,
+          isRecreate: true,
         };
         const opCreate: AlterOperation = {
           operationType: 'createConstraint',
@@ -441,7 +459,7 @@ export class AlterPlan {
       // console.log('*****************RECREATED NEEDED', op, operationType, isAllowed);
       // console.log(this.dialect);
 
-      if (this.opts.noDropTable) {
+      if (this.opts.noDropTable && !this.opts.allowTableRecreateWhenNoDrop) {
         // skip this operation, as it cannot be achieved
         return [];
       }
@@ -532,8 +550,14 @@ export class AlterPlan {
       if (this.opts.noDropColumn && op.operationType == 'dropColumn') return false;
       if (this.opts.noDropTable && op.operationType == 'dropTable') return false;
       if (this.opts.noDropTable && op.operationType == 'recreateTable') return false;
-      if (this.opts.noDropConstraint && op.operationType == 'dropConstraint') return false;
-      if (this.opts.noDropSqlObject && op.operationType == 'dropSqlObject') return false;
+      if (this.opts.noDropConstraint && op.operationType == 'dropConstraint' && !op.isRecreate) return false;
+      // if (
+      //   this.opts.noDropSqlObject &&
+      //   op.operationType == 'dropSqlObject' &&
+      //   // allow to drop previously deleted SQL objects
+      //   !hasDeletedPrefix(op.oldObject.pureName, this.opts, this.opts.deletedSqlObjectPrefix)
+      // )
+      //   return false;
       return true;
     });
   }
@@ -595,6 +619,9 @@ export function runAlterOperation(op: AlterOperation, processor: AlterProcessor)
     case 'renameTable':
       processor.renameTable(op.object, op.newName);
       break;
+    case 'renameSqlObject':
+      processor.renameSqlObject(op.object, op.newName);
+      break;
     case 'renameConstraint':
       processor.renameConstraint(op.object, op.newName);
       break;

packages/tools/src/computeDiffRows.ts

@@ -1,6 +1,14 @@
-import { DbDiffOptions, testEqualColumns, testEqualTables, testEqualSqlObjects } from './diffTools';
-import type { DatabaseInfo, EngineDriver, SqlObjectInfo, TableInfo } from 'dbgate-types';
+import {
+  DbDiffOptions,
+  testEqualColumns,
+  testEqualTables,
+  testEqualSqlObjects,
+  createAlterDatabasePlan,
+} from './diffTools';
+import type { DatabaseInfo, EngineDriver, NamedObjectInfo, SqlObjectInfo, TableInfo } from 'dbgate-types';
 import _ from 'lodash';
+import { extendDatabaseInfo } from './structureTools';
+import { AlterOperation, runAlterOperation } from './alterPlan';
 
 export function computeDiffRowsCore(sourceList, targetList, testEqual) {
   const res = [];
@@ -124,6 +132,58 @@ export function computeTableDiffColumns(
   }));
 }
 
+export interface DiffOperationItemDisplay {
+  operationType: string;
+  name: string;
+  sqlScript: string;
+  identifier?: string;
+}
+
+export function getOperationDisplay(operation: AlterOperation, driver: EngineDriver): DiffOperationItemDisplay {
+  const op = operation as any;
+  const name =
+    op?.newName ??
+    op?.newObject?.columnName ??
+    op?.newObject?.constraintName ??
+    op?.newObject?.pureName ??
+    op?.oldObject?.columnName ??
+    op?.oldObject?.constraintName ??
+    op?.oldObject?.pureName ??
+    op?.table?.pureName ??
+    op?.object?.columnName ??
+    op?.object?.constraintName ??
+    op?.object?.pureName;
+
+  const dmp = driver.createDumper();
+  runAlterOperation(operation, dmp);
+
+  return {
+    operationType: operation.operationType,
+    name,
+    sqlScript: dmp.s,
+    identifier: dmp.s,
+  };
+}
+
+export function computeObjectDiffOperations(
+  sourceObject: { objectTypeField: string },
+  targetObject: { objectTypeField: string },
+  sourceDb: DatabaseInfo,
+  targetDb: DatabaseInfo,
+  opts: DbDiffOptions,
+  driver: EngineDriver
+): DiffOperationItemDisplay[] {
+  if (!driver) return [];
+  const srcdb = sourceObject
+    ? extendDatabaseInfo({ [sourceObject.objectTypeField]: [sourceObject] } as unknown as DatabaseInfo)
+    : extendDatabaseInfo({} as unknown as DatabaseInfo);
+  const dstdb = targetObject
+    ? extendDatabaseInfo({ [targetObject.objectTypeField]: [targetObject] } as unknown as DatabaseInfo)
+    : extendDatabaseInfo({} as unknown as DatabaseInfo);
+  const plan = createAlterDatabasePlan(dstdb, srcdb, opts, targetDb, sourceDb, driver);
+  return plan.operations.map(item => getOperationDisplay(item, driver));
+}
+
 export function getCreateObjectScript(obj: TableInfo | SqlObjectInfo, driver: EngineDriver) {
   if (!obj || !driver) return '';
   if (obj.objectTypeField == 'tables') {

packages/tools/src/database-info-alter-processor.ts

@@ -112,6 +112,11 @@ export class DatabaseInfoAlterProcessor {
     this.db.tables.find(x => x.pureName == table.pureName && x.schemaName == table.schemaName).pureName = newName;
   }
 
+  renameSqlObject(obj: SqlObjectInfo, newName: string) {
+    this.db[obj.objectTypeField].find(x => x.pureName == obj.pureName && x.schemaName == obj.schemaName).pureName =
+      newName;
+  }
+
   renameColumn(column: ColumnInfo, newName: string) {
     const table = this.db.tables.find(x => x.pureName == column.pureName && x.schemaName == column.schemaName);
     table.columns.find(x => x.columnName == column.columnName).columnName = newName;

packages/tools/src/diffTools.ts

@@ -1,12 +1,18 @@
 import type {
+  CheckInfo,
   ColumnInfo,
+  ColumnReference,
   ConstraintInfo,
   DatabaseInfo,
   EngineDriver,
+  ForeignKeyInfo,
+  IndexInfo,
   NamedObjectInfo,
+  PrimaryKeyInfo,
   SqlDialect,
   SqlObjectInfo,
   TableInfo,
+  UniqueInfo,
   ViewInfo,
 } from 'dbgate-types';
 import uuidv1 from 'uuid/v1';
@@ -19,6 +25,7 @@ import _isEqual from 'lodash/isEqual';
 import _pick from 'lodash/pick';
 import _compact from 'lodash/compact';
 import _isString from 'lodash/isString';
+import { detectChangesInPreloadedRows } from './structureTools';
 
 type DbDiffSchemaMode = 'strict' | 'ignore' | 'ignoreImplicit';
 
@@ -35,9 +42,18 @@ export interface DbDiffOptions {
   ignoreConstraintNames?: boolean;
 
   noDropTable?: boolean;
+  allowTableRecreateWhenNoDrop?: boolean;
+  deletedTablePrefix?: string;
+
   noDropColumn?: boolean;
+  deletedColumnPrefix?: string;
+
   noDropConstraint?: boolean;
+
+  // unlike tables, sql objects could be recreated, when this option is set
   noDropSqlObject?: boolean;
+  deletedSqlObjectPrefix?: string;
+
   noRenameTable?: boolean;
   noRenameColumn?: boolean;
 
@@ -112,6 +128,13 @@ export function removeTablePairingId(table: TableInfo): TableInfo {
   };
 }
 
+function simplifySqlExpression(sql: string) {
+  return (sql || '')
+    .replace(/[\s\(\)\[\]\"]/g, '')
+    .toLowerCase()
+    .trim();
+}
+
 function generateObjectPairingId(obj) {
   if (obj.objectTypeField)
     return {
@@ -136,7 +159,36 @@ export function generateDbPairingId(db: DatabaseInfo): DatabaseInfo {
   };
 }
 
-function testEqualNames(a: string, b: string, opts: DbDiffOptions) {
+function getNameWithoutDeletedPrefix(name: string, opts: DbDiffOptions, deletedPrefix?: string) {
+  if (deletedPrefix) {
+    if (opts.ignoreCase) {
+      if ((name || '').toLowerCase().startsWith(deletedPrefix.toLowerCase())) {
+        return name.slice(deletedPrefix.length);
+      }
+    } else {
+      if ((name || '').startsWith(deletedPrefix)) {
+        return name.slice(deletedPrefix.length);
+      }
+    }
+  }
+
+  return name;
+}
+
+export function hasDeletedPrefix(name: string, opts: DbDiffOptions, deletedPrefix?: string) {
+  if (deletedPrefix) {
+    if (opts.ignoreCase) {
+      return (name || '').toLowerCase().startsWith(deletedPrefix.toLowerCase());
+    } else {
+      return (name || '').startsWith(deletedPrefix);
+    }
+  }
+  return false;
+}
+
+function testEqualNames(a: string, b: string, opts: DbDiffOptions, deletedPrefix?: string) {
+  a = getNameWithoutDeletedPrefix(a, opts, deletedPrefix);
+  b = getNameWithoutDeletedPrefix(b, opts, deletedPrefix);
   if (opts.ignoreCase) return (a || '').toLowerCase() == (b || '').toLowerCase();
   return a == b;
 }
@@ -149,9 +201,12 @@ function testEqualSchemas(lschema: string, rschema: string, opts: DbDiffOptions)
   return testEqualNames(lschema, rschema, opts);
 }
 
-function testEqualFullNames(lft: NamedObjectInfo, rgt: NamedObjectInfo, opts: DbDiffOptions) {
+function testEqualFullNames(lft: NamedObjectInfo, rgt: NamedObjectInfo, opts: DbDiffOptions, deletedPrefix?: string) {
   if (lft == null || rgt == null) return lft == rgt;
-  return testEqualSchemas(lft.schemaName, rgt.schemaName, opts) && testEqualNames(lft.pureName, rgt.pureName, opts);
+  return (
+    testEqualSchemas(lft.schemaName, rgt.schemaName, opts) &&
+    testEqualNames(lft.pureName, rgt.pureName, opts, deletedPrefix)
+  );
 }
 
 function testEqualDefaultValues(value1: string | null | undefined, value2: string | null | undefined) {
@@ -159,7 +214,7 @@ function testEqualDefaultValues(value1: string | null | undefined, value2: strin
   if (value2 == null) return value1 == null || value1 == 'NULL';
   if (_isString(value1) && _isString(value2)) {
     value1 = value1.trim();
-    value2 = value1.trim();
+    value2 = value2.trim();
     if (value1.startsWith("'") && value1.endsWith("'") && value2.startsWith("'") && value2.endsWith("'")) {
       return value1 == value2;
     }
@@ -305,20 +360,85 @@ export function testEqualColumns(
   return true;
 }
 
+function testEqualColumnRefs(a: ColumnReference[], b: ColumnReference[], opts: DbDiffOptions) {
+  if (a.length != b.length) return false;
+  for (let i = 0; i < a.length; i++) {
+    if (!testEqualNames(a[i].columnName, b[i].columnName, opts)) return false;
+    if (!testEqualNames(a[i].refColumnName, b[i].refColumnName, opts)) return false;
+  }
+  return true;
+}
+
+function testEqualPrimaryKeys(a: PrimaryKeyInfo, b: PrimaryKeyInfo, opts: DbDiffOptions) {
+  if (!testEqualColumnRefs(a.columns, b.columns, opts)) return false;
+  return true;
+}
+
+function testEqualForeignKeys(a: ForeignKeyInfo, b: ForeignKeyInfo, opts: DbDiffOptions) {
+  if (!testEqualColumnRefs(a.columns, b.columns, opts)) return false;
+  if (!opts.ignoreConstraintNames && !testEqualNames(a.constraintName, b.constraintName, opts)) return false;
+  return true;
+}
+
+function testEqualIndex(a: IndexInfo, b: IndexInfo, opts: DbDiffOptions) {
+  if (!testEqualColumnRefs(a.columns, b.columns, opts)) return false;
+  if (!!a.isUnique != !!b.isUnique) return false;
+  if (simplifySqlExpression(a.filterDefinition) != simplifySqlExpression(b.filterDefinition)) return false;
+
+  if (!opts.ignoreConstraintNames && !testEqualNames(a.constraintName, b.constraintName, opts)) return false;
+  return true;
+}
+
+function testEqualUnique(a: UniqueInfo, b: UniqueInfo, opts: DbDiffOptions) {
+  if (!testEqualColumnRefs(a.columns, b.columns, opts)) return false;
+
+  if (!opts.ignoreConstraintNames && !testEqualNames(a.constraintName, b.constraintName, opts)) return false;
+  return true;
+}
+
+function testEqualCheck(a: CheckInfo, b: CheckInfo, opts: DbDiffOptions) {
+  if (a.definition != b.definition) return false;
+  if (!opts.ignoreConstraintNames && !testEqualNames(a.constraintName, b.constraintName, opts)) return false;
+  return true;
+}
+
 function testEqualConstraints(a: ConstraintInfo, b: ConstraintInfo, opts: DbDiffOptions = {}) {
-  const omitList = [];
-  if (opts.ignoreForeignKeyActions) {
-    omitList.push('updateAction');
-    omitList.push('deleteAction');
+  if (a.constraintType != b.constraintType) {
+    console.debug(`Constraint ${a.pureName}: different constraint type: ${a.constraintType}, ${b.constraintType}`);
+    return false;
   }
-  if (opts.ignoreConstraintNames) {
-    omitList.push('constraintName');
-  }
-  if (opts.schemaMode == 'ignore') {
-    omitList.push('schemaName');
-    omitList.push('refSchemaName');
+
+  switch (a.constraintType) {
+    case 'primaryKey':
+    case 'sortingKey':
+      return testEqualPrimaryKeys(a as PrimaryKeyInfo, b as PrimaryKeyInfo, opts);
+    case 'foreignKey':
+      return testEqualForeignKeys(a as ForeignKeyInfo, b as ForeignKeyInfo, opts);
+    case 'index':
+      return testEqualIndex(a as IndexInfo, b as IndexInfo, opts);
+    case 'unique':
+      return testEqualUnique(a as UniqueInfo, b as UniqueInfo, opts);
+    case 'check':
+      return testEqualCheck(a as CheckInfo, b as CheckInfo, opts);
   }
 
+  console.debug(`Unknown constraint type: ${a.pureName}`);
+
+  return false;
+
+  // const omitList = ['pairingId'];
+  // if (opts.ignoreForeignKeyActions) {
+  //   omitList.push('updateAction');
+  //   omitList.push('deleteAction');
+  // }
+  // if (opts.ignoreConstraintNames) {
+  //   omitList.push('constraintName');
+  // }
+  // if (opts.schemaMode == 'ignore') {
+  //   omitList.push('schemaName');
+  //   omitList.push('refSchemaName');
+  // }
+
   // if (a.constraintType == 'primaryKey' && b.constraintType == 'primaryKey') {
   //   console.log('PK1', stableStringify(_.omit(a, omitList)));
   //   console.log('PK2', stableStringify(_.omit(b, omitList)));
@@ -334,7 +454,10 @@ function testEqualConstraints(a: ConstraintInfo, b: ConstraintInfo, opts: DbDiff
   //   console.log('IX2', stableStringify(_omit(b, omitList)));
   // }
 
-  return stableStringify(_omit(a, omitList)) == stableStringify(_omit(b, omitList));
+  // const aStringified = stableStringify(_omit(a, omitList));
+  // const bStringified = stableStringify(_omit(b, omitList));
+
+  // return aStringified == bStringified;
 }
 
 export function testEqualTypes(a: ColumnInfo, b: ColumnInfo, opts: DbDiffOptions = {}) {
@@ -342,7 +465,7 @@ export function testEqualTypes(a: ColumnInfo, b: ColumnInfo, opts: DbDiffOptions
     return true;
   }
 
-  if ((a.dataType || '').toLowerCase() != (b.dataType || '').toLowerCase()) {
+  if (simplifySqlExpression(a.dataType) != simplifySqlExpression(b.dataType)) {
     console.debug(
       `Column ${a.pureName}.${a.columnName}, ${b.pureName}.${b.columnName}: different data type: ${a.dataType}, ${b.dataType}`
     );
@@ -385,7 +508,7 @@ function createPairs(oldList, newList, additionalCondition = null) {
 
 function planTablePreload(plan: AlterPlan, oldTable: TableInfo, newTable: TableInfo) {
   const key = newTable.preloadedRowsKey || newTable.primaryKey?.columns?.map(x => x.columnName);
-  if (newTable.preloadedRows?.length > 0 && key?.length > 0) {
+  if (newTable.preloadedRows?.length > 0 && key?.length > 0 && detectChangesInPreloadedRows(oldTable, newTable)) {
     plan.fillPreloadedRows(
       newTable,
       oldTable?.preloadedRows,
@@ -417,7 +540,15 @@ function planAlterTable(plan: AlterPlan, oldTable: TableInfo, newTable: TableInf
   if (!opts.noDropConstraint) {
     constraintPairs.filter(x => x[1] == null).forEach(x => plan.dropConstraint(x[0]));
   }
-  if (!opts.noDropColumn) {
+  if (opts.deletedColumnPrefix) {
+    columnPairs
+      .filter(x => x[1] == null)
+      .forEach(x => {
+        if (!hasDeletedPrefix(x[0].columnName, opts, opts.deletedColumnPrefix)) {
+          plan.renameColumn(x[0], opts.deletedColumnPrefix + x[0].columnName);
+        }
+      });
+  } else if (!opts.noDropColumn) {
     columnPairs.filter(x => x[1] == null).forEach(x => plan.dropColumn(x[0]));
   }
 
@@ -425,18 +556,33 @@ function planAlterTable(plan: AlterPlan, oldTable: TableInfo, newTable: TableInf
     plan.renameTable(oldTable, newTable.pureName);
   }
 
+  if (hasDeletedPrefix(oldTable.pureName, opts, opts.deletedTablePrefix)) {
+    plan.renameTable(oldTable, newTable.pureName);
+  }
+
   columnPairs.filter(x => x[0] == null).forEach(x => plan.createColumn(x[1]));
 
   columnPairs
     .filter(x => x[0] && x[1])
     .forEach(x => {
-      if (!testEqualColumns(x[0], x[1], true, true, opts)) {
-        if (testEqualColumns(x[0], x[1], false, true, opts) && !opts.noRenameColumn) {
+      let srccol: ColumnInfo = x[0];
+      let dstcol: ColumnInfo = x[1];
+      if (hasDeletedPrefix(srccol.columnName, opts, opts.deletedColumnPrefix)) {
+        plan.renameColumn(srccol, dstcol.columnName);
+        // rename is already done
+        srccol = {
+          ...srccol,
+          columnName: dstcol.columnName,
+        };
+      }
+
+      if (!testEqualColumns(srccol, dstcol, true, true, opts)) {
+        if (testEqualColumns(srccol, dstcol, false, true, opts) && !opts.noRenameColumn) {
           // console.log('PLAN RENAME COLUMN')
-          plan.renameColumn(x[0], x[1].columnName);
+          plan.renameColumn(srccol, dstcol.columnName);
         } else {
           // console.log('PLAN CHANGE COLUMN', x[0], x[1]);
-          plan.changeColumn(x[0], x[1]);
+          plan.changeColumn(srccol, dstcol);
         }
       }
     });
@@ -452,8 +598,6 @@ function planAlterTable(plan: AlterPlan, oldTable: TableInfo, newTable: TableInf
 
   constraintPairs.filter(x => x[0] == null).forEach(x => plan.createConstraint(x[1]));
 
-  planTablePreload(plan, oldTable, newTable);
-
   planChangeTableOptions(plan, oldTable, newTable, opts);
 
   // console.log('oldTable', oldTable);
@@ -487,12 +631,22 @@ export function testEqualTables(
 ) {
   const plan = new AlterPlan(wholeOldDb, wholeNewDb, driver.dialect, opts);
   planAlterTable(plan, a, b, opts);
-  // console.log('plan.operations', a, b, plan.operations);
-  return plan.operations.length == 0;
+  // if (plan.operations.length > 0) {
+  //   console.log('************** plan.operations', a, b, plan.operations);
+  // }
+  if (plan.operations.length > 0) {
+    return false;
+  }
+
+  if (detectChangesInPreloadedRows(a, b)) {
+    return false;
+  }
+
+  return true;
 }
 
 export function testEqualSqlObjects(a: SqlObjectInfo, b: SqlObjectInfo, opts: DbDiffOptions) {
-  return a.createSql == b.createSql;
+  return a.createSql?.trim() == b.createSql?.trim();
 }
 
 export function createAlterTablePlan(
@@ -511,6 +665,7 @@ export function createAlterTablePlan(
     plan.dropTable(oldTable);
   } else {
     planAlterTable(plan, oldTable, newTable, opts);
+    planTablePreload(plan, oldTable, newTable);
   }
   plan.transformPlan();
   return plan;
@@ -565,23 +720,35 @@ export function createAlterDatabasePlan(
       const newobj = (newDb[objectTypeField] || []).find(x => x.pairingId == oldobj.pairingId);
       if (objectTypeField == 'tables') {
         if (newobj == null) {
-          if (!opts.noDropTable) {
+          if (opts.deletedTablePrefix && !hasDeletedPrefix(oldobj.pureName, opts, opts.deletedTablePrefix)) {
+            plan.renameTable(oldobj, opts.deletedTablePrefix + oldobj.pureName);
+          } else if (!opts.noDropTable) {
             plan.dropTable(oldobj);
           }
         } else {
           planAlterTable(plan, oldobj, newobj, opts);
+          planTablePreload(plan, oldobj, newobj);
         }
       } else {
         if (newobj == null) {
-          if (!opts.noDropSqlObject) {
+          if (
+            opts.deletedSqlObjectPrefix &&
+            driver.dialect.renameSqlObject &&
+            !hasDeletedPrefix(oldobj.pureName, opts, opts.deletedSqlObjectPrefix)
+          ) {
+            plan.renameSqlObject(oldobj, opts.deletedSqlObjectPrefix + oldobj.pureName);
+          } else if (!opts.noDropSqlObject) {
             plan.dropSqlObject(oldobj);
           }
-        } else if (!testEqualSqlObjects(oldobj.createSql, newobj.createSql, opts)) {
-          plan.recreates.sqlObjects += 1;
-          if (!opts.noDropSqlObject) {
+        } else {
+          if (opts.deletedSqlObjectPrefix && hasDeletedPrefix(oldobj.pureName, opts, opts.deletedSqlObjectPrefix)) {
             plan.dropSqlObject(oldobj);
+            plan.createSqlObject(newobj);
+          } else if (!testEqualSqlObjects(oldobj, newobj, opts)) {
+            plan.recreates.sqlObjects += 1;
+            plan.dropSqlObject(oldobj);
+            plan.createSqlObject(newobj);
           }
-          plan.createSqlObject(newobj);
         }
       }
     }
@@ -634,9 +801,13 @@ export function getAlterDatabaseScript(
   opts: DbDiffOptions,
   wholeOldDb: DatabaseInfo,
   wholeNewDb: DatabaseInfo,
-  driver: EngineDriver
+  driver: EngineDriver,
+  transformPlan: (plan: AlterPlan) => void = null
 ) {
   const plan = createAlterDatabasePlan(oldDb, newDb, opts, wholeOldDb, wholeNewDb, driver);
+  if (transformPlan) {
+    transformPlan(plan);
+  }
   const dmp = driver.createDumper({ useHardSeparator: true });
   plan.run(dmp);
   return {
@@ -653,13 +824,22 @@ export function matchPairedObjects(db1: DatabaseInfo, db2: DatabaseInfo, opts: D
 
   for (const objectTypeField of ['tables', 'views', 'procedures', 'matviews', 'functions']) {
     for (const obj2 of res[objectTypeField] || []) {
-      const obj1 = db1[objectTypeField].find(x => testEqualFullNames(x, obj2, opts));
+      const obj1 = db1[objectTypeField].find(x =>
+        testEqualFullNames(
+          x,
+          obj2,
+          opts,
+          objectTypeField == 'tables' ? opts.deletedTablePrefix : opts.deletedSqlObjectPrefix
+        )
+      );
       if (obj1) {
         obj2.pairingId = obj1.pairingId;
 
         if (objectTypeField == 'tables') {
           for (const col2 of obj2.columns) {
-            const col1 = obj1.columns.find(x => testEqualNames(x.columnName, col2.columnName, opts));
+            const col1 = obj1.columns.find(x =>
+              testEqualNames(x.columnName, col2.columnName, opts, opts.deletedColumnPrefix)
+            );
             if (col1) col2.pairingId = col1.pairingId;
           }
 
@@ -699,7 +879,6 @@ export function matchPairedObjects(db1: DatabaseInfo, db2: DatabaseInfo, opts: D
 
 export const modelCompareDbDiffOptions: DbDiffOptions = {
   ignoreCase: true,
-  schemaMode: 'ignore',
   ignoreConstraintNames: true,
   ignoreForeignKeyActions: true,
   ignoreDataTypes: true,

packages/tools/src/driverBase.ts

@@ -76,7 +76,7 @@ export const driverBase = {
     for (const sqlItem of splitQuery(sql, this.getQuerySplitterOptions('script'))) {
       try {
         if (options?.logScriptItems) {
-          logger.info({ sql: getLimitedQuery(sqlItem as string) }, `Execute script item`);
+          logger.info({ sql: getLimitedQuery(sqlItem as string) }, 'Execute script item');
         }
         await this.query(pool, sqlItem, { discardResult: true, ...options?.queryOptions });
       } catch (err) {

packages/tools/src/stringTools.ts

@@ -312,6 +312,14 @@ export function safeJsonParse(json, defaultValue?, logError = false) {
   }
 }
 
+export function safeCompileRegExp(regex: string, flags: string) {
+  try {
+    return new RegExp(regex, flags);
+  } catch (err) {
+    return null;
+  }
+}
+
 export function shouldOpenMultilineDialog(value) {
   if (_isString(value)) {
     if (value.includes('\n')) {
@@ -521,3 +529,23 @@ export function getLimitedQuery(sql: string): string {
   }
   return sql;
 }
+
+export function pinoLogRecordToMessageRecord(logRecord, defaultSeverity = 'info') {
+  const { level, time, msg, ...rest } = logRecord;
+
+  const levelToSeverity = {
+    10: 'debug',
+    20: 'debug',
+    30: 'info',
+    40: 'info',
+    50: 'error',
+    60: 'error',
+  };
+
+  return {
+    ...rest,
+    time,
+    message: msg,
+    severity: levelToSeverity[level] ?? defaultSeverity,
+  };
+}

packages/tools/src/structureTools.ts

@@ -1,5 +1,14 @@
-import type { DatabaseInfo, TableInfo, ApplicationDefinition, ViewInfo, CollectionInfo } from 'dbgate-types';
+import type {
+  DatabaseInfo,
+  TableInfo,
+  ApplicationDefinition,
+  ViewInfo,
+  CollectionInfo,
+  NamedObjectInfo,
+} from 'dbgate-types';
 import _flatten from 'lodash/flatten';
+import _uniq from 'lodash/uniq';
+import _keys from 'lodash/keys';
 
 export function addTableDependencies(db: DatabaseInfo): DatabaseInfo {
   if (!db.tables) {
@@ -142,3 +151,145 @@ export function isViewInfo(obj: { objectTypeField?: string }): obj is ViewInfo {
 export function isCollectionInfo(obj: { objectTypeField?: string }): obj is CollectionInfo {
   return obj.objectTypeField == 'collections';
 }
+
+export function filterStructureBySchema(db: DatabaseInfo, schema: string) {
+  if (!db) {
+    return db;
+  }
+
+  return {
+    ...db,
+    tables: (db.tables || []).filter(x => x.schemaName == schema),
+    views: (db.views || []).filter(x => x.schemaName == schema),
+    collections: (db.collections || []).filter(x => x.schemaName == schema),
+    matviews: (db.matviews || []).filter(x => x.schemaName == schema),
+    procedures: (db.procedures || []).filter(x => x.schemaName == schema),
+    functions: (db.functions || []).filter(x => x.schemaName == schema),
+    triggers: (db.triggers || []).filter(x => x.schemaName == schema),
+  };
+}
+
+export function getSchemasUsedByStructure(db: DatabaseInfo) {
+  if (!db) {
+    return db;
+  }
+
+  return _uniq([
+    ...(db.tables || []).map(x => x.schemaName),
+    ...(db.views || []).map(x => x.schemaName),
+    ...(db.collections || []).map(x => x.schemaName),
+    ...(db.matviews || []).map(x => x.schemaName),
+    ...(db.procedures || []).map(x => x.schemaName),
+    ...(db.functions || []).map(x => x.schemaName),
+    ...(db.triggers || []).map(x => x.schemaName),
+  ]);
+}
+
+export function replaceSchemaInStructure(db: DatabaseInfo, schema: string) {
+  if (!db) {
+    return db;
+  }
+
+  return {
+    ...db,
+    tables: (db.tables || []).map(tbl => ({
+      ...tbl,
+      schemaName: schema,
+      columns: (tbl.columns || []).map(column => ({ ...column, schemaName: schema })),
+      primaryKey: tbl.primaryKey ? { ...tbl.primaryKey, schemaName: schema } : undefined,
+      sortingKey: tbl.sortingKey ? { ...tbl.sortingKey, schemaName: schema } : undefined,
+      foreignKeys: (tbl.foreignKeys || []).map(fk => ({ ...fk, refSchemaName: schema, schemaName: schema })),
+      indexes: (tbl.indexes || []).map(idx => ({ ...idx, schemaName: schema })),
+      uniques: (tbl.uniques || []).map(idx => ({ ...idx, schemaName: schema })),
+      checks: (tbl.checks || []).map(idx => ({ ...idx, schemaName: schema })),
+    })),
+    views: (db.views || []).map(x => ({ ...x, schemaName: schema })),
+    collections: (db.collections || []).map(x => ({ ...x, schemaName: schema })),
+    matviews: (db.matviews || []).map(x => ({ ...x, schemaName: schema })),
+    procedures: (db.procedures || []).map(x => ({ ...x, schemaName: schema })),
+    functions: (db.functions || []).map(x => ({ ...x, schemaName: schema })),
+    triggers: (db.triggers || []).map(x => ({ ...x, schemaName: schema })),
+  };
+}
+
+export function skipNamesInStructureByRegex(db: DatabaseInfo, regex: RegExp) {
+  if (!db) {
+    return db;
+  }
+
+  return {
+    ...db,
+    tables: (db.tables || []).filter(tbl => !regex.test(tbl.pureName)),
+    views: (db.views || []).filter(tbl => !regex.test(tbl.pureName)),
+    collections: (db.collections || []).filter(tbl => !regex.test(tbl.pureName)),
+    matviews: (db.matviews || []).filter(tbl => !regex.test(tbl.pureName)),
+    procedures: (db.procedures || []).filter(tbl => !regex.test(tbl.pureName)),
+    functions: (db.functions || []).filter(tbl => !regex.test(tbl.pureName)),
+    triggers: (db.triggers || []).filter(tbl => !regex.test(tbl.pureName)),
+  };
+}
+
+export function detectChangesInPreloadedRows(oldTable: TableInfo, newTable: TableInfo): boolean {
+  const key =
+    newTable?.preloadedRowsKey ||
+    oldTable?.preloadedRowsKey ||
+    newTable?.primaryKey?.columns?.map(x => x.columnName) ||
+    oldTable?.primaryKey?.columns?.map(x => x.columnName);
+  const oldRows = oldTable?.preloadedRows || [];
+  const newRows = newTable?.preloadedRows || [];
+  const insertOnly = newTable?.preloadedRowsInsertOnly || oldTable?.preloadedRowsInsertOnly;
+
+  if (newRows.length != oldRows.length) {
+    return true;
+  }
+
+  for (const row of newRows) {
+    const old = oldRows?.find(r => key.every(col => r[col] == row[col]));
+    const rowKeys = _keys(row);
+    if (old) {
+      const updated = [];
+      for (const col of rowKeys) {
+        if (row[col] != old[col] && !insertOnly?.includes(col)) {
+          updated.push(col);
+        }
+      }
+      if (updated.length > 0) {
+        return true;
+      }
+    } else {
+      return true;
+    }
+  }
+
+  for (const row of oldRows || []) {
+    const newr = oldRows?.find(r => key.every(col => r[col] == row[col]));
+    if (!newr) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+export function removePreloadedRowsFromStructure(db: DatabaseInfo): DatabaseInfo {
+  if (!db) {
+    return db;
+  }
+
+  return {
+    ...db,
+    tables: (db.tables || []).map(tbl => ({
+      ...tbl,
+      preloadedRows: undefined,
+      preloadedRowsKey: undefined,
+      preloadedRowsInsertOnly: undefined,
+    })),
+  };
+}
+
+export function skipDbGateInternalObjects(db: DatabaseInfo) {
+  return {
+    ...db,
+    tables: (db.tables || []).filter(tbl => tbl.pureName != 'dbgate_deploy_journal'),
+  };
+}

packages/tools/src/yamlModelConv.ts

@@ -22,12 +22,22 @@ export interface DatabaseModelFile {
   text: string;
   json: {};
 }
+
+export interface IndexInfoYaml {
+  name: string;
+  unique?: boolean;
+  filter?: string;
+  columns: string[];
+  included?: string[];
+}
+
 export interface TableInfoYaml {
   name: string;
   // schema?: string;
   columns: ColumnInfoYaml[];
   primaryKey?: string[];
   sortingKey?: string[];
+  indexes?: IndexInfoYaml[];
 
   insertKey?: string[];
   insertOnly?: string[];
@@ -97,6 +107,20 @@ export function tableInfoToYaml(table: TableInfo): TableInfoYaml {
     res.sortingKey = tableCopy.sortingKey.columns.map(x => x.columnName);
   }
   // const foreignKeys = (tableCopy.foreignKeys || []).filter(x => !x['_dumped']).map(foreignKeyInfoToYaml);
+  if (tableCopy.indexes?.length > 0) {
+    res.indexes = tableCopy.indexes.map(index => {
+      const idx: IndexInfoYaml = {
+        name: index.constraintName,
+        unique: index.isUnique,
+        filter: index.filterDefinition,
+        columns: index.columns.filter(x => !x.isIncludedColumn).map(x => x.columnName),
+      };
+      if (index.columns.some(x => x.isIncludedColumn)) {
+        idx.included = index.columns.filter(x => x.isIncludedColumn).map(x => x.columnName);
+      }
+      return idx;
+    });
+  }
   return res;
 }
 
@@ -130,6 +154,17 @@ export function tableInfoFromYaml(table: TableInfoYaml, allTables: TableInfoYaml
     foreignKeys: _compact(
       table.columns.filter(x => x.references).map(col => convertForeignKeyFromYaml(col, table, allTables))
     ),
+    indexes: table.indexes?.map(index => ({
+      constraintName: index.name,
+      pureName: table.name,
+      isUnique: index.unique,
+      constraintType: 'index',
+      filterDefinition: index.filter,
+      columns: [
+        ...index.columns.map(columnName => ({ columnName })),
+        ...(index.included || []).map(columnName => ({ columnName, isIncludedColumn: true })),
+      ],
+    })),
   };
   if (table.primaryKey) {
     res.primaryKey = {

packages/types/alter-processor.d.ts

@@ -10,6 +10,7 @@ export interface AlterProcessor {
   changeConstraint(oldConstraint: ConstraintInfo, newConstraint: ConstraintInfo);
   dropConstraint(constraint: ConstraintInfo);
   renameTable(table: TableInfo, newName: string);
+  renameSqlObject(obj: SqlObjectInfo, newName: string);
   renameColumn(column: ColumnInfo, newName: string);
   renameConstraint(constraint: ConstraintInfo, newName: string);
   recreateTable(oldTable: TableInfo, newTable: TableInfo);

packages/types/dbinfo.d.ts

@@ -3,7 +3,6 @@ export interface NamedObjectInfo {
   schemaName?: string;
   contentHash?: string;
   engine?: string;
-  undropPureName?: string;
 }
 
 export interface ColumnReference {
@@ -35,7 +34,9 @@ export interface ForeignKeyInfo extends ColumnsConstraintInfo {
 export interface IndexInfo extends ColumnsConstraintInfo {
   isUnique: boolean;
   // indexType: 'normal' | 'clustered' | 'xml' | 'spatial' | 'fulltext';
-  indexType: string;
+  indexType?: string;
+  // condition for filtered index (SQL Server)  
+  filterDefinition?: string;
 }
 
 export interface UniqueInfo extends ColumnsConstraintInfo {}

packages/types/dialect.d.ts

@@ -34,6 +34,9 @@ export interface SqlDialect {
   dropUnique?: boolean;
   createCheck?: boolean;
   dropCheck?: boolean;
+  renameSqlObject?: boolean;
+  multipleSchema?: boolean;
+  filteredIndexes?: boolean;
 
   specificNullabilityImplementation?: boolean;
   omitForeignKeys?: boolean;

packages/web/src/EnterLicensePage.svelte

@@ -9,11 +9,11 @@
   import { apiCall } from './utility/api';
   import FormStyledButton from './buttons/FormStyledButton.svelte';
   import getElectron from './utility/getElectron';
-  import { openWebLink } from './utility/exportFileTools';
   import SpecialPageLayout from './widgets/SpecialPageLayout.svelte';
   import hasPermission from './utility/hasPermission';
   import ErrorInfo from './elements/ErrorInfo.svelte';
   import { isOneOfPage } from './utility/pageDefs';
+  import { openWebLink } from './utility/simpleTools';
 
   const config = useConfig();
   const values = writable({ amoid: null, databaseServer: null });

packages/web/src/LoginPage.svelte

@@ -202,11 +202,6 @@
           on:click={async e => {
             const state = `dbg-dblogin:${strmid}:${selectedConnection?.conid}:${$values.amoid}`;
             sessionStorage.setItem('dbloginAuthState', state);
-            // openWebLink(
-            //   `connections/dblogin?conid=${selectedConnection?.conid}&state=${encodeURIComponent(state)}&redirectUri=${
-            //     location.origin + location.pathname
-            //   }`
-            // );
             internalRedirectTo(
               `/connections/dblogin-web?conid=${selectedConnection?.conid}&state=${encodeURIComponent(state)}&redirectUri=${extractRedirectUri()}`
             );

packages/web/src/appobj/ArchiveFolderAppObject.svelte

@@ -18,6 +18,8 @@
   import ErrorMessageModal from '../modals/ErrorMessageModal.svelte';
   import { apiCall } from '../utility/api';
   import hasPermission from '../utility/hasPermission';
+  import { isProApp } from '../utility/proTools';
+  import { extractShellConnection } from '../impexp/createImpExpScript';
 
   export let data;
 
@@ -65,10 +67,7 @@
         
 await dbgateApi.deployDb(${JSON.stringify(
           {
-            connection: {
-              ..._.omit($currentDatabase.connection, '_id', 'displayName'),
-              database: $currentDatabase.name,
-            },
+            connection: extractShellConnection($currentDatabase.connection, $currentDatabase.name),
             modelFolder: `archive:${data.name}`,
           },
           undefined,
@@ -136,12 +135,13 @@ await dbgateApi.deployDb(${JSON.stringify(
       data.name != 'default' &&
         $currentDatabase && [
           { text: 'Data duplicator', onClick: handleOpenDuplicatorTab },
-          { text: 'Generate deploy DB SQL - experimental', onClick: handleGenerateDeploySql },
-          { text: 'Shell: Deploy DB - experimental', onClick: handleGenerateDeployScript },
+          { text: 'Generate deploy DB SQL', onClick: handleGenerateDeploySql },
+          { text: 'Shell: Deploy DB', onClick: handleGenerateDeployScript },
         ],
 
       data.name != 'default' &&
         hasPermission('dbops/model/compare') &&
+        isProApp() &&
         _.get($currentDatabase, 'connection._id') && {
           onClick: handleCompareWithCurrentDb,
           text: `Compare with ${_.get($currentDatabase, 'name')}`,

packages/web/src/appobj/DatabaseAppObject.svelte

@@ -170,14 +170,18 @@
     };
 
     const handleExportModel = async () => {
-      const resp = await apiCall('database-connections/export-model', {
+      showModal(ExportDbModelModal, {
         conid: connection._id,
         database: name,
       });
-      currentArchive.set(resp.archiveFolder);
-      selectedWidget.set('archive');
-      visibleWidgetSideBar.set(true);
-      showSnackbarSuccess(`Saved to archive ${resp.archiveFolder}`);
+      // const resp = await apiCall('database-connections/export-model', {
+      //   conid: connection._id,
+      //   database: name,
+      // });
+      // currentArchive.set(resp.archiveFolder);
+      // selectedWidget.set('archive');
+      // visibleWidgetSideBar.set(true);
+      // showSnackbarSuccess(`Saved to archive ${resp.archiveFolder}`);
     };
 
     const handleCompareWithCurrentDb = () => {
@@ -198,13 +202,13 @@
       );
     };
 
-    const handleOpenJsonModel = async () => {
-      const db = await getDatabaseInfo({
-        conid: connection._id,
-        database: name,
-      });
-      openJsonDocument(db, name);
-    };
+    // const handleOpenJsonModel = async () => {
+    //   const db = await getDatabaseInfo({
+    //     conid: connection._id,
+    //     database: name,
+    //   });
+    //   openJsonDocument(db, name);
+    // };
 
     const handleGenerateScript = async () => {
       const data = await apiCall('database-connections/export-keys', {
@@ -277,6 +281,57 @@
       saveScriptToDatabase({ conid: connection._id, database: name }, sql, false);
     }
 
+    const handleGenerateDropAllObjectsScript = () => {
+      showModal(ConfirmModal, {
+        message: `This will generate script, after executing this script all objects in ${name} will be dropped. Continue?`,
+
+        onConfirm: () => {
+          openNewTab(
+            {
+              title: 'Shell #',
+              icon: 'img shell',
+              tabComponent: 'ShellTab',
+            },
+            {
+              editor: `// @require ${extractPackageName(connection.engine)}
+        
+await dbgateApi.dropAllDbObjects(${JSON.stringify(
+                {
+                  connection: extractShellConnection(connection, name),
+                },
+                undefined,
+                2
+              )})`,
+            }
+          );
+        },
+      });
+    };
+
+    const handleImportWithDbDuplicator = () => {
+      showModal(ChooseArchiveFolderModal, {
+        message: 'Choose archive folder for import from',
+        onConfirm: archiveFolder => {
+          openNewTab(
+            {
+              title: archiveFolder,
+              icon: 'img duplicator',
+              tabComponent: 'DataDuplicatorTab',
+              props: {
+                conid: connection?._id,
+                database: name,
+              },
+            },
+            {
+              editor: {
+                archiveFolder,
+              },
+            }
+          );
+        },
+      });
+    };
+
     const driver = findEngineDriver(connection, getExtensions());
 
     const commands = _.flatten((apps || []).map(x => x.commands || []));
@@ -325,14 +380,16 @@
         hasPermission(`dbops/sql-generator`) && { onClick: handleSqlGenerator, text: 'SQL Generator' },
       driver?.supportsDatabaseProfiler &&
         hasPermission(`dbops/profiler`) && { onClick: handleDatabaseProfiler, text: 'Database profiler' },
+      // isSqlOrDoc &&
+      //   isSqlOrDoc &&
+      //   hasPermission(`dbops/model/view`) && { onClick: handleOpenJsonModel, text: 'Open model as JSON' },
       isSqlOrDoc &&
-        isSqlOrDoc &&
-        hasPermission(`dbops/model/view`) && { onClick: handleOpenJsonModel, text: 'Open model as JSON' },
-      isSqlOrDoc &&
-        hasPermission(`dbops/model/view`) && { onClick: handleExportModel, text: 'Export DB model - experimental' },
+        isProApp() &&
+        hasPermission(`dbops/model/view`) && { onClick: handleExportModel, text: 'Export DB model' },
       isSqlOrDoc &&
         _.get($currentDatabase, 'connection._id') &&
         hasPermission('dbops/model/compare') &&
+        isProApp() &&
         (_.get($currentDatabase, 'connection._id') != _.get(connection, '_id') ||
           (_.get($currentDatabase, 'connection._id') == _.get(connection, '_id') &&
             _.get($currentDatabase, 'name') != _.get(connection, 'name'))) && {
@@ -346,8 +403,23 @@
         (_.get($currentDatabase, 'connection._id') == _.get(connection, '_id') &&
           _.get($currentDatabase, 'name') == name)) && { onClick: handleDisconnect, text: 'Disconnect' },
 
+      { divider: true },
+
+      driver?.databaseEngineTypes?.includes('sql') &&
+        hasPermission(`dbops/dropdb`) && {
+          onClick: handleGenerateDropAllObjectsScript,
+          text: 'Shell: Drop all objects',
+        },
+
+      driver?.databaseEngineTypes?.includes('sql') &&
+        hasPermission(`dbops/import`) && {
+          onClick: handleImportWithDbDuplicator,
+          text: 'Import with DB duplicator',
+        },
+
+      { divider: true },
+
       commands.length > 0 && [
-        { divider: true },
         commands.map((cmd: any) => ({
           text: cmd.name,
           onClick: () => {
@@ -388,7 +460,7 @@
   import openNewTab from '../utility/openNewTab';
   import AppObjectCore from './AppObjectCore.svelte';
   import { showSnackbarError, showSnackbarSuccess } from '../utility/snackbar';
-  import { extractDbNameFromComposite, findEngineDriver, getConnectionLabel } from 'dbgate-tools';
+  import { extractDbNameFromComposite, extractPackageName, findEngineDriver, getConnectionLabel } from 'dbgate-tools';
   import InputTextModal from '../modals/InputTextModal.svelte';
   import { getDatabaseInfo, useUsedApps } from '../utility/metadataLoaders';
   import { openJsonDocument } from '../tabs/JsonTab.svelte';
@@ -406,6 +478,10 @@
   import { openImportExportTab } from '../utility/importExportTools';
   import newTable from '../tableeditor/newTable';
   import { loadSchemaList, switchCurrentDatabase } from '../utility/common';
+  import { isProApp } from '../utility/proTools';
+  import ExportDbModelModal from '../modals/ExportDbModelModal.svelte';
+  import ChooseArchiveFolderModal from '../modals/ChooseArchiveFolderModal.svelte';
+  import { extractShellConnection } from '../impexp/createImpExpScript';
 
   export let data;
   export let passProps;

packages/web/src/appobj/SavedFileAppObject.svelte

@@ -81,6 +81,14 @@
     currentConnection: true,
   };
 
+  const modtrans: FileTypeHandler = {
+    icon: 'img transform',
+    format: 'text',
+    tabComponent: 'ModelTransformTab',
+    folder: 'modtrans',
+    currentConnection: false,
+  };
+
   export const SAVED_FILE_HANDLERS = {
     sql,
     shell,
@@ -91,6 +99,7 @@
     diagrams,
     perspectives,
     jobs,
+    modtrans,
   };
 
   export const extractKey = data => data.file;

packages/web/src/clientAuth.ts

@@ -2,6 +2,7 @@ import { apiCall, enableApi, getAuthCategory } from './utility/api';
 import { getConfig } from './utility/metadataLoaders';
 import { isAdminPage } from './utility/pageDefs';
 import getElectron from './utility/getElectron';
+import { isProApp } from './utility/proTools';
 
 export function isOauthCallback() {
   const params = new URLSearchParams(location.search);
@@ -127,6 +128,9 @@ export async function handleAuthOnStartup(config) {
   }
 
   function checkInvalidLicense() {
+    if (!isProApp()) {
+      return;
+    }
     if (!config.isLicenseValid) {
       if (config.storageDatabase || getElectron()) {
         if (isAdminPage()) {
@@ -142,6 +146,9 @@ export async function handleAuthOnStartup(config) {
   }
 
   function checkTrialDaysLeft() {
+    if (!isProApp()) {
+      return;
+    }
     if (
       config.trialDaysLeft != null &&
       config.trialDaysLeft <= 14 &&

packages/web/src/commands/stdCommands.ts

@@ -33,7 +33,6 @@ import { removeLocalStorage } from '../utility/storageCache';
 import { showSnackbarSuccess } from '../utility/snackbar';
 import { apiCall } from '../utility/api';
 import runCommand from './runCommand';
-import { openWebLink } from '../utility/exportFileTools';
 import { getSettings } from '../utility/metadataLoaders';
 import { isMac, switchCurrentDatabase } from '../utility/common';
 import { doLogout } from '../clientAuth';
@@ -45,6 +44,8 @@ import ConfirmModal from '../modals/ConfirmModal.svelte';
 import localforage from 'localforage';
 import { openImportExportTab } from '../utility/importExportTools';
 import newTable from '../tableeditor/newTable';
+import { isProApp } from '../utility/proTools';
+import { openWebLink } from '../utility/simpleTools';
 
 // function themeCommand(theme: ThemeDefinition) {
 //   return {
@@ -184,6 +185,50 @@ registerCommand({
     findEngineDriver(getCurrentDatabase()?.connection, getExtensions())?.databaseEngineTypes?.includes('sql'),
 });
 
+if (isProApp()) {
+  registerCommand({
+    id: 'new.modelTransform',
+    category: 'New',
+    icon: 'img transform',
+    name: 'Model transform',
+    menuName: 'New model transform',
+    onClick: () => {
+      openNewTab(
+        {
+          title: 'Model transform #',
+          icon: 'img transform',
+          tabComponent: 'ModelTransformTab',
+        },
+        {
+          editor: JSON.stringify(
+            [
+              {
+                transform: 'dataTypeMapperTransform',
+                arguments: ['json', 'nvarchar(max)'],
+              },
+              {
+                transform: 'sqlTextReplacementTransform',
+                arguments: [
+                  {
+                    oldval1: 'newval1',
+                    oldval2: 'newval2',
+                  },
+                ],
+              },
+              {
+                transform: 'autoIndexForeignKeysTransform',
+                arguments: [],
+              },
+            ],
+            null,
+            2
+          ),
+        }
+      );
+    },
+  });
+}
+
 registerCommand({
   id: 'new.perspective',
   category: 'New',
@@ -298,20 +343,22 @@ registerCommand({
   },
 });
 
-registerCommand({
-  id: 'new.modelCompare',
-  category: 'New',
-  icon: 'icon compare',
-  name: 'Compare DB',
-  toolbar: true,
-  onClick: () => {
-    openNewTab({
-      title: 'Compare',
-      icon: 'img compare',
-      tabComponent: 'CompareModelTab',
-    });
-  },
-});
+if (isProApp()) {
+  registerCommand({
+    id: 'new.modelCompare',
+    category: 'New',
+    icon: 'icon compare',
+    name: 'Compare DB',
+    toolbar: true,
+    onClick: () => {
+      openNewTab({
+        title: 'Compare',
+        icon: 'img compare',
+        tabComponent: 'CompareModelTab',
+      });
+    },
+  });
+}
 
 registerCommand({
   id: 'new.jsonl',

packages/web/src/elements/Link.svelte

@@ -1,7 +1,7 @@
 <script lang="ts">
-  import { openWebLink } from '../utility/exportFileTools';
   import contextMenu from '../utility/contextMenu';
   import { internalRedirectTo } from '../clientAuth';
+  import { openWebLink } from '../utility/simpleTools';
 
   export let href = undefined;
   export let onClick = undefined;

packages/web/src/elements/ObjectFieldsEditor.svelte

@@ -6,6 +6,7 @@
   import FormProviderCore from '../forms/FormProviderCore.svelte';
   import FormTextField from '../forms/FormTextField.svelte';
   import FormSelectField from '../forms/FormSelectField.svelte';
+  import stableStringify from 'json-stable-stringify';
 
   export let title;
   export let fieldDefinitions;
@@ -18,7 +19,9 @@
 
   const valuesStore = writable(values || {});
 
-  $: onChangeValues($valuesStore);
+  $: if (stableStringify($valuesStore) != stableStringify(values)) {
+    onChangeValues($valuesStore);
+  }
 </script>
 
 <div class="wrapper">

packages/web/src/forms/FormArchiveFolderSelect.svelte

@@ -22,6 +22,7 @@
       label: folder.name,
     })),
     ...additionalFolders
+      .filter(x => x != '@create')
       .filter(x => !($folders || []).find(y => y.name == x))
       .map(folder => ({
         value: folder,

packages/web/src/forms/FormElectronFileSelectorRaw.svelte

@@ -9,19 +9,32 @@
   export let name;
   export let disabled = false;
   export let defaultFileName = '';
+  export let dialogProperties = undefined;
+  export let isSaveDialog = false;
+  export let dialogFilters = [{ name: 'All Files', extensions: ['*'] }];
 
   const { values, setFieldValue } = getFormContext();
 
   async function handleBrowse() {
     const electron = getElectron();
     if (!electron) return;
-    const filePaths = await electron.showOpenDialog({
-      defaultPath: values[name],
-      properties: ['showHiddenFiles', 'openFile'],
-      filters: [{ name: 'All Files', extensions: ['*'] }],
-    });
-    const filePath = filePaths && filePaths[0];
-    if (filePath) setFieldValue(name, filePath);
+
+    if (isSaveDialog) {
+      const filePath = await electron.showSaveDialog({
+        defaultPath: values[name],
+        properties: dialogProperties ?? ['showHiddenFiles', 'showOverwriteConfirmation'],
+        filters: dialogFilters,
+      });
+      if (filePath) setFieldValue(name, filePath);
+    } else {
+      const filePaths = await electron.showOpenDialog({
+        defaultPath: values[name],
+        properties: dialogProperties ?? ['showHiddenFiles', 'openFile'],
+        filters: dialogFilters,
+      });
+      const filePath = filePaths && filePaths[0];
+      if (filePath) setFieldValue(name, filePath);
+    }
   }
 </script>
 

packages/web/src/icons/FontIcon.svelte

@@ -208,6 +208,7 @@
     'img error-inv': 'mdi mdi-close-circle color-icon-inv-red',
     'img warn': 'mdi mdi-alert color-icon-gold',
     'img info': 'mdi mdi-information color-icon-blue',
+    'img debug': 'mdi mdi-monitor color-icon-green',
     // 'img statusbar-ok': 'mdi mdi-check-circle color-on-statusbar-green',
     'img circular': 'mdi mdi-circular-saw color-icon-red',
 
@@ -283,6 +284,7 @@
     'img duplicator': 'mdi mdi-content-duplicate color-icon-green',
     'img import': 'mdi mdi-database-import color-icon-green',
     'img export': 'mdi mdi-database-export color-icon-green',
+    'img transform': 'mdi mdi-rotate-orbit color-icon-blue',
   };
 </script>
 

packages/web/src/impexp/FormSchemaSelect.svelte

@@ -6,9 +6,10 @@
 
   export let conidName;
   export let databaseName;
+  export let allowAllSchemas = false;
 
   const { values } = getFormContext();
-  $: schemaList = useSchemaList({ conid: $values[conidName], database: values[databaseName] });
+  $: schemaList = useSchemaList({ conid: $values[conidName], database: $values[databaseName] });
 
   $: schemaOptions = (_.isArray($schemaList) ? $schemaList : []).map(schema => ({
     value: schema.schemaName,
@@ -17,5 +18,8 @@
 </script>
 
 {#if schemaOptions.length > 0}
-  <FormSelectField {...$$restProps} options={schemaOptions} />
+  <FormSelectField
+    {...$$restProps}
+    options={allowAllSchemas ? [{ value: '__all', label: '(All schemas)' }, ...schemaOptions] : schemaOptions}
+  />
 {/if}

packages/web/src/impexp/createImpExpScript.ts

@@ -39,7 +39,10 @@ export function extractShellConnection(connection, database) {
 
   return config.allowShellConnection
     ? {
-        ..._.omit(connection, ['_id', 'displayName', 'databases', 'connectionColor']),
+        ..._.omitBy(
+          _.omit(connection, ['_id', 'displayName', 'databases', 'connectionColor', 'status', 'unsaved']),
+          v => !v
+        ),
         database,
       }
     : {
@@ -192,7 +195,7 @@ export function normalizeExportColumnMap(colmap) {
   return null;
 }
 
-export default async function createImpExpScript(extensions, values,  forceScript = false) {
+export default async function createImpExpScript(extensions, values, forceScript = false) {
   const config = getCurrentConfig();
   const script =
     config.allowShellScripting || forceScript

packages/web/src/jsontree/JSONNested.svelte

@@ -4,20 +4,30 @@
   import JSONNode from './JSONNode.svelte';
   import JSONKey from './JSONKey.svelte';
 
-  export let key, keys, colon = ':', label = '', isParentExpanded, isParentArray, isArray = false, bracketOpen, bracketClose;
+  export let key,
+    keys,
+    colon = ':',
+    label = '',
+    isParentExpanded,
+    isParentArray,
+    isArray = false,
+    bracketOpen,
+    bracketClose;
   export let previewKeys = keys;
   export let getKey = key => key;
   export let getValue = key => key;
   export let getPreviewValue = getValue;
-  export let expanded = false, expandable = true;
+  export let expanded = false,
+    expandable = true;
   export let elementValue = null;
+  export let onRootExpandedChanged = null;
 
   const context = getContext('json-tree-context-key');
-  setContext('json-tree-context-key', { ...context, colon })
-  const elementData=getContext('json-tree-element-data');
+  setContext('json-tree-context-key', { ...context, colon });
+  const elementData = getContext('json-tree-element-data');
   const slicedKeyCount = getContext('json-tree-sliced-key-count');
 
-  $: slicedKeys = expanded ? keys: previewKeys.slice(0, slicedKeyCount || 5);
+  $: slicedKeys = expanded ? keys : previewKeys.slice(0, slicedKeyCount || 5);
 
   $: if (!isParentExpanded) {
     expanded = false;
@@ -25,6 +35,9 @@
 
   function toggleExpand() {
     expanded = !expanded;
+    if (onRootExpandedChanged) {
+      onRootExpandedChanged(expanded);
+    }
   }
 
   function expand() {
@@ -34,10 +47,41 @@
   let domElement;
 
   $: if (domElement && elementData && elementValue) {
-    elementData.set(domElement, elementValue)
+    elementData.set(domElement, elementValue);
   }
-
 </script>
+
+<li class:indent={isParentExpanded} class:jsonValueHolder={!!elementValue} bind:this={domElement}>
+  <label>
+    {#if expandable && isParentExpanded}
+      <JSONArrow on:click={toggleExpand} {expanded} />
+    {/if}
+    <JSONKey {key} colon={context.colon} {isParentExpanded} {isParentArray} on:click={toggleExpand} />
+    <span on:click={toggleExpand}><span>{label}</span>{bracketOpen}</span>
+  </label>
+  {#if isParentExpanded}
+    <ul class:collapse={!expanded} on:click={expand}>
+      {#each slicedKeys as key, index}
+        <JSONNode
+          key={getKey(key)}
+          isParentExpanded={expanded}
+          isParentArray={isArray}
+          value={expanded ? getValue(key) : getPreviewValue(key)}
+        />
+        {#if !expanded && index < previewKeys.length - 1}
+          <span class="comma">,</span>
+        {/if}
+      {/each}
+      {#if slicedKeys.length < previewKeys.length}
+        <span>…</span>
+      {/if}
+    </ul>
+  {:else}
+    <span>…</span>
+  {/if}
+  <span>{bracketClose}</span>
+</li>
+
 <style>
   label {
     display: inline-block;
@@ -60,28 +104,3 @@
     position: relative;
   }
 </style>
-<li class:indent={isParentExpanded} class:jsonValueHolder={!!elementValue} bind:this={domElement}>
-  <label>
-    {#if expandable && isParentExpanded}
-      <JSONArrow on:click={toggleExpand} {expanded} />
-    {/if}
-    <JSONKey {key} colon={context.colon} {isParentExpanded} {isParentArray} on:click={toggleExpand} />
-    <span on:click={toggleExpand}><span>{label}</span>{bracketOpen}</span>
-  </label>
-    {#if isParentExpanded}
-      <ul class:collapse={!expanded} on:click={expand}>
-        {#each slicedKeys as key, index}
-          <JSONNode key={getKey(key)} isParentExpanded={expanded} isParentArray={isArray} value={expanded ? getValue(key) : getPreviewValue(key)} />
-          {#if !expanded && index < previewKeys.length - 1}
-            <span class="comma">,</span>
-          {/if}
-        {/each}
-        {#if slicedKeys.length < previewKeys.length }
-          <span>…</span>
-        {/if}
-      </ul>
-    {:else}
-      <span>…</span>
-    {/if}
-  <span>{bracketClose}</span>
-</li>

packages/web/src/jsontree/JSONNode.svelte

@@ -15,6 +15,7 @@
   export let isParentArray;
   export let expanded = !!getContext('json-tree-default-expanded');
   export let labelOverride = null;
+  export let onRootExpandedChanged = null;
 
   $: nodeType = objType(value);
   $: componentType = getComponent(nodeType);
@@ -79,4 +80,5 @@
   {valueGetter}
   {expanded}
   {labelOverride}
+  {onRootExpandedChanged}
 />

packages/web/src/jsontree/JSONObjectNode.svelte

@@ -4,6 +4,7 @@
   export let key, value, isParentExpanded, isParentArray, nodeType;
   export let expanded = false;
   export let labelOverride = null;
+  export let onRootExpandedChanged = null;
 
   $: keys = Object.getOwnPropertyNames(value);
 
@@ -24,4 +25,5 @@
   bracketOpen={'{'}
   bracketClose={'}'}
   elementValue={value}
+  {onRootExpandedChanged}
 />

packages/web/src/jsontree/JSONTree.svelte

@@ -17,6 +17,7 @@
   export let labelOverride = null;
   export let slicedKeyCount = null;
   export let disableContextMenu = null;
+  export let onRootExpandedChanged = null;
 
   export let isDeleted = false;
   export let isInserted = false;
@@ -66,7 +67,15 @@
   class:isInserted
   class:isModified
 >
-  <JSONNode {key} {value} isParentExpanded={true} isParentArray={false} {expanded} {labelOverride} />
+  <JSONNode
+    {key}
+    {value}
+    isParentExpanded={true}
+    isParentArray={false}
+    {expanded}
+    {labelOverride}
+    {onRootExpandedChanged}
+  />
 </ul>
 
 <style>

packages/web/src/modals/ChooseArchiveFolderModal.svelte

@@ -0,0 +1,33 @@
+<script lang="ts">
+  import FormStyledButton from '../buttons/FormStyledButton.svelte';
+  import FormArchiveFolderSelect from '../forms/FormArchiveFolderSelect.svelte';
+
+  import FormProvider from '../forms/FormProvider.svelte';
+  import FormSubmit from '../forms/FormSubmit.svelte';
+  import ModalBase from './ModalBase.svelte';
+  import { closeCurrentModal } from './modalTools';
+
+  export let message = '';
+  export let onConfirm;
+</script>
+
+<FormProvider>
+  <ModalBase {...$$restProps}>
+    <svelte:fragment slot="header">Choose archive folder</svelte:fragment>
+
+    <div>{message}</div>
+
+    <FormArchiveFolderSelect label="Archive folder" name="archiveFolder" isNative />
+
+    <svelte:fragment slot="footer">
+      <FormSubmit
+        value="OK"
+        on:click={e => {
+          closeCurrentModal();
+          onConfirm(e.detail.archiveFolder);
+        }}
+      />
+      <FormStyledButton type="button" value="Close" on:click={closeCurrentModal} />
+    </svelte:fragment>
+  </ModalBase>
+</FormProvider>

packages/web/src/modals/ExportDbModelModal.svelte

@@ -0,0 +1,5 @@
+<script lang="ts">
+  import ModalBase from './ModalBase.svelte';
+</script>
+
+<ModalBase {...$$restProps}></ModalBase>

packages/web/src/modals/RunScriptModal.svelte

@@ -67,7 +67,12 @@
     </svelte:fragment>
 
     <div class="messages">
-      <SocketMessageView eventName={runid ? `runner-info-${runid}` : null} {executeNumber} showNoMessagesAlert />
+      <SocketMessageView
+        eventName={runid ? `runner-info-${runid}` : null}
+        {executeNumber}
+        showNoMessagesAlert
+        showCaller
+      />
     </div>
 
     <svelte:fragment slot="footer">

packages/web/src/plugins/PluginsProvider.svelte

@@ -42,13 +42,17 @@
     return res;
   }
 
+  function filterByEdition(arr) {
+    return arr.filter(x => !x.premiumOnly || isProApp());
+  }
+
   export function buildExtensions(plugins) {
     const extensions = {
       plugins,
-      fileFormats: buildFileFormats(plugins),
-      themes: buildThemes(plugins),
-      drivers: buildDrivers(plugins),
-      quickExports: buildQuickExports(plugins),
+      fileFormats: filterByEdition(buildFileFormats(plugins)),
+      themes: filterByEdition(buildThemes(plugins)),
+      drivers: filterByEdition(buildDrivers(plugins)),
+      quickExports: filterByEdition(buildQuickExports(plugins)),
     };
     return extensions;
   }
@@ -63,6 +67,7 @@
   import * as dbgateTools from 'dbgate-tools';
   import * as sqlTree from 'dbgate-sqltree';
   import { apiCall } from '../utility/api';
+  import { isProApp } from '../utility/proTools';
 
   let pluginsDict = {};
   const installedPlugins = useInstalledPlugins();

packages/web/src/query/MessageView.svelte

@@ -1,102 +1,132 @@
-<script lang="ts" context="module">
-  function formatDuration(duration) {
-    if (duration == 0) return '0';
-    if (duration < 1000) {
-      return `${Math.round(duration)} ms`;
-    }
-    if (duration < 10000) {
-      return `${Math.round(duration / 100) / 10} s`;
-    }
-    return `${Math.round(duration / 1000)} s`;
-  }
-</script>
-
 <script lang="ts">
-  import { createEventDispatcher } from 'svelte';
-  import moment from 'moment';
+  import { writable } from 'svelte/store';
+  import MessageViewRow from './MessageViewRow.svelte';
+  import RowsFilterSwitcher from '../forms/RowsFilterSwitcher.svelte';
+  import SearchInput from '../elements/SearchInput.svelte';
+  import { filterName } from 'dbgate-tools';
 
   export let items: any[];
   export let showProcedure = false;
   export let showLine = false;
+  export let showCaller = false;
   export let startLine = 0;
+  export let onMessageClick = null;
+
+  export let filter = '';
 
   $: time0 = items[0] && new Date(items[0].time).getTime();
 
-  const dispatch = createEventDispatcher();
+  // $: console.log('MESSAGE ROWS', items);
+  const values = writable({
+    hideDebug: false,
+    hideInfo: false,
+    hideError: false,
+  });
+
+  function filterRow(row, filter, values) {
+    return (
+      (!filter || filterName(filter, JSON.stringify(row))) &&
+      ((!values.hideDebug && row.severity == 'debug') ||
+        (!values.hideInfo && row.severity == 'info') ||
+        (!values.hideError && row.severity == 'error') ||
+        (!values.hideDebug && !values.hideInfo && !values.hideError))
+    );
+  }
 </script>
 
 <div class="main">
-  <table>
-    <tr>
-      <td class="header">Number</td>
-      <td class="header">Message</td>
-      <td class="header">Time</td>
-      <td class="header">Delta</td>
-      <td class="header">Duration</td>
-      {#if showProcedure}
-        <td class="header">Procedure</td>
-      {/if}
-      {#if showLine}
-        <td class="header">Line</td>
-      {/if}
-    </tr>
-    {#each items as row, index}
-      <tr
-        class:isError={row.severity == 'error'}
-        class:isActive={row.line}
-        on:click={() => dispatch('messageclick', row)}
-      >
-        <td>{index + 1}</td>
-        <td>{row.message}</td>
-        <td>{moment(row.time).format('HH:mm:ss')}</td>
-        <td>{formatDuration(new Date(row.time).getTime() - time0)}</td>
-        <td>
-          {index > 0
-            ? formatDuration(new Date(row.time).getTime() - new Date(items[index - 1].time).getTime())
-            : 'n/a'}</td
-        >
-        {#if showProcedure}
-          <td>{row.procedure || ''}</td>
-        {/if}
-        {#if showLine}
-          <td>{row.line == null ? '' : row.line + 1 + startLine}</td>
-        {/if}
-      </tr>
-    {/each}
-  </table>
+  <div class="topbar">
+    <RowsFilterSwitcher
+      icon="img debug"
+      label="Debug"
+      {values}
+      field="hideDebug"
+      count={items.filter(x => x.severity == 'debug').length}
+    />
+    <RowsFilterSwitcher
+      icon="img info"
+      label="Info"
+      {values}
+      field="hideInfo"
+      count={items.filter(x => x.severity == 'info').length}
+    />
+    <RowsFilterSwitcher
+      icon="img error"
+      label="Error"
+      {values}
+      field="hideError"
+      count={items.filter(x => x.severity == 'error').length}
+    />
+    <SearchInput placeholder="Filter log messages" bind:value={filter} />
+  </div>
+  <div class="tablewrap">
+    <table>
+      <thead>
+        <tr>
+          <td class="header">Number</td>
+          <td class="header">Message</td>
+          <td class="header">Time</td>
+          <td class="header">Delta</td>
+          <td class="header">Duration</td>
+          {#if showProcedure}
+            <td class="header">Procedure</td>
+          {/if}
+          {#if showLine}
+            <td class="header">Line</td>
+          {/if}
+          {#if showCaller}
+            <td class="header">Caller</td>
+          {/if}
+        </tr>
+      </thead>
+      {#each items.filter(row => filterRow(row, filter, $values)) as row, index}
+        <MessageViewRow
+          {row}
+          {index}
+          {showProcedure}
+          {showLine}
+          {showCaller}
+          {time0}
+          {startLine}
+          previousRow={index > 0 ? items[index - 1] : null}
+          {onMessageClick}
+        />
+      {/each}
+    </table>
+  </div>
 </div>
 
 <style>
   .main {
     flex: 1;
     display: flex;
-    position: relative;
-    overflow-y: scroll;
+    flex-direction: column;
     background-color: var(--theme-bg-0);
   }
+  .tablewrap {
+    flex: 1;
+    position: relative;
+    overflow: scroll;
+  }
   table {
     position: absolute;
+    top: 0;
     left: 0;
     right: 0;
-    top: 0;
+    bottom: 0;
     width: 100%;
     border-spacing: 0;
     border-collapse: collapse;
+    overflow: scroll;
   }
-  td.header {
-    text-align: left;
-    border-bottom: 2px solid var(--theme-border);
-    background-color: var(--theme-bg-1);
-    padding: 5px;
+  .topbar {
+    display: flex;
+    width: 100%;
   }
-  td:not(.header) {
-    border-top: 1px solid var(--theme-border);
-    padding: 5px;
-  }
-  tr.isActive:hover {
-    background: var(--theme-bg-2);
-  }
-  tr.isError {
-    color: var(--theme-icon-red);
+  table thead {
+    position: sticky;
+    top: 0;
+    z-index: 100;
+    background: var(--theme-bg-1);
   }
 </style>

packages/web/src/query/MessageViewRow.svelte

@@ -0,0 +1,101 @@
+<script lang="ts" context="module">
+  function formatDuration(duration) {
+    if (duration == 0) return '0';
+    if (duration < 1000) {
+      return `${Math.round(duration)} ms`;
+    }
+    if (duration < 10000) {
+      return `${Math.round(duration / 100) / 10} s`;
+    }
+    return `${Math.round(duration / 1000)} s`;
+  }
+</script>
+
+<script lang="ts">
+  import moment from 'moment';
+  import JSONTree from '../jsontree/JSONTree.svelte';
+  import FontIcon from '../icons/FontIcon.svelte';
+  import { plusExpandIcon } from '../icons/expandIcons';
+
+  export let row;
+  export let index;
+  export let showProcedure = false;
+  export let showLine = false;
+  export let showCaller = false;
+  export let time0;
+  export let startLine;
+
+  export let previousRow = null;
+  export let onMessageClick = null;
+
+  let isExpanded = false;
+</script>
+
+<tr
+  class:isError={row.severity == 'error'}
+  class:isDebug={row.severity == 'debug'}
+  class:isActive={row.line}
+  on:click={() => onMessageClick?.(row)}
+>
+  <td>{index + 1}</td>
+  <td>
+    <span on:click={() => (isExpanded = !isExpanded)} class="expand-button">
+      <FontIcon icon={plusExpandIcon(isExpanded)} />
+    </span>
+    {row.message}
+  </td>
+  <td>{moment(row.time).format('HH:mm:ss')}</td>
+  <td>{formatDuration(new Date(row.time).getTime() - time0)}</td>
+  <td> {previousRow ? formatDuration(new Date(row.time).getTime() - new Date(previousRow.time).getTime()) : 'n/a'}</td>
+  {#if showProcedure}
+    <td>{row.procedure || ''}</td>
+  {/if}
+  {#if showLine}
+    <td>{row.line == null ? '' : row.line + 1 + startLine}</td>
+  {/if}
+  {#if showCaller}
+    <td>{row.caller || ''}</td>
+  {/if}
+</tr>
+{#if isExpanded}
+  <tr>
+    <td colspan={5 + (showProcedure ? 1 : 0) + (showLine ? 1 : 0) + (showCaller ? 1 : 0)}>
+      <JSONTree
+        value={row}
+        expanded
+        onRootExpandedChanged={() => {
+          isExpanded = false;
+        }}
+      />
+    </td>
+  </tr>
+{/if}
+
+<style>
+  .expand-button {
+    cursor: pointer;
+  }
+
+  td.header {
+    text-align: left;
+    border-bottom: 2px solid var(--theme-border);
+    background-color: var(--theme-bg-1);
+    padding: 5px;
+  }
+  td:not(.header) {
+    border-top: 1px solid var(--theme-border);
+    padding: 5px;
+  }
+  tr.isActive {
+    cursor: pointer;
+  }
+  tr.isActive:hover {
+    background: var(--theme-bg-2);
+  }
+  tr.isError {
+    color: var(--theme-icon-red);
+  }
+  tr.isDebug {
+    color: var(--theme-font-3);
+  }
+</style>

packages/web/src/query/RunnerOutputPane.svelte

@@ -11,7 +11,12 @@
 <HorizontalSplitter>
   <div class="container" slot="1">
     <WidgetTitle>Messages</WidgetTitle>
-    <SocketMessageView eventName={runnerId ? `runner-info-${runnerId}` : null} {executeNumber} showNoMessagesAlert />
+    <SocketMessageView
+      eventName={runnerId ? `runner-info-${runnerId}` : null}
+      {executeNumber}
+      showNoMessagesAlert
+      showCaller
+    />
   </div>
   <div class="container" slot="2">
     <WidgetTitle>Output files</WidgetTitle>

packages/web/src/query/SocketMessageView.svelte

@@ -10,11 +10,13 @@
 
   export let showProcedure = false;
   export let showLine = false;
+  export let showCaller = false;
   export let eventName;
   export let executeNumber;
   export let showNoMessagesAlert = false;
   export let startLine = 0;
   export let onChangeErrors = null;
+  export let onMessageClick = null;
 
   const cachedMessagesRef = createRef([]);
   const lastErrorMessageCountRef = createRef(0);
@@ -68,5 +70,5 @@
 {#if showNoMessagesAlert && (!displayedMessages || displayedMessages.length == 0)}
   <ErrorInfo message="No messages" icon="img alert" />
 {:else}
-  <MessageView items={displayedMessages} on:messageclick {showProcedure} {showLine} {startLine} />
+  <MessageView items={displayedMessages} {onMessageClick} {showProcedure} {showLine} {showCaller} {startLine} />
 {/if}

packages/web/src/tableeditor/IndexEditorModal.svelte

@@ -2,21 +2,26 @@
   import CheckboxField from '../forms/CheckboxField.svelte';
   import FormCheckboxField from '../forms/FormCheckboxField.svelte';
   import SelectField from '../forms/SelectField.svelte';
+  import TextField from '../forms/TextField.svelte';
 
   import ColumnsConstraintEditorModal from './ColumnsConstraintEditorModal.svelte';
 
   export let constraintInfo;
   export let setTableInfo;
   export let tableInfo;
+  export let driver;
 
   let isUnique = constraintInfo?.isUnique;
 
   function getExtractConstraintProps() {
     return {
       isUnique,
+      filterDefinition,
     };
   }
 
+  let filterDefinition = constraintInfo?.filterDefinition;
+
   $: isReadOnly = !setTableInfo;
 </script>
 
@@ -60,6 +65,22 @@
         index
       </div>
     </div>
+
+    <div class="largeFormMarker">
+      {#if driver?.dialect?.filteredIndexes}
+        <div class="row">
+          <div class="label col-3">Filtered index condition</div>
+          <div class="col-9">
+            <TextField
+              value={filterDefinition}
+              on:input={e => (filterDefinition = e.target['value'])}
+              focused
+              disabled={isReadOnly}
+            />
+          </div>
+        </div>
+      {/if}
+    </div>
   </svelte:fragment>
 </ColumnsConstraintEditorModal>
 

packages/web/src/tableeditor/TableEditor.svelte

@@ -141,6 +141,7 @@
       setTableInfo,
       tableInfo,
       dbInfo,
+      driver,
     });
   }
 
@@ -286,7 +287,7 @@
       title={`Indexes (${indexes?.length || 0})`}
       emptyMessage={isWritable ? 'No index defined' : null}
       clickable
-      on:clickrow={e => showModal(IndexEditorModal, { constraintInfo: e.detail, tableInfo, setTableInfo })}
+      on:clickrow={e => showModal(IndexEditorModal, { constraintInfo: e.detail, tableInfo, setTableInfo, driver })}
       columns={[
         {
           fieldName: 'columns',

packages/web/src/tabs/CompareModelTab.svelte

@@ -1,609 +0,0 @@
-<script lang="ts" context="module">
-  const getCurrentEditor = () => getActiveComponent('CompareModelTab');
-
-  registerCommand({
-    id: 'compareModels.reportDiff',
-    category: 'Compare models',
-    toolbarName: 'Report',
-    name: 'Report diff',
-    icon: 'icon report',
-    toolbar: true,
-    isRelatedToTab: true,
-    onClick: () => getCurrentEditor().showReport(),
-    testEnabled: () => getCurrentEditor() != null,
-  });
-
-  registerCommand({
-    id: 'compareModels.swap',
-    category: 'Compare models',
-    toolbarName: 'Swap',
-    name: 'Swap source & target',
-    icon: 'icon swap',
-    toolbar: true,
-    isRelatedToTab: true,
-    onClick: () => getCurrentEditor().swap(),
-    testEnabled: () => getCurrentEditor() != null,
-  });
-
-  registerCommand({
-    id: 'compareModels.deploy',
-    category: 'Compare models',
-    toolbarName: 'Deploy',
-    name: 'Deploy',
-    icon: 'icon deploy',
-    group: 'save',
-    toolbar: true,
-    isRelatedToTab: true,
-    onClick: () => getCurrentEditor().deploy(),
-    testEnabled: () => getCurrentEditor() != null,
-  });
-
-  registerCommand({
-    id: 'compareModels.refresh',
-    category: 'Compare models',
-    toolbarName: 'Refresh',
-    name: 'Refresh models',
-    icon: 'icon reload',
-    toolbar: true,
-    isRelatedToTab: true,
-    onClick: () => getCurrentEditor().refreshModels(),
-    testEnabled: () => getCurrentEditor() != null,
-  });
-
-  function stateOrder(state) {
-    switch (state) {
-      case 'added':
-        return 1;
-      case 'changed':
-        return 2;
-      case 'removed':
-        return 3;
-      case 'equal':
-        return 4;
-    }
-    return 5;
-  }
-
-  function getAlterObjectScript(objectTypeField, oldObject, newObject, opts, db, driver) {
-    if ((!oldObject && !newObject) || !driver) {
-      return { sql: '' };
-    }
-
-    if (objectTypeField == 'tables') {
-      return getAlterTableScript(oldObject, newObject, opts, db, db, driver);
-    }
-    const dmp = driver.createDumper();
-    if (oldObject) dmp.dropSqlObject(oldObject);
-    if (newObject) dmp.createSqlObject(newObject);
-    return { sql: dmp.s };
-  }
-
-  function filterDiffRowsByFlag(rows, values, skip = null) {
-    let res = rows;
-
-    if (skip != 'added') {
-      res = res.filter(row => !values?.hideAdded || row.state != 'added');
-    }
-    if (skip != 'removed') {
-      res = res.filter(row => !values?.hideRemoved || row.state != 'removed');
-    }
-    if (skip != 'changed') {
-      res = res.filter(row => !values?.hideChanged || row.state != 'changed');
-    }
-    if (skip != 'equal') {
-      res = res.filter(row => !values?.hideEqual || row.state != 'equal');
-    }
-
-    for (const objectTypeField of _.keys(DbDiffCompareDefs)) {
-      if (skip == objectTypeField) {
-        continue;
-      }
-      if (values && values[`hide_${objectTypeField}`]) {
-        res = res.filter(row => row.objectTypeField != objectTypeField);
-      }
-    }
-
-    return res;
-  }
-
-  function filterDiffRows(rows, values, filter) {
-    let res = rows.filter(row => filterName(filter, row.sourcePureName, row.targetPureName));
-
-    res = filterDiffRowsByFlag(rows, values);
-
-    return res;
-  }
-</script>
-
-<script lang="ts">
-  import {
-    findEngineDriver,
-    generateDbPairingId,
-    getAlterTableScript,
-    matchPairedObjects,
-    computeDbDiffRows,
-    computeTableDiffColumns,
-    getCreateObjectScript,
-    modelCompareDbDiffOptions,
-    filterName,
-    DbDiffCompareDefs,
-    getAlterDatabaseScript,
-    DatabaseAnalyser,
-  } from 'dbgate-tools';
-
-  import _, { startsWith } from 'lodash';
-  import { derived, writable } from 'svelte/store';
-  import registerCommand from '../commands/registerCommand';
-  import DiffView from '../elements/DiffView.svelte';
-  import InlineButton from '../buttons/InlineButton.svelte';
-  import ScrollableTableControl from '../elements/ScrollableTableControl.svelte';
-  import SearchInput from '../elements/SearchInput.svelte';
-  import TabControl from '../elements/TabControl.svelte';
-  import TableControl from '../elements/TableControl.svelte';
-  import VerticalSplitter from '../elements/VerticalSplitter.svelte';
-  import FormFieldTemplateTiny from '../forms/FormFieldTemplateTiny.svelte';
-  import FormProviderCore from '../forms/FormProviderCore.svelte';
-  import FormSelectField from '../forms/FormSelectField.svelte';
-  import RowsFilterSwitcher from '../forms/RowsFilterSwitcher.svelte';
-  import FontIcon from '../icons/FontIcon.svelte';
-  import FormConnectionSelect from '../impexp/FormConnectionSelect.svelte';
-  import FormDatabaseSelect from '../impexp/FormDatabaseSelect.svelte';
-  import ConfirmSqlModal from '../modals/ConfirmSqlModal.svelte';
-  import ErrorMessageModal from '../modals/ErrorMessageModal.svelte';
-  import { showModal } from '../modals/modalTools';
-  import SqlEditor from '../query/SqlEditor.svelte';
-  import useEditorData from '../query/useEditorData';
-  import { extensions } from '../stores';
-  import { apiCall } from '../utility/api';
-  import { changeTab } from '../utility/common';
-  import contextMenu, { getContextMenu, registerMenu } from '../utility/contextMenu';
-  import createActivator, { getActiveComponent } from '../utility/createActivator';
-  import { saveFileToDisk } from '../utility/exportFileTools';
-  import { useArchiveFolders, useConnectionInfo, useDatabaseInfo } from '../utility/metadataLoaders';
-  import resolveApi from '../utility/resolveApi';
-  import { showSnackbarSuccess } from '../utility/snackbar';
-  import ToolStripContainer from '../buttons/ToolStripContainer.svelte';
-  import ToolStripCommandButton from '../buttons/ToolStripCommandButton.svelte';
-
-  export let tabid;
-
-  let pairIndex = 0;
-  let filter = '';
-
-  export const activator = createActivator('CompareModelTab', true);
-
-  $: dbDiffOptions = $values?.sourceConid == '__model' ? modelCompareDbDiffOptions : {};
-
-  $: sourceDbValue = useDatabaseInfo({ conid: $values?.sourceConid, database: $values?.sourceDatabase });
-  $: targetDbValue = useDatabaseInfo({ conid: $values?.targetConid, database: $values?.targetDatabase });
-
-  // $: console.log('$sourceDbValue', $sourceDbValue);
-  // $: console.log('$targetDbValue', $targetDbValue);
-
-  $: sourceDb = generateDbPairingId($sourceDbValue);
-  $: targetDb = generateDbPairingId($targetDbValue);
-
-  $: connection = useConnectionInfo({ conid: $values?.targetConid });
-  $: driver = findEngineDriver($connection, $extensions);
-
-  // $: console.log('sourceDb', sourceDb);
-  // $: console.log('targetDb', targetDb);
-  // $: console.log('$connection', $connection);
-  // $: console.log('$extensions', $extensions);
-  // $: console.log('driver', driver);
-
-  $: targetDbPaired = matchPairedObjects(sourceDb, targetDb, dbDiffOptions);
-  $: diffRowsAll = _.sortBy(computeDbDiffRows(sourceDb, targetDbPaired, dbDiffOptions, driver), x =>
-    stateOrder(x.state)
-  );
-
-  // $: console.log('diffRowsAll', diffRowsAll);
-
-  $: diffRows = filterDiffRows(diffRowsAll, $values, filter);
-  $: diffColumns = computeTableDiffColumns(
-    diffRows[pairIndex]?.source,
-    diffRows[pairIndex]?.target,
-    dbDiffOptions,
-    driver
-  );
-
-  $: sqlPreview = getAlterObjectScript(
-    diffRows[pairIndex]?.objectTypeField,
-    diffRows[pairIndex]?.target,
-    diffRows[pairIndex]?.source,
-    dbDiffOptions,
-    targetDb,
-    driver
-  ).sql;
-
-  $: archiveFolders = useArchiveFolders();
-
-  $: changeTab(tabid, tab => ({
-    ...tab,
-    title: `${$values?.sourceDatabase || '???'}=>${$values?.targetDatabase || '???'}`,
-    props: {
-      ...tab.props,
-      conid: $values?.targetConid,
-      database: $values?.targetDatabase,
-    },
-  }));
-
-  export async function showReport() {
-    saveFileToDisk(async filePath => {
-      await apiCall('database-connections/generate-db-diff-report', {
-        filePath,
-        sourceConid: $values?.sourceConid,
-        sourceDatabase: $values?.sourceDatabase,
-        targetConid: $values?.targetConid,
-        targetDatabase: $values?.targetDatabase,
-      });
-    });
-  }
-
-  export function swap() {
-    $values = {
-      ...$values,
-      sourceConid: $values?.targetConid,
-      sourceDatabase: $values?.targetDatabase,
-      targetConid: $values?.sourceConid,
-      targetDatabase: $values?.sourceDatabase,
-    };
-  }
-
-  function handleCheckAll() {
-    const isAnyChecked = diffRows.some(row => $values[`isChecked_${row.identifier}`]);
-    if (isAnyChecked) {
-      $values = _.omitBy($values, (v, k) => k.startsWith('isChecked_'));
-    } else {
-      $values = {
-        ...$values,
-        ..._.fromPairs(diffRows.filter(row => row.state != 'equal').map(row => [`isChecked_${row.identifier}`, true])),
-      };
-    }
-  }
-
-  export function refreshModels() {
-    apiCall('database-connections/sync-model', {
-      conid: $values?.targetConid,
-      database: $values?.targetDatabase,
-    });
-    apiCall('database-connections/sync-model', {
-      conid: $values?.sourceConid,
-      database: $values?.sourceDatabase,
-    });
-  }
-
-  async function handleConfirmSql(sql) {
-    const conid = $values?.targetConid;
-    const database = $values?.targetDatabase;
-
-    const resp = await apiCall('database-connections/run-script', { conid, database, sql, useTransaction: true });
-    const { errorMessage } = resp || {};
-    if (errorMessage) {
-      showModal(ErrorMessageModal, { title: 'Error when saving', message: errorMessage });
-    } else {
-      $values = _.omitBy($values, (v, k) => k.startsWith('isChecked_'));
-      await apiCall('database-connections/sync-model', { conid, database });
-      showSnackbarSuccess('Saved to database');
-    }
-  }
-
-  function getDeploySql() {
-    const leftDb = DatabaseAnalyser.createEmptyStructure();
-    const rightDb = DatabaseAnalyser.createEmptyStructure();
-
-    for (const diffRow of diffRows.filter(row => $values[`isChecked_${row.identifier}`])) {
-      if (diffRow.source) leftDb[diffRow.objectTypeField].push(diffRow.source);
-      if (diffRow.target) rightDb[diffRow.objectTypeField].push(diffRow.target);
-    }
-
-    return getAlterDatabaseScript(rightDb, leftDb, dbDiffOptions, targetDb, sourceDb, driver).sql;
-    // getAlterDatabaseScript();
-    // return diffRows
-    //   .filter(row => $values[`isChecked_${row.identifier}`])
-    //   .map(row => getAlterTableScript(row?.target, row?.source, dbDiffOptions, sourceDb, targetDb, driver).sql)
-    //   .join('\n');
-  }
-
-  export function deploy() {
-    const sql = getDeploySql();
-    showModal(ConfirmSqlModal, {
-      sql,
-      onConfirm: () => {
-        handleConfirmSql(sql);
-      },
-      engine: driver.engine,
-    });
-  }
-
-  const { editorState, editorValue, setEditorData } = useEditorData({
-    tabid,
-  });
-
-  const values = {
-    ...editorValue,
-    update: setEditorData,
-    set: setEditorData,
-  };
-
-  registerMenu(
-    { command: 'compareModels.deploy' },
-    { divider: true },
-    { command: 'compareModels.refresh' },
-    { command: 'compareModels.swap' },
-    { divider: true },
-    { command: 'compareModels.reportDiff' }
-  );
-
-  const menu = getContextMenu();
-</script>
-
-<ToolStripContainer>
-  <div class="wrapper" use:contextMenu={menu}>
-    <VerticalSplitter>
-      <div slot="1" class="flexcol">
-        <FormProviderCore {values}>
-          <div class="topbar">
-            <div class="col-3">
-              <FormConnectionSelect
-                name="sourceConid"
-                label="Source server"
-                templateProps={{ noMargin: true }}
-                isNative
-                allowChooseModel
-                notSelected
-              />
-            </div>
-            <div class="col-3">
-              {#if $values?.sourceConid == '__model'}
-                <FormSelectField
-                  name="sourceDatabase"
-                  label="Source DB model"
-                  templateProps={{ noMargin: true }}
-                  isNative
-                  options={($archiveFolders || []).map(x => ({ label: x.name, value: `archive:${x.name}` }))}
-                  notSelected
-                />
-              {:else}
-                <FormDatabaseSelect
-                  conidName="sourceConid"
-                  name="sourceDatabase"
-                  label="Source database"
-                  templateProps={{ noMargin: true }}
-                  isNative
-                  notSelected
-                />
-              {/if}
-            </div>
-            <div class="deployButton">
-              <InlineButton on:click={deploy}>
-                <div class="arrow">
-                  <FontIcon icon="icon arrow-right-bold" />
-                </div>
-                Deploy (experimental)
-              </InlineButton>
-            </div>
-            <div class="col-3">
-              <FormConnectionSelect
-                name="targetConid"
-                label="Target server"
-                templateProps={{ noMargin: true }}
-                isNative
-                notSelected
-              />
-            </div>
-            <div class="col-3">
-              <FormDatabaseSelect
-                conidName="targetConid"
-                name="targetDatabase"
-                label="Target database"
-                templateProps={{ noMargin: true }}
-                isNative
-                notSelected
-              />
-            </div>
-          </div>
-          <div class="filters">
-            <SearchInput placeholder="Search tables or objects" bind:value={filter} />
-
-            <RowsFilterSwitcher
-              icon="img add"
-              label="Added"
-              {values}
-              field="hideAdded"
-              count={filterDiffRowsByFlag(
-                diffRowsAll.filter(x => x.state == 'added'),
-                $values,
-                'added'
-              ).length}
-            />
-            <RowsFilterSwitcher
-              icon="img minus"
-              label="Removed"
-              {values}
-              field="hideRemoved"
-              count={filterDiffRowsByFlag(
-                diffRowsAll.filter(x => x.state == 'removed'),
-                $values,
-                'removed'
-              ).length}
-            />
-            <RowsFilterSwitcher
-              icon="img changed"
-              label="Changed"
-              {values}
-              field="hideChanged"
-              count={filterDiffRowsByFlag(
-                diffRowsAll.filter(x => x.state == 'changed'),
-                $values,
-                'changed'
-              ).length}
-            />
-            <RowsFilterSwitcher
-              icon="img equal"
-              label="Equal"
-              {values}
-              field="hideEqual"
-              count={filterDiffRowsByFlag(
-                diffRowsAll.filter(x => x.state == 'equal'),
-                $values,
-                'equal'
-              ).length}
-            />
-
-            {#each _.keys(DbDiffCompareDefs) as objectTypeField}
-              <RowsFilterSwitcher
-                icon={DbDiffCompareDefs[objectTypeField].icon}
-                label={DbDiffCompareDefs[objectTypeField].plural}
-                {values}
-                field={'hide_' + objectTypeField}
-                count={filterDiffRowsByFlag(
-                  diffRowsAll.filter(x => x.objectTypeField == objectTypeField),
-                  $values,
-                  objectTypeField
-                ).length}
-              />
-            {/each}
-          </div>
-        </FormProviderCore>
-
-        <div class="tableWrapper">
-          <ScrollableTableControl
-            rows={diffRows}
-            bind:selectedIndex={pairIndex}
-            selectable
-            disableFocusOutline
-            columns={[
-              { fieldName: 'isChecked', header: '', width: '50px', slot: 1, headerSlot: 2 },
-              { fieldName: 'type', header: 'Type', width: '100px', slot: 3 },
-              { fieldName: 'sourceSchemaName', header: 'Schema' },
-              { fieldName: 'sourcePureName', header: 'Name' },
-              { fieldName: 'state', header: 'Action', width: '100px' },
-              { fieldName: 'targetSchemaName', header: 'Schema' },
-              { fieldName: 'targetPureName', header: 'Name' },
-            ]}
-          >
-            <input
-              type="checkbox"
-              slot="1"
-              let:row
-              disabled={row.state == 'equal'}
-              checked={!!$values[`isChecked_${row['identifier']}`]}
-              on:change={e => {
-                // @ts-ignore
-                $values = { ...$values, [`isChecked_${row.identifier}`]: e.target.checked };
-              }}
-            />
-            <svelte:fragment slot="2">
-              <InlineButton on:click={handleCheckAll}>
-                <FontIcon icon="icon check-all" />
-              </InlineButton>
-            </svelte:fragment>
-            <svelte:fragment slot="3" let:row>
-              <FontIcon icon={row.typeIcon} />
-              {row.typeName}
-            </svelte:fragment>
-          </ScrollableTableControl>
-        </div>
-      </div>
-
-      <svelte:fragment slot="2">
-        <TabControl
-          tabs={[
-            {
-              label: 'DDL',
-              slot: 1,
-            },
-            {
-              label: 'Synchronize script',
-              slot: 2,
-            },
-            {
-              label: 'Columns',
-              slot: 3,
-            },
-          ]}
-        >
-          <svelte:fragment slot="1">
-            <DiffView
-              leftTitle={diffRows[pairIndex]?.target?.pureName}
-              rightTitle={diffRows[pairIndex]?.source?.pureName}
-              leftText={getCreateObjectScript(diffRows[pairIndex]?.target, driver)}
-              rightText={getCreateObjectScript(diffRows[pairIndex]?.source, driver)}
-            />
-          </svelte:fragment>
-
-          <svelte:fragment slot="2">
-            <SqlEditor readOnly value={sqlPreview} />
-          </svelte:fragment>
-
-          <svelte:fragment slot="3">
-            <ScrollableTableControl
-              rows={diffColumns}
-              disableFocusOutline
-              columns={[
-                { fieldName: 'sourceColumnName', header: 'Name', width: '100px' },
-                { fieldName: 'sourceDataType', header: 'Type' },
-                { fieldName: 'sourceNotNull', header: 'Not null', slot: 1 },
-                { fieldName: 'state', header: 'Action', width: '100px' },
-                { fieldName: 'targetColumnName', header: 'Schema' },
-                { fieldName: 'targetDataType', header: 'Name' },
-                { fieldName: 'targetNotNull', header: 'Not null', slot: 2 },
-              ]}
-            >
-              <input type="checkbox" disabled slot="1" let:row checked={!!row.sourceNotNull} />
-              <input type="checkbox" disabled slot="2" let:row checked={!!row.targetNotNull} />
-            </ScrollableTableControl>
-          </svelte:fragment>
-        </TabControl>
-      </svelte:fragment>
-    </VerticalSplitter>
-  </div>
-  <svelte:fragment slot="toolstrip">
-    <ToolStripCommandButton command="compareModels.reportDiff" />
-    <ToolStripCommandButton command="compareModels.swap" />
-    <ToolStripCommandButton command="compareModels.deploy" />
-    <ToolStripCommandButton command="compareModels.refresh" />
-  </svelte:fragment>
-</ToolStripContainer>
-
-<style>
-  .wrapper {
-    overflow: auto;
-    flex: 1;
-  }
-
-  .flexcol {
-    flex: 1;
-    display: flex;
-    flex-direction: column;
-  }
-  .topbar {
-    display: flex;
-    margin: 10px 0px;
-    width: 100%;
-  }
-  .arrow {
-    font-size: 30px;
-    color: var(--theme-icon-blue);
-    align-self: center;
-    position: relative;
-    /* top: 10px; */
-  }
-
-  .deployButton {
-    margin-left: 20px;
-    margin-right: 20px;
-  }
-
-  .tableWrapper {
-    position: relative;
-    width: 100%;
-    flex: 1;
-  }
-
-  .filters {
-    display: flex;
-    flex-wrap: wrap;
-  }
-</style>

packages/web/src/tabs/DataDuplicatorTab.svelte

@@ -22,6 +22,16 @@
     testEnabled: () => getCurrentEditor()?.canKill(),
     onClick: () => getCurrentEditor().kill(),
   });
+  registerCommand({
+    id: 'dataDuplicator.generateScript',
+    category: 'Data duplicator',
+    icon: 'img shell',
+    name: 'Generate Script',
+    toolbar: true,
+    isRelatedToTab: true,
+    testEnabled: () => getCurrentEditor()?.canRun(),
+    onClick: () => getCurrentEditor().generateScript(),
+  });
 </script>
 
 <script lang="ts">
@@ -52,7 +62,6 @@
   import useEffect from '../utility/useEffect';
   import useTimerLabel from '../utility/useTimerLabel';
   import appObjectTypes from '../appobj';
-  import RowHeaderCell from '../datagrid/RowHeaderCell.svelte';
 
   export let conid;
   export let database;
@@ -124,6 +133,7 @@
       options: {
         rollbackAfterFinish: !!$editorState.value?.rollbackAfterFinish,
         skipRowsWithUnresolvedRefs: !!$editorState.value?.skipRowsWithUnresolvedRefs,
+        setNullForUnresolvedNullableRefs: !!$editorState.value?.setNullForUnresolvedNullableRefs,
       },
     });
     return script.getScript();
@@ -145,6 +155,18 @@
     timerLabel.start();
   }
 
+  export async function generateScript() {
+    const code = await createScript();
+    openNewTab(
+      {
+        title: 'Shell #',
+        icon: 'img shell',
+        tabComponent: 'ShellTab',
+      },
+      { editor: code }
+    );
+  }
+
   $: effect = useEffect(() => registerRunnerDone(runnerId));
 
   function registerRunnerDone(rid) {
@@ -286,6 +308,29 @@
               }}
             />
           </FormFieldTemplateLarge>
+
+          <FormFieldTemplateLarge
+            label="Set NULL for nullable unresolved references"
+            type="checkbox"
+            labelProps={{
+              onClick: () => {
+                setEditorData(old => ({
+                  ...old,
+                  setNullForUnresolvedNullableRefs: !$editorState.value?.setNullForUnresolvedNullableRefs,
+                }));
+              },
+            }}
+          >
+            <CheckboxField
+              checked={$editorState.value?.setNullForUnresolvedNullableRefs}
+              on:change={e => {
+                setEditorData(old => ({
+                  ...old,
+                  setNullForUnresolvedNullableRefs: e.target.checked,
+                }));
+              }}
+            />
+          </FormFieldTemplateLarge>
         </ObjectConfigurationControl>
 
         <ObjectConfigurationControl title="Imported files">
@@ -386,13 +431,19 @@
       </div>
     </svelte:fragment>
     <svelte:fragment slot="2">
-      <SocketMessageView eventName={runnerId ? `runner-info-${runnerId}` : null} {executeNumber} showNoMessagesAlert />
+      <SocketMessageView
+        eventName={runnerId ? `runner-info-${runnerId}` : null}
+        {executeNumber}
+        showNoMessagesAlert
+        showCaller
+      />
     </svelte:fragment>
   </VerticalSplitter>
 
   <svelte:fragment slot="toolstrip">
     <ToolStripCommandButton command="dataDuplicator.run" />
     <ToolStripCommandButton command="dataDuplicator.kill" />
+    <ToolStripCommandButton command="dataDuplicator.generateScript" />
   </svelte:fragment>
 </ToolStripContainer>
 

packages/web/src/tabs/ImportExportTab.svelte

@@ -255,6 +255,7 @@
               eventName={runnerId ? `runner-info-${runnerId}` : null}
               {executeNumber}
               showNoMessagesAlert
+              showCaller
             />
           </WidgetColumnBarItem>
           <WidgetColumnBarItem title="Preview" name="preview" skip={!$previewReaderStore}>
@@ -274,7 +275,7 @@
     {:else}
       <ToolStripButton on:click={handleExecute} icon="icon run">Run</ToolStripButton>
     {/if}
-    <ToolStripButton icon="img sql-file" on:click={handleGenerateScript}>Generate script</ToolStripButton>
+    <ToolStripButton icon="img shell" on:click={handleGenerateScript}>Generate script</ToolStripButton>
     <ToolStripSaveButton idPrefix="job" />
   </svelte:fragment>
 </ToolStripContainer>

packages/web/src/tabs/QueryTab.svelte

@@ -415,7 +415,7 @@
   }
 
   let isInitialized = false;
-  let queryParameterStyle = localStorage.getItem(`tabdata_queryParamStyle_${tabid}`) ?? ':';
+  let queryParameterStyle = localStorage.getItem(`tabdata_queryParamStyle_${tabid}`) ?? '';
 </script>
 
 <ToolStripContainer bind:this={domToolStrip}>
@@ -469,7 +469,7 @@
         <svelte:fragment slot="0">
           <SocketMessageView
             eventName={sessionId ? `session-info-${sessionId}` : null}
-            on:messageClick={handleMesageClick}
+            onMessageClick={handleMesageClick}
             {executeNumber}
             startLine={executeStartLine}
             showProcedure

packages/web/src/tabs/TableStructureTab.svelte

@@ -150,7 +150,7 @@
     schemaList={$schemaList}
     {driver}
     {resetCounter}
-    isCreateTable={objectTypeField == 'tables' && !$editorValue?.base}
+    isCreateTable={objectTypeField == 'tables' && $editorValue && !$editorValue?.base}
     setTableInfo={objectTypeField == 'tables' && !$connection?.isReadOnly && hasPermission(`dbops/model/edit`)
       ? tableInfoUpdater =>
           setEditorData(tbl =>

packages/web/src/tabs/index.js

@@ -16,7 +16,6 @@ import * as CommandListTab from './CommandListTab.svelte';
 import * as YamlEditorTab from './YamlEditorTab.svelte';
 import * as JsonEditorTab from './JsonEditorTab.svelte';
 import * as JsonLinesEditorTab from './JsonLinesEditorTab.svelte';
-import * as CompareModelTab from './CompareModelTab.svelte';
 import * as JsonTab from './JsonTab.svelte';
 import * as ChangelogTab from './ChangelogTab.svelte';
 import * as DiagramTab from './DiagramTab.svelte';
@@ -51,7 +50,6 @@ export default {
   YamlEditorTab,
   JsonEditorTab,
   JsonLinesEditorTab,
-  CompareModelTab,
   JsonTab,
   ChangelogTab,
   DiagramTab,

packages/web/src/utility/api.ts

@@ -9,10 +9,10 @@ import { showModal } from '../modals/modalTools';
 import DatabaseLoginModal, { isDatabaseLoginVisible } from '../modals/DatabaseLoginModal.svelte';
 import _ from 'lodash';
 import uuidv1 from 'uuid/v1';
-import { openWebLink } from './exportFileTools';
 import { callServerPing } from './connectionsPinger';
 import { batchDispatchCacheTriggers, dispatchCacheChange } from './cache';
 import { isAdminPage, isOneOfPage } from './pageDefs';
+import { openWebLink } from './simpleTools';
 
 export const strmid = uuidv1();
 

packages/web/src/utility/exportFileTools.ts

@@ -211,16 +211,6 @@ export async function saveFileToDisk(
   }
 }
 
-export function openWebLink(href) {
-  const electron = getElectron();
-
-  if (electron) {
-    electron.send('open-link', href);
-  } else {
-    window.open(href, '_blank');
-  }
-}
-
 export async function downloadFromApi(route: string, donloadName: string) {
   fetch(`${resolveApi()}/${route}`, {
     method: 'GET',
@@ -240,4 +230,3 @@ export async function downloadFromApi(route: string, donloadName: string) {
       });
     });
 }
-

packages/web/src/utility/metadataLoaders.ts

@@ -6,9 +6,9 @@ import { extendDatabaseInfo } from 'dbgate-tools';
 import { setLocalStorage } from '../utility/storageCache';
 import { apiCall, apiOff, apiOn } from './api';
 
-const databaseInfoLoader = ({ conid, database }) => ({
+const databaseInfoLoader = ({ conid, database, modelTransFile }) => ({
   url: 'database-connections/structure',
-  params: { conid, database },
+  params: { conid, database, modelTransFile },
   reloadTrigger: { key: `database-structure-changed`, conid, database },
   transform: extendDatabaseInfo,
 });

packages/web/src/utility/simpleTools.ts

@@ -0,0 +1,11 @@
+import getElectron from './getElectron';
+
+export function openWebLink(href) {
+  const electron = getElectron();
+
+  if (electron) {
+    electron.send('open-link', href);
+  } else {
+    window.open(href, '_blank');
+  }
+}

packages/web/src/widgets/SavedFilesList.svelte

@@ -22,6 +22,7 @@
   const diagramFiles = useFiles({ folder: 'diagrams' });
   const jobFiles = useFiles({ folder: 'jobs' });
   const perspectiveFiles = useFiles({ folder: 'perspectives' });
+  const modelTransformFiles = useFiles({ folder: 'modtrans' });
 
   $: files = [
     ...($sqlFiles || []),
@@ -33,13 +34,30 @@
     ...($diagramFiles || []),
     ...($perspectiveFiles || []),
     ...($jobFiles || []),
+    ...($modelTransformFiles || []),
   ];
 
   function handleRefreshFiles() {
     apiCall('files/refresh', {
-      folders: ['sql', 'shell', 'markdown', 'charts', 'query', 'sqlite', 'diagrams', 'perspectives', 'jobs'],
+      folders: [
+        'sql',
+        'shell',
+        'markdown',
+        'charts',
+        'query',
+        'sqlite',
+        'diagrams',
+        'perspectives',
+        'jobs',
+        'modtrans',
+      ],
     });
   }
+
+  function dataFolderTitle(folder) {
+    if (folder == 'modtrans') return 'Model transforms';
+    return _.startCase(folder);
+  }
 </script>
 
 <WidgetsInnerContainer>
@@ -51,5 +69,5 @@
     </InlineButton>
   </SearchBoxWrapper>
 
-  <AppObjectList list={files} module={savedFileAppObject} groupFunc={data => _.startCase(data.folder)} {filter} />
+  <AppObjectList list={files} module={savedFileAppObject} groupFunc={data => dataFolderTitle(data.folder)} {filter} />
 </WidgetsInnerContainer>

plugins/dbgate-plugin-clickhouse/package.json

@@ -25,14 +25,13 @@
     "prepublishOnly": "yarn build"
   },
   "devDependencies": {
-    "byline": "^5.0.0",
     "dbgate-plugin-tools": "^1.0.8",
-    "dbgate-tools": "^5.0.0-alpha.1",
-    "json-stable-stringify": "^1.0.1",
     "webpack": "^5.91.0",
     "webpack-cli": "^5.1.4"
   },
   "dependencies": {
-    "@clickhouse/client": "^1.5.0"
+    "@clickhouse/client": "^1.5.0",
+    "dbgate-tools": "^5.0.0-alpha.1",
+    "json-stable-stringify": "^1.0.1"
   }
 }

plugins/dbgate-plugin-clickhouse/webpack-backend.config.js

@@ -18,6 +18,12 @@ var config = {
   //   optimization: {
   //     minimize: false,
   //   },
+
+  externals: {
+    '@clickhouse/client': 'commonjs @clickhouse/client',
+    'json-stable-stringify': 'commonjs json-stable-stringify',
+    'dbgate-tools': 'commonjs dbgate-tools',
+  },
 };
 
 module.exports = config;

plugins/dbgate-plugin-csv/package.json

@@ -32,11 +32,13 @@
     "prepublishOnly": "yarn build"
   },
   "devDependencies": {
-    "csv": "^6.3.10",
     "dbgate-plugin-tools": "^1.0.7",
-    "line-reader": "^0.4.0",
-    "lodash": "^4.17.21",
     "webpack": "^5.91.0",
     "webpack-cli": "^5.1.4"
+  },
+  "dependencies": {
+    "csv": "^6.3.10",
+    "line-reader": "^0.4.0",
+    "lodash": "^4.17.21"
   }
-}
+}