oreolek/nmig
1
0
Fork 0
Code Releases Activity

Merge branch 'release/v4-0-0'

Browse source
This commit is contained in:
Anatoly 2019-04-12 18:00:35 +03:00
parent 72a2b65f28 fdf2c0a1f3
commit 562c0a0c28
12 changed files with 220 additions and 166 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -89,7 +89,7 @@ from MySQL to PostgreSQL as easy and smooth as possible.</p>
<b>Note:</b> "logs_directory" will be created during script execution.</p>
<h3>VERSION</h3>
<p>Current version is 3.5.0<br />
<p>Current version is 4.0.0<br />
(major version . improvements . bug fixes)</p>
<h3>KNOWN ISSUES</h3>

21
config/config.json
View file

@ -81,16 +81,23 @@
],
"exclude_tables": [],
"migrate_only_data_description" : [
"In order to skip schema migration, and just migrate data into a preset schema",
" - set this parameter true."
"include_tables_description": [
"List (Array) of tables, that will not be migrated.",
"By default, nmig will migrate all tables."
],
"migrate_only_data" : false,
"include_tables": [],
"migrate_only_data_description" : [
"In order to skip schema migration, and just migrate data into a preset tables",
"fill following array with preset table names.",
"If all of your tables are preset, then use '*'"
],
"migrate_only_data" : [],
"delimiter_description" : [
"Specifies the character, that separates columns within each record.",
"This must be a single one-byte character.",
"The default is comma."
"Specifies the character, that separates columns within each record.",
"This must be a single one-byte character.",
"The default is comma."
],
"delimiter" : ",",

17
config/extra_config.json
View file

@ -24,8 +24,8 @@
"tables" : [
{
"name" : {
"original" : "admins",
"new" : "renamed_admins"
"original" : "salary",
"new" : "renamed_salary"
},
"columns" : [
@ -35,21 +35,16 @@
},
{
"original" : "username",
"new" : "renamed_username"
},
{
"original" : "group_id",
"new" : "renamed_group_id"
"original" : "employee_id",
"new" : "renamed_employee_id"
}
]
},
{
"name" : {
"original" : "groups",
"new" : "renamed_groups"
"original" : "contract",
"new" : "renamed_contract"
},
"columns" : [

197
package-lock.json generated
View file

@ -1,6 +1,6 @@
{
"name": "nmig",
"version": "3.5.0",
"version": "4.0.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
@ -14,15 +14,15 @@
}
},
"@types/node": {
"version": "10.12.0",
"resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.0.tgz",
"integrity": "sha512-3TUHC3jsBAB7qVRGxT6lWyYo2v96BMmD2PTcl47H25Lu7UXtFH/2qqmKiVrnel6Ne//0TFYf6uvNX+HW2FRkLQ==",
"version": "11.13.4",
"resolved": "https://registry.npmjs.org/@types/node/-/node-11.13.4.tgz",
"integrity": "sha512-+rabAZZ3Yn7tF/XPGHupKIL5EcAbrLxnTr/hgQICxbeuAfWtT0UZSfULE+ndusckBItcv4o6ZeOJplQikVcLvQ==",
"dev": true
},
"@types/pg": {
"version": "7.4.11",
"resolved": "https://registry.npmjs.org/@types/pg/-/pg-7.4.11.tgz",
"integrity": "sha512-Eksj2yOBNHnNqLuU1AqwF1qXgdaYDcWVH1ZQlxS7k1i34+JZd/ZNd15ugIpTVQxmBBMqjliJstmrnusjXy08Tg==",
"version": "7.4.14",
"resolved": "https://registry.npmjs.org/@types/pg/-/pg-7.4.14.tgz",
"integrity": "sha512-2e4XapP9V/X42IGByC5IHzCzHqLLCNJid8iZBbkk6lkaDMvli8Rk62YE9wjGcLD5Qr5Zaw1ShkQyXy91PI8C0Q==",
"dev": true,
"requires": {
"@types/node": "*",
@ -39,9 +39,9 @@
}
},
"@types/tape": {
"version": "4.2.32",
"resolved": "https://registry.npmjs.org/@types/tape/-/tape-4.2.32.tgz",
"integrity": "sha512-xil0KO5wkPoixdBWGIGolPv9dekf6dVkjjJLAFYchfKcd4DICou67rgGCIO7wAh3i5Ff/6j9IDgZz+GU9cMaqQ==",
"version": "4.2.33",
"resolved": "https://registry.npmjs.org/@types/tape/-/tape-4.2.33.tgz",
"integrity": "sha512-ltfyuY5BIkYlGuQfwqzTDT8f0q8Z5DGppvUnWGs39oqDmMd6/UWhNpX3ZMh/VYvfxs3rFGHMrLC/eGRdLiDGuw==",
"dev": true,
"requires": {
"@types/node": "*"
@ -69,9 +69,9 @@
}
},
"buffer-writer": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.1.tgz",
"integrity": "sha1-Iqk2kB4wKa/NdUfrRIfOtpejvwg="
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz",
"integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw=="
},
"concat-map": {
"version": "0.0.1",
@ -91,13 +91,12 @@
"dev": true
},
"define-properties": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.2.tgz",
"integrity": "sha1-g6c/L+pWmJj7c3GTyPhzyvbUXJQ=",
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz",
"integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==",
"dev": true,
"requires": {
"foreach": "^2.0.5",
"object-keys": "^1.0.8"
"object-keys": "^1.0.12"
}
},
"defined": {
@ -107,27 +106,28 @@
"dev": true
},
"es-abstract": {
"version": "1.12.0",
"resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.12.0.tgz",
"integrity": "sha512-C8Fx/0jFmV5IPoMOFPA9P9G5NtqW+4cOPit3MIuvR2t7Ag2K15EJTpxnHAYTzL+aYQJIESYeXZmDBfOBE1HcpA==",
"version": "1.13.0",
"resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.13.0.tgz",
"integrity": "sha512-vDZfg/ykNxQVwup/8E1BZhVzFfBxs9NqMzGcvIJrqg5k2/5Za2bWo40dK2J1pgLngZ7c+Shh8lwYtLGyrwPutg==",
"dev": true,
"requires": {
"es-to-primitive": "^1.1.1",
"es-to-primitive": "^1.2.0",
"function-bind": "^1.1.1",
"has": "^1.0.1",
"is-callable": "^1.1.3",
"is-regex": "^1.0.4"
"has": "^1.0.3",
"is-callable": "^1.1.4",
"is-regex": "^1.0.4",
"object-keys": "^1.0.12"
}
},
"es-to-primitive": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.1.1.tgz",
"integrity": "sha1-RTVSSKiJeQNLZ5Lhm7gfK3l13Q0=",
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.0.tgz",
"integrity": "sha512-qZryBOJjV//LaxLTV6UC//WewneB3LcXOL9NP++ozKVXsIIIpm/2c13UDiD9Jp2eThsecw9m3jPqDwTyobcdbg==",
"dev": true,
"requires": {
"is-callable": "^1.1.1",
"is-callable": "^1.1.4",
"is-date-object": "^1.0.1",
"is-symbol": "^1.0.1"
"is-symbol": "^1.0.2"
}
},
"for-each": {
@ -139,12 +139,6 @@
"is-callable": "^1.1.3"
}
},
"foreach": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz",
"integrity": "sha1-C+4AUBiusmDQo6865ljdATbsG5k=",
"dev": true
},
"fs.realpath": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@ -158,9 +152,9 @@
"dev": true
},
"glob": {
"version": "7.1.2",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz",
"integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==",
"version": "7.1.3",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz",
"integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==",
"dev": true,
"requires": {
"fs.realpath": "^1.0.0",
@ -180,6 +174,12 @@
"function-bind": "^1.1.1"
}
},
"has-symbols": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz",
"integrity": "sha1-uhqPGvKg/DllD1yFA2dwQSIGO0Q=",
"dev": true
},
"inflight": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
@ -217,10 +217,13 @@
}
},
"is-symbol": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.1.tgz",
"integrity": "sha1-PMWfAAJRlLarLjjbrmaJJWtmBXI=",
"dev": true
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.2.tgz",
"integrity": "sha512-HS8bZ9ox60yCJLH9snBpIwv9pYUAkcuLhSA1oero1UB5y9aiQpRA8y2ex945AOtCZL1lJDeIk3G5LthswI46Lw==",
"dev": true,
"requires": {
"has-symbols": "^1.0.0"
}
},
"isarray": {
"version": "1.0.0",
@ -243,9 +246,9 @@
"dev": true
},
"moment": {
"version": "2.22.2",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.22.2.tgz",
"integrity": "sha1-PCV/mDn8DpP/UxSWMiOeuQeD/2Y=",
"version": "2.24.0",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz",
"integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==",
"dev": true
},
"mysql": {
@ -266,9 +269,9 @@
"dev": true
},
"object-keys": {
"version": "1.0.12",
"resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.0.12.tgz",
"integrity": "sha512-FTMyFUm2wBcGHnH2eXmz7tC6IwlqQZ6mVZ+6dm6vZ4IQIHjs6FdNsQBuKGPuUUUY6NfJw2PshC08Tn6LzLDOag==",
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz",
"integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==",
"dev": true
},
"once": {
@ -281,9 +284,9 @@
}
},
"packet-reader": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-0.3.1.tgz",
"integrity": "sha1-zWLmCvjX/qinBexP+ZCHHEaHHyc="
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz",
"integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ=="
},
"path-is-absolute": {
"version": "1.0.1",
@ -292,21 +295,21 @@
"dev": true
},
"path-parse": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.5.tgz",
"integrity": "sha1-PBrfhx6pzWyUMbbqK9dKD/BVxME=",
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz",
"integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==",
"dev": true
},
"pg": {
"version": "7.5.0",
"resolved": "https://registry.npmjs.org/pg/-/pg-7.5.0.tgz",
"integrity": "sha512-VFyAnp8xsMZp8nwZnMp7lmU5QcWDOZSI3IDNcWv6pblsiOXis5o7lD7/zzVK1Z1JTBiIDDGQAMbFMkiUzCL59A==",
"version": "7.9.0",
"resolved": "https://registry.npmjs.org/pg/-/pg-7.9.0.tgz",
"integrity": "sha512-GkzteBFpsIoIBCSuomqik3IGvhqAtTr32jclR24RmUg170Jrn6ypwR97YalFHrsE1iaW8T0aAH13dmij8QUQ0g==",
"requires": {
"buffer-writer": "1.0.1",
"packet-reader": "0.3.1",
"buffer-writer": "2.0.0",
"packet-reader": "1.0.0",
"pg-connection-string": "0.1.3",
"pg-pool": "~2.0.3",
"pg-types": "~1.12.1",
"pg-pool": "^2.0.4",
"pg-types": "~2.0.0",
"pgpass": "1.x",
"semver": "4.3.2"
}
@ -317,23 +320,29 @@
"integrity": "sha1-2hhHsglA5C7hSSvq9l1J2RskXfc="
},
"pg-copy-streams": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/pg-copy-streams/-/pg-copy-streams-1.2.0.tgz",
"integrity": "sha1-ez+d7gtsX8IGj1nED6IY4MHXQkk="
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/pg-copy-streams/-/pg-copy-streams-2.2.0.tgz",
"integrity": "sha512-w1gy/KAD49vaCCUfqcjh4fy1V4TF6fvtZ+EaRAU0QKtADMxihb+qYRgbWTDbi6Lh6v31CsWC0ru8vCDHofrRQQ=="
},
"pg-int8": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
"integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw=="
},
"pg-pool": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.3.tgz",
"integrity": "sha1-wCIDLIlJ8xKk+R+2QJzgQHa+Mlc="
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.6.tgz",
"integrity": "sha512-hod2zYQxM8Gt482q+qONGTYcg/qVcV32VHVPtktbBJs0us3Dj7xibISw0BAAXVMCzt8A/jhfJvpZaxUlqtqs0g=="
},
"pg-types": {
"version": "1.12.1",
"resolved": "https://registry.npmjs.org/pg-types/-/pg-types-1.12.1.tgz",
"integrity": "sha1-1kCH45A7WP+q0nnnWVxSIIoUw9I=",
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.0.1.tgz",
"integrity": "sha512-b7y6QM1VF5nOeX9ukMQ0h8a9z89mojrBHXfJeSug4mhL0YpxNBm83ot2TROyoAmX/ZOX3UbwVO4EbH7i1ZZNiw==",
"requires": {
"postgres-array": "~1.0.0",
"pg-int8": "1.0.1",
"postgres-array": "~2.0.0",
"postgres-bytea": "~1.0.0",
"postgres-date": "~1.0.0",
"postgres-date": "~1.0.4",
"postgres-interval": "^1.1.0"
}
},
@ -346,9 +355,9 @@
}
},
"postgres-array": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-1.0.3.tgz",
"integrity": "sha512-5wClXrAP0+78mcsNX3/ithQ5exKvCyK5lr5NEEEeGwwM6NJdQgzIJBVxLvRW+huFpX92F2QnZ5CcokH0VhK2qQ=="
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
"integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA=="
},
"postgres-bytea": {
"version": "1.0.0",
@ -356,14 +365,14 @@
"integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU="
},
"postgres-date": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.3.tgz",
"integrity": "sha1-4tiXAu/bJY/52c7g/pG9BpdSV6g="
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.4.tgz",
"integrity": "sha512-bESRvKVuTrjoBluEcpv2346+6kgB7UlnqWZsnbnCccTNq/pqfj1j6oBaN5+b/NrDXepYUT/HKadqv3iS9lJuVA=="
},
"postgres-interval": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.1.2.tgz",
"integrity": "sha512-fC3xNHeTskCxL1dC8KOtxXt7YeFmlbTYtn7ul8MkVERuTmf7pI4DrkAxcw3kh1fQ9uz4wQmd03a1mRiXUZChfQ==",
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
"integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
"requires": {
"xtend": "^4.0.0"
}
@ -388,12 +397,12 @@
}
},
"resolve": {
"version": "1.7.1",
"resolved": "https://registry.npmjs.org/resolve/-/resolve-1.7.1.tgz",
"integrity": "sha512-c7rwLofp8g1U+h1KNyHL/jicrKg1Ek4q+Lr33AL65uZTinUZHe30D5HlyN5V9NW0JX1D5dXQ4jqW5l7Sy/kGfw==",
"version": "1.10.0",
"resolved": "https://registry.npmjs.org/resolve/-/resolve-1.10.0.tgz",
"integrity": "sha512-3sUr9aq5OfSg2S9pNtPA9hL1FVEAjvfOC4leW0SNf/mpnaakz2a9femSd6LqAww2RaFctwyf1lCqnTHuF1rxDg==",
"dev": true,
"requires": {
"path-parse": "^1.0.5"
"path-parse": "^1.0.6"
}
},
"resumer": {
@ -448,21 +457,21 @@
}
},
"tape": {
"version": "4.9.1",
"resolved": "https://registry.npmjs.org/tape/-/tape-4.9.1.tgz",
"integrity": "sha512-6fKIXknLpoe/Jp4rzHKFPpJUHDHDqn8jus99IfPnHIjyz78HYlefTGD3b5EkbQzuLfaEvmfPK3IolLgq2xT3kw==",
"version": "4.10.1",
"resolved": "https://registry.npmjs.org/tape/-/tape-4.10.1.tgz",
"integrity": "sha512-G0DywYV1jQeY3axeYnXUOt6ktnxS9OPJh97FGR3nrua8lhWi1zPflLxcAHavZ7Jf3qUfY7cxcVIVFa4mY2IY1w==",
"dev": true,
"requires": {
"deep-equal": "~1.0.1",
"defined": "~1.0.0",
"for-each": "~0.3.3",
"function-bind": "~1.1.1",
"glob": "~7.1.2",
"glob": "~7.1.3",
"has": "~1.0.3",
"inherits": "~2.0.3",
"minimist": "~1.2.0",
"object-inspect": "~1.6.0",
"resolve": "~1.7.1",
"resolve": "~1.10.0",
"resumer": "~0.0.0",
"string.prototype.trim": "~1.1.2",
"through": "~2.3.8"
@ -474,9 +483,9 @@
"integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
},
"typescript": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.1.3.tgz",
"integrity": "sha512-+81MUSyX+BaSo+u2RbozuQk/UWx6hfG0a5gHu4ANEM4sU96XbuIyAB+rWBW1u70c6a5QuZfuYICn3s2UjuHUpA==",
"version": "3.4.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.4.3.tgz",
"integrity": "sha512-FFgHdPt4T/duxx6Ndf7hwgMZZjZpB+U0nMNGVCYPq0rEzWKjEDobm4J6yb3CS7naZ0yURFqdw9Gwc7UOh/P9oQ==",
"dev": true
},
"util-deprecate": {

16
package.json
View file

@ -1,6 +1,6 @@
{
"name": "nmig",
"version": "3.5.0",
"version": "4.0.0",
"description": "The database migration app",
"author": "Anatoly Khaytovich<anatolyuss@gmail.com>",
"license": "GPL-3.0",
@ -13,16 +13,16 @@
},
"dependencies": {
"mysql": "^2.16.0",
"pg": "^7.5.0",
"pg-copy-streams": "^1.2.0"
"pg": "^7.9.0",
"pg-copy-streams": "^2.2.0"
},
"devDependencies": {
"@types/mysql": "^2.15.5",
"@types/node": "^10.12.0",
"@types/pg": "^7.4.11",
"@types/tape": "^4.2.32",
"tape": "^4.9.1",
"typescript": "^3.1.3"
"@types/node": "^11.13.4",
"@types/pg": "^7.4.14",
"@types/tape": "^4.2.33",
"tape": "^4.10.1",
"typescript": "^3.4.3"
},
"scripts": {
"build": "tsc",

38
src/ConstraintsProcessor.ts
View file

@ -34,28 +34,18 @@ import { dropDataChunkIdColumn } from './ConsistencyEnforcer';
import Conversion from './Conversion';
/**
* Continues migration process after data loading, when migrate_only_data is true.
* Continues migration process after data loading.
*/
async function continueProcessAfterDataLoadingShort(conversion: Conversion): Promise<void> {
const promises: Promise<void>[] = conversion._tablesToMigrate.map(async (tableName: string) => {
await dropDataChunkIdColumn(conversion, tableName);
return sequencesProcessor.setSequenceValue(conversion, tableName);
});
await Promise.all(promises);
await dataPoolManager.dropDataPoolTable(conversion);
await runVacuumFullAndAnalyze(conversion);
await migrationStateManager.dropStateLogsTable(conversion);
generateReport(conversion, 'NMIG migration is accomplished.');
}
/**
* Continues migration process after data loading, when migrate_only_data is false.
*/
async function continueProcessAfterDataLoadingLong(conversion: Conversion): Promise<void> {
export default async function(conversion: Conversion): Promise<void> {
const isTableConstraintsLoaded: boolean = await migrationStateManager.get(conversion, 'per_table_constraints_loaded');
const promises: Promise<void>[] = conversion._tablesToMigrate.map(async (tableName: string) => {
if (!isTableConstraintsLoaded) {
if (conversion.shouldMigrateOnlyDataFor(tableName)) {
await dropDataChunkIdColumn(conversion, tableName);
return sequencesProcessor.setSequenceValue(conversion, tableName);
}
await dropDataChunkIdColumn(conversion, tableName);
await processEnum(conversion, tableName);
await processNull(conversion, tableName);
@ -77,15 +67,3 @@ async function continueProcessAfterDataLoadingLong(conversion: Conversion): Prom
await migrationStateManager.dropStateLogsTable(conversion);
generateReport(conversion, 'NMIG migration is accomplished.');
}
/**
* Continues migration process after data loading.
*/
export default async function(conversion: Conversion): Promise<void> {
if (conversion._migrateOnlyData) {
await continueProcessAfterDataLoadingShort(conversion);
return;
}
await continueProcessAfterDataLoadingLong(conversion);
}

27
src/Conversion.ts
View file

@ -22,6 +22,7 @@ import * as path from 'path';
import { EventEmitter } from 'events';
import { Pool as MySQLPool } from 'mysql';
import { Pool as PgPool } from 'pg';
import { Encoding } from './Encoding';
export default class Conversion {
/**
@ -57,7 +58,7 @@ export default class Conversion {
/**
* JavaScript encoding type.
*/
public readonly _encoding: string;
public readonly _encoding: Encoding;
/**
* The path to the "all.log" file.
@ -75,9 +76,10 @@ export default class Conversion {
public readonly _delimiter: string;
/**
* Defines if only the data should be migrated (into a preset schema).
* Defines preset tables.
* The only thing to do with these tables is a data migration, since the schema is preset.
*/
public readonly _migrateOnlyData: boolean;
public readonly _migrateOnlyData: string[];
/**
* A path to the "logs_directory".
@ -105,10 +107,15 @@ export default class Conversion {
public readonly _noVacuum: string[];
/**
* List of tables, that will not be migrated.List (Array) of tables, that will not be migrated.
* List of tables, that will not be migrated.
*/
public readonly _excludeTables: string[];
/**
* List of tables, that will be migrated.
*/
public readonly _includeTables: string[];
/**
* The timestamp, at which the migration began.
*/
@ -208,9 +215,10 @@ export default class Conversion {
this._notCreatedViewsPath = path.join(this._logsDirPath, 'not_created_views');
this._noVacuum = this._config.no_vacuum === undefined ? [] : this._config.no_vacuum;
this._excludeTables = this._config.exclude_tables === undefined ? [] : this._config.exclude_tables;
this._includeTables = this._config.include_tables === undefined ? [] : this._config.include_tables;
this._timeBegin = new Date();
this._encoding = this._config.encoding === undefined ? 'utf8' : this._config.encoding;
this._dataChunkSize = this._config.data_chunk_size === undefined ? 1 : +this._config.data_chunk_size;
this._dataChunkSize = this._config.data_chunk_size === undefined ? 1 : Math.ceil(+this._config.data_chunk_size);
this._dataChunkSize = this._dataChunkSize <= 0 ? 1 : this._dataChunkSize;
this._0777 = '0777';
this._mysqlVersion = '5.6.21'; // Simply a default value.
@ -236,12 +244,19 @@ export default class Conversion {
this._maxDbConnectionPoolSize = this._maxDbConnectionPoolSize > 0 ? this._maxDbConnectionPoolSize : 10;
this._loaderMaxOldSpaceSize = this._config.loader_max_old_space_size;
this._loaderMaxOldSpaceSize = Conversion._isIntNumeric(this._loaderMaxOldSpaceSize) ? this._loaderMaxOldSpaceSize : 'DEFAULT';
this._migrateOnlyData = this._config.migrate_only_data === undefined ? false : this._config.migrate_only_data;
this._migrateOnlyData = this._config.migrate_only_data === undefined ? [] : this._config.migrate_only_data;
this._delimiter = this._config.delimiter !== undefined && this._config.delimiter.length === 1
? this._config.delimiter
: ',';
}
/**
* Checks if there are actions to take on given table other than data migration.
*/
public shouldMigrateOnlyDataFor(tableName: string): boolean {
return this._migrateOnlyData.indexOf(tableName) !== -1 || this._migrateOnlyData.indexOf('*') !== -1;
}
/**
* Checks if given value is integer number.
*/

9
src/DataChunksProcessor.ts
View file

@ -43,7 +43,7 @@ export default async (conversion: Conversion, tableName: string, haveDataChunksP
const dbAccess: DBAccess = new DBAccess(conversion);
const sizeQueryResult: DBAccessQueryResult = await dbAccess.query(logTitle, sql, DBVendors.MYSQL, true, true);
const tableSizeInMb: number = +sizeQueryResult.data[0].size_in_mb;
const tableSizeInMb: number = Math.ceil(+sizeQueryResult.data[0].size_in_mb);
const strSelectFieldList: string = arrangeColumnsData(conversion._dicTables[tableName].arrTableColumns, conversion._mysqlVersion);
sql = `SELECT COUNT(1) AS rows_count FROM \`${ originalTableName }\`;`;
const countResult: DBAccessQueryResult = await dbAccess.query(
@ -56,8 +56,7 @@ export default async (conversion: Conversion, tableName: string, haveDataChunksP
);
const rowsCnt: number = countResult.data[0].rows_count;
let chunksCnt: number = tableSizeInMb / conversion._dataChunkSize;
chunksCnt = chunksCnt < 1 ? 1 : chunksCnt;
const chunksCnt: number = Math.ceil(tableSizeInMb / conversion._dataChunkSize);
const rowsInChunk: number = Math.ceil(rowsCnt / chunksCnt);
const arrDataPoolPromises: Promise<void>[] = [];
const msg: string = `\t--[prepareDataChunks] Total rows to insert into "${ conversion._schema }"."${ tableName }": ${ rowsCnt }`;
@ -78,8 +77,8 @@ export default async (conversion: Conversion, tableName: string, haveDataChunksP
if (chunksCnt === 1) {
currentChunkSizeInMb = tableSizeInMb;
} else if (offset + rowsInChunk >= rowsCnt) {
currentChunkSizeInMb = tableSizeInMb % chunksCnt;
currentChunkSizeInMb = currentChunkSizeInMb || conversion._dataChunkSize;
// !!!Note, it can happen only on the last iteration, when defining a size of the last chunk.
currentChunkSizeInMb = tableSizeInMb - conversion._dataChunkSize * (chunksCnt - 1);
} else {
currentChunkSizeInMb = conversion._dataChunkSize;
}

24
src/DataPipeManager.ts
View file

@ -100,6 +100,27 @@ function fillBandwidth(conversion: Conversion): number[] {
}
}
if (dataChunkIndexes.length === 0 && conversion._dataPool.length !== 0) {
// This is the case where there are chunks, larger than "conversion._dataChunkSize".
// It may happen with maximum one chunk per table.
// See calculations from DataChunksProcessor.ts for the reference.
//
// Each call to "fillBandwidth()" will return an index of one (and only one!!!) such chunk.
// Eventually, all of the chunks, including the "bigger" ones, will be processed.
const firstUnprocessedChunkIndex: number = conversion
._dataPool
.findIndex((item: any) => item._processed === false);
if (firstUnprocessedChunkIndex === -1) {
const msg: string = 'Something went wrong with DataPipeManager.';
log(conversion, msg, undefined, true);
process.exit();
}
dataChunkIndexes.push(firstUnprocessedChunkIndex);
conversion._dataPool[firstUnprocessedChunkIndex]._processed = true;
}
return dataChunkIndexes;
}
@ -114,9 +135,8 @@ async function pipeData(conversion: Conversion, dataLoaderPath: string, options:
return processConstraints(conversion);
}
const chunksToLoad: any[] = fillBandwidth(conversion).map((index: number) => conversion._dataPool[index]);
const loaderProcess: ChildProcess = fork(dataLoaderPath, options);
const bandwidth: number[] = fillBandwidth(conversion);
const chunksToLoad: any[] = bandwidth.map((index: number) => conversion._dataPool[index]);
loaderProcess.on('message', async (signal: any) => {
if (typeof signal === 'object') {

21
src/Encoding.ts Normal file
View file

@ -0,0 +1,21 @@
/*
* This file is a part of "NMIG" - the database migration tool.
*
* Copyright (C) 2016 - present, Anatoly Khaytovich <anatolyuss@gmail.com>
*
* This program is free software= you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License.
*
* This program is distributed in the hope that it will be useful;
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program (please see the "LICENSE.md" file).
* If not; see <http=//www.gnu.org/licenses/gpl.txt>.
*
* @author Anatoly Khaytovich <anatolyuss@gmail.com>
*/
export type Encoding = 'ascii' | 'utf8' | 'utf-8' | 'utf16le' | 'ucs2' | 'ucs-2' | 'base64' | 'latin1' | 'binary' | 'hex';

12
src/StructureLoader.ts
View file

@ -62,7 +62,17 @@ export default async (conversion: Conversion): Promise<Conversion> => {
await getMySqlVersion(conversion);
const dbAccess: DBAccess = new DBAccess(conversion);
const haveTablesLoaded: boolean = await migrationStateManager.get(conversion, 'tables_loaded');
const sql: string = `SHOW FULL TABLES IN \`${ conversion._mySqlDbName }\`;`;
let sql: string = `SHOW FULL TABLES IN \`${ conversion._mySqlDbName }\` WHERE 1 = 1`;
if (conversion._includeTables.length !== 0) {
sql += ` AND Tables_in_${ conversion._mySqlDbName } IN(${ conversion._includeTables.map((table: string) => `"${table}"`).join(',') })`;
}
if (conversion._excludeTables.length !== 0) {
sql += ` AND Tables_in_${ conversion._mySqlDbName } NOT IN(${ conversion._excludeTables.map((table: string) => `"${table}"`).join(',') })`;
}
sql += ';';
const result: DBAccessQueryResult = await dbAccess.query('StructureLoader::default', sql, DBVendors.MYSQL, true, false);
let tablesCnt: number = 0;
let viewsCnt: number = 0;

2
src/TableProcessor.ts
View file

@ -87,7 +87,7 @@ export async function createTable(conversion: Conversion, tableName: string): Pr
conversion._dicTables[tableName].arrTableColumns = columns.data;
if (conversion._migrateOnlyData) {
if (conversion.shouldMigrateOnlyDataFor(tableName)) {
// Although the schema is preset, the data chunk id column must be added.
// This is due to the need to enforce data consistency in case of failures.
const sqlAddDataChunkIdColumn: string = `ALTER TABLE "${ conversion._schema }"."${ tableName }"