merge virtual columns support. Closes #107

Author: ifsnop

.travis.yml

@@ -1,4 +1,5 @@
-sudo: false
+dist: trusty
+sudo: required
 
 language: php
 
@@ -8,7 +9,7 @@ php:
   - 5.6
   - 5.5
   - 5.4
-  - 5.3
+#  - 5.3
   - hhvm
   - nightly
 
@@ -17,8 +18,23 @@ services:
 
 before_script:
   - curl -s http://getcomposer.org/installer | php
-  - php composer.phar install --dev
+  - php composer.phar install
+  - sudo service mysql stop || echo "mysql not stopped"
+  - echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections
+  - wget http://dev.mysql.com/get/mysql-apt-config_0.7.3-1_all.deb
+  - sudo dpkg --install mysql-apt-config_0.7.3-1_all.deb
+  - sudo apt-get update -q
+  - sudo apt-get install -q -y --force-yes -o Dpkg::Options::=--force-confnew mysql-server
+  - sudo mysql_upgrade
+  - sudo service mysql stop || echo "mysql not stopped"
+  - sudo mysqld_safe --skip-grant-tables &
+  - sleep 4
+  - sudo mysql -e "use mysql; update user set authentication_string=PASSWORD('') where User='root'; update user set plugin='mysql_native_password';FLUSH PRIVILEGES;"
+  - sudo mysqladmin shutdown
+  - sleep 1
+  - sudo service mysql start
   - mysql -V
+  - tests/create_users.sh
 
 script:
   - php src/Ifsnop/Mysqldump/Mysqldump.php

README.md

@@ -17,14 +17,16 @@ MySQLDump - PHP
 
 This is a php version of mysqldump cli that comes with MySQL, without dependencies, output compression and sane defaults.
 
-Out of the box, MySQLDump-PHP supports backing up table structures, the data itself, views and triggers.
+Out of the box, MySQLDump-PHP supports backing up table structures, the data itself, views, triggers and events.
 
 MySQLDump-PHP is the only library that supports:
 * output binary blobs as hex.
 * resolves view dependencies (using Stand-In tables).
-* output compared against original mysqldump. Linked to travis-ci testing system.
+* output compared against original mysqldump. Linked to travis-ci testing system (testing from php 5.3 to 7.1 & hhvm)
 * dumps stored procedures.
+* dumps events.
 * does extended-insert and/or complete-insert.
+* supports virtual columns from MySQL 5.7.
 
 ## Important
 
@@ -111,32 +113,37 @@ Refer to the [wiki](https://github.com/ifsnop/mysqldump-php/wiki/full-example) f
         $pdoSettings = array()
     )
 
-    $dumpSettingsDefault = array(
+   $dumpSettingsDefault = array(
         'include-tables' => array(),
         'exclude-tables' => array(),
-        'compress' => 'None',
-        'no-data' => false,
+        'compress' => Mysqldump::NONE,
+        'init_commands' => array(),
+        'no-data' => array(),
         'reset-auto-increment' => false,
+        'add-drop-database' => false,
         'add-drop-table' => false,
-        'single-transaction' => true,
-        'lock-tables' => false,
+        'add-drop-trigger' => true,
         'add-locks' => true,
-        'extended-insert' => true,
         'complete-insert' => false,
+        'databases' => false,
+        'default-character-set' => Mysqldump::UTF8,
         'disable-keys' => true,
-        'where' => '',
+        'extended-insert' => true,
+        'events' => false,
+        'hex-blob' => true, /* faster than escaped content */
+        'net_buffer_length' => self::MAXLINESIZE,
+        'no-autocommit' => true,
         'no-create-info' => false,
-        'skip-triggers' => false,
-        'add-drop-trigger' => true,
+        'lock-tables' => true,
         'routines' => false,
-        'hex-blob' => true,
-        'databases' => false,
-        'add-drop-database' => false,
+        'single-transaction' => true,
+        'skip-triggers' => false,
         'skip-tz-utc' => false,
-        'no-autocommit' => true,
-        'default-character-set' => 'utf8',
         'skip-comments' => false,
         'skip-dump-date' => false,
+        'where' => '',
+        /* deprecated */
+        'disable-foreign-keys-check' => true
     );
 
     $pdoSettingsDefaults = array(
@@ -256,7 +263,7 @@ it is identical tests are OK.
 
 ## TODO
 
-...
+Write more tests.
 
 ## Contributing
 
@@ -269,11 +276,13 @@ This project is open-sourced software licensed under the [GPL license](http://ww
 
 ## Credits
 
+After more than 8 years, there is barely anything left from the original source code, but:
+
 Originally based on James Elliott's script from 2009.
 http://code.google.com/p/db-mysqldump/
 
 Adapted and extended by Michael J. Calkins.
 https://github.com/clouddueling
 
-Currently maintained and developed by Diego Torres.
+Currently maintained, developed and improved by Diego Torres.
 https://github.com/ifsnop

src/Ifsnop/Mysqldump/Mysqldump.php

@@ -658,7 +658,8 @@ private function getTableColumnTypes($tableName) {
                 'is_numeric'=> $types['is_numeric'],
                 'is_blob' => $types['is_blob'],
                 'type' => $types['type'],
-                'type_sql' => $col['Type']
+                'type_sql' => $col['Type'],
+                'is_virtual' => $types['is_virtual']
             );
         }
 
@@ -861,7 +862,7 @@ private function listValues($tableName)
         $lineSize = 0;
 
         $colStmt = $this->getColumnStmt($tableName);
-        $stmt = "SELECT $colStmt FROM `$tableName`";
+        $stmt = "SELECT " . implode(",", $colStmt) . " FROM `$tableName`";
 
         if ($this->dumpSettings['where']) {
             $stmt .= " WHERE {$this->dumpSettings['where']}";
@@ -875,9 +876,9 @@ private function listValues($tableName)
 
                 if ($this->dumpSettings['complete-insert']) {
                     $lineSize += $this->compressManager->write(
-                        "INSERT INTO `$tableName` (`" .
-                        implode("`, `", array_keys($this->tableColumnTypes[$tableName])) .
-                        "`) VALUES (" . implode(",", $vals) . ")"
+                        "INSERT INTO `$tableName` (" .
+                        implode(", ", $colStmt) .
+                        ") VALUES (" . implode(",", $vals) . ")"
                     );
                 } else {
                     $lineSize += $this->compressManager->write(
@@ -1007,11 +1008,13 @@ function getColumnStmt($tableName)
                 $colStmt[] = "LPAD(HEX(`${colName}`),2,'0') AS `${colName}`";
             } else if ($colType['is_blob'] && $this->dumpSettings['hex-blob']) {
                 $colStmt[] = "HEX(`${colName}`) AS `${colName}`";
+            } else if ($colType['is_virtual']) {
+                $this->dumpSettings['complete-insert'] = true;
+                continue;
             } else {
                 $colStmt[] = "`${colName}`";
             }
         }
-        $colStmt = implode($colStmt, ",");
 
         return $colStmt;
     }
@@ -1857,6 +1860,8 @@ public function parseColumnType($colType)
         }
         $colInfo['is_numeric'] = in_array($colInfo['type'], $this->mysqlTypes['numerical']);
         $colInfo['is_blob'] = in_array($colInfo['type'], $this->mysqlTypes['blob']);
+        // for virtual 'Extra' -> "STORED GENERATED"
+        $colInfo['is_virtual'] = strpos($colType['Extra'], "STORED GENERATED") === false ? false : true;
 
         return $colInfo;
     }

tests/create_users.sh

@@ -1,22 +1,24 @@
 #!/bin/bash
 
-mysql -e "CREATE USER 'travis'@'%';"
-mysql -e "CREATE DATABASE test001;"
-mysql -e "CREATE DATABASE test002;"
-mysql -e "CREATE DATABASE test005;"
-mysql -e "CREATE DATABASE test006a;"
-mysql -e "CREATE DATABASE test006b;"
-mysql -e "CREATE DATABASE test008;"
-mysql -e "CREATE DATABASE test009;"
-mysql -e "CREATE DATABASE test010;"
-mysql -e "GRANT ALL PRIVILEGES ON test001.* TO 'travis'@'%' WITH GRANT OPTION;"
-mysql -e "GRANT ALL PRIVILEGES ON test002.* TO 'travis'@'%' WITH GRANT OPTION;"
-mysql -e "GRANT ALL PRIVILEGES ON test005.* TO 'travis'@'%' WITH GRANT OPTION;"
-mysql -e "GRANT ALL PRIVILEGES ON test006a.* TO 'travis'@'%' WITH GRANT OPTION;"
-mysql -e "GRANT ALL PRIVILEGES ON test006b.* TO 'travis'@'%' WITH GRANT OPTION;"
-mysql -e "GRANT ALL PRIVILEGES ON test008.* TO 'travis'@'%' WITH GRANT OPTION;"
-mysql -e "GRANT ALL PRIVILEGES ON test009.* TO 'travis'@'%' WITH GRANT OPTION;"
-mysql -e "GRANT ALL PRIVILEGES ON test010.* TO 'travis'@'%' WITH GRANT OPTION;"
-mysql -e "GRANT SUPER,LOCK TABLES ON *.* TO 'travis'@'%';"
-mysql -e "GRANT SELECT ON mysql.proc to 'travis'@'%';"
-mysql -e "FLUSH PRIVILEGES;"
+mysql -u root -e "CREATE USER 'travis'@'%';"
+mysql -u root -e "CREATE DATABASE test001;"
+mysql -u root -e "CREATE DATABASE test002;"
+mysql -u root -e "CREATE DATABASE test005;"
+mysql -u root -e "CREATE DATABASE test006a;"
+mysql -u root -e "CREATE DATABASE test006b;"
+mysql -u root -e "CREATE DATABASE test008;"
+mysql -u root -e "CREATE DATABASE test009;"
+mysql -u root -e "CREATE DATABASE test010;"
+mysql -u root -e "CREATE DATABASE test011;"
+mysql -u root -e "GRANT ALL PRIVILEGES ON test001.* TO 'travis'@'%' WITH GRANT OPTION;"
+mysql -u root -e "GRANT ALL PRIVILEGES ON test002.* TO 'travis'@'%' WITH GRANT OPTION;"
+mysql -u root -e "GRANT ALL PRIVILEGES ON test005.* TO 'travis'@'%' WITH GRANT OPTION;"
+mysql -u root -e "GRANT ALL PRIVILEGES ON test006a.* TO 'travis'@'%' WITH GRANT OPTION;"
+mysql -u root -e "GRANT ALL PRIVILEGES ON test006b.* TO 'travis'@'%' WITH GRANT OPTION;"
+mysql -u root -e "GRANT ALL PRIVILEGES ON test008.* TO 'travis'@'%' WITH GRANT OPTION;"
+mysql -u root -e "GRANT ALL PRIVILEGES ON test009.* TO 'travis'@'%' WITH GRANT OPTION;"
+mysql -u root -e "GRANT ALL PRIVILEGES ON test010.* TO 'travis'@'%' WITH GRANT OPTION;"
+mysql -u root -e "GRANT ALL PRIVILEGES ON test011.* TO 'travis'@'%' WITH GRANT OPTION;"
+mysql -u root -e "GRANT SUPER,LOCK TABLES ON *.* TO 'travis'@'%';"
+mysql -u root -e "GRANT SELECT ON mysql.proc to 'travis'@'%';"
+mysql -u root -e "FLUSH PRIVILEGES;"

tests/test.php

@@ -83,4 +83,18 @@
     array("events" => true));
 $dump->start("mysqldump-php_test010.sql");
 
+$dump = new IMysqldump\Mysqldump(
+    "mysql:unix_socket=/var/run/mysqld/mysqld.sock;dbname=test011",
+    "travis",
+    "",
+    array('complete-insert' =>  false));
+$dump->start("mysqldump-php_test011a.sql");
+
+$dump = new IMysqldump\Mysqldump(
+    "mysql:unix_socket=/var/run/mysqld/mysqld.sock;dbname=test011",
+    "travis",
+    "",
+    array('complete-insert' =>  true));
+$dump->start("mysqldump-php_test011b.sql");
+
 exit;

tests/test.sh

@@ -18,7 +18,7 @@ for i in 000; do
 done
 }
 
-for i in $(seq 0 30) ; do
+for i in $(seq 0 35) ; do
     ret[$i]=0
 done
 
@@ -31,6 +31,7 @@ mysql -utravis < test006.src.sql; ret[((index++))]=$?
 mysql -utravis < test008.src.sql; ret[((index++))]=$?
 mysql -utravis < test009.src.sql; ret[((index++))]=$?
 mysql -utravis < test010.src.sql; ret[((index++))]=$?
+mysql -utravis < test011.src.sql; ret[((index++))]=$?
 
 checksum_test001 > test001.src.checksum
 checksum_test002 > test002.src.checksum
@@ -82,6 +83,7 @@ cat test002.src.sql | grep ^INSERT > test002.filtered.sql
 cat test005.src.sql | grep ^INSERT > test005.filtered.sql
 cat test008.src.sql | grep FOREIGN > test008.filtered.sql
 cat test010.src.sql | grep CREATE | grep EVENT > test010.filtered.sql
+cat test011.src.sql | grep INSERT > test011.filtered.sql
 cat mysqldump_test001.sql | grep ^INSERT > mysqldump_test001.filtered.sql
 cat mysqldump_test002.sql | grep ^INSERT > mysqldump_test002.filtered.sql
 cat mysqldump_test005.sql | grep ^INSERT > mysqldump_test005.filtered.sql
@@ -90,6 +92,8 @@ cat mysqldump-php_test002.sql | grep ^INSERT > mysqldump-php_test002.filtered.sq
 cat mysqldump-php_test005.sql | grep ^INSERT > mysqldump-php_test005.filtered.sql
 cat mysqldump-php_test008.sql | grep FOREIGN > mysqldump-php_test008.filtered.sql
 cat mysqldump-php_test010.sql | grep CREATE | grep EVENT > mysqldump-php_test010.filtered.sql
+cat mysqldump-php_test011a.sql | grep INSERT > mysqldump-php_test011a.filtered.sql
+cat mysqldump-php_test011b.sql | grep INSERT > mysqldump-php_test011b.filtered.sql
 
 diff test001.filtered.sql mysqldump_test001.filtered.sql
 ret[((index++))]=$?
@@ -114,22 +118,28 @@ ret[((index++))]=$?
 diff test008.filtered.sql mysqldump-php_test008.filtered.sql
 ret[((index++))]=$?
 
-#test 24 - reset-auto-increment
+#test reset-auto-increment
 test009=`cat mysqldump-php_test009.sql | grep -i ENGINE | grep AUTO_INCREMENT`
 if [[ -z $test009 ]]; then ret[((index++))]=0; else ret[((index++))]=1; fi
 
 # test backup events
 diff test010.filtered.sql mysqldump-php_test010.filtered.sql
 ret[((index++))]=$?
 
+# test virtual column support, with simple inserts forced to complete (a) and complete inserts (b)
+diff test011.filtered.sql mysqldump-php_test011a.filtered.sql
+ret[((index++))]=$?
+diff test011.filtered.sql mysqldump-php_test011b.filtered.sql
+ret[((index++))]=$?
+
 rm *.checksum 2> /dev/null
 rm *.filtered.sql 2> /dev/null
 rm mysqldump* 2> /dev/null
 
 echo "Done $index tests"
 
 retvalue=0
-for i in $(seq 0 30) ; do
+for i in $(seq 0 35) ; do
     if [[ ${ret[$i]} -ne 0 ]]; then
         echo "test $i returned ${ret[$i]}"
         retvalue=${ret[$i]}

tests/test011.src.sql

@@ -0,0 +1,52 @@
+DROP DATABASE IF EXISTS `test011`;
+CREATE DATABASE `test011`;
+USE `test011`;
+
+-- MySQL dump 10.13  Distrib 5.7.15, for Linux (x86_64)
+--
+-- Host: localhost    Database: test
+-- ------------------------------------------------------
+-- Server version       5.7.15
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `test011`
+--
+
+DROP TABLE IF EXISTS `test011`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `test011` (
+  `id` int(11) NOT NULL,
+  `hash` char(32) CHARACTER SET ascii COLLATE ascii_bin GENERATED ALWAYS AS (md5(`id`)) STORED NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 ROW_FORMAT=DYNAMIC;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `test011`
+--
+
+LOCK TABLES `test011` WRITE;
+/*!40000 ALTER TABLE `test011` DISABLE KEYS */;
+INSERT INTO `test011` (`id`) VALUES (159413),(294775);
+/*!40000 ALTER TABLE `test011` ENABLE KEYS */;
+UNLOCK TABLES;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;