Merge pull request #766 from LearningLocker/issue/storedDate

ht2 · ht2 · commit a04c8a8f739b · 2016-02-23T10:28:43.000Z
Issue/stored date - requires migration
diff --git a/app/commands/ConvertTimestamp.php b/app/commands/ConvertTimestamp.php
@@ -43,7 +43,8 @@ public function fire()
   {
     Statement::chunk(1000, function($statements){
       foreach ($statements as $s){
-        $s->timestamp = new \MongoDate(strtotime($s->statement['timestamp']));
+        $timestamp = new Carbon\Carbon($s->statement['timestamp']);
+        $s->timestamp = new \MongoDate($timestamp->timestamp, $timestamp->micro);
         $s->save();
       }
       $this->info(count($statements) . ' converted.');
diff --git a/app/config/database.php b/app/config/database.php
@@ -5,13 +5,13 @@
   'default' => 'mongodb',
   'connections' => [
 		'mongodb' => [
-		    'driver'   => 'mongodb',
-		    'host'     => 'localhost',
-		    'port'     => 27017,
-		    'username' => '',
-		    'password' => '',
-		    'database' => 'll_staging' // Default name (removing this makes Travis fail).
-		],
+        'driver'   => 'mongodb',
+        'host'     => 'localhost',
+        'port'     => 27017,
+        'username' => '',
+        'password' => '',
+        'database' => 'll_staging' // Default name (removing this makes Travis fail).
+    ],
 	],
 	'migrations' => 'migrations',
 ];
diff --git a/app/controllers/LrsController.php b/app/controllers/LrsController.php
@@ -198,7 +198,7 @@ public function statements($lrs_id){
         'scopes' => ['all']
       ]),
       'scopes' => ['all']
-    ]))->orderBy('statement.stored', 'DESC')->paginate(15);
+    ]))->orderBy('stored', 'DESC')->paginate(15);
     return View::make('partials.statements.list', array_merge($this->getLrs($lrs_id), [
       'statements' => $statements,
       'statement_nav' => true,
diff --git a/app/controllers/api/Exports.php b/app/controllers/api/Exports.php
@@ -23,6 +23,9 @@ public function __construct(Export $export) {
    * @return StreamedResponse.
    */
   public function showJson($id) {
+    // should top timeouts on long queries
+    set_time_limit(0);
+
     $opts = $this->getOptions();
     $model = $this->repo->show($id, $opts);
 
@@ -46,6 +49,9 @@ public function showJson($id) {
    * @return StreamedResponse.
    */
   public function showCsv($id) {
+    // should top timeouts on long queries
+    set_time_limit(0);
+
     $opts = $this->getOptions();
     $model = $this->repo->show($id, $opts);
 
@@ -92,4 +98,4 @@ private function quoteCSV($field) {
       return $field;
     }
   }
-}
+}
diff --git a/app/database/migrations/2015_12_23_093122_add_stored_to_statement_root.php b/app/database/migrations/2015_12_23_093122_add_stored_to_statement_root.php
@@ -0,0 +1,130 @@
+<?php
+
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Database\Migrations\Migration;
+
+class AddStoredToStatementRoot extends Migration {
+
+	/**
+	 * Run the migrations.
+	 *
+	 * @return void
+	 */
+	public function up()
+	{
+		set_time_limit(0);
+
+		$db = \DB::getMongoDB();
+    $statementsCollection = new MongoCollection($db, 'statements');
+    
+    $statementsCollection->createIndex(['stored' => 1], ['background'=>1, 'socketTimeoutMS'=>-1]);
+    $statementsCollection->createIndex(['stored' => -1], ['background'=>1, 'socketTimeoutMS'=>-1]);
+    $statementsCollection->createIndex(['lrs_id' => 1, 'stored' => 1], ['background'=>1, 'socketTimeoutMS'=>-1]);
+    $statementsCollection->createIndex(['lrs_id' => 1, 'stored' => -1], ['background'=>1, 'socketTimeoutMS'=>-1]);
+
+    $statementsCursor = $statementsCollection->find();
+
+    $remaining = $statementsCursor->count();
+    print($remaining . ' statements total' . PHP_EOL);
+
+    $maxBatchSize = 10000;
+
+    while($statementsCursor->hasNext()) {
+	    $batch = new MongoUpdateBatch($statementsCollection);
+	    $batchSize = 0;
+
+	    while($batchSize < $maxBatchSize && $statementsCursor->hasNext()) {
+	    	$batchSize++;
+	    	$statement = $statementsCursor->next();
+	    	$statementStored = new Carbon\Carbon($statement['statement']['stored']);
+
+	    	$query = [
+				  'q' => ['_id' => $statement['_id']],
+				  'u' => ['$set' => ["stored" => new \MongoDate($statementStored->timestamp, $statementStored->micro)]],
+				  'multi' => false,
+				  'upsert' => false,
+				];
+
+	    	if(isset($statement['refs'])) {
+	    		foreach ($statement['refs'] as $key => $refStatement) {
+	    			if(isset($refStatement['timestamp']) && !$refStatement['timestamp'] instanceof MongoDate) {
+              $timestamp = new Carbon\Carbon($refStatement['timestamp']);
+              $query['u']['$set']['refs.'.$key.'.timestamp'] = new \MongoDate($timestamp->timestamp, $timestamp->micro);
+            } 
+	    			if(isset($refStatement['stored']) && !$stored['stored'] instanceof MongoDate) {
+              $stored = new Carbon\Carbon($refStatement['stored']);
+              $query['u']['$set']['refs.'.$key.'.stored'] = new \MongoDate($stored->timestamp, $stored->micro);
+            }
+	    		}
+	    	}
+
+				$batch->add((object) $query);
+	    }
+	    $batch->execute();
+	    $remaining -= $batchSize;
+
+	    print($remaining . ' remaining' . PHP_EOL);
+    }
+	}
+
+	/**
+	 * Reverse the migrations.
+	 *
+	 * @return void
+	 */
+	public function down()
+	{
+		$db = \DB::getMongoDB();
+    $statementsCollection = new MongoCollection($db, 'statements');
+    
+    $statementsCollection->deleteIndex('stored');
+    $statementsCollection->deleteIndex(['lrs_id' => 1, 'stored' => -1]);
+
+    $statementsCollection->update([], ['$unset' => ["stored" => ""]], ['multiple' => true]);
+
+    $statementsCursor = $statementsCollection->find();
+    $remaining = $statementsCursor->count();
+    print($remaining . ' statements total' . PHP_EOL);
+
+    $maxBatchSize = 10000;
+
+    while($statementsCursor->hasNext()) {
+	    $batch = new MongoUpdateBatch($statementsCollection);
+	    $batchSize = 0;
+	    $shouldExecute = false;
+
+	    while($batchSize < $maxBatchSize && $statementsCursor->hasNext()) {
+	    	$batchSize++;
+	    	$statement = $statementsCursor->next();
+
+	    	if(isset($statement['refs'])) {
+		    	$query = [
+					  'q' => ['_id' => $statement['_id']],
+					  'u' => ['$set' => []],
+					  'multi' => false,
+					  'upsert' => false,
+					];
+	    		foreach ($statement['refs'] as $key => $refStatement) {
+            if(isset($refStatement['timestamp']) && $refStatement['timestamp'] instanceof MongoDate ) {
+              $query['u']['$set']['refs.'.$key.'.timestamp'] = date('Y-m-d\TH:i:s.uP', $refStatement['timestamp']->sec);
+            }
+            if(isset($refStatement['stored']) && $refStatement['stored'] instanceof MongoDate ) {
+              $query['u']['$set']['refs.'.$key.'.stored'] = date('Y-m-d\TH:i:s.uP', $refStatement['stored']->sec);
+            }
+	    		}
+          
+          if(!empty($query['u']['$set'])) {
+						$batch->add((object) $query);
+						$shouldExecute = true;
+					}
+	    	}
+	    }
+
+	    if($shouldExecute) $batch->execute();
+	    $remaining -= $batchSize;
+
+	    print($remaining . ' remaining' . PHP_EOL);
+    }
+	}
+
+}
diff --git a/app/locker/data/analytics/Analytics.php b/app/locker/data/analytics/Analytics.php
@@ -196,7 +196,8 @@ private function setFilter(array $options) {
    * @return MongoDate object
    **/
   private function setMongoDate($date) {
-    return new \MongoDate(strtotime($date));
+    $carbon = new \Carbon\Carbon($date);
+    return new \MongoDate($carbon->timestamp, $carbon->micro);
   }
 
   /**
diff --git a/app/locker/repository/Report/EloquentRepository.php b/app/locker/repository/Report/EloquentRepository.php
@@ -105,6 +105,6 @@ public function statements($id, array $opts) {
     return (new QueryRepository)->where(
       $report->lrs_id,
       Helpers::replaceHtmlEntity($report->where)
-    )->orderBy('statement.stored', 'DESC');
+    )->orderBy('stored', 'DESC');
   }
 }
diff --git a/app/locker/repository/Statement/EloquentIndexer.php b/app/locker/repository/Statement/EloquentIndexer.php
@@ -42,10 +42,24 @@ public function index(IndexOptions $opts) {
         return $this->addWhere($builder, 'context.registration', $value);
       },
       'since' => function ($value, $builder, IndexOptions $opts) {
-        return $this->addWhere($builder, 'stored', $value, '>');
+        $key = 'stored';
+        $op = '>';
+        return $builder->where(function ($query) use ($key, $value, $op) {
+          $date = new \MongoDate(strtotime($value));
+          return $query
+            ->orWhere($key, $op, $date)
+            ->orWhere('refs.'.$key, $op, $date);
+        });
       },
       'until' => function ($value, $builder, IndexOptions $opts) {
-        return $this->addWhere($builder, 'stored', $value, '<=');
+        $key = 'stored';
+        $op = '<=';
+        return $builder->where(function ($query) use ($key, $value, $op) {
+          $date = new \MongoDate(strtotime($value));
+          return $query
+            ->orWhere($key, $op, $date)
+            ->orWhere('refs.'.$key, $op, $date);
+        });
       },
       'active' => function ($value, $builder, IndexOptions $opts) {
         return $builder->where('active', $value);
@@ -133,7 +147,7 @@ public function format(Builder $builder, IndexOptions $opts) {
 
     // Returns the models.
     return json_decode($builder
-      ->orderBy('statement.stored', $opts->getOpt('ascending') === true ? 'ASC' : 'DESC')
+      ->orderBy('stored', $opts->getOpt('ascending') === true ? 'ASC' : 'DESC')
       ->skip($opts->getOpt('offset'))
       ->take($opts->getOpt('limit'))
       ->get()
diff --git a/app/locker/repository/Statement/EloquentInserter.php b/app/locker/repository/Statement/EloquentInserter.php
@@ -85,15 +85,17 @@ private function matchableStatement(\stdClass $statement) {
    * @return [String => Mixed] $model
    */
   private function constructModel(\stdClass $statement, StoreOptions $opts) {
+    $timestamp = new \Carbon\Carbon($statement->timestamp);
+    $stored    = new \Carbon\Carbon($statement->stored);
     return [
       'lrs' => ['_id' => $opts->getOpt('lrs_id')], // Deprecated.
       'lrs_id' => $opts->getOpt('lrs_id'),
       'client_id' => $opts->getOpt('client')->_id,
       'statement' => Helpers::replaceFullStop(json_decode(json_encode($statement), true)),
       'active' => false,
       'voided' => false,
-      'timestamp' => new \MongoDate(strtotime($statement->timestamp))
-    ];
+      'timestamp' => new \MongoDate($timestamp->timestamp, $timestamp->micro),
+      'stored'    => new \MongoDate($stored->timestamp, $stored->micro),    ];
   }
 
   /**
diff --git a/app/locker/repository/Statement/EloquentLinker.php b/app/locker/repository/Statement/EloquentLinker.php
@@ -139,7 +139,10 @@ private function setRefs(\stdClass $statement, array $refs, StoreOptions $opts)
       ->where('statement.id', $statement->id)
       ->update([
         'refs' => array_map(function ($ref) {
-          return Helpers::replaceFullStop(json_decode(json_encode($ref->statement), true));
+          $statement = Helpers::replaceFullStop(json_decode(json_encode($ref->statement), true));
+          $statement['stored'] = new \MongoDate( strtotime($statement['stored']));
+          $statement['timestamp'] = new \MongoDate( strtotime($statement['timestamp']));
+          return $statement;
         }, $refs)
       ]);
   }
diff --git a/app/models/Statement.php b/app/models/Statement.php
@@ -6,7 +6,7 @@ class Statement extends Eloquent {
 
   protected $collection = 'statements';
   protected $hidden = ['_id', 'created_at', 'updated_at'];
-  protected $fillable = ['statement', 'active', 'voided', 'refs', 'lrs_id', 'timestamp'];
+  protected $fillable = ['statement', 'active', 'voided', 'refs', 'lrs_id', 'timestamp', 'stored'];
 
   public function lrs(){
     return $this->belongsTo('Lrs');
diff --git a/app/tests/StatementsTestCase.php b/app/tests/StatementsTestCase.php
@@ -48,7 +48,8 @@ protected function createStatement(\Lrs $lrs, \Client $client, array $statement)
       'voided' => false,
       'refs' => []
     ]);
-    $model->timestamp = new \MongoDate(strtotime($model->statement['timestamp']));
+    $timestamp = new \Carbon\Carbon($model->statement['timestamp']);
+    $model->timestamp = new \MongoDate($timestamp->timestamp, $timestamp->micro);
     $model->save();
     return $model;
   }

Original file line number	Diff line number	Diff line change
`@@ -43,7 +43,8 @@ public function fire()`
`43`	`43`	`{`
`44`	`44`	`Statement::chunk(1000, function($statements){`
`45`	`45`	`foreach ($statements as $s){`
`46`		`- $s->timestamp = new \MongoDate(strtotime($s->statement['timestamp']));`
	`46`	`+ $timestamp = new Carbon\Carbon($s->statement['timestamp']);`
	`47`	`+ $s->timestamp = new \MongoDate($timestamp->timestamp, $timestamp->micro);`
`47`	`48`	`$s->save();`
`48`	`49`	`}`
`49`	`50`	`$this->info(count($statements) . ' converted.');`
Original file line number	Diff line number	Diff line change
`@@ -196,7 +196,8 @@ private function setFilter(array $options) {`
`196`	`196`	`* @return MongoDate object`
`197`	`197`	`**/`
`198`	`198`	`private function setMongoDate($date) {`
`199`		`- return new \MongoDate(strtotime($date));`
	`199`	`+ $carbon = new \Carbon\Carbon($date);`
	`200`	`+ return new \MongoDate($carbon->timestamp, $carbon->micro);`
`200`	`201`	`}`
`201`	`202`
`202`	`203`	`/**`
Original file line number	Diff line number	Diff line change
`@@ -105,6 +105,6 @@ public function statements($id, array $opts) {`
`105`	`105`	`return (new QueryRepository)->where(`
`106`	`106`	`$report->lrs_id,`
`107`	`107`	`Helpers::replaceHtmlEntity($report->where)`
`108`		`- )->orderBy('statement.stored', 'DESC');`
	`108`	`+ )->orderBy('stored', 'DESC');`
`109`	`109`	`}`
`110`	`110`	`}`