With large imports it has a large impact on performances and isn't really useful.
Instead, write an IMPORT event, which let client using the history service resync its DB.
-> 15k link import done in 6 seconds.
Fixes #985
* - UPDATED: link updated
* - DELETED: link deleted
* - SETTINGS: the settings have been updated through the UI.
* - UPDATED: link updated
* - DELETED: link deleted
* - SETTINGS: the settings have been updated through the UI.
+ * - IMPORT: bulk links import
*
* Note: new events are put at the beginning of the file and history array.
*/
*
* Note: new events are put at the beginning of the file and history array.
*/
*/
const SETTINGS = 'SETTINGS';
*/
const SETTINGS = 'SETTINGS';
+ /**
+ * @var string Action key: a bulk import has been processed.
+ */
+ const IMPORT = 'IMPORT';
+
/**
* @var string History file path.
*/
/**
* @var string History file path.
*/
$this->addEvent(self::SETTINGS);
}
$this->addEvent(self::SETTINGS);
}
+ /**
+ * Add Event: bulk import.
+ *
+ * Note: we don't store links add/update one by one since it can have a huge impact on performances.
+ */
+ public function importLinks()
+ {
+ $this->addEvent(self::IMPORT);
+ }
+
/**
* Save a new event and write it in the history file.
*
/**
* Save a new event and write it in the history file.
*
* @param int $importCount how many links were imported
* @param int $overwriteCount how many links were overwritten
* @param int $skipCount how many links were skipped
* @param int $importCount how many links were imported
* @param int $overwriteCount how many links were overwritten
* @param int $skipCount how many links were skipped
+ * @param int $duration how many seconds did the import take
*
* @return string Summary of the bookmark import status
*/
*
* @return string Summary of the bookmark import status
*/
$filesize,
$importCount=0,
$overwriteCount=0,
$filesize,
$importCount=0,
$overwriteCount=0,
+ $skipCount=0,
+ $duration=0
)
{
$status = 'File '.$filename.' ('.$filesize.' bytes) ';
if ($importCount == 0 && $overwriteCount == 0 && $skipCount == 0) {
$status .= 'has an unknown file format. Nothing was imported.';
} else {
)
{
$status = 'File '.$filename.' ('.$filesize.' bytes) ';
if ($importCount == 0 && $overwriteCount == 0 && $skipCount == 0) {
$status .= 'has an unknown file format. Nothing was imported.';
} else {
- $status .= 'was successfully processed: '.$importCount.' links imported, ';
+ $status .= 'was successfully processed in '. $duration .' seconds: ';
+ $status .= $importCount.' links imported, ';
$status .= $overwriteCount.' links overwritten, ';
$status .= $skipCount.' links skipped.';
}
$status .= $overwriteCount.' links overwritten, ';
$status .= $skipCount.' links skipped.';
}
*/
public static function import($post, $files, $linkDb, $conf, $history)
{
*/
public static function import($post, $files, $linkDb, $conf, $history)
{
$filename = $files['filetoupload']['name'];
$filesize = $files['filetoupload']['size'];
$data = file_get_contents($files['filetoupload']['tmp_name']);
$filename = $files['filetoupload']['name'];
$filesize = $files['filetoupload']['size'];
$data = file_get_contents($files['filetoupload']['tmp_name']);
$linkDb[$existingLink['id']] = $newLink;
$importCount++;
$overwriteCount++;
$linkDb[$existingLink['id']] = $newLink;
$importCount++;
$overwriteCount++;
- $history->updateLink($newLink);
$newLink['shorturl'] = link_small_hash($newLink['created'], $newLink['id']);
$linkDb[$newLink['id']] = $newLink;
$importCount++;
$newLink['shorturl'] = link_small_hash($newLink['created'], $newLink['id']);
$linkDb[$newLink['id']] = $newLink;
$importCount++;
- $history->addLink($newLink);
}
$linkDb->save($conf->get('resource.page_cache'));
}
$linkDb->save($conf->get('resource.page_cache'));
+ $history->importLinks();
+
+ $duration = time() - $start;
return self::importStatus(
$filename,
$filesize,
$importCount,
$overwriteCount,
return self::importStatus(
$filename,
$filesize,
$importCount,
$overwriteCount,
+ $skipCount,
+ $duration
public function testImportInternetExplorerEncoding()
{
$files = file2array('internet_explorer_encoding.htm');
public function testImportInternetExplorerEncoding()
{
$files = file2array('internet_explorer_encoding.htm');
- $this->assertEquals(
- 'File internet_explorer_encoding.htm (356 bytes) was successfully processed:'
+ $this->assertStringMatchesFormat(
+ 'File internet_explorer_encoding.htm (356 bytes) was successfully processed in %d seconds:'
.' 1 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import([], $files, $this->linkDb, $this->conf, $this->history)
);
.' 1 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import([], $files, $this->linkDb, $this->conf, $this->history)
);
public function testImportNested()
{
$files = file2array('netscape_nested.htm');
public function testImportNested()
{
$files = file2array('netscape_nested.htm');
- $this->assertEquals(
- 'File netscape_nested.htm (1337 bytes) was successfully processed:'
+ $this->assertStringMatchesFormat(
+ 'File netscape_nested.htm (1337 bytes) was successfully processed in %d seconds:'
.' 8 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import([], $files, $this->linkDb, $this->conf, $this->history)
);
.' 8 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import([], $files, $this->linkDb, $this->conf, $this->history)
);
public function testImportDefaultPrivacyNoPost()
{
$files = file2array('netscape_basic.htm');
public function testImportDefaultPrivacyNoPost()
{
$files = file2array('netscape_basic.htm');
- $this->assertEquals(
- 'File netscape_basic.htm (482 bytes) was successfully processed:'
+ $this->assertStringMatchesFormat(
+ 'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
.' 2 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import([], $files, $this->linkDb, $this->conf, $this->history)
);
.' 2 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import([], $files, $this->linkDb, $this->conf, $this->history)
);
{
$post = array('privacy' => 'default');
$files = file2array('netscape_basic.htm');
{
$post = array('privacy' => 'default');
$files = file2array('netscape_basic.htm');
- $this->assertEquals(
- 'File netscape_basic.htm (482 bytes) was successfully processed:'
+ $this->assertStringMatchesFormat(
+ 'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
.' 2 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
);
.' 2 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
);
{
$post = array('privacy' => 'public');
$files = file2array('netscape_basic.htm');
{
$post = array('privacy' => 'public');
$files = file2array('netscape_basic.htm');
- $this->assertEquals(
- 'File netscape_basic.htm (482 bytes) was successfully processed:'
+ $this->assertStringMatchesFormat(
+ 'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
.' 2 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
);
.' 2 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
);
{
$post = array('privacy' => 'private');
$files = file2array('netscape_basic.htm');
{
$post = array('privacy' => 'private');
$files = file2array('netscape_basic.htm');
- $this->assertEquals(
- 'File netscape_basic.htm (482 bytes) was successfully processed:'
+ $this->assertStringMatchesFormat(
+ 'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
.' 2 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
);
.' 2 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
);
// import links as private
$post = array('privacy' => 'private');
// import links as private
$post = array('privacy' => 'private');
- $this->assertEquals(
- 'File netscape_basic.htm (482 bytes) was successfully processed:'
+ $this->assertStringMatchesFormat(
+ 'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
.' 2 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
);
.' 2 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
);
'privacy' => 'public',
'overwrite' => 'true'
);
'privacy' => 'public',
'overwrite' => 'true'
);
- $this->assertEquals(
- 'File netscape_basic.htm (482 bytes) was successfully processed:'
+ $this->assertStringMatchesFormat(
+ 'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
.' 2 links imported, 2 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
);
.' 2 links imported, 2 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
);
// import links as public
$post = array('privacy' => 'public');
// import links as public
$post = array('privacy' => 'public');
- $this->assertEquals(
- 'File netscape_basic.htm (482 bytes) was successfully processed:'
+ $this->assertStringMatchesFormat(
+ 'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
.' 2 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
);
.' 2 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
);
'privacy' => 'private',
'overwrite' => 'true'
);
'privacy' => 'private',
'overwrite' => 'true'
);
- $this->assertEquals(
- 'File netscape_basic.htm (482 bytes) was successfully processed:'
+ $this->assertStringMatchesFormat(
+ 'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
.' 2 links imported, 2 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
);
.' 2 links imported, 2 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
);
{
$post = array('privacy' => 'public');
$files = file2array('netscape_basic.htm');
{
$post = array('privacy' => 'public');
$files = file2array('netscape_basic.htm');
- $this->assertEquals(
- 'File netscape_basic.htm (482 bytes) was successfully processed:'
+ $this->assertStringMatchesFormat(
+ 'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
.' 2 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
);
.' 2 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
);
// re-import as private, DO NOT enable overwriting
$post = array('privacy' => 'private');
// re-import as private, DO NOT enable overwriting
$post = array('privacy' => 'private');
- $this->assertEquals(
- 'File netscape_basic.htm (482 bytes) was successfully processed:'
+ $this->assertStringMatchesFormat(
+ 'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
.' 0 links imported, 0 links overwritten, 2 links skipped.',
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
);
.' 0 links imported, 0 links overwritten, 2 links skipped.',
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
);
'default_tags' => 'tag1,tag2 tag3'
);
$files = file2array('netscape_basic.htm');
'default_tags' => 'tag1,tag2 tag3'
);
$files = file2array('netscape_basic.htm');
- $this->assertEquals(
- 'File netscape_basic.htm (482 bytes) was successfully processed:'
+ $this->assertStringMatchesFormat(
+ 'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
.' 2 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
);
.' 2 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
);
'default_tags' => 'tag1&,tag2 "tag3"'
);
$files = file2array('netscape_basic.htm');
'default_tags' => 'tag1&,tag2 "tag3"'
);
$files = file2array('netscape_basic.htm');
- $this->assertEquals(
- 'File netscape_basic.htm (482 bytes) was successfully processed:'
+ $this->assertStringMatchesFormat(
+ 'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
.' 2 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
);
.' 2 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
);
public function testImportSameDate()
{
$files = file2array('same_date.htm');
public function testImportSameDate()
{
$files = file2array('same_date.htm');
- $this->assertEquals(
- 'File same_date.htm (453 bytes) was successfully processed:'
+ $this->assertStringMatchesFormat(
+ 'File same_date.htm (453 bytes) was successfully processed in %d seconds:'
.' 3 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import(array(), $files, $this->linkDb, $this->conf, $this->history)
);
.' 3 links imported, 0 links overwritten, 0 links skipped.',
NetscapeBookmarkUtils::import(array(), $files, $this->linkDb, $this->conf, $this->history)
);
'overwrite' => 'true',
];
$files = file2array('netscape_basic.htm');
'overwrite' => 'true',
];
$files = file2array('netscape_basic.htm');
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history);
$history = $this->history->getHistory();
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history);
$history = $this->history->getHistory();
- $this->assertEquals($nbLinks, count($history));
- foreach ($history as $value) {
- $this->assertEquals(History::CREATED, $value['event']);
- $this->assertTrue(new DateTime('-5 seconds') < $value['datetime']);
- $this->assertTrue(is_int($value['id']));
- }
+ $this->assertEquals(1, count($history));
+ $this->assertEquals(History::IMPORT, $history[0]['event']);
+ $this->assertTrue(new DateTime('-5 seconds') < $history[0]['datetime']);
// re-import as private, enable overwriting
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history);
$history = $this->history->getHistory();
// re-import as private, enable overwriting
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history);
$history = $this->history->getHistory();
- $this->assertEquals($nbLinks * 2, count($history));
- for ($i = 0 ; $i < $nbLinks ; $i++) {
- $this->assertEquals(History::UPDATED, $history[$i]['event']);
- $this->assertTrue(new DateTime('-5 seconds') < $history[$i]['datetime']);
- $this->assertTrue(is_int($history[$i]['id']));
- }
+ $this->assertEquals(2, count($history));
+ $this->assertEquals(History::IMPORT, $history[0]['event']);
+ $this->assertTrue(new DateTime('-5 seconds') < $history[0]['datetime']);
+ $this->assertEquals(History::IMPORT, $history[1]['event']);
+ $this->assertTrue(new DateTime('-5 seconds') < $history[1]['datetime']);